tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
2013514f7b58fb4dc56dcca99f6823f58f4b3131
hakmem/core/hakmem_tiny_stats.c
Moe Charm (CI) acc64f2438 Phase ML1: Pool v1 memset 89.73% overhead 軽量化 (+15.34% improvement) 
## Summary
- ChatGPT により bench_profile.h の setenv segfault を修正（RTLD_NEXT 経由に切り替え）
- core/box/pool_zero_mode_box.h 新設：ENV キャッシュ経由で ZERO_MODE を統一管理
- core/hakmem_pool.c で zero mode に応じた memset 制御（FULL/header/off）
- A/B テスト結果：ZERO_MODE=header で +15.34% improvement（1M iterations, C6-heavy）

## Files Modified
- core/box/pool_api.inc.h: pool_zero_mode_box.h include
- core/bench_profile.h: glibc setenv → malloc+putenv（segfault 回避）
- core/hakmem_pool.c: zero mode 参照・制御ロジック
- core/box/pool_zero_mode_box.h (新設): enum/getter
- CURRENT_TASK.md: Phase ML1 結果記載

## Test Results
| Iterations | ZERO_MODE=full | ZERO_MODE=header | Improvement |
|-----------|----------------|-----------------|------------|
| 10K       | 3.06 M ops/s   | 3.17 M ops/s    | +3.65%     |
| 1M        | 23.71 M ops/s  | 27.34 M ops/s   | **+15.34%** |

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2025-12-10 09:08:18 +09:00

683 lines
31 KiB
C
Raw Blame History

// hakmem_tiny_stats.c
// Phase 2, Module 1: Statistics and Debug Functions
// Extracted from hakmem_tiny.c (lines 4348-4728, non-contiguous)
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <stdatomic.h>
#include <unistd.h>
#include "hakmem_tiny.h"
#include "hakmem_tiny_config.h" // extern g_tiny_class_sizes
#include "hakmem_tiny_stats_api.h"
#include "hakmem_stats_master.h" // Phase 4d: Master stats control
static int g_dump_atexit_only = -1; // env: HAKMEM_TINY_DUMP_ATEXIT_ONLY=1
// Forward declaration of local dump (defined later in this file)
static void hak_tiny_refill_counters_dump(void);
static void hak_tiny_stats_init_flags(void) {
if (g_dump_atexit_only == -1) {
const char* e = getenv("HAKMEM_TINY_DUMP_ATEXIT_ONLY");
g_dump_atexit_only = (e && atoi(e) != 0) ? 1 : 0;
}
}
void hak_tiny_dump_all_counters_now(void) {
// Dump both minimal and extended (if compiled)
hak_tiny_refill_counters_dump();
hak_tiny_debug_counters_dump();
}
#include "hakmem_tiny_superslab.h"
#include "hakmem_config.h"
#include "hakmem_tiny_stats.h"
// ============================================================================
// Phase 8.1: Public Statistics API (lines 4348-4415)
// ============================================================================
void hak_tiny_get_stats(uint64_t* alloc_count, uint64_t* free_count, uint64_t* slab_count) {
if (!g_tiny_initialized) return;
#ifdef HAKMEM_ENABLE_STATS
// Flush TLS batches to global counters for accurate stats
stats_flush_all();
#endif
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
if (alloc_count) alloc_count[i] = g_tiny_pool.alloc_count[i];
if (free_count) free_count[i] = g_tiny_pool.free_count[i];
if (slab_count) slab_count[i] = g_tiny_pool.slab_count[i];
}
}
void hak_tiny_print_stats(void) {
if (!g_tiny_initialized) {
printf("Tiny Pool not initialized\n");
return;
}
printf("\n");
printf("Tiny Pool Statistics\n");
printf("========================================\n");
printf("Class | Size | Allocs | Frees | Slabs\n");
printf("------|--------|---------|---------|-------\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
printf(" %d | %4zuB | %7lu | %7lu | %5lu\n",
i,
g_tiny_class_sizes[i],
(unsigned long)g_tiny_pool.alloc_count[i],
(unsigned long)g_tiny_pool.free_count[i],
(unsigned long)g_tiny_pool.slab_count[i]);
}
printf("========================================\n");
printf("\n");
#if HAKMEM_BUILD_DEBUG
printf("TLS Debug Counters (hit/miss/spill) per class\n");
printf("---------------------------------------------\n");
printf("Class | Hit | Miss | SpillSS | SpillOwner | SpillMag | SpillReq\n");
printf("------+-----------+-----------+-----------+-----------+-----------+-----------\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
printf(" %d | %9llu | %9llu | %9llu | %9llu | %9llu | %9llu\n",
i,
(unsigned long long)g_tls_hit_count[i],
(unsigned long long)g_tls_miss_count[i],
(unsigned long long)g_tls_spill_ss_count[i],
(unsigned long long)g_tls_spill_owner_count[i],
(unsigned long long)g_tls_spill_mag_count[i],
(unsigned long long)g_tls_spill_requeue_count[i]);
}
printf("---------------------------------------------\n\n");
printf("Observation Snapshot: removed (obs pipeline retired)\n\n");
#endif
}
// ============================================================================
// Phase 8.2: Memory Profiling Debug (toggle with HAKMEM_DEBUG_MEMORY)
// ============================================================================
#ifdef HAKMEM_DEBUG_MEMORY
// NOTE: count_active_superslabs and hak_tiny_print_memory_profile are currently disabled
#else
// Stub function when debug is disabled
void hak_tiny_print_memory_profile(void) {
// No-op in release builds
}
#endif // HAKMEM_DEBUG_MEMORY
// ============================================================================
// Debug Print Functions (always available, gated by HAKMEM_DEBUG_COUNTERS)
// ============================================================================
// Debug print for Ultra Tiny counters
#if HAKMEM_DEBUG_COUNTERS
void hak_tiny_ultra_debug_dump(void) {
// NOTE: Ultra Tiny counters (pop_hits, refills, resets, sll_count) are currently not tracked
// Uncomment when these variables are implemented
/*
fprintf(stderr, "\n[Ultra Tiny Debug]\n");
fprintf(stderr, "class, pop_hits, refills, resets, sll_count\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%u\n",
i,
(unsigned long long)g_ultra_pop_hits[i],
(unsigned long long)g_ultra_refill_calls[i],
(unsigned long long)g_ultra_resets[i],
(unsigned)g_tls_sll[i].count);
}
*/
}
#else
void hak_tiny_ultra_debug_dump(void) { /* no-op in release builds */ }
#endif
// Debug print for normal path counters (SLL/MAG/FRONT/SUPER)
void hak_tiny_path_debug_dump(void) {
#if HAKMEM_DEBUG_COUNTERS
const char* on = getenv("HAKMEM_TINY_PATH_DEBUG");
if (!(on && atoi(on) != 0)) return;
// NOTE: Path debug counters (sll_pop, mag_pop, etc.) are currently not tracked
// Uncomment when these variables are implemented
/*
fprintf(stderr, "\n[Tiny Path Debug]\n");
fprintf(stderr, "class, sll_pop, mag_pop, front_pop, superslab, refills\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu\n",
i,
(unsigned long long)g_path_sll_pop[i],
(unsigned long long)g_path_mag_pop[i],
(unsigned long long)g_path_front_pop[i],
(unsigned long long)g_path_superslab[i],
(unsigned long long)g_path_refill_calls[i]);
}
*/
(void)on;
#else
(void)getenv; // suppress unused warnings when compiled out
#endif
}
// Debug print for extended counters (slow/bin/bump/spec)
void hak_tiny_debug_counters_dump(void) {
#if HAKMEM_DEBUG_COUNTERS
#if HAKMEM_BUILD_RELEASE
return;
#else
if (!hak_stats_check("HAKMEM_TINY_COUNTERS_DUMP", "counters")) {
return;
}
#endif
// NOTE: Extended counters (alloc_slow, bitmap_scans, etc.) are currently not tracked
// Uncomment when these variables are implemented
/*
fprintf(stderr, "\n[Tiny Extended Counters]\n");
fprintf(stderr, "class, alloc_slow, ss_refill, bitmap_scans, bin_pops, bump_hits, bump_arms, spec_calls, spec_hits\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n",
i,
(unsigned long long)g_alloc_slow_calls[i],
(unsigned long long)g_superslab_refill_calls_dbg[i],
(unsigned long long)g_bitmap_scan_calls[i],
(unsigned long long)g_bgbin_pops[i],
(unsigned long long)g_bump_hits[i],
(unsigned long long)g_bump_arms[i],
(unsigned long long)g_spec_calls[i],
(unsigned long long)g_spec_hits[i]);
}
*/
// SuperSlab adopt/publish debug
extern unsigned long long g_ss_publish_dbg[];
extern unsigned long long g_ss_adopt_dbg[];
fprintf(stderr, "\n[SS Adopt/Publish Counters]\n");
fprintf(stderr, "class, ss_publish, ss_adopt\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu\n", i,
(unsigned long long)g_ss_publish_dbg[i],
(unsigned long long)g_ss_adopt_dbg[i]);
}
// Refill-stage counters
extern unsigned long long g_rf_total_calls[];
extern unsigned long long g_rf_hit_bench[];
extern unsigned long long g_rf_hit_hot[];
extern unsigned long long g_rf_hit_ready[];
extern unsigned long long g_rf_hit_slab[];
extern unsigned long long g_rf_hit_ss[];
extern unsigned long long g_rf_hit_reg[];
extern unsigned long long g_rf_mmap_calls[];
fprintf(stderr, "\n[Refill Stage Counters]\n");
fprintf(stderr, "class, total, ready, bench, hot, slab, ss, reg, mmap\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_rf_total_calls[i],
(unsigned long long)g_rf_hit_ready[i],
(unsigned long long)g_rf_hit_bench[i],
(unsigned long long)g_rf_hit_hot[i],
(unsigned long long)g_rf_hit_slab[i],
(unsigned long long)g_rf_hit_ss[i],
(unsigned long long)g_rf_hit_reg[i],
(unsigned long long)g_rf_mmap_calls[i]);
}
// Refill item sources (freelist vs carve)
extern unsigned long long g_rf_freelist_items[];
extern unsigned long long g_rf_carve_items[];
fprintf(stderr, "\n[Refill Item Sources]\n");
fprintf(stderr, "class, freelist_items, carve_items\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu\n", i,
(unsigned long long)g_rf_freelist_items[i],
(unsigned long long)g_rf_carve_items[i]);
}
// Refill item sources (freelist vs carve)
extern unsigned long long g_rf_freelist_items[];
extern unsigned long long g_rf_carve_items[];
fprintf(stderr, "\n[Refill Item Sources]\n");
fprintf(stderr, "class, freelist_items, carve_items\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu\n", i,
(unsigned long long)g_rf_freelist_items[i],
(unsigned long long)g_rf_carve_items[i]);
}
// Diagnostic: refill early return counters
extern unsigned long long g_rf_early_no_ss[];
extern unsigned long long g_rf_early_no_meta[];
extern unsigned long long g_rf_early_no_room[];
extern unsigned long long g_rf_early_want_zero[];
fprintf(stderr, "\n[Refill Early Returns - Diagnostic]\n");
fprintf(stderr, "class, no_ss, no_meta, no_room, want_zero\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_rf_early_no_ss[i],
(unsigned long long)g_rf_early_no_meta[i],
(unsigned long long)g_rf_early_no_room[i],
(unsigned long long)g_rf_early_want_zero[i]);
}
// Slab-ring counters
extern unsigned long long g_slab_publish_dbg[];
extern unsigned long long g_slab_adopt_dbg[];
extern unsigned long long g_slab_requeue_dbg[];
extern unsigned long long g_slab_miss_dbg[];
fprintf(stderr, "\n[Slab Adopt/Publish Counters]\n");
fprintf(stderr, "class, slab_publish, slab_adopt, slab_requeue, slab_miss\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_slab_publish_dbg[i],
(unsigned long long)g_slab_adopt_dbg[i],
(unsigned long long)g_slab_requeue_dbg[i],
(unsigned long long)g_slab_miss_dbg[i]);
}
// Publish-side counters
extern unsigned long long g_pub_bench_hits[];
extern unsigned long long g_pub_hot_hits[];
extern unsigned long long g_pub_mail_hits[];
fprintf(stderr, "\n[Publish Hits]\n");
fprintf(stderr, "class, pub_mail, pub_bench, pub_hot\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu\n", i,
(unsigned long long)g_pub_mail_hits[i],
(unsigned long long)g_pub_bench_hits[i],
(unsigned long long)g_pub_hot_hits[i]);
}
// Front Gate Breakdown (SFC/SLL/Quick/Mag)
extern unsigned long long g_front_sfc_hit[];
extern unsigned long long g_front_sll_hit[];
extern unsigned long long g_front_quick_hit[];
extern unsigned long long g_front_mag_hit[];
fprintf(stderr, "\n[Front Gate Breakdown]\n");
fprintf(stderr, "class, sfc_hit, sll_hit, quick_hit, mag_hit\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_front_sfc_hit[i],
(unsigned long long)g_front_sll_hit[i],
(unsigned long long)g_front_quick_hit[i],
(unsigned long long)g_front_mag_hit[i]);
}
// Free Triggers (first-free / remote transition)
extern unsigned long long g_first_free_transitions[];
extern unsigned long long g_remote_free_transitions[];
fprintf(stderr, "\n[Free Triggers]\n");
fprintf(stderr, "class, first_free, remote_transition\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu\n", i,
(unsigned long long)g_first_free_transitions[i],
(unsigned long long)g_remote_free_transitions[i]);
}
// Adopt/Registry Gate
extern unsigned long long g_adopt_gate_calls[];
extern unsigned long long g_adopt_gate_success[];
extern unsigned long long g_reg_scan_attempts[];
extern unsigned long long g_reg_scan_hits[];
fprintf(stderr, "\n[Adopt/Registry Gate]\n");
fprintf(stderr, "class, adopt_calls, adopt_success, reg_scans, reg_hits\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_adopt_gate_calls[i],
(unsigned long long)g_adopt_gate_success[i],
(unsigned long long)g_reg_scan_attempts[i],
(unsigned long long)g_reg_scan_hits[i]);
}
// SuperSlab Registry (per-class sizes)
extern int g_super_reg_class_size[];
fprintf(stderr, "\n[SuperSlab Registry]\n");
fprintf(stderr, "class, reg_size\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%d\n", i, g_super_reg_class_size[i]);
}
extern unsigned long long g_fast_push_hits[];
extern unsigned long long g_fast_push_full[];
extern unsigned long long g_fast_push_disabled[];
extern unsigned long long g_fast_push_zero_cap[];
extern unsigned long long g_fast_push_gate_disabled[];
extern unsigned long long g_fast_push_gate_zero_cap[];
extern unsigned long long g_fast_spare_attempts[];
extern unsigned long long g_fast_spare_disabled[];
extern unsigned long long g_fast_spare_empty[];
extern unsigned long long g_fast_spare_lookup_fail[];
extern unsigned long long g_fast_spare_bad_index[];
extern unsigned long long g_fast_lookup_ss[];
extern unsigned long long g_fast_lookup_slab[];
extern unsigned long long g_fast_lookup_none;
fprintf(stderr, "\n[Fast Cache Debug]\n");
fprintf(stderr, "class, push_hits, push_full, push_disabled, push_zero_cap, gate_disabled, gate_zero_cap, spare_attempts, spare_disabled, spare_empty, spare_lookup_fail, spare_bad_index, lookup_ss, lookup_slab\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_fast_push_hits[i],
(unsigned long long)g_fast_push_full[i],
(unsigned long long)g_fast_push_disabled[i],
(unsigned long long)g_fast_push_zero_cap[i],
(unsigned long long)g_fast_push_gate_disabled[i],
(unsigned long long)g_fast_push_gate_zero_cap[i],
(unsigned long long)g_fast_spare_attempts[i],
(unsigned long long)g_fast_spare_disabled[i],
(unsigned long long)g_fast_spare_empty[i],
(unsigned long long)g_fast_spare_lookup_fail[i],
(unsigned long long)g_fast_spare_bad_index[i],
(unsigned long long)g_fast_lookup_ss[i],
(unsigned long long)g_fast_lookup_slab[i]);
}
fprintf(stderr, "lookup_none,%llu\n", (unsigned long long)g_fast_lookup_none);
extern uint64_t g_ss_cache_hits[];
extern uint64_t g_ss_cache_misses[];
extern uint64_t g_ss_cache_puts[];
extern uint64_t g_ss_cache_drops[];
extern uint64_t g_ss_cache_precharged[];
extern uint64_t g_superslabs_reused;
extern uint64_t g_superslabs_cached;
fprintf(stderr, "\n[SS Cache Stats]\n");
fprintf(stderr, "class, cache_hits, cache_misses, cache_puts, cache_drops, precharged\n");
for (int i = 0; i < 8; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_ss_cache_hits[i],
(unsigned long long)g_ss_cache_misses[i],
(unsigned long long)g_ss_cache_puts[i],
(unsigned long long)g_ss_cache_drops[i],
(unsigned long long)g_ss_cache_precharged[i]);
}
fprintf(stderr, "cache_reused=%llu cache_cached=%llu\n",
(unsigned long long)g_superslabs_reused,
(unsigned long long)g_superslabs_cached);
// Free pipeline
extern unsigned long long g_free_via_ss_local[];
extern unsigned long long g_free_via_ss_remote[];
extern unsigned long long g_free_via_tls_sll[];
extern unsigned long long g_free_via_mag[];
extern unsigned long long g_free_via_fast_tls[];
extern unsigned long long g_free_via_fastcache[];
extern unsigned long long g_fast_spare_flush[];
fprintf(stderr, "\n[Free Pipeline]\n");
fprintf(stderr, "class, ss_local, ss_remote, fast_tls, fast_cache, tls_sll, magazine, fast_spare_flush\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_free_via_ss_local[i],
(unsigned long long)g_free_via_ss_remote[i],
(unsigned long long)g_free_via_fast_tls[i],
(unsigned long long)g_free_via_fastcache[i],
(unsigned long long)g_free_via_tls_sll[i],
(unsigned long long)g_free_via_mag[i],
(unsigned long long)g_fast_spare_flush[i]);
}
// Publish pipeline
extern unsigned long long g_pub_notify_calls[];
extern unsigned long long g_pub_same_empty[];
extern unsigned long long g_remote_transitions[];
extern unsigned long long g_mailbox_register_calls[];
extern unsigned long long g_mailbox_slow_discoveries[];
fprintf(stderr, "\n[Publish Pipeline]\n");
fprintf(stderr, "class, notify_calls, same_empty_pubs, remote_transitions, mailbox_reg_calls, mailbox_slow_disc\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_pub_notify_calls[i],
(unsigned long long)g_pub_same_empty[i],
(unsigned long long)g_remote_transitions[i],
(unsigned long long)g_mailbox_register_calls[i],
(unsigned long long)g_mailbox_slow_discoveries[i]);
}
extern unsigned long long g_fast_push_hits[];
extern unsigned long long g_fast_push_full[];
extern unsigned long long g_fast_push_disabled[];
extern unsigned long long g_fast_push_zero_cap[];
extern unsigned long long g_fast_push_gate_disabled[];
extern unsigned long long g_fast_push_gate_zero_cap[];
extern unsigned long long g_fast_spare_attempts[];
extern unsigned long long g_fast_spare_disabled[];
extern unsigned long long g_fast_spare_empty[];
extern unsigned long long g_fast_spare_lookup_fail[];
extern unsigned long long g_fast_spare_bad_index[];
extern unsigned long long g_fast_lookup_ss[];
extern unsigned long long g_fast_lookup_slab[];
extern unsigned long long g_fast_lookup_none;
fprintf(stderr, "\n[Fast Cache Debug]\n");
fprintf(stderr, "class, push_hits, push_full, push_disabled, push_zero_cap, gate_disabled, gate_zero_cap, spare_attempts, spare_disabled, spare_empty, spare_lookup_fail, spare_bad_index, lookup_ss, lookup_slab\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_fast_push_hits[i],
(unsigned long long)g_fast_push_full[i],
(unsigned long long)g_fast_push_disabled[i],
(unsigned long long)g_fast_push_zero_cap[i],
(unsigned long long)g_fast_push_gate_disabled[i],
(unsigned long long)g_fast_push_gate_zero_cap[i],
(unsigned long long)g_fast_spare_attempts[i],
(unsigned long long)g_fast_spare_disabled[i],
(unsigned long long)g_fast_spare_empty[i],
(unsigned long long)g_fast_spare_lookup_fail[i],
(unsigned long long)g_fast_spare_bad_index[i],
(unsigned long long)g_fast_lookup_ss[i],
(unsigned long long)g_fast_lookup_slab[i]);
}
fprintf(stderr, "lookup_none,%llu\n", (unsigned long long)g_fast_lookup_none);
// Refill timing (ns)
extern unsigned long long g_rf_time_total_ns[];
extern unsigned long long g_rf_time_hot_ns[];
extern unsigned long long g_rf_time_bench_ns[];
extern unsigned long long g_rf_time_mail_ns[];
extern unsigned long long g_rf_time_slab_ns[];
extern unsigned long long g_rf_time_ss_ns[];
extern unsigned long long g_rf_time_reg_ns[];
extern unsigned long long g_rf_time_mmap_ns[];
fprintf(stderr, "\n[Refill Time (ns)]\n");
fprintf(stderr, "class, total, hot, bench, mail, slab, ss, reg, mmap\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_rf_time_total_ns[i],
(unsigned long long)g_rf_time_hot_ns[i],
(unsigned long long)g_rf_time_bench_ns[i],
(unsigned long long)g_rf_time_mail_ns[i],
(unsigned long long)g_rf_time_slab_ns[i],
(unsigned long long)g_rf_time_ss_ns[i],
(unsigned long long)g_rf_time_reg_ns[i],
(unsigned long long)g_rf_time_mmap_ns[i]);
}
#endif
}
// Always-available: Refill stage counters dump (env: HAKMEM_TINY_REFILL_DUMP=1 or reuse HAKMEM_TINY_COUNTERS_DUMP)
static void hak_tiny_refill_counters_dump(void) {
hak_tiny_stats_init_flags();
#if HAKMEM_BUILD_RELEASE
return;
#else
if (!hak_stats_check("HAKMEM_TINY_REFILL_DUMP", "refill") &&
!hak_stats_check("HAKMEM_TINY_COUNTERS_DUMP", "counters")) {
return;
}
#endif
extern unsigned long long g_rf_total_calls[];
extern unsigned long long g_rf_hit_bench[];
extern unsigned long long g_rf_hit_hot[];
extern unsigned long long g_rf_hit_mail[];
extern unsigned long long g_rf_hit_slab[];
extern unsigned long long g_rf_hit_ss[];
extern unsigned long long g_rf_hit_reg[];
extern unsigned long long g_rf_mmap_calls[];
fprintf(stderr, "\n[Refill Stage Counters]\n");
fprintf(stderr, "class, total, mail, bench, hot, slab, ss, reg, mmap\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_rf_total_calls[i],
(unsigned long long)g_rf_hit_mail[i],
(unsigned long long)g_rf_hit_bench[i],
(unsigned long long)g_rf_hit_hot[i],
(unsigned long long)g_rf_hit_slab[i],
(unsigned long long)g_rf_hit_ss[i],
(unsigned long long)g_rf_hit_reg[i],
(unsigned long long)g_rf_mmap_calls[i]);
}
// Diagnostic: refill early return counters
extern unsigned long long g_rf_early_no_ss[];
extern unsigned long long g_rf_early_no_meta[];
extern unsigned long long g_rf_early_no_room[];
extern unsigned long long g_rf_early_want_zero[];
fprintf(stderr, "\n[Refill Early Returns - Diagnostic]\n");
fprintf(stderr, "class, no_ss, no_meta, no_room, want_zero\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_rf_early_no_ss[i],
(unsigned long long)g_rf_early_no_meta[i],
(unsigned long long)g_rf_early_no_room[i],
(unsigned long long)g_rf_early_want_zero[i]);
}
// Publish-side counters (always available)
extern unsigned long long g_pub_bench_hits[];
extern unsigned long long g_pub_hot_hits[];
extern unsigned long long g_pub_mail_hits[];
fprintf(stderr, "\n[Publish Hits]\n");
fprintf(stderr, "class, pub_mail, pub_bench, pub_hot\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu\n", i,
(unsigned long long)g_pub_mail_hits[i],
(unsigned long long)g_pub_bench_hits[i],
(unsigned long long)g_pub_hot_hits[i]);
}
extern uint64_t g_ss_cache_hits[];
extern uint64_t g_ss_cache_misses[];
extern uint64_t g_ss_cache_puts[];
extern uint64_t g_ss_cache_drops[];
extern uint64_t g_ss_cache_precharged[];
extern uint64_t g_superslabs_reused;
extern uint64_t g_superslabs_cached;
fprintf(stderr, "\n[SS Cache Stats]\n");
fprintf(stderr, "class, cache_hits, cache_misses, cache_puts, cache_drops, precharged\n");
for (int i = 0; i < 8; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_ss_cache_hits[i],
(unsigned long long)g_ss_cache_misses[i],
(unsigned long long)g_ss_cache_puts[i],
(unsigned long long)g_ss_cache_drops[i],
(unsigned long long)g_ss_cache_precharged[i]);
}
fprintf(stderr, "cache_reused=%llu cache_cached=%llu\n",
(unsigned long long)g_superslabs_reused,
(unsigned long long)g_superslabs_cached);
// Free pipeline
extern unsigned long long g_free_via_ss_local[];
extern unsigned long long g_free_via_ss_remote[];
extern unsigned long long g_free_via_tls_sll[];
extern unsigned long long g_free_via_mag[];
extern unsigned long long g_free_via_fast_tls[];
extern unsigned long long g_free_via_fastcache[];
extern unsigned long long g_fast_spare_flush[];
fprintf(stderr, "\n[Free Pipeline]\n");
fprintf(stderr, "class, ss_local, ss_remote, fast_tls, fast_cache, tls_sll, magazine, fast_spare_flush\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_free_via_ss_local[i],
(unsigned long long)g_free_via_ss_remote[i],
(unsigned long long)g_free_via_fast_tls[i],
(unsigned long long)g_free_via_fastcache[i],
(unsigned long long)g_free_via_tls_sll[i],
(unsigned long long)g_free_via_mag[i],
(unsigned long long)g_fast_spare_flush[i]);
}
// Publish pipeline
extern unsigned long long g_pub_notify_calls[];
extern unsigned long long g_pub_same_empty[];
extern unsigned long long g_remote_transitions[];
extern unsigned long long g_mailbox_register_calls[];
extern unsigned long long g_mailbox_slow_discoveries[];
fprintf(stderr, "\n[Publish Pipeline]\n");
fprintf(stderr, "class, notify_calls, same_empty_pubs, remote_transitions, mailbox_reg_calls, mailbox_slow_disc\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_pub_notify_calls[i],
(unsigned long long)g_pub_same_empty[i],
(unsigned long long)g_remote_transitions[i],
(unsigned long long)g_mailbox_register_calls[i],
(unsigned long long)g_mailbox_slow_discoveries[i]);
}
extern unsigned long long g_fast_push_hits[];
extern unsigned long long g_fast_push_full[];
extern unsigned long long g_fast_push_disabled[];
extern unsigned long long g_fast_push_zero_cap[];
extern unsigned long long g_fast_push_gate_disabled[];
extern unsigned long long g_fast_push_gate_zero_cap[];
extern unsigned long long g_fast_spare_attempts[];
extern unsigned long long g_fast_spare_disabled[];
extern unsigned long long g_fast_spare_empty[];
extern unsigned long long g_fast_spare_lookup_fail[];
extern unsigned long long g_fast_spare_bad_index[];
extern unsigned long long g_fast_lookup_ss[];
extern unsigned long long g_fast_lookup_slab[];
extern unsigned long long g_fast_lookup_none;
fprintf(stderr, "\n[Fast Cache Debug]\n");
fprintf(stderr, "class, push_hits, push_full, push_disabled, push_zero_cap, gate_disabled, gate_zero_cap, spare_attempts, spare_disabled, spare_empty, spare_lookup_fail, spare_bad_index, lookup_ss, lookup_slab\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_fast_push_hits[i],
(unsigned long long)g_fast_push_full[i],
(unsigned long long)g_fast_push_disabled[i],
(unsigned long long)g_fast_push_zero_cap[i],
(unsigned long long)g_fast_push_gate_disabled[i],
(unsigned long long)g_fast_push_gate_zero_cap[i],
(unsigned long long)g_fast_spare_attempts[i],
(unsigned long long)g_fast_spare_disabled[i],
(unsigned long long)g_fast_spare_empty[i],
(unsigned long long)g_fast_spare_lookup_fail[i],
(unsigned long long)g_fast_spare_bad_index[i],
(unsigned long long)g_fast_lookup_ss[i],
(unsigned long long)g_fast_lookup_slab[i]);
}
fprintf(stderr, "lookup_none,%llu\n", (unsigned long long)g_fast_lookup_none);
// Refill timing (ns)
extern unsigned long long g_rf_time_total_ns[];
extern unsigned long long g_rf_time_hot_ns[];
extern unsigned long long g_rf_time_bench_ns[];
extern unsigned long long g_rf_time_mail_ns[];
extern unsigned long long g_rf_time_slab_ns[];
extern unsigned long long g_rf_time_ss_ns[];
extern unsigned long long g_rf_time_reg_ns[];
extern unsigned long long g_rf_time_mmap_ns[];
fprintf(stderr, "\n[Refill Time (ns)]\n");
fprintf(stderr, "class, total, hot, bench, mail, slab, ss, reg, mmap\n");
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
fprintf(stderr, "%d,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", i,
(unsigned long long)g_rf_time_total_ns[i],
(unsigned long long)g_rf_time_hot_ns[i],
(unsigned long long)g_rf_time_bench_ns[i],
(unsigned long long)g_rf_time_mail_ns[i],
(unsigned long long)g_rf_time_slab_ns[i],
(unsigned long long)g_rf_time_ss_ns[i],
(unsigned long long)g_rf_time_reg_ns[i],
(unsigned long long)g_rf_time_mmap_ns[i]);
}
}
__attribute__((destructor))
static void hak_tiny_stats_auto_dump(void) {
// Dump at exit if enabled or atexit-only requested
hak_tiny_stats_init_flags();
if (g_dump_atexit_only || hak_stats_dump_enabled()) {
// Force dump regardless of individual envs when atexit-only
// or when master HAKMEM_STATS(_DUMP) requests global dump.
hak_tiny_dump_all_counters_now();
} else {
hak_tiny_refill_counters_dump();
}
}
Reference in New Issue View Git Blame Copy Permalink