diff --git a/core/box/bench_fast_box.c b/core/box/bench_fast_box.c index 5342bf74..689b5ee6 100644 --- a/core/box/bench_fast_box.c +++ b/core/box/bench_fast_box.c @@ -120,6 +120,14 @@ int bench_fast_init(void) { // Set guard to prevent recursion during initialization bench_fast_init_in_progress = 1; + // Phase 8-Step2: Prewarm Unified Cache (initialize before benchmark) + // This enables PGO builds to remove lazy init checks in hot paths + #ifdef USE_HAKMEM + extern void unified_cache_init(void); + unified_cache_init(); + fprintf(stderr, "[BENCH_FAST] Unified Cache prewarmed\n"); + #endif + fprintf(stderr, "[BENCH_FAST] Starting preallocation...\n"); int total = 0; diff --git a/core/box/tiny_front_config_box.h b/core/box/tiny_front_config_box.h index cbb665e2..7a9fcaa1 100644 --- a/core/box/tiny_front_config_box.h +++ b/core/box/tiny_front_config_box.h @@ -53,6 +53,7 @@ #define TINY_FRONT_SFC_ENABLED 1 // Enabled (SFC cascade) #define TINY_FRONT_FASTCACHE_ENABLED 0 // Disabled (use Unified Cache) #define TINY_FRONT_TLS_SLL_ENABLED 1 // Enabled (TLS SLL freelist) +#define TINY_FRONT_UNIFIED_CACHE_ENABLED 1 // Enabled (Unified Cache - tcache-style) #define TINY_FRONT_UNIFIED_GATE_ENABLED 1 // Enabled (Front Gate Unification) #define TINY_FRONT_METRICS_ENABLED 0 // Disabled (no runtime overhead) #define TINY_FRONT_DIAG_ENABLED 0 // Disabled (no diagnostics) @@ -100,16 +101,21 @@ static inline int tiny_tls_sll_enabled(void) { return g_tls_sll_enable; } +// Phase 8-Step1: Unified Cache enabled wrapper +// Forward declaration - actual function is in tiny_unified_cache.c +int unified_cache_enabled(void); + // Config macros (runtime function calls) // These expand to actual function calls in normal mode -#define TINY_FRONT_ULTRA_SLIM_ENABLED ultra_slim_mode_enabled() -#define TINY_FRONT_HEAP_V2_ENABLED tiny_heap_v2_enabled() -#define TINY_FRONT_SFC_ENABLED tiny_sfc_enabled() -#define TINY_FRONT_FASTCACHE_ENABLED tiny_fastcache_enabled() -#define TINY_FRONT_TLS_SLL_ENABLED tiny_tls_sll_enabled() -#define TINY_FRONT_UNIFIED_GATE_ENABLED front_gate_unified_enabled() -#define TINY_FRONT_METRICS_ENABLED tiny_metrics_enabled() -#define TINY_FRONT_DIAG_ENABLED tiny_diag_enabled() +#define TINY_FRONT_ULTRA_SLIM_ENABLED ultra_slim_mode_enabled() +#define TINY_FRONT_HEAP_V2_ENABLED tiny_heap_v2_enabled() +#define TINY_FRONT_SFC_ENABLED tiny_sfc_enabled() +#define TINY_FRONT_FASTCACHE_ENABLED tiny_fastcache_enabled() +#define TINY_FRONT_TLS_SLL_ENABLED tiny_tls_sll_enabled() +#define TINY_FRONT_UNIFIED_CACHE_ENABLED unified_cache_enabled() +#define TINY_FRONT_UNIFIED_GATE_ENABLED front_gate_unified_enabled() +#define TINY_FRONT_METRICS_ENABLED tiny_metrics_enabled() +#define TINY_FRONT_DIAG_ENABLED tiny_diag_enabled() #endif // HAKMEM_TINY_FRONT_PGO diff --git a/core/front/tiny_unified_cache.c b/core/front/tiny_unified_cache.c index 6aedf0e9..2d8c9f75 100644 --- a/core/front/tiny_unified_cache.c +++ b/core/front/tiny_unified_cache.c @@ -31,6 +31,26 @@ __thread uint64_t g_unified_cache_push[TINY_NUM_CLASSES] = {0}; __thread uint64_t g_unified_cache_full[TINY_NUM_CLASSES] = {0}; #endif +// ============================================================================ +// Phase 8-Step1-Fix: unified_cache_enabled() implementation (non-static) +// ============================================================================ + +// Enable flag (default: ON, disable with HAKMEM_TINY_UNIFIED_CACHE=0) +int unified_cache_enabled(void) { + static int g_enable = -1; + if (__builtin_expect(g_enable == -1, 0)) { + const char* e = getenv("HAKMEM_TINY_UNIFIED_CACHE"); + g_enable = (e && *e && *e == '0') ? 0 : 1; // default ON +#if !HAKMEM_BUILD_RELEASE + if (g_enable) { + fprintf(stderr, "[Unified-INIT] unified_cache_enabled() = %d\n", g_enable); + fflush(stderr); + } +#endif + } + return g_enable; +} + // ============================================================================ // Init (called at thread start or lazy on first access) // ============================================================================ diff --git a/core/front/tiny_unified_cache.h b/core/front/tiny_unified_cache.h index 5a355b3f..08ceb5ff 100644 --- a/core/front/tiny_unified_cache.h +++ b/core/front/tiny_unified_cache.h @@ -27,6 +27,7 @@ #include #include "../hakmem_build_flags.h" #include "../hakmem_tiny_config.h" // For TINY_NUM_CLASSES +#include "../box/tiny_front_config_box.h" // Phase 8-Step1: Config macros // ============================================================================ // Unified Cache Structure (per class) @@ -61,21 +62,9 @@ extern __thread uint64_t g_unified_cache_full[TINY_NUM_CLASSES]; // Free full // ENV Control (cached, lazy init) // ============================================================================ -// Enable flag (default: 0, OFF) -static inline int unified_cache_enabled(void) { - static int g_enable = -1; - if (__builtin_expect(g_enable == -1, 0)) { - const char* e = getenv("HAKMEM_TINY_UNIFIED_CACHE"); - g_enable = (e && *e && *e == '0') ? 0 : 1; // default ON -#if !HAKMEM_BUILD_RELEASE - if (g_enable) { - fprintf(stderr, "[Unified-INIT] unified_cache_enabled() = %d\n", g_enable); - fflush(stderr); - } -#endif - } - return g_enable; -} +// Phase 8-Step1-Fix: Forward declaration only (implementation in .c file) +// Enable flag (default: 0, OFF) - implemented in tiny_unified_cache.c +int unified_cache_enabled(void); // Per-class capacity (default: Hot_2048 strategy - optimized for 256B workload) // Phase 23 Capacity Optimization Result: Hot_2048 = 14.63M ops/s (+43% vs baseline) @@ -135,17 +124,23 @@ void* unified_cache_refill(int class_idx); // Pop from unified cache (alloc fast path) // Returns: BASE pointer (caller must convert to USER with +1) static inline void* unified_cache_pop(int class_idx) { + // Phase 8-Step1: Use config macro for dead code elimination in PGO mode // Fast path: Unified cache disabled → return NULL immediately - if (__builtin_expect(!unified_cache_enabled(), 0)) return NULL; + #include "../box/tiny_front_config_box.h" + if (__builtin_expect(!TINY_FRONT_UNIFIED_CACHE_ENABLED, 0)) return NULL; TinyUnifiedCache* cache = &g_unified_cache[class_idx]; // 1 cache miss (TLS) + // Phase 8-Step3: Lazy init check (conditional in PGO mode) + // PGO builds assume bench_fast_init() prewarmed cache → remove check (-1 branch) + #if !HAKMEM_TINY_FRONT_PGO // Lazy init check (once per thread, per class) if (__builtin_expect(cache->slots == NULL, 0)) { unified_cache_init(); // First call in this thread // Re-check after init (may fail if allocation failed) if (cache->slots == NULL) return NULL; } + #endif // Empty check if (__builtin_expect(cache->head == cache->tail, 0)) { @@ -170,17 +165,22 @@ static inline void* unified_cache_pop(int class_idx) { // Input: BASE pointer (caller must pass BASE, not USER) // Returns: 1=SUCCESS, 0=FULL static inline int unified_cache_push(int class_idx, void* base) { + // Phase 8-Step1: Use config macro for dead code elimination in PGO mode // Fast path: Unified cache disabled → return 0 (not handled) - if (__builtin_expect(!unified_cache_enabled(), 0)) return 0; + if (__builtin_expect(!TINY_FRONT_UNIFIED_CACHE_ENABLED, 0)) return 0; TinyUnifiedCache* cache = &g_unified_cache[class_idx]; // 1 cache miss (TLS) + // Phase 8-Step3: Lazy init check (conditional in PGO mode) + // PGO builds assume bench_fast_init() prewarmed cache → remove check (-1 branch) + #if !HAKMEM_TINY_FRONT_PGO // Lazy init check (once per thread, per class) if (__builtin_expect(cache->slots == NULL, 0)) { unified_cache_init(); // First call in this thread // Re-check after init (may fail if allocation failed) if (cache->slots == NULL) return 0; } + #endif uint16_t next_tail = (cache->tail + 1) & cache->mask; @@ -211,16 +211,21 @@ static inline int unified_cache_push(int class_idx, void* base) { // Returns: BASE pointer (caller converts to USER), or NULL if failed // Design: Self-contained, bypasses all other frontend layers (Ring/FC/SFC/SLL) static inline void* unified_cache_pop_or_refill(int class_idx) { + // Phase 8-Step1: Use config macro for dead code elimination in PGO mode // Fast path: Unified cache disabled → return NULL (caller uses legacy cascade) - if (__builtin_expect(!unified_cache_enabled(), 0)) return NULL; + if (__builtin_expect(!TINY_FRONT_UNIFIED_CACHE_ENABLED, 0)) return NULL; TinyUnifiedCache* cache = &g_unified_cache[class_idx]; // 1 cache miss (TLS) + // Phase 8-Step3: Lazy init check (conditional in PGO mode) + // PGO builds assume bench_fast_init() prewarmed cache → remove check (-1 branch) + #if !HAKMEM_TINY_FRONT_PGO // Lazy init check (once per thread, per class) if (__builtin_expect(cache->slots == NULL, 0)) { unified_cache_init(); if (cache->slots == NULL) return NULL; } + #endif // Try pop from cache (fast path) if (__builtin_expect(cache->head != cache->tail, 1)) {