From e96e9a4bf947dad1bd48c125e417faf7fa845ea2 Mon Sep 17 00:00:00 2001 From: "Moe Charm (CI)" Date: Fri, 5 Dec 2025 20:50:24 +0900 Subject: [PATCH] Feat: Add TLS carve experiment for warm C7 --- core/front/tiny_unified_cache.c | 129 +++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 44 deletions(-) diff --git a/core/front/tiny_unified_cache.c b/core/front/tiny_unified_cache.c index 550788c8..26ec29aa 100644 --- a/core/front/tiny_unified_cache.c +++ b/core/front/tiny_unified_cache.c @@ -19,6 +19,7 @@ #include #include #include +#include #include // ============================================================================ @@ -97,6 +98,45 @@ _Atomic uint64_t g_dbg_warm_pop_attempts = 0; _Atomic uint64_t g_dbg_warm_pop_hits = 0; _Atomic uint64_t g_dbg_warm_pop_empty = 0; _Atomic uint64_t g_dbg_warm_pop_carve_zero = 0; + +// Debug-only: cached ENV for Warm TLS Bind (C7) +static int g_warm_tls_bind_mode_c7 = -1; + +static inline int warm_tls_bind_mode_c7(void) { + if (__builtin_expect(g_warm_tls_bind_mode_c7 == -1, 0)) { + const char* e = getenv("HAKMEM_WARM_TLS_BIND_C7"); + // 0/empty: disabled, 1: bind only, 2: bind + TLS carve one block + g_warm_tls_bind_mode_c7 = (e && *e) ? atoi(e) : 0; + } + return g_warm_tls_bind_mode_c7; +} + +static inline void* warm_tls_carve_one_block(int class_idx) { + TinyTLSSlab* tls = &g_tls_slabs[class_idx]; + TinySlabMeta* meta = tls->meta; + + if (!meta || !tls->ss || tls->slab_base == NULL) return NULL; + if (meta->class_idx != (uint8_t)class_idx) return NULL; + if (tls->slab_idx < 0 || tls->slab_idx >= ss_slabs_capacity(tls->ss)) return NULL; + + if (meta->freelist) { + void* block = meta->freelist; + meta->freelist = tiny_next_read(class_idx, block); + meta->used++; + ss_active_add(tls->ss, 1); + return block; + } + + if (meta->used < meta->capacity) { + size_t block_size = tiny_stride_for_class(meta->class_idx); + void* block = tiny_block_at_index(tls->slab_base, meta->used, block_size); + meta->used++; + ss_active_add(tls->ss, 1); + return block; + } + + return NULL; +} #endif // Forward declaration for Warm Pool stats printer (defined later in this file) @@ -518,47 +558,46 @@ hak_base_ptr_t unified_cache_refill(int class_idx) { SuperSlab* warm_ss = tiny_warm_pool_pop(class_idx); if (warm_ss) { #if !HAKMEM_BUILD_RELEASE - // FUTURE: TLS Bind Box Integration - // Currently we carve directly from warm_ss via slab_carve_from_ss(). - // To unify logic, we should eventually: - // 1. Choose a slab index (via tiny_page_box or heuristic). - // 2. Bind it to TLS via ss_tls_bind_one(..., warm_ss, slab_idx, ...). - // 3. Fall through to TLS-based allocation. - - // EXPERIMENTAL: Test TLS Bind Box connectivity for C7 (Debug only) - static int g_warm_tls_bind_c7 = -1; - if (g_warm_tls_bind_c7 == -1) { - const char* e = getenv("HAKMEM_WARM_TLS_BIND_C7"); - g_warm_tls_bind_c7 = (e && *e && *e != '0') ? 1 : 0; - } - - if (g_warm_tls_bind_c7 && class_idx == 7) { - // Find a slab index in this SuperSlab that matches our class - int cap = ss_slabs_capacity(warm_ss); - int slab_idx = -1; - - // Simple heuristic: find first slab belonging to this class - // Note: In real logic, we should pick the *best* slab (e.g. from PageBox) - for (int i = 0; i < cap; i++) { - if (tiny_get_class_from_ss(warm_ss, i) == class_idx) { - slab_idx = i; - break; + // Debug-only: Warm TLS Bind experiment (C7 only) + if (class_idx == 7) { + int warm_mode = warm_tls_bind_mode_c7(); + if (warm_mode >= 1) { + int cap = ss_slabs_capacity(warm_ss); + int slab_idx = -1; + + // Simple heuristic: first slab matching class + for (int i = 0; i < cap; i++) { + if (tiny_get_class_from_ss(warm_ss, i) == class_idx) { + slab_idx = i; + break; + } } - } - - if (slab_idx >= 0) { - TinyTLSSlab* tls = &g_tls_slabs[class_idx]; - // Try to bind. If successful, we have "connected" the path. - // For now, we still fall through to slab_carve_from_ss() to do the actual - // work, but the side effect (TLS updated) confirms connectivity. - // In a future step, we would 'break' here and let the TLS path handle it. - uint32_t tid = (uint32_t)(uintptr_t)pthread_self(); - if (ss_tls_bind_one(class_idx, tls, warm_ss, slab_idx, tid)) { - static int logged = 0; - if (!logged) { - fprintf(stderr, "[WARM_TLS_BIND] C7 bind success: ss=%p slab=%d\n", - (void*)warm_ss, slab_idx); - logged = 1; + + if (slab_idx >= 0) { + TinyTLSSlab* tls = &g_tls_slabs[class_idx]; + uint32_t tid = (uint32_t)(uintptr_t)pthread_self(); + if (ss_tls_bind_one(class_idx, tls, warm_ss, slab_idx, tid)) { + static int logged = 0; + if (!logged) { + fprintf(stderr, "[WARM_TLS_BIND] C7 bind success: ss=%p slab=%d\n", + (void*)warm_ss, slab_idx); + logged = 1; + } + + // Mode 2: carve a single block via TLS fast path + if (warm_mode == 2) { + void* tls_block = warm_tls_carve_one_block(class_idx); + if (tls_block) { + fprintf(stderr, + "[WARM_TLS_BIND] C7 TLS carve success: ss=%p slab=%d block=%p\n", + (void*)warm_ss, slab_idx, tls_block); + out[0] = tls_block; + produced = 1; + } else { + fprintf(stderr, + "[WARM_TLS_BIND] C7 TLS carve failed, fallback\n"); + } + } } } } @@ -567,10 +606,12 @@ hak_base_ptr_t unified_cache_refill(int class_idx) { atomic_fetch_add_explicit(&g_dbg_warm_pop_hits, 1, memory_order_relaxed); #endif // HOT PATH: Warm pool hit, try to carve directly - produced = slab_carve_from_ss(class_idx, warm_ss, out, room); - if (produced > 0) { - // Update active counter for carved blocks - ss_active_add(warm_ss, (uint32_t)produced); + if (produced == 0) { + produced = slab_carve_from_ss(class_idx, warm_ss, out, room); + if (produced > 0) { + // Update active counter for carved blocks + ss_active_add(warm_ss, (uint32_t)produced); + } } if (produced > 0) {