Feat: Add TLS carve experiment for warm C7
This commit is contained in:
@ -19,6 +19,7 @@
|
|||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
|
#include <stdio.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@ -97,6 +98,45 @@ _Atomic uint64_t g_dbg_warm_pop_attempts = 0;
|
|||||||
_Atomic uint64_t g_dbg_warm_pop_hits = 0;
|
_Atomic uint64_t g_dbg_warm_pop_hits = 0;
|
||||||
_Atomic uint64_t g_dbg_warm_pop_empty = 0;
|
_Atomic uint64_t g_dbg_warm_pop_empty = 0;
|
||||||
_Atomic uint64_t g_dbg_warm_pop_carve_zero = 0;
|
_Atomic uint64_t g_dbg_warm_pop_carve_zero = 0;
|
||||||
|
|
||||||
|
// Debug-only: cached ENV for Warm TLS Bind (C7)
|
||||||
|
static int g_warm_tls_bind_mode_c7 = -1;
|
||||||
|
|
||||||
|
static inline int warm_tls_bind_mode_c7(void) {
|
||||||
|
if (__builtin_expect(g_warm_tls_bind_mode_c7 == -1, 0)) {
|
||||||
|
const char* e = getenv("HAKMEM_WARM_TLS_BIND_C7");
|
||||||
|
// 0/empty: disabled, 1: bind only, 2: bind + TLS carve one block
|
||||||
|
g_warm_tls_bind_mode_c7 = (e && *e) ? atoi(e) : 0;
|
||||||
|
}
|
||||||
|
return g_warm_tls_bind_mode_c7;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void* warm_tls_carve_one_block(int class_idx) {
|
||||||
|
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
|
||||||
|
TinySlabMeta* meta = tls->meta;
|
||||||
|
|
||||||
|
if (!meta || !tls->ss || tls->slab_base == NULL) return NULL;
|
||||||
|
if (meta->class_idx != (uint8_t)class_idx) return NULL;
|
||||||
|
if (tls->slab_idx < 0 || tls->slab_idx >= ss_slabs_capacity(tls->ss)) return NULL;
|
||||||
|
|
||||||
|
if (meta->freelist) {
|
||||||
|
void* block = meta->freelist;
|
||||||
|
meta->freelist = tiny_next_read(class_idx, block);
|
||||||
|
meta->used++;
|
||||||
|
ss_active_add(tls->ss, 1);
|
||||||
|
return block;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (meta->used < meta->capacity) {
|
||||||
|
size_t block_size = tiny_stride_for_class(meta->class_idx);
|
||||||
|
void* block = tiny_block_at_index(tls->slab_base, meta->used, block_size);
|
||||||
|
meta->used++;
|
||||||
|
ss_active_add(tls->ss, 1);
|
||||||
|
return block;
|
||||||
|
}
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// Forward declaration for Warm Pool stats printer (defined later in this file)
|
// Forward declaration for Warm Pool stats printer (defined later in this file)
|
||||||
@ -518,27 +558,14 @@ hak_base_ptr_t unified_cache_refill(int class_idx) {
|
|||||||
SuperSlab* warm_ss = tiny_warm_pool_pop(class_idx);
|
SuperSlab* warm_ss = tiny_warm_pool_pop(class_idx);
|
||||||
if (warm_ss) {
|
if (warm_ss) {
|
||||||
#if !HAKMEM_BUILD_RELEASE
|
#if !HAKMEM_BUILD_RELEASE
|
||||||
// FUTURE: TLS Bind Box Integration
|
// Debug-only: Warm TLS Bind experiment (C7 only)
|
||||||
// Currently we carve directly from warm_ss via slab_carve_from_ss().
|
if (class_idx == 7) {
|
||||||
// To unify logic, we should eventually:
|
int warm_mode = warm_tls_bind_mode_c7();
|
||||||
// 1. Choose a slab index (via tiny_page_box or heuristic).
|
if (warm_mode >= 1) {
|
||||||
// 2. Bind it to TLS via ss_tls_bind_one(..., warm_ss, slab_idx, ...).
|
|
||||||
// 3. Fall through to TLS-based allocation.
|
|
||||||
|
|
||||||
// EXPERIMENTAL: Test TLS Bind Box connectivity for C7 (Debug only)
|
|
||||||
static int g_warm_tls_bind_c7 = -1;
|
|
||||||
if (g_warm_tls_bind_c7 == -1) {
|
|
||||||
const char* e = getenv("HAKMEM_WARM_TLS_BIND_C7");
|
|
||||||
g_warm_tls_bind_c7 = (e && *e && *e != '0') ? 1 : 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (g_warm_tls_bind_c7 && class_idx == 7) {
|
|
||||||
// Find a slab index in this SuperSlab that matches our class
|
|
||||||
int cap = ss_slabs_capacity(warm_ss);
|
int cap = ss_slabs_capacity(warm_ss);
|
||||||
int slab_idx = -1;
|
int slab_idx = -1;
|
||||||
|
|
||||||
// Simple heuristic: find first slab belonging to this class
|
// Simple heuristic: first slab matching class
|
||||||
// Note: In real logic, we should pick the *best* slab (e.g. from PageBox)
|
|
||||||
for (int i = 0; i < cap; i++) {
|
for (int i = 0; i < cap; i++) {
|
||||||
if (tiny_get_class_from_ss(warm_ss, i) == class_idx) {
|
if (tiny_get_class_from_ss(warm_ss, i) == class_idx) {
|
||||||
slab_idx = i;
|
slab_idx = i;
|
||||||
@ -548,10 +575,6 @@ hak_base_ptr_t unified_cache_refill(int class_idx) {
|
|||||||
|
|
||||||
if (slab_idx >= 0) {
|
if (slab_idx >= 0) {
|
||||||
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
|
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
|
||||||
// Try to bind. If successful, we have "connected" the path.
|
|
||||||
// For now, we still fall through to slab_carve_from_ss() to do the actual
|
|
||||||
// work, but the side effect (TLS updated) confirms connectivity.
|
|
||||||
// In a future step, we would 'break' here and let the TLS path handle it.
|
|
||||||
uint32_t tid = (uint32_t)(uintptr_t)pthread_self();
|
uint32_t tid = (uint32_t)(uintptr_t)pthread_self();
|
||||||
if (ss_tls_bind_one(class_idx, tls, warm_ss, slab_idx, tid)) {
|
if (ss_tls_bind_one(class_idx, tls, warm_ss, slab_idx, tid)) {
|
||||||
static int logged = 0;
|
static int logged = 0;
|
||||||
@ -560,6 +583,22 @@ hak_base_ptr_t unified_cache_refill(int class_idx) {
|
|||||||
(void*)warm_ss, slab_idx);
|
(void*)warm_ss, slab_idx);
|
||||||
logged = 1;
|
logged = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Mode 2: carve a single block via TLS fast path
|
||||||
|
if (warm_mode == 2) {
|
||||||
|
void* tls_block = warm_tls_carve_one_block(class_idx);
|
||||||
|
if (tls_block) {
|
||||||
|
fprintf(stderr,
|
||||||
|
"[WARM_TLS_BIND] C7 TLS carve success: ss=%p slab=%d block=%p\n",
|
||||||
|
(void*)warm_ss, slab_idx, tls_block);
|
||||||
|
out[0] = tls_block;
|
||||||
|
produced = 1;
|
||||||
|
} else {
|
||||||
|
fprintf(stderr,
|
||||||
|
"[WARM_TLS_BIND] C7 TLS carve failed, fallback\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -567,11 +606,13 @@ hak_base_ptr_t unified_cache_refill(int class_idx) {
|
|||||||
atomic_fetch_add_explicit(&g_dbg_warm_pop_hits, 1, memory_order_relaxed);
|
atomic_fetch_add_explicit(&g_dbg_warm_pop_hits, 1, memory_order_relaxed);
|
||||||
#endif
|
#endif
|
||||||
// HOT PATH: Warm pool hit, try to carve directly
|
// HOT PATH: Warm pool hit, try to carve directly
|
||||||
|
if (produced == 0) {
|
||||||
produced = slab_carve_from_ss(class_idx, warm_ss, out, room);
|
produced = slab_carve_from_ss(class_idx, warm_ss, out, room);
|
||||||
if (produced > 0) {
|
if (produced > 0) {
|
||||||
// Update active counter for carved blocks
|
// Update active counter for carved blocks
|
||||||
ss_active_add(warm_ss, (uint32_t)produced);
|
ss_active_add(warm_ss, (uint32_t)produced);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (produced > 0) {
|
if (produced > 0) {
|
||||||
// Success! Return SuperSlab to warm pool for next use
|
// Success! Return SuperSlab to warm pool for next use
|
||||||
|
|||||||
Reference in New Issue
Block a user