tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: Add TLS carve experiment for warm C7

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-05 20:50:24 +09:00
parent 3e1d7c3798
commit e96e9a4bf9
1 changed files with 85 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
core/front/tiny_unified_cache.c
View File

@ -19,6 +19,7 @@
#include <stdlib.h>
#include <string.h>
#include <stdatomic.h>
#include <stdio.h>
#include <time.h>
// ============================================================================
@ -97,6 +98,45 @@ _Atomic uint64_t g_dbg_warm_pop_attempts = 0;
_Atomic uint64_t g_dbg_warm_pop_hits = 0;
_Atomic uint64_t g_dbg_warm_pop_empty = 0;
_Atomic uint64_t g_dbg_warm_pop_carve_zero = 0;
// Debug-only: cached ENV for Warm TLS Bind (C7)
static int g_warm_tls_bind_mode_c7 = -1;
static inline int warm_tls_bind_mode_c7(void) {
if (__builtin_expect(g_warm_tls_bind_mode_c7 == -1, 0)) {
const char* e = getenv("HAKMEM_WARM_TLS_BIND_C7");
// 0/empty: disabled, 1: bind only, 2: bind + TLS carve one block
g_warm_tls_bind_mode_c7 = (e && *e) ? atoi(e) : 0;
}
return g_warm_tls_bind_mode_c7;
}
static inline void* warm_tls_carve_one_block(int class_idx) {
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
TinySlabMeta* meta = tls->meta;
if (!meta || !tls->ss || tls->slab_base == NULL) return NULL;
if (meta->class_idx != (uint8_t)class_idx) return NULL;
if (tls->slab_idx < 0 || tls->slab_idx >= ss_slabs_capacity(tls->ss)) return NULL;
if (meta->freelist) {
void* block = meta->freelist;
meta->freelist = tiny_next_read(class_idx, block);
meta->used++;
ss_active_add(tls->ss, 1);
return block;
}
if (meta->used < meta->capacity) {
size_t block_size = tiny_stride_for_class(meta->class_idx);
void* block = tiny_block_at_index(tls->slab_base, meta->used, block_size);
meta->used++;
ss_active_add(tls->ss, 1);
return block;
}
return NULL;
}
#endif
// Forward declaration for Warm Pool stats printer (defined later in this file)
@ -518,47 +558,46 @@ hak_base_ptr_t unified_cache_refill(int class_idx) {
SuperSlab* warm_ss = tiny_warm_pool_pop(class_idx);
if (warm_ss) {
#if !HAKMEM_BUILD_RELEASE
// FUTURE: TLS Bind Box Integration
// Currently we carve directly from warm_ss via slab_carve_from_ss().
// To unify logic, we should eventually:
// 1. Choose a slab index (via tiny_page_box or heuristic).
// 2. Bind it to TLS via ss_tls_bind_one(..., warm_ss, slab_idx, ...).
// 3. Fall through to TLS-based allocation.
// EXPERIMENTAL: Test TLS Bind Box connectivity for C7 (Debug only)
static int g_warm_tls_bind_c7 = -1;
if (g_warm_tls_bind_c7 == -1) {
const char* e = getenv("HAKMEM_WARM_TLS_BIND_C7");
g_warm_tls_bind_c7 = (e && *e && *e != '0') ? 1 : 0;
}
if (g_warm_tls_bind_c7 && class_idx == 7) {
// Find a slab index in this SuperSlab that matches our class
int cap = ss_slabs_capacity(warm_ss);
int slab_idx = -1;
// Simple heuristic: find first slab belonging to this class
// Note: In real logic, we should pick the *best* slab (e.g. from PageBox)
for (int i = 0; i < cap; i++) {
if (tiny_get_class_from_ss(warm_ss, i) == class_idx) {
slab_idx = i;
break;
// Debug-only: Warm TLS Bind experiment (C7 only)
if (class_idx == 7) {
int warm_mode = warm_tls_bind_mode_c7();
if (warm_mode >= 1) {
int cap = ss_slabs_capacity(warm_ss);
int slab_idx = -1;
// Simple heuristic: first slab matching class
for (int i = 0; i < cap; i++) {
if (tiny_get_class_from_ss(warm_ss, i) == class_idx) {
slab_idx = i;
break;
}
}
}
if (slab_idx >= 0) {
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
// Try to bind. If successful, we have "connected" the path.
// For now, we still fall through to slab_carve_from_ss() to do the actual
// work, but the side effect (TLS updated) confirms connectivity.
// In a future step, we would 'break' here and let the TLS path handle it.
uint32_t tid = (uint32_t)(uintptr_t)pthread_self();
if (ss_tls_bind_one(class_idx, tls, warm_ss, slab_idx, tid)) {
static int logged = 0;
if (!logged) {
fprintf(stderr, "[WARM_TLS_BIND] C7 bind success: ss=%p slab=%d\n",
(void*)warm_ss, slab_idx);
logged = 1;
if (slab_idx >= 0) {
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
uint32_t tid = (uint32_t)(uintptr_t)pthread_self();
if (ss_tls_bind_one(class_idx, tls, warm_ss, slab_idx, tid)) {
static int logged = 0;
if (!logged) {
fprintf(stderr, "[WARM_TLS_BIND] C7 bind success: ss=%p slab=%d\n",
(void*)warm_ss, slab_idx);
logged = 1;
}
// Mode 2: carve a single block via TLS fast path
if (warm_mode == 2) {
void* tls_block = warm_tls_carve_one_block(class_idx);
if (tls_block) {
fprintf(stderr,
"[WARM_TLS_BIND] C7 TLS carve success: ss=%p slab=%d block=%p\n",
(void*)warm_ss, slab_idx, tls_block);
out[0] = tls_block;
produced = 1;
} else {
fprintf(stderr,
"[WARM_TLS_BIND] C7 TLS carve failed, fallback\n");
}
}
}
}
}
@ -567,10 +606,12 @@ hak_base_ptr_t unified_cache_refill(int class_idx) {
atomic_fetch_add_explicit(&g_dbg_warm_pop_hits, 1, memory_order_relaxed);
#endif
// HOT PATH: Warm pool hit, try to carve directly
produced = slab_carve_from_ss(class_idx, warm_ss, out, room);
if (produced > 0) {
// Update active counter for carved blocks
ss_active_add(warm_ss, (uint32_t)produced);
if (produced == 0) {
produced = slab_carve_from_ss(class_idx, warm_ss, out, room);
if (produced > 0) {
// Update active counter for carved blocks
ss_active_add(warm_ss, (uint32_t)produced);
}
}
if (produced > 0) {