tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor shared_pool Stage 0.5 EMPTY scan into helper box

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-11-30 11:38:04 +09:00
parent 5545e1633a
commit a592727b38
1 changed files with 75 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
core/hakmem_shared_pool.c
View File

@ -519,6 +519,76 @@ static SharedSSMeta* sp_meta_find_or_create(SuperSlab* ss) {
return meta; return meta;
} }
// ============================================================================
// Phase 12-1.x: Acquire Helper Boxes (Stage 0.5/1/2/3)
// ============================================================================
// Stage 0.5: EMPTY slab direct scan（registry ベースの EMPTY 再利用）
static inline int
sp_acquire_from_empty_scan(int class_idx, SuperSlab** ss_out, int* slab_idx_out, int dbg_acquire)
{
static int empty_reuse_enabled = -1;
if (__builtin_expect(empty_reuse_enabled == -1, 0)) {
const char* e = getenv("HAKMEM_SS_EMPTY_REUSE");
empty_reuse_enabled = (e && *e && *e == '0') ? 0 : 1; // default ON
}
if (!empty_reuse_enabled) {
return -1;
}
extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS];
extern int g_super_reg_class_size[TINY_NUM_CLASSES];
int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0;
static int scan_limit = -1;
if (__builtin_expect(scan_limit == -1, 0)) {
const char* e = getenv("HAKMEM_SS_EMPTY_SCAN_LIMIT");
scan_limit = (e && *e) ? atoi(e) : 16; // default: scan first 16 SuperSlabs
}
if (scan_limit > reg_size) scan_limit = reg_size;
for (int i = 0; i < scan_limit; i++) {
SuperSlab* ss = g_super_reg_by_class[class_idx][i];
if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue;
if (ss->empty_count == 0) continue; // No EMPTY slabs in this SS
uint32_t mask = ss->empty_mask;
while (mask) {
int empty_idx = __builtin_ctz(mask);
mask &= (mask - 1); // clear lowest bit
TinySlabMeta* meta = &ss->slabs[empty_idx];
if (meta->capacity > 0 && meta->used == 0) {
tiny_tls_slab_reuse_guard(ss);
ss_clear_slab_empty(ss, empty_idx);
meta->class_idx = (uint8_t)class_idx;
ss->class_map[empty_idx] = (uint8_t)class_idx;
#if !HAKMEM_BUILD_RELEASE
if (dbg_acquire == 1) {
fprintf(stderr,
"[SP_ACQUIRE_STAGE0.5_EMPTY] class=%d reusing EMPTY slab (ss=%p slab=%d empty_count=%u)\n",
class_idx, (void*)ss, empty_idx, ss->empty_count);
}
#else
(void)dbg_acquire;
#endif
*ss_out = ss;
*slab_idx_out = empty_idx;
sp_stage_stats_init();
if (g_sp_stage_stats_enabled) {
atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1);
}
return 0;
}
}
}
return -1;
}
// ---------- Layer 3: Free List Management ---------- // ---------- Layer 3: Free List Management ----------
// Push empty slot to per-class free list // Push empty slot to per-class free list
@ -871,70 +941,11 @@ shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out)
} }
stage1_retry_after_tension_drain: stage1_retry_after_tension_drain:
// ========== Stage 0.5 (NEW - Phase 12-1.1): EMPTY slab direct scan ========== // ========== Stage 0.5 (Phase 12-1.1): EMPTY slab direct scan ==========
// Scan existing SuperSlabs for EMPTY slabs (highest reuse priority) // Scan existing SuperSlabs for EMPTY slabs (highest reuse priority) to
// This avoids Stage 3 (mmap) when freed slabs are available // avoid Stage 3 (mmap) when freed slabs are available.
// ENV: HAKMEM_SS_EMPTY_REUSE=0 to disable (default ON, +557% performance) if (sp_acquire_from_empty_scan(class_idx, ss_out, slab_idx_out, dbg_acquire) == 0) {
static int empty_reuse_enabled = -1; return 0;
if (__builtin_expect(empty_reuse_enabled == -1, 0)) {
const char* e = getenv("HAKMEM_SS_EMPTY_REUSE");
empty_reuse_enabled = (e && *e && *e == '0') ? 0 : 1; // default ON
}
if (empty_reuse_enabled) {
extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS]; // from hakmem_super_registry.h
extern int g_super_reg_class_size[TINY_NUM_CLASSES];
int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0;
static int scan_limit = -1;
if (__builtin_expect(scan_limit == -1, 0)) {
const char* e = getenv("HAKMEM_SS_EMPTY_SCAN_LIMIT");
scan_limit = (e && *e) ? atoi(e) : 16; // default: scan first 16 SuperSlabs
}
if (scan_limit > reg_size) scan_limit = reg_size;
for (int i = 0; i < scan_limit; i++) {
SuperSlab* ss = g_super_reg_by_class[class_idx][i];
if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue;
if (ss->empty_count == 0) continue; // No EMPTY slabs in this SS
// Found SuperSlab with EMPTY slabs - scan empty_mask
uint32_t mask = ss->empty_mask;
while (mask) {
int empty_idx = __builtin_ctz(mask);
mask &= (mask - 1); // clear lowest bit
// Validate this slab is truly EMPTY and reusable
TinySlabMeta* meta = &ss->slabs[empty_idx];
if (meta->capacity > 0 && meta->used == 0) {
// P0.3: Guard against TLS SLL orphaned pointers before reusing slab
tiny_tls_slab_reuse_guard(ss);
// Clear EMPTY state (will be re-marked on next free)
ss_clear_slab_empty(ss, empty_idx);
// Bind this slab to class_idx
meta->class_idx = (uint8_t)class_idx;
// P1.1: Update class_map for EMPTY slab reuse
ss->class_map[empty_idx] = (uint8_t)class_idx;
#if !HAKMEM_BUILD_RELEASE
if (dbg_acquire == 1) {
fprintf(stderr, "[SP_ACQUIRE_STAGE0.5_EMPTY] class=%d reusing EMPTY slab (ss=%p slab=%d empty_count=%u)\n",
class_idx, (void*)ss, empty_idx, ss->empty_count);
}
#endif
*ss_out = ss;
*slab_idx_out = empty_idx;
sp_stage_stats_init();
if (g_sp_stage_stats_enabled) {
atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1); // Count as Stage 1 hit
}
return 0; // ✅ Stage 0.5 (EMPTY scan) success
}
}
}
} }
// ========== Stage 1 (Lock-Free): Try to reuse EMPTY slots ========== // ========== Stage 1 (Lock-Free): Try to reuse EMPTY slots ==========