diff --git a/core/hakmem_shared_pool.c b/core/hakmem_shared_pool.c index 484ba5b1..92beb38c 100644 --- a/core/hakmem_shared_pool.c +++ b/core/hakmem_shared_pool.c @@ -519,6 +519,76 @@ static SharedSSMeta* sp_meta_find_or_create(SuperSlab* ss) { return meta; } +// ============================================================================ +// Phase 12-1.x: Acquire Helper Boxes (Stage 0.5/1/2/3) +// ============================================================================ + +// Stage 0.5: EMPTY slab direct scan（registry ベースの EMPTY 再利用） +static inline int +sp_acquire_from_empty_scan(int class_idx, SuperSlab** ss_out, int* slab_idx_out, int dbg_acquire) +{ + static int empty_reuse_enabled = -1; + if (__builtin_expect(empty_reuse_enabled == -1, 0)) { + const char* e = getenv("HAKMEM_SS_EMPTY_REUSE"); + empty_reuse_enabled = (e && *e && *e == '0') ? 0 : 1; // default ON + } + + if (!empty_reuse_enabled) { + return -1; + } + + extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS]; + extern int g_super_reg_class_size[TINY_NUM_CLASSES]; + + int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0; + static int scan_limit = -1; + if (__builtin_expect(scan_limit == -1, 0)) { + const char* e = getenv("HAKMEM_SS_EMPTY_SCAN_LIMIT"); + scan_limit = (e && *e) ? atoi(e) : 16; // default: scan first 16 SuperSlabs + } + if (scan_limit > reg_size) scan_limit = reg_size; + + for (int i = 0; i < scan_limit; i++) { + SuperSlab* ss = g_super_reg_by_class[class_idx][i]; + if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue; + if (ss->empty_count == 0) continue; // No EMPTY slabs in this SS + + uint32_t mask = ss->empty_mask; + while (mask) { + int empty_idx = __builtin_ctz(mask); + mask &= (mask - 1); // clear lowest bit + + TinySlabMeta* meta = &ss->slabs[empty_idx]; + if (meta->capacity > 0 && meta->used == 0) { + tiny_tls_slab_reuse_guard(ss); + ss_clear_slab_empty(ss, empty_idx); + + meta->class_idx = (uint8_t)class_idx; + ss->class_map[empty_idx] = (uint8_t)class_idx; + +#if !HAKMEM_BUILD_RELEASE + if (dbg_acquire == 1) { + fprintf(stderr, + "[SP_ACQUIRE_STAGE0.5_EMPTY] class=%d reusing EMPTY slab (ss=%p slab=%d empty_count=%u)\n", + class_idx, (void*)ss, empty_idx, ss->empty_count); + } +#else + (void)dbg_acquire; +#endif + + *ss_out = ss; + *slab_idx_out = empty_idx; + sp_stage_stats_init(); + if (g_sp_stage_stats_enabled) { + atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1); + } + return 0; + } + } + } + return -1; +} + // ---------- Layer 3: Free List Management ---------- // Push empty slot to per-class free list @@ -871,70 +941,11 @@ shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out) } stage1_retry_after_tension_drain: - // ========== Stage 0.5 (NEW - Phase 12-1.1): EMPTY slab direct scan ========== - // Scan existing SuperSlabs for EMPTY slabs (highest reuse priority) - // This avoids Stage 3 (mmap) when freed slabs are available - // ENV: HAKMEM_SS_EMPTY_REUSE=0 to disable (default ON, +557% performance) - static int empty_reuse_enabled = -1; - if (__builtin_expect(empty_reuse_enabled == -1, 0)) { - const char* e = getenv("HAKMEM_SS_EMPTY_REUSE"); - empty_reuse_enabled = (e && *e && *e == '0') ? 0 : 1; // default ON - } - - if (empty_reuse_enabled) { - extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS]; // from hakmem_super_registry.h - extern int g_super_reg_class_size[TINY_NUM_CLASSES]; - - int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0; - static int scan_limit = -1; - if (__builtin_expect(scan_limit == -1, 0)) { - const char* e = getenv("HAKMEM_SS_EMPTY_SCAN_LIMIT"); - scan_limit = (e && *e) ? atoi(e) : 16; // default: scan first 16 SuperSlabs - } - if (scan_limit > reg_size) scan_limit = reg_size; - - for (int i = 0; i < scan_limit; i++) { - SuperSlab* ss = g_super_reg_by_class[class_idx][i]; - if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue; - if (ss->empty_count == 0) continue; // No EMPTY slabs in this SS - - // Found SuperSlab with EMPTY slabs - scan empty_mask - uint32_t mask = ss->empty_mask; - while (mask) { - int empty_idx = __builtin_ctz(mask); - mask &= (mask - 1); // clear lowest bit - - // Validate this slab is truly EMPTY and reusable - TinySlabMeta* meta = &ss->slabs[empty_idx]; - if (meta->capacity > 0 && meta->used == 0) { - // P0.3: Guard against TLS SLL orphaned pointers before reusing slab - tiny_tls_slab_reuse_guard(ss); - - // Clear EMPTY state (will be re-marked on next free) - ss_clear_slab_empty(ss, empty_idx); - - // Bind this slab to class_idx - meta->class_idx = (uint8_t)class_idx; - // P1.1: Update class_map for EMPTY slab reuse - ss->class_map[empty_idx] = (uint8_t)class_idx; - - #if !HAKMEM_BUILD_RELEASE - if (dbg_acquire == 1) { - fprintf(stderr, "[SP_ACQUIRE_STAGE0.5_EMPTY] class=%d reusing EMPTY slab (ss=%p slab=%d empty_count=%u)\n", - class_idx, (void*)ss, empty_idx, ss->empty_count); - } - #endif - - *ss_out = ss; - *slab_idx_out = empty_idx; - sp_stage_stats_init(); - if (g_sp_stage_stats_enabled) { - atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1); // Count as Stage 1 hit - } - return 0; // ✅ Stage 0.5 (EMPTY scan) success - } - } - } + // ========== Stage 0.5 (Phase 12-1.1): EMPTY slab direct scan ========== + // Scan existing SuperSlabs for EMPTY slabs (highest reuse priority) to + // avoid Stage 3 (mmap) when freed slabs are available. + if (sp_acquire_from_empty_scan(class_idx, ss_out, slab_idx_out, dbg_acquire) == 0) { + return 0; } // ========== Stage 1 (Lock-Free): Try to reuse EMPTY slots ==========