#include "hakmem_shared_pool_internal.h" #include "hakmem_debug_master.h" #include "box/ss_slab_meta_box.h" #include "box/ss_hot_cold_box.h" #include "box/ss_tier_box.h" // P-Tier: Utilization-aware tiering #include "hakmem_env_cache.h" // Priority-2: ENV cache #include "superslab/superslab_inline.h" // superslab_ref_get guard for TLS pins #include "box/ss_release_guard_box.h" // Box: SuperSlab Release Guard #include "box/ss_slab_reset_box.h" // Box: Reset slab metadata on reuse path #include "box/ss_stats_box.h" // Observability: Superslab/slab counters #include "box/ss_budget_box.h" // Budget guard (global/class caps) #include #include #include static inline void c7_release_log_once(SuperSlab* ss, int slab_idx) { #if HAKMEM_BUILD_RELEASE static _Atomic uint32_t rel_c7_release_logs = 0; uint32_t n = atomic_fetch_add_explicit(&rel_c7_release_logs, 1, memory_order_relaxed); if (n < 8) { TinySlabMeta* meta = &ss->slabs[slab_idx]; fprintf(stderr, "[REL_C7_RELEASE] ss=%p slab=%d used=%u cap=%u carved=%u\n", (void*)ss, slab_idx, (unsigned)meta->used, (unsigned)meta->capacity, (unsigned)meta->carved); } #else static _Atomic uint32_t dbg_c7_release_logs = 0; uint32_t n = atomic_fetch_add_explicit(&dbg_c7_release_logs, 1, memory_order_relaxed); if (n < 8) { TinySlabMeta* meta = &ss->slabs[slab_idx]; fprintf(stderr, "[DBG_C7_RELEASE] ss=%p slab=%d used=%u cap=%u carved=%u\n", (void*)ss, slab_idx, (unsigned)meta->used, (unsigned)meta->capacity, (unsigned)meta->carved); } #endif } void shared_pool_release_slab(SuperSlab* ss, int slab_idx) { // Phase 12: SP-SLOT Box - Slot-based Release // // Flow: // 1. Validate inputs and check meta->used == 0 // 2. Find SharedSSMeta for this SuperSlab // 3. Mark slot ACTIVE → EMPTY // 4. Push to per-class free list (enables same-class reuse) // 5. If all slots EMPTY → superslab_free() → LRU cache if (!ss) { return; } if (slab_idx < 0 || slab_idx >= SLABS_PER_SUPERSLAB_MAX) { return; } // Phase 9-2 FIX: Promote Legacy SuperSlabs to Shared Pool on first recycle // If we are recycling a slot from a Legacy SS, we must remove it from the // Legacy list (g_superslab_heads) to prevent Legacy Backend from allocating // from it simultaneously (Double Allocation Race). // This effectively transfers ownership to Shared Pool. extern void remove_superslab_from_legacy_head(SuperSlab* ss); remove_superslab_from_legacy_head(ss); // BUGFIX: Re-check used count after removal. Legacy Backend might have // allocated from this slab while we were waiting for the lock in remove(). TinySlabMeta* slab_meta = &ss->slabs[slab_idx]; if (atomic_load_explicit(&slab_meta->used, memory_order_acquire) != 0) { // Legacy Backend stole this slab. It's now an orphan (removed from list). // We abort recycling. It will be recycled when Legacy frees it later. return; } // Debug logging #if !HAKMEM_BUILD_RELEASE // Priority-2: Use cached ENV int dbg = HAK_ENV_SS_FREE_DEBUG(); #else static const int dbg = 0; #endif (void)dbg; // P0 instrumentation: count lock acquisitions lock_stats_init(); if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_stats_enabled, 1); atomic_fetch_add(&g_lock_release_slab_count, 1); } pthread_mutex_lock(&g_shared_pool.alloc_lock); // TinySlabMeta* slab_meta = &ss->slabs[slab_idx]; // Already declared above if (slab_meta->used != 0) { // Not actually empty (double check under lock) if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } pthread_mutex_unlock(&g_shared_pool.alloc_lock); return; } uint8_t class_idx = slab_meta->class_idx; if (class_idx == 7) { c7_release_log_once(ss, slab_idx); } // Guard: if SuperSlab is pinned (TLS/remote references), defer release to avoid // class_map=255 while pointers are still in-flight. uint32_t ss_refs_guard = superslab_ref_get(ss); if (ss_refs_guard != 0) { #if !HAKMEM_BUILD_RELEASE if (dbg == 1) { fprintf(stderr, "[SP_SLOT_RELEASE_SKIP_PINNED] ss=%p slab_idx=%d class=%d refcount=%u\n", (void*)ss, slab_idx, class_idx, (unsigned)ss_refs_guard); } #endif if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } pthread_mutex_unlock(&g_shared_pool.alloc_lock); return; } #if !HAKMEM_BUILD_RELEASE if (dbg == 1) { fprintf(stderr, "[SP_SLOT_RELEASE] ss=%p slab_idx=%d class=%d used=0 (marking EMPTY)\n", (void*)ss, slab_idx, class_idx); } #endif if (class_idx == 7) { ss_slab_reset_meta_for_tiny(ss, slab_idx, class_idx); #if HAKMEM_BUILD_RELEASE static _Atomic uint32_t rel_c7_reset_logs = 0; uint32_t rn = atomic_fetch_add_explicit(&rel_c7_reset_logs, 1, memory_order_relaxed); if (rn < 4) { TinySlabMeta* m = &ss->slabs[slab_idx]; fprintf(stderr, "[REL_C7_RELEASE_RESET] ss=%p slab=%d used=%u cap=%u carved=%u freelist=%p\n", (void*)ss, slab_idx, (unsigned)m->used, (unsigned)m->capacity, (unsigned)m->carved, m->freelist); } #else static _Atomic uint32_t dbg_c7_reset_logs = 0; uint32_t rn = atomic_fetch_add_explicit(&dbg_c7_reset_logs, 1, memory_order_relaxed); if (rn < 4) { TinySlabMeta* m = &ss->slabs[slab_idx]; fprintf(stderr, "[DBG_C7_RELEASE_RESET] ss=%p slab=%d used=%u cap=%u carved=%u freelist=%p\n", (void*)ss, slab_idx, (unsigned)m->used, (unsigned)m->capacity, (unsigned)m->carved, m->freelist); } #endif } // Find SharedSSMeta for this SuperSlab SharedSSMeta* sp_meta = NULL; uint32_t count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed); for (uint32_t i = 0; i < count; i++) { // RACE FIX: Load pointer atomically SuperSlab* meta_ss = atomic_load_explicit(&g_shared_pool.ss_metadata[i].ss, memory_order_relaxed); if (meta_ss == ss) { sp_meta = &g_shared_pool.ss_metadata[i]; break; } } if (!sp_meta) { // SuperSlab not in SP-SLOT system yet - create metadata sp_meta = sp_meta_find_or_create(ss); if (!sp_meta) { pthread_mutex_unlock(&g_shared_pool.alloc_lock); return; // Failed to create metadata } } // Mark slot as EMPTY (ACTIVE → EMPTY) uint32_t slab_bit = (1u << slab_idx); SlotState slot_state = atomic_load_explicit( &sp_meta->slots[slab_idx].state, memory_order_acquire); if (slot_state != SLOT_ACTIVE && (ss->slab_bitmap & slab_bit)) { // Legacy path import: rebuild slot states from SuperSlab bitmap/class_map sp_meta_sync_slots_from_ss(sp_meta, ss); slot_state = atomic_load_explicit( &sp_meta->slots[slab_idx].state, memory_order_acquire); } if (slot_state != SLOT_ACTIVE || sp_slot_mark_empty(sp_meta, slab_idx) != 0) { if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } pthread_mutex_unlock(&g_shared_pool.alloc_lock); return; // Slot wasn't ACTIVE } // Update SuperSlab metadata uint32_t bit = (1u << slab_idx); if (ss->slab_bitmap & bit) { ss->slab_bitmap &= ~bit; slab_meta->class_idx = 255; // UNASSIGNED // P1.1: Mark class_map as UNASSIGNED when releasing slab ss->class_map[slab_idx] = 255; // Reset slab metadata to a pristine state for all classes (C0–C7) ss_slab_reset_meta_for_tiny(ss, slab_idx, -1); if (ss->active_slabs > 0) { ss->active_slabs--; if (ss->active_slabs == 0 && g_shared_pool.active_count > 0) { g_shared_pool.active_count--; } } if (class_idx < TINY_NUM_CLASSES_SS && g_shared_pool.class_active_slots[class_idx] > 0) { g_shared_pool.class_active_slots[class_idx]--; } } // P0-4: Push to lock-free per-class free list (enables reuse by same class) // Note: push BEFORE releasing mutex (slot state already updated under lock) if (class_idx < TINY_NUM_CLASSES_SS) { sp_freelist_push_lockfree(class_idx, sp_meta, slab_idx); #if !HAKMEM_BUILD_RELEASE if (dbg == 1) { fprintf(stderr, "[SP_SLOT_FREELIST_LOCKFREE] class=%d pushed slot (ss=%p slab=%d) active_slots=%u/%u\n", class_idx, (void*)ss, slab_idx, sp_meta->active_slots, sp_meta->total_slots); } #endif } // P-Tier: Check tier transition after releasing slab // This may transition HOT → DRAINING if utilization dropped below threshold // or DRAINING → FREE if utilization reached 0 ss_tier_check_transition(ss); // P-Tier Step B: Eager FREE eviction // If tier transitioned to FREE (total_active_blocks == 0), immediately try to // release the SuperSlab regardless of active_slots. This prevents registry bloat. SSTier current_tier = ss_tier_get(ss); if (current_tier == SS_TIER_FREE) { // Double-check: total_active_blocks should be 0 for FREE tier uint32_t active_blocks = atomic_load_explicit(&ss->total_active_blocks, memory_order_acquire); if (active_blocks == 0 && ss_release_guard_superslab_can_free(ss)) { #if !HAKMEM_BUILD_RELEASE if (dbg == 1) { fprintf(stderr, "[SP_TIER_FREE_EAGER] ss=%p tier=FREE active_slots=%u -> immediate free\n", (void*)ss, sp_meta->active_slots); } #endif // Force all remaining slots to EMPTY state for clean metadata for (uint32_t i = 0; i < sp_meta->total_slots; i++) { SlotState st = atomic_load_explicit(&sp_meta->slots[i].state, memory_order_relaxed); if (st == SLOT_ACTIVE) { atomic_store_explicit(&sp_meta->slots[i].state, SLOT_EMPTY, memory_order_relaxed); } } sp_meta->active_slots = 0; if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } // Clear meta->ss before unlocking (race prevention) atomic_store_explicit(&sp_meta->ss, NULL, memory_order_release); pthread_mutex_unlock(&g_shared_pool.alloc_lock); // Free SuperSlab immediately (bypasses normal active_slots==0 check) extern void superslab_free(SuperSlab* ss); ss_stats_on_ss_free_class(class_idx); ss_budget_on_free(class_idx); superslab_free(ss); return; } } // Check if SuperSlab is now completely empty (all slots EMPTY or UNUSED) if (sp_meta->active_slots == 0) { #if !HAKMEM_BUILD_RELEASE if (dbg == 1) { fprintf(stderr, "[SP_SLOT_COMPLETELY_EMPTY] ss=%p active_slots=0 (calling superslab_free)\n", (void*)ss); } #endif if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } // RACE FIX: Set meta->ss to NULL BEFORE unlocking mutex // This prevents Stage 2 from accessing freed SuperSlab atomic_store_explicit(&sp_meta->ss, NULL, memory_order_release); pthread_mutex_unlock(&g_shared_pool.alloc_lock); // Remove from legacy backend list (moved to top of function) // extern void remove_superslab_from_legacy_head(SuperSlab* ss); // remove_superslab_from_legacy_head(ss); // Free SuperSlab: // 1. Try LRU cache (hak_ss_lru_push) - lazy deallocation // 2. Or munmap if LRU is full - eager deallocation // BUGFIX: Double check total_active_blocks and refcount. Legacy Backend might have // allocated from ANOTHER slab in this SS just before we removed it. // If so, we must NOT free the SS. if (ss_release_guard_superslab_can_free(ss)) { extern void superslab_free(SuperSlab* ss); ss_stats_on_ss_free_class(class_idx); ss_budget_on_free(class_idx); superslab_free(ss); } else { #if !HAKMEM_BUILD_RELEASE if (dbg == 1) { uint32_t active_blocks = atomic_load_explicit(&ss->total_active_blocks, memory_order_acquire); uint32_t ss_refs = superslab_ref_get(ss); fprintf(stderr, "[SP_SLOT_RELEASE] SKIP free ss=%p: total_active_blocks=%u refcount=%u\n", (void*)ss, (unsigned)active_blocks, (unsigned)ss_refs); } #endif } return; } if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } pthread_mutex_unlock(&g_shared_pool.alloc_lock); }