diff --git a/core/hakmem_shared_pool.c b/core/hakmem_shared_pool.c index 86151bb3..daaeb6c2 100644 --- a/core/hakmem_shared_pool.c +++ b/core/hakmem_shared_pool.c @@ -102,9 +102,7 @@ SharedSuperSlabPool g_shared_pool = { .free_slots_lockfree = {{.head = ATOMIC_VAR_INIT(NULL)}}, // Legacy: mutex-protected free lists .free_slots = {{.entries = {{0}}, .count = 0}}, - // Phase 12: SP-SLOT fields - .ss_metadata = NULL, - .ss_meta_capacity = 0, + // Phase 12: SP-SLOT fields (ss_metadata is fixed-size array, auto-zeroed) .ss_meta_count = 0 }; @@ -218,31 +216,18 @@ static int sp_slot_mark_empty(SharedSSMeta* meta, int slot_idx) { // Ensure ss_metadata array has capacity for at least min_count entries // Caller must hold alloc_lock -// Returns: 0 on success, -1 on allocation failure +// Returns: 0 on success, -1 if capacity exceeded +// RACE FIX: No realloc! Fixed-size array prevents race with lock-free Stage 2 static int sp_meta_ensure_capacity(uint32_t min_count) { - if (g_shared_pool.ss_meta_capacity >= min_count) { - return 0; - } - - uint32_t new_cap = g_shared_pool.ss_meta_capacity ? g_shared_pool.ss_meta_capacity : 16; - while (new_cap < min_count) { - new_cap *= 2; - } - - SharedSSMeta* new_meta = (SharedSSMeta*)realloc( - g_shared_pool.ss_metadata, - new_cap * sizeof(SharedSSMeta) - ); - if (!new_meta) { + if (min_count > MAX_SS_METADATA_ENTRIES) { + static int warn_once = 0; + if (warn_once == 0) { + fprintf(stderr, "[SP_META_CAPACITY_ERROR] Exceeded MAX_SS_METADATA_ENTRIES=%d\n", + MAX_SS_METADATA_ENTRIES); + warn_once = 1; + } return -1; } - - // Zero new entries - memset(new_meta + g_shared_pool.ss_meta_capacity, 0, - (new_cap - g_shared_pool.ss_meta_capacity) * sizeof(SharedSSMeta)); - - g_shared_pool.ss_metadata = new_meta; - g_shared_pool.ss_meta_capacity = new_cap; return 0; } @@ -252,20 +237,29 @@ static int sp_meta_ensure_capacity(uint32_t min_count) { static SharedSSMeta* sp_meta_find_or_create(SuperSlab* ss) { if (!ss) return NULL; + // RACE FIX: Load count atomically for consistency (even under mutex) + uint32_t count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed); + // Search existing metadata - for (uint32_t i = 0; i < g_shared_pool.ss_meta_count; i++) { - if (g_shared_pool.ss_metadata[i].ss == ss) { + for (uint32_t i = 0; i < count; i++) { + // RACE FIX: Load pointer atomically for consistency + SuperSlab* meta_ss = atomic_load_explicit(&g_shared_pool.ss_metadata[i].ss, memory_order_relaxed); + if (meta_ss == ss) { return &g_shared_pool.ss_metadata[i]; } } // Create new metadata entry - if (sp_meta_ensure_capacity(g_shared_pool.ss_meta_count + 1) != 0) { + if (sp_meta_ensure_capacity(count + 1) != 0) { return NULL; } - SharedSSMeta* meta = &g_shared_pool.ss_metadata[g_shared_pool.ss_meta_count]; - meta->ss = ss; + // RACE FIX: Read current count atomically (even under mutex for consistency) + uint32_t current_count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed); + SharedSSMeta* meta = &g_shared_pool.ss_metadata[current_count]; + + // RACE FIX: Store SuperSlab pointer atomically (visible to lock-free Stage 2) + atomic_store_explicit(&meta->ss, ss, memory_order_relaxed); meta->total_slots = (uint8_t)ss_slabs_capacity(ss); meta->active_slots = 0; @@ -277,7 +271,10 @@ static SharedSSMeta* sp_meta_find_or_create(SuperSlab* ss) { meta->slots[i].slab_idx = (uint8_t)i; } - g_shared_pool.ss_meta_count++; + // RACE FIX: Atomic increment with release semantics + // This ensures all writes to metadata[current_count] (lines 268-278) are visible + // before the count increment is visible to lock-free Stage 2 readers + atomic_fetch_add_explicit(&g_shared_pool.ss_meta_count, 1, memory_order_release); return meta; } @@ -563,7 +560,8 @@ shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out) // Activate slot under mutex (slot state transition requires protection) if (sp_slot_mark_active(reuse_meta, reuse_slot_idx, class_idx) == 0) { - SuperSlab* ss = reuse_meta->ss; + // RACE FIX: Load SuperSlab pointer atomically (consistency) + SuperSlab* ss = atomic_load_explicit(&reuse_meta->ss, memory_order_relaxed); if (dbg_acquire == 1) { fprintf(stderr, "[SP_ACQUIRE_STAGE1_LOCKFREE] class=%d reusing EMPTY slot (ss=%p slab=%d)\n", @@ -602,9 +600,10 @@ shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out) // ========== Stage 2 (Lock-Free): Try to claim UNUSED slots ========== // P0-5: Lock-free atomic CAS claiming (no mutex needed for slot state transition!) - // Read ss_meta_count atomically (safe: only grows, never shrinks) + // RACE FIX: Read ss_meta_count atomically (now properly declared as _Atomic) + // No cast needed! memory_order_acquire synchronizes with release in sp_meta_find_or_create uint32_t meta_count = atomic_load_explicit( - (_Atomic uint32_t*)&g_shared_pool.ss_meta_count, + &g_shared_pool.ss_meta_count, memory_order_acquire ); @@ -614,8 +613,13 @@ shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out) // Try lock-free claiming (UNUSED → ACTIVE via CAS) int claimed_idx = sp_slot_claim_lockfree(meta, class_idx); if (claimed_idx >= 0) { - // Successfully claimed slot! Now acquire mutex ONLY for metadata update - SuperSlab* ss = meta->ss; + // RACE FIX: Load SuperSlab pointer atomically (critical for lock-free Stage 2) + // Use memory_order_acquire to synchronize with release in sp_meta_find_or_create + SuperSlab* ss = atomic_load_explicit(&meta->ss, memory_order_acquire); + if (!ss) { + // SuperSlab was freed between claiming and loading - skip this entry + continue; + } if (dbg_acquire == 1) { fprintf(stderr, "[SP_ACQUIRE_STAGE2_LOCKFREE] class=%d claimed UNUSED slot (ss=%p slab=%d)\n", @@ -788,8 +792,11 @@ shared_pool_release_slab(SuperSlab* ss, int slab_idx) // Find SharedSSMeta for this SuperSlab SharedSSMeta* sp_meta = NULL; - for (uint32_t i = 0; i < g_shared_pool.ss_meta_count; i++) { - if (g_shared_pool.ss_metadata[i].ss == ss) { + uint32_t count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed); + for (uint32_t i = 0; i < count; i++) { + // RACE FIX: Load pointer atomically + SuperSlab* meta_ss = atomic_load_explicit(&g_shared_pool.ss_metadata[i].ss, memory_order_relaxed); + if (meta_ss == ss) { sp_meta = &g_shared_pool.ss_metadata[i]; break; } @@ -849,6 +856,11 @@ shared_pool_release_slab(SuperSlab* ss, int slab_idx) if (g_lock_stats_enabled == 1) { atomic_fetch_add(&g_lock_release_count, 1); } + + // RACE FIX: Set meta->ss to NULL BEFORE unlocking mutex + // This prevents Stage 2 from accessing freed SuperSlab + atomic_store_explicit(&sp_meta->ss, NULL, memory_order_release); + pthread_mutex_unlock(&g_shared_pool.alloc_lock); // Free SuperSlab: diff --git a/core/hakmem_shared_pool.h b/core/hakmem_shared_pool.h index b387ff55..b763ead4 100644 --- a/core/hakmem_shared_pool.h +++ b/core/hakmem_shared_pool.h @@ -50,7 +50,7 @@ typedef struct { // Per-SuperSlab metadata for slot management #define MAX_SLOTS_PER_SS 32 // Typical: 1MB SS has 32 slabs of 32KB each typedef struct SharedSSMeta { - SuperSlab* ss; // Physical SuperSlab pointer + _Atomic(SuperSlab*) ss; // Physical SuperSlab pointer (atomic for lock-free Stage 2) SharedSlot slots[MAX_SLOTS_PER_SS]; // Slot state for each slab uint8_t active_slots; // Number of SLOT_ACTIVE slots uint8_t total_slots; // Total available slots (from ss_slabs_capacity) @@ -120,9 +120,10 @@ typedef struct SharedSuperSlabPool { FreeSlotList free_slots[TINY_NUM_CLASSES_SS]; // SharedSSMeta array for all SuperSlabs in pool - SharedSSMeta* ss_metadata; // Dynamic array - uint32_t ss_meta_capacity; // Allocated entries - uint32_t ss_meta_count; // Used entries + // RACE FIX: Fixed-size array (no realloc!) to avoid race with lock-free Stage 2 +#define MAX_SS_METADATA_ENTRIES 2048 + SharedSSMeta ss_metadata[MAX_SS_METADATA_ENTRIES]; // Fixed-size array + _Atomic uint32_t ss_meta_count; // Used entries (atomic for lock-free Stage 2) } SharedSuperSlabPool; // Global singleton