#include "free_local_box.h" #include "free_publish_box.h" #include "hakmem_tiny.h" #include "tiny_next_ptr_box.h" // Phase E1-CORRECT: Box API #include "ss_hot_cold_box.h" // Phase 12-1.1: EMPTY slab marking #include "tiny_region_id.h" // HEADER_MAGIC / HEADER_CLASS_MASK // Local prototypes (fail-fast helpers live in tiny_failfast.c) int tiny_refill_failfast_level(void); void tiny_failfast_abort_ptr(const char* stage, SuperSlab* ss, int slab_idx, void* ptr, const char* reason); void tiny_failfast_log(const char* stage, int class_idx, SuperSlab* ss, TinySlabMeta* meta, void* ptr, void* prev); void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void* ptr, uint32_t my_tid) { extern _Atomic uint64_t g_free_local_box_calls; atomic_fetch_add_explicit(&g_free_local_box_calls, 1, memory_order_relaxed); if (!(ss && ss->magic == SUPERSLAB_MAGIC)) return; if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) return; (void)my_tid; // ✅ Phase E1-CORRECT: ALL classes have headers, calculate BASE pointer once void* base = (void*)((uint8_t*)ptr - 1); // Targeted header integrity check (env: HAKMEM_TINY_SLL_DIAG, C7 focus) do { static int g_free_diag_en = -1; static _Atomic uint32_t g_free_diag_shot = 0; if (__builtin_expect(g_free_diag_en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_SLL_DIAG"); g_free_diag_en = (e && *e && *e != '0') ? 1 : 0; } if (__builtin_expect(g_free_diag_en && meta && meta->class_idx == 7, 0)) { uint8_t hdr = *(uint8_t*)base; uint8_t expect = (uint8_t)(HEADER_MAGIC | (meta->class_idx & HEADER_CLASS_MASK)); if (hdr != expect) { uint32_t shot = atomic_fetch_add_explicit(&g_free_diag_shot, 1, memory_order_relaxed); if (shot < 8) { fprintf(stderr, "[C7_FREE_HDR_DIAG] ss=%p slab=%d base=%p hdr=0x%02x expect=0x%02x freelist=%p used=%u\n", (void*)ss, slab_idx, base, hdr, expect, meta ? meta->freelist : NULL, meta ? meta->used : 0); } } } } while (0); if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { int actual_idx = slab_index_for(ss, base); if (actual_idx != slab_idx) { tiny_failfast_abort_ptr("free_local_box_idx", ss, slab_idx, ptr, "slab_idx_mismatch"); } else { uint8_t cls = (meta && meta->class_idx < TINY_NUM_CLASSES) ? meta->class_idx : 0; size_t blk = g_tiny_class_sizes[cls]; uint8_t* slab_base = tiny_slab_base_for(ss, slab_idx); uintptr_t delta = (uintptr_t)base - (uintptr_t)slab_base; if (blk == 0 || (delta % blk) != 0) { tiny_failfast_abort_ptr("free_local_box_align", ss, slab_idx, ptr, "misaligned"); } else if (meta && delta / blk >= meta->capacity) { tiny_failfast_abort_ptr("free_local_box_range", ss, slab_idx, ptr, "out_of_capacity"); } } } void* prev = meta->freelist; // Detect suspicious prev before writing next (env-gated) do { static int g_prev_diag_en = -1; static _Atomic uint32_t g_prev_diag_shot = 0; if (__builtin_expect(g_prev_diag_en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_SLL_DIAG"); g_prev_diag_en = (e && *e && *e != '0') ? 1 : 0; } if (__builtin_expect(g_prev_diag_en && prev && ((uintptr_t)prev < 4096 || (uintptr_t)prev > 0x00007fffffffffffULL), 0)) { uint8_t cls_dbg = (meta && meta->class_idx < TINY_NUM_CLASSES) ? meta->class_idx : 0xFF; uint32_t shot = atomic_fetch_add_explicit(&g_prev_diag_shot, 1, memory_order_relaxed); if (shot < 8) { fprintf(stderr, "[FREELIST_PREV_INVALID] cls=%u slab=%d ptr=%p base=%p prev=%p freelist=%p used=%u\n", cls_dbg, slab_idx, ptr, base, prev, meta ? meta->freelist : NULL, meta ? meta->used : 0); } } } while (0); // FREELIST CORRUPTION DEBUG: Validate pointer before writing if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { uint8_t cls = (meta && meta->class_idx < TINY_NUM_CLASSES) ? meta->class_idx : 0; size_t blk = g_tiny_class_sizes[cls]; uint8_t* base_ss = (uint8_t*)ss; uint8_t* slab_base = tiny_slab_base_for(ss, slab_idx); // Verify prev pointer is valid (if not NULL) if (prev != NULL) { uintptr_t prev_addr = (uintptr_t)prev; uintptr_t slab_addr = (uintptr_t)slab_base; // Check if prev is within this slab if (prev_addr < (uintptr_t)base_ss || prev_addr >= (uintptr_t)base_ss + (2*1024*1024)) { fprintf(stderr, "[FREE_CORRUPT] prev=%p outside SuperSlab ss=%p slab=%d\n", prev, ss, slab_idx); tiny_failfast_abort_ptr("free_local_prev_range", ss, slab_idx, ptr, "prev_outside_ss"); } // Check alignment of prev if ((prev_addr - slab_addr) % blk != 0) { fprintf(stderr, "[FREE_CORRUPT] prev=%p misaligned (cls=%u slab=%d blk=%zu offset=%zu)\n", prev, cls, slab_idx, blk, (size_t)(prev_addr - slab_addr)); fprintf(stderr, "[FREE_CORRUPT] Writing from ptr=%p, freelist was=%p\n", ptr, prev); tiny_failfast_abort_ptr("free_local_prev_misalign", ss, slab_idx, ptr, "prev_misaligned"); } } fprintf(stderr, "[FREE_VERIFY] cls=%u slab=%d ptr=%p prev=%p (offset_ptr=%zu offset_prev=%zu)\n", cls, slab_idx, ptr, prev, (size_t)((uintptr_t)base - (uintptr_t)slab_base), prev ? (size_t)((uintptr_t)prev - (uintptr_t)slab_base) : 0); } // Use per-slab class for freelist linkage (BASE pointers only) uint8_t cls = (meta && meta->class_idx < TINY_NUM_CLASSES) ? meta->class_idx : 0; tiny_next_write(cls, base, prev); // Phase E1-CORRECT: Box API with shared pool meta->freelist = base; // FREELIST CORRUPTION DEBUG: Verify write succeeded if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { void* readback = tiny_next_read(cls, ptr); // Phase E1-CORRECT: Box API if (readback != prev) { fprintf(stderr, "[FREE_CORRUPT] Wrote prev=%p to ptr=%p but read back %p!\n", prev, ptr, readback); fprintf(stderr, "[FREE_CORRUPT] Memory corruption detected during freelist push\n"); tiny_failfast_abort_ptr("free_local_readback", ss, slab_idx, ptr, "write_corrupted"); } } tiny_failfast_log("free_local_box", cls, ss, meta, base, prev); // BUGFIX: Memory barrier to ensure freelist visibility before used decrement // Without this, other threads can see new freelist but old used count (race) atomic_thread_fence(memory_order_release); // Optional freelist mask update on first push do { static int g_mask_en = -1; if (__builtin_expect(g_mask_en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_FREELIST_MASK"); g_mask_en = (e && *e && *e != '0') ? 1 : 0; } if (__builtin_expect(g_mask_en, 0) && prev == NULL) { uint32_t bit = (1u << slab_idx); atomic_fetch_or_explicit(&ss->freelist_mask, bit, memory_order_release); } } while (0); // Track local free (debug helpers may be no-op) tiny_remote_track_on_local_free(ss, slab_idx, ptr, "local_free", my_tid); meta->used--; ss_active_dec_one(ss); // Phase 12-1.1: EMPTY slab detection (immediate reuse optimization) if (meta->used == 0) { // Slab became EMPTY → mark for highest-priority reuse ss_mark_slab_empty(ss, slab_idx); // DEBUG LOGGING - Track when used reaches 0 static int dbg = -1; if (__builtin_expect(dbg == -1, 0)) { const char* e = getenv("HAKMEM_SS_FREE_DEBUG"); dbg = (e && *e && *e != '0') ? 1 : 0; } if (dbg == 1) { fprintf(stderr, "[FREE_LOCAL_BOX] EMPTY detected: cls=%u ss=%p slab=%d empty_mask=0x%x empty_count=%u\n", cls, (void*)ss, slab_idx, ss->empty_mask, ss->empty_count); } } if (prev == NULL) { // First-free → advertise slab to adopters using per-slab class uint8_t cls0 = (meta && meta->class_idx < TINY_NUM_CLASSES) ? meta->class_idx : 0; tiny_free_publish_first_free((int)cls0, ss, slab_idx); } }