diff --git a/core/box/free_local_box.c b/core/box/free_local_box.c index f72cd613..72f6b4b1 100644 --- a/core/box/free_local_box.c +++ b/core/box/free_local_box.c @@ -1,6 +1,7 @@ #include "free_local_box.h" #include "free_publish_box.h" #include "hakmem_tiny.h" +#include "tiny_next_ptr_box.h" // Phase E1-CORRECT: Box API void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void* ptr, uint32_t my_tid) { extern _Atomic uint64_t g_free_local_box_calls; @@ -9,14 +10,17 @@ void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void* if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) return; (void)my_tid; + // ✅ Phase E1-CORRECT: ALL classes have headers, calculate BASE pointer once + void* base = (void*)((uint8_t*)ptr - 1); + if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { - int actual_idx = slab_index_for(ss, ptr); + int actual_idx = slab_index_for(ss, base); if (actual_idx != slab_idx) { tiny_failfast_abort_ptr("free_local_box_idx", ss, slab_idx, ptr, "slab_idx_mismatch"); } else { size_t blk = g_tiny_class_sizes[ss->size_class]; uint8_t* slab_base = tiny_slab_base_for(ss, slab_idx); - uintptr_t delta = (uintptr_t)ptr - (uintptr_t)slab_base; + uintptr_t delta = (uintptr_t)base - (uintptr_t)slab_base; if (blk == 0 || (delta % blk) != 0) { tiny_failfast_abort_ptr("free_local_box_align", ss, slab_idx, ptr, "misaligned"); } else if (meta && delta / blk >= meta->capacity) { @@ -56,16 +60,16 @@ void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void* fprintf(stderr, "[FREE_VERIFY] cls=%u slab=%d ptr=%p prev=%p (offset_ptr=%zu offset_prev=%zu)\n", ss->size_class, slab_idx, ptr, prev, - (size_t)((uintptr_t)ptr - (uintptr_t)slab_base), + (size_t)((uintptr_t)base - (uintptr_t)slab_base), prev ? (size_t)((uintptr_t)prev - (uintptr_t)slab_base) : 0); } - *(void**)ptr = prev; + tiny_next_write(ss->size_class, ptr, prev); // Phase E1-CORRECT: Box API meta->freelist = ptr; // FREELIST CORRUPTION DEBUG: Verify write succeeded if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { - void* readback = *(void**)ptr; + void* readback = tiny_next_read(ss->size_class, ptr); // Phase E1-CORRECT: Box API if (readback != prev) { fprintf(stderr, "[FREE_CORRUPT] Wrote prev=%p to ptr=%p but read back %p!\n", prev, ptr, readback); diff --git a/core/hakmem_tiny.c b/core/hakmem_tiny.c index dcdf70fa..77e32d8d 100644 --- a/core/hakmem_tiny.c +++ b/core/hakmem_tiny.c @@ -440,12 +440,14 @@ extern int g_use_superslab; #if !HAKMEM_BUILD_RELEASE static inline void tiny_debug_track_alloc_ret(int cls, void* ptr) { if (!ptr) return; + // ✅ Phase E1-CORRECT: ptr is USER pointer, convert to BASE + void* base_ptr = ptr ? (void*)((uint8_t*)ptr - 1) : NULL; if (g_use_superslab && __builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { SuperSlab* ss = hak_super_lookup(ptr); if (!(ss && ss->magic == SUPERSLAB_MAGIC)) { tiny_failfast_abort_ptr("alloc_ret_lookup", ss, -1, ptr, "lookup_fail"); } else { - int slab_idx = slab_index_for(ss, ptr); + int slab_idx = slab_index_for(ss, base_ptr); if (slab_idx < 0) { tiny_failfast_abort_ptr("alloc_ret_slabidx", ss, slab_idx, ptr, "slab_idx_mismatch"); } else { @@ -455,7 +457,7 @@ static inline void tiny_debug_track_alloc_ret(int cls, void* ptr) { } size_t blk = g_tiny_class_sizes[cls]; uintptr_t base = (uintptr_t)tiny_slab_base_for(ss, slab_idx); - uintptr_t delta = (uintptr_t)ptr - base; + uintptr_t delta = (uintptr_t)base_ptr - base; if (blk == 0 || (delta % blk) != 0) { tiny_failfast_abort_ptr("alloc_ret_align", ss, slab_idx, ptr, "misaligned"); } else if (delta / blk >= ss->slabs[slab_idx].capacity) { @@ -468,7 +470,7 @@ static inline void tiny_debug_track_alloc_ret(int cls, void* ptr) { if (!g_use_superslab) return; SuperSlab* ss = hak_super_lookup(ptr); if (!(ss && ss->magic == SUPERSLAB_MAGIC)) return; - int slab_idx = slab_index_for(ss, ptr); + int slab_idx = slab_index_for(ss, base_ptr); if (slab_idx >= 0) { tiny_remote_track_on_alloc(ss, slab_idx, ptr, "alloc_ret", 0); } diff --git a/core/hakmem_tiny_free.inc b/core/hakmem_tiny_free.inc index bbf212e7..d2e3ef99 100644 --- a/core/hakmem_tiny_free.inc +++ b/core/hakmem_tiny_free.inc @@ -6,6 +6,7 @@ #include "tiny_tls_guard.h" #include "box/free_publish_box.h" #include "box/tls_sll_box.h" // Box TLS-SLL: C7-safe push/pop/splice +#include "box/tiny_next_ptr_box.h" // Box API: next pointer read/write #include "mid_tcache.h" extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES]; extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES]; @@ -29,9 +30,8 @@ static inline int tiny_drain_to_sll_budget(void) { static inline void tiny_drain_freelist_to_sll_once(SuperSlab* ss, int slab_idx, int class_idx) { int budget = tiny_drain_to_sll_budget(); if (__builtin_expect(budget <= 0, 1)) return; - // CRITICAL: C7 (1KB) is headerless - MUST NOT drain to TLS SLL - // Reason: SLL stores next pointer in first 8 bytes (user data for C7) - if (__builtin_expect(class_idx == 7, 0)) return; + // Phase E1-CORRECT: C7 now has headers, can use TLS SLL like other classes + // (removed early return for class_idx == 7) if (!(ss && ss->magic == SUPERSLAB_MAGIC)) return; if (slab_idx < 0) return; TinySlabMeta* m = &ss->slabs[slab_idx]; @@ -67,7 +67,7 @@ static inline void tiny_drain_freelist_to_sll_once(SuperSlab* ss, int slab_idx, class_idx, p, old_head, moved, budget); } - m->freelist = *(void**)p; + m->freelist = tiny_next_read(class_idx, p); // Phase E1-CORRECT: Box API // Use Box TLS-SLL API (C7-safe push) // Note: C7 already rejected at line 34, so this always succeeds @@ -119,9 +119,8 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { return; } // Optional: cross-lookup TinySlab owner and detect class mismatch early - // Class7(1KB)はヘッダ無しのため、releaseでも軽量ガードを常時適用 - // それ以外は従来どおり g_tiny_safe_free で切替 - if (__builtin_expect(g_tiny_safe_free || class_idx == 7, 0)) { + // Phase E1-CORRECT: All classes have headers now, standard safe_free guard + if (__builtin_expect(g_tiny_safe_free, 0)) { TinySlab* ts = hak_tiny_owner_slab(ptr); if (ts) { int ts_cls = ts->class_idx; @@ -135,17 +134,20 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { } tiny_debug_ring_record(TINY_RING_EVENT_FREE_ENTER, (uint16_t)class_idx, ptr, 0); // Detect cross-thread: cross-thread free MUST go via superslab path - int slab_idx = slab_index_for(ss, ptr); + // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation + void* base = (void*)((uint8_t*)ptr - 1); + int slab_idx = slab_index_for(ss, base); int ss_cap = ss_slabs_capacity(ss); if (__builtin_expect(slab_idx < 0 || slab_idx >= ss_cap, 0)) { tiny_debug_ring_record(TINY_RING_EVENT_SUPERSLAB_ADOPT_FAIL, (uint16_t)0xFEu, ss, (uintptr_t)slab_idx); return; } TinySlabMeta* meta = &ss->slabs[slab_idx]; - if (__builtin_expect(g_tiny_safe_free || class_idx == 7, 0)) { + if (__builtin_expect(g_tiny_safe_free, 0)) { size_t blk = g_tiny_class_sizes[class_idx]; - uint8_t* base = tiny_slab_base_for(ss, slab_idx); - uintptr_t delta = (uintptr_t)ptr - (uintptr_t)base; + uint8_t* slab_base = tiny_slab_base_for(ss, slab_idx); + // Phase E1-CORRECT: All classes have headers, validate block base (ptr-1) not user ptr + uintptr_t delta = (uintptr_t)((uint8_t*)ptr - 1) - (uintptr_t)slab_base; int cap_ok = (meta->capacity > 0) ? 1 : 0; int align_ok = (delta % blk) == 0; int range_ok = cap_ok && (delta / blk) < meta->capacity; @@ -155,7 +157,7 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { if (range_ok) code |= 0x1u; uintptr_t aux = tiny_remote_pack_diag(code, ss_base, ss_size, (uintptr_t)ptr); tiny_debug_ring_record(TINY_RING_EVENT_REMOTE_INVALID, (uint16_t)class_idx, ptr, aux); - if (g_tiny_safe_free_strict || class_idx == 7) { raise(SIGUSR2); return; } + if (g_tiny_safe_free_strict) { raise(SIGUSR2); return; } return; } } @@ -191,10 +193,10 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { if (__builtin_expect(g_debug_fast0, 0)) { tiny_debug_ring_record(TINY_RING_EVENT_FRONT_BYPASS, (uint16_t)class_idx, ptr, (uintptr_t)slab_idx); - // Always operate on block base for C0-C6 (header lives at base) - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); void* prev = meta->freelist; - *(void**)base = prev; + tiny_next_write(class_idx, base, prev); // Box API: uses offset 1 for headers meta->freelist = base; meta->used--; ss_active_dec_one(ss); @@ -207,8 +209,8 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { } if (g_fast_enable && g_fast_cap[class_idx] != 0) { - // Push block base into array stack for C0–C3, otherwise into TLS fast list - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); int pushed = 0; if (__builtin_expect(g_fastcache_enable && class_idx <= 3, 1)) { pushed = fastcache_push(class_idx, base); @@ -230,7 +232,8 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { } // TinyHotMag front push(8/16/32B, A/B) if (__builtin_expect(g_hotmag_enable && class_idx <= 2, 1)) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (hotmag_push(class_idx, base)) { tiny_debug_ring_record(TINY_RING_EVENT_FREE_RETURN_MAG, (uint16_t)class_idx, ptr, 1); HAK_STAT_FREE(class_idx); @@ -238,7 +241,8 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { } } if (tls->count < tls->cap) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); tls_list_push(tls, base, class_idx); tiny_debug_ring_record(TINY_RING_EVENT_FREE_LOCAL, (uint16_t)class_idx, ptr, 0); @@ -250,7 +254,8 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { tiny_tls_refresh_params(class_idx, tls); } { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); tls_list_push(tls, base, class_idx); } @@ -343,10 +348,8 @@ void hak_tiny_free(void* ptr) { TinySlab* slab = hak_tiny_owner_slab(ptr); if (slab) class_idx = slab->class_idx; } - // CRITICAL: C7 (1KB) is headerless - MUST NOT use TLS SLL - // Reason: SLL stores next pointer in first 8 bytes (user data for C7) - // Fix: Exclude C7 from ultra free path - if (class_idx >= 0 && class_idx != 7) { + // Phase E1-CORRECT: C7 now has headers, can use TLS SLL like other classes + if (class_idx >= 0) { // Ultra free: push directly to TLS SLL without magazine init int sll_cap = ultra_sll_cap_for_class(class_idx); if ((int)g_tls_sll_count[class_idx] < sll_cap) { @@ -379,11 +382,12 @@ void hak_tiny_free(void* ptr) { // Use Box TLS-SLL API (C7-safe push) // Note: C7 already rejected at line 334 { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (tls_sll_push(class_idx, base, (uint32_t)sll_cap)) { // CORRUPTION DEBUG: Verify write succeeded if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { - void* readback = *(void**)base; + void* readback = tiny_next_read(class_idx, base); // Phase E1-CORRECT: Box API (void)readback; void* new_head = g_tls_sll_head[class_idx]; if (new_head != base) { @@ -440,7 +444,8 @@ void hak_tiny_free(void* ptr) { tiny_debug_ring_record(TINY_RING_EVENT_FREE_ENTER, (uint16_t)fast_class_idx, ptr, 1); } if (fast_class_idx >= 0 && g_fast_enable && g_fast_cap[fast_class_idx] != 0) { - void* base2 = (fast_class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base2 = (void*)((uint8_t*)ptr - 1); if (tiny_fast_push(fast_class_idx, base2)) { tiny_debug_ring_record(TINY_RING_EVENT_FREE_FAST, (uint16_t)fast_class_idx, ptr, 0); HAK_STAT_FREE(fast_class_idx); diff --git a/core/hakmem_tiny_ultra_simple.inc b/core/hakmem_tiny_ultra_simple.inc index fe3c8eea..e8f55daf 100644 --- a/core/hakmem_tiny_ultra_simple.inc +++ b/core/hakmem_tiny_ultra_simple.inc @@ -145,7 +145,9 @@ void hak_tiny_free_ultra_simple(void* ptr) { if (__builtin_expect(g_use_superslab != 0, 1)) { SuperSlab* ss = hak_super_lookup(ptr); if (__builtin_expect(ss != NULL && ss->magic == SUPERSLAB_MAGIC, 0)) { - int slab_idx = slab_index_for(ss, ptr); + // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation + void* base = (void*)((uint8_t*)ptr - 1); + int slab_idx = slab_index_for(ss, base); TinySlabMeta* meta = &ss->slabs[slab_idx]; uint32_t self_tid = tiny_self_u32(); if (__builtin_expect(meta->owner_tid == self_tid, 1)) { diff --git a/core/tiny_free_fast.inc.h b/core/tiny_free_fast.inc.h index 4f761433..05915cf1 100644 --- a/core/tiny_free_fast.inc.h +++ b/core/tiny_free_fast.inc.h @@ -120,7 +120,8 @@ static inline int tiny_free_fast_ss(SuperSlab* ss, int slab_idx, void* ptr, uint // Fast path: Same-thread free (2-3 instructions) int class_idx = ss->size_class; - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); #if HAKMEM_DEBUG_COUNTERS // Track same-thread frees (compile-time gated) @@ -161,7 +162,8 @@ static inline int tiny_free_fast_legacy(TinySlab* slab, void* ptr) { // Fast path: Same-thread free int class_idx = slab->class_idx; - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); // Box 5-NEW/5-OLD integration: Push to TLS freelist (SFC or SLL) extern int g_sfc_enabled; @@ -215,7 +217,9 @@ static inline void tiny_free_fast(void* ptr) { if (__builtin_expect(g_use_superslab != 0, 1)) { SuperSlab* ss = hak_super_lookup(ptr); if (__builtin_expect(ss != NULL && ss->magic == SUPERSLAB_MAGIC, 0)) { - int slab_idx = slab_index_for(ss, ptr); + // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation + void* base = (void*)((uint8_t*)ptr - 1); + int slab_idx = slab_index_for(ss, base); uint32_t self_tid = tiny_self_u32(); // Box 6 Boundary: Try same-thread fast path diff --git a/core/tiny_free_magazine.inc.h b/core/tiny_free_magazine.inc.h index f85d479c..438e490a 100644 --- a/core/tiny_free_magazine.inc.h +++ b/core/tiny_free_magazine.inc.h @@ -19,7 +19,8 @@ if (g_quick_enable && class_idx <= 4) { TinyQuickSlot* qs = &g_tls_quick[class_idx]; if (__builtin_expect(qs->top < QUICK_CAP, 1)) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); qs->items[qs->top++] = base; HAK_STAT_FREE(class_idx); return; @@ -29,7 +30,8 @@ // Fast path: TLS SLL push for hottest classes if (!g_tls_list_enable && g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)cap)) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap); if (tls_sll_push(class_idx, base, sll_cap)) { // BUGFIX: Decrement used counter (was missing, causing Fail-Fast on next free) @@ -49,7 +51,8 @@ (void)bulk_mag_to_sll_if_room(class_idx, mag, cap / 2); } if (mag->top < cap + g_spill_hyst) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER mag->items[mag->top].owner = NULL; // SuperSlab owner not a TinySlab; leave NULL @@ -74,24 +77,25 @@ int limit = g_bg_spill_max_batch; if (limit > cap/2) limit = cap/2; if (limit > 32) limit = 32; // keep free-path bounded - void* head = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* head = (void*)((uint8_t*)ptr - 1); #if HAKMEM_TINY_HEADER_CLASSIDX - const size_t next_off = (class_idx == 7) ? 0 : 1; + const size_t next_off = 1; // Phase E1-CORRECT: Always 1 #else const size_t next_off = 0; #endif - #include "tiny_nextptr.h" - tiny_next_store(head, class_idx, NULL); + #include "box/tiny_next_ptr_box.h" + tiny_next_write(class_idx, head, NULL); void* tail = head; // current tail int taken = 1; while (taken < limit && mag->top > 0) { void* p2 = mag->items[--mag->top].ptr; #if HAKMEM_TINY_HEADER_CLASSIDX - const size_t next_off2 = (class_idx == 7) ? 0 : 1; + const size_t next_off2 = 1; // Phase E1-CORRECT: Always 1 #else const size_t next_off2 = 0; #endif - tiny_next_store(p2, class_idx, head); + tiny_next_write(class_idx, p2, head); head = p2; taken++; } @@ -119,13 +123,15 @@ SuperSlab* owner_ss = hak_super_lookup(it.ptr); if (owner_ss && owner_ss->magic == SUPERSLAB_MAGIC) { // Direct freelist push (same as old hak_tiny_free_superslab) - int slab_idx = slab_index_for(owner_ss, it.ptr); + // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation + void* base = (void*)((uint8_t*)it.ptr - 1); + int slab_idx = slab_index_for(owner_ss, base); // BUGFIX: Validate slab_idx before array access (prevents OOB) if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(owner_ss)) { continue; // Skip invalid index } TinySlabMeta* meta = &owner_ss->slabs[slab_idx]; - *(void**)it.ptr = meta->freelist; + tiny_next_write(class_idx, it.ptr, meta->freelist); meta->freelist = it.ptr; meta->used--; // Decrement SuperSlab active counter (spill returns blocks to SS) @@ -152,7 +158,8 @@ // Finally, try FastCache push first (≤128B) — compile-out if HAKMEM_TINY_NO_FRONT_CACHE #if !defined(HAKMEM_TINY_NO_FRONT_CACHE) if (g_fastcache_enable && class_idx <= 4) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (fastcache_push(class_idx, base)) { HAK_TP1(front_push, class_idx); HAK_STAT_FREE(class_idx); @@ -163,7 +170,8 @@ // Then TLS SLL if room, else magazine if (g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)mag->cap)) { uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap); - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (!tls_sll_push(class_idx, base, sll_cap2)) { // fallback to magazine mag->items[mag->top].ptr = base; @@ -173,7 +181,8 @@ mag->top++; } } else { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER mag->items[mag->top].owner = slab; @@ -195,7 +204,8 @@ if (pthread_equal(slab->owner_tid, tiny_self_pt())) { int class_idx = slab->class_idx; - if (g_tls_list_enable && class_idx != 7) { + // Phase E1-CORRECT: C7 now has headers, can use TLS list like other classes + if (g_tls_list_enable) { TinyTLSList* tls = &g_tls_lists[class_idx]; uint32_t seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed); if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) { @@ -203,14 +213,16 @@ } // TinyHotMag front push(8/16/32B, A/B) if (__builtin_expect(g_hotmag_enable && class_idx <= 2, 1)) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (hotmag_push(class_idx, base)) { HAK_STAT_FREE(class_idx); return; } } if (tls->count < tls->cap) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); tls_list_push_fast(tls, base, class_idx); HAK_STAT_FREE(class_idx); @@ -221,7 +233,8 @@ tiny_tls_refresh_params(class_idx, tls); } { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); tls_list_push_fast(tls, base, class_idx); } @@ -247,7 +260,8 @@ if (!g_tls_list_enable && g_tls_sll_enable && class_idx <= 5) { uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap); if (g_tls_sll_count[class_idx] < sll_cap) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (tls_sll_push(class_idx, base, sll_cap)) { HAK_STAT_FREE(class_idx); return; @@ -261,7 +275,8 @@ // Remote-drain can be handled opportunistically on future calls. if (mag->top < cap) { { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER mag->items[mag->top].owner = slab; @@ -299,14 +314,16 @@ SuperSlab* ss_owner = hak_super_lookup(it.ptr); if (ss_owner && ss_owner->magic == SUPERSLAB_MAGIC) { // SuperSlab spill - return to freelist - int slab_idx = slab_index_for(ss_owner, it.ptr); + // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation + void* base = (void*)((uint8_t*)it.ptr - 1); + int slab_idx = slab_index_for(ss_owner, base); // BUGFIX: Validate slab_idx before array access (prevents OOB) if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss_owner)) { HAK_STAT_FREE(class_idx); continue; // Skip invalid index } TinySlabMeta* meta = &ss_owner->slabs[slab_idx]; - *(void**)it.ptr = meta->freelist; + tiny_next_write(class_idx, it.ptr, meta->freelist); meta->freelist = it.ptr; meta->used--; // 空SuperSlab処理はフラッシュ/バックグラウンドで対応(ホットパス除外) @@ -389,12 +406,14 @@ if (g_quick_enable && class_idx <= 4) { TinyQuickSlot* qs = &g_tls_quick[class_idx]; if (__builtin_expect(qs->top < QUICK_CAP, 1)) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); qs->items[qs->top++] = base; } else if (g_tls_sll_enable) { uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap); if (g_tls_sll_count[class_idx] < sll_cap2) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (!tls_sll_push(class_idx, base, sll_cap2)) { if (!tiny_optional_push(class_idx, base)) { mag->items[mag->top].ptr = base; @@ -404,8 +423,9 @@ mag->top++; } } - } else if (!tiny_optional_push(class_idx, (class_idx == 7 ? ptr : (void*)((uint8_t*)ptr - 1)))) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + } else if (!tiny_optional_push(class_idx, (void*)((uint8_t*)ptr - 1))) { // Phase E1-CORRECT + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER mag->items[mag->top].owner = slab; @@ -413,7 +433,8 @@ mag->top++; } } else { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (!tiny_optional_push(class_idx, base)) { mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER @@ -428,7 +449,8 @@ if (g_tls_sll_enable && class_idx <= 5) { uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap); if (g_tls_sll_count[class_idx] < sll_cap2) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (!tls_sll_push(class_idx, base, sll_cap2)) { if (!tiny_optional_push(class_idx, base)) { mag->items[mag->top].ptr = base; @@ -438,8 +460,9 @@ mag->top++; } } - } else if (!tiny_optional_push(class_idx, (class_idx == 7 ? ptr : (void*)((uint8_t*)ptr - 1)))) { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + } else if (!tiny_optional_push(class_idx, (void*)((uint8_t*)ptr - 1))) { // Phase E1-CORRECT + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER mag->items[mag->top].owner = slab; @@ -447,7 +470,8 @@ mag->top++; } } else { - void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); if (!tiny_optional_push(class_idx, base)) { mag->items[mag->top].ptr = base; #if HAKMEM_TINY_MAG_OWNER @@ -465,7 +489,8 @@ HAK_STAT_FREE(class_idx); // Phase 3 return; } else { - void* base = (slab && slab->class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header + void* base = (void*)((uint8_t*)ptr - 1); tiny_remote_push(slab, base); } }