From 5b3162965086b3fb97d55d470b897dd2a92ab650 Mon Sep 17 00:00:00 2001 From: HakmemBot Date: Tue, 11 Nov 2025 10:00:36 +0900 Subject: [PATCH] tiny: fix TLS list next_off scope; default TLS_LIST=1; add sentinel guards; header-aware TLS ops; release quiet for benches --- core/box/front_gate_box.c | 7 +-- core/box/front_gate_box.d | 3 ++ core/box/front_gate_classifier.c | 24 +++++++++- core/box/front_gate_classifier.d | 11 ++++- core/box/hak_wrappers.inc.h | 23 +++++++++ core/box/tls_sll_box.h | 28 +++++++++-- core/hakmem_build_flags.h | 4 +- core/hakmem_tiny.c | 6 +-- core/hakmem_tiny.d | 3 ++ core/hakmem_tiny_free.inc | 8 ++-- core/hakmem_tiny_hotmag.inc.h | 10 +++- core/hakmem_tiny_lifecycle.inc | 22 +++++---- core/hakmem_tiny_refill.inc.h | 21 +++++---- core/hakmem_tiny_slow.inc | 13 ++++-- core/hakmem_tiny_tls_list.h | 80 +++++++++++++++++++++++--------- core/hakmem_tiny_tls_ops.h | 33 ++++++++----- core/tiny_free_magazine.inc.h | 6 +-- core/tiny_superslab_alloc.inc.h | 4 +- hakmem.d | 4 +- 19 files changed, 231 insertions(+), 79 deletions(-) diff --git a/core/box/front_gate_box.c b/core/box/front_gate_box.c index f899c2c9..6117c810 100644 --- a/core/box/front_gate_box.c +++ b/core/box/front_gate_box.c @@ -62,11 +62,12 @@ void front_gate_after_refill(int class_idx, int refilled_count) { } void front_gate_push_tls(int class_idx, void* ptr) { - // Use Box TLS-SLL API (C7-safe) - if (!tls_sll_push(class_idx, ptr, UINT32_MAX)) { + // Normalize to base for header classes (C0–C6) + void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); + // Use Box TLS-SLL API (C7-safe; expects base pointer) + if (!tls_sll_push(class_idx, base, UINT32_MAX)) { // C7 rejected or capacity exceeded - should not happen in front gate // but handle gracefully (silent discard) return; } } - diff --git a/core/box/front_gate_box.d b/core/box/front_gate_box.d index 81ccb980..64a39671 100644 --- a/core/box/front_gate_box.d +++ b/core/box/front_gate_box.d @@ -3,6 +3,7 @@ core/box/front_gate_box.o: core/box/front_gate_box.c \ core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \ core/tiny_alloc_fast_sfc.inc.h core/hakmem_tiny.h core/box/tls_sll_box.h \ core/box/../ptr_trace.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_build_flags.h core/box/../tiny_region_id.h \ core/box/../hakmem_build_flags.h core/box/front_gate_box.h: core/hakmem_tiny.h: @@ -15,3 +16,5 @@ core/box/tls_sll_box.h: core/box/../ptr_trace.h: core/box/../hakmem_tiny_config.h: core/box/../hakmem_build_flags.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_build_flags.h: diff --git a/core/box/front_gate_classifier.c b/core/box/front_gate_classifier.c index aa135847..222076aa 100644 --- a/core/box/front_gate_classifier.c +++ b/core/box/front_gate_classifier.c @@ -13,6 +13,7 @@ #include "../hakmem_tiny_superslab.h" #include "../superslab/superslab_inline.h" // For ss_slabs_capacity #include "../hakmem_build_flags.h" +#include "../hakmem_internal.h" // AllocHeader, HAKMEM_MAGIC, HEADER_SIZE, hak_is_memory_readable #include "../hakmem_tiny_config.h" // For TINY_NUM_CLASSES, SLAB_SIZE #include "../hakmem_super_registry.h" // For hak_super_lookup (Box REG) @@ -209,8 +210,27 @@ ptr_classification_t classify_ptr(void* ptr) { return result; } - // Step 3: Not Tiny or Pool - return UNKNOWN - // Caller should check AllocHeader (16-byte) or delegate to system free + // Step 3: Try AllocHeader (HAKMEM header) for Mid/Large/Mmap + do { + if (!ptr) break; + // Quick page-safety check: avoid crossing page for header read + uintptr_t off = (uintptr_t)ptr & 0xFFFu; + int safe_same_page = (off >= HEADER_SIZE); + void* raw = (char*)ptr - HEADER_SIZE; + if (!safe_same_page) { + if (!hak_is_memory_readable(raw)) break; + } + AllocHeader* hdr = (AllocHeader*)raw; + if (hdr->magic == HAKMEM_MAGIC) { + result.kind = PTR_KIND_MID_LARGE; // HAKMEM-owned (non-Tiny) +#if !HAKMEM_BUILD_RELEASE + g_classify_unknown_hit++; // reuse for stats without adding a new counter +#endif + return result; + } + } while (0); + + // Step 4: Not recognized → UNKNOWN (route to libc or slow path) result.kind = PTR_KIND_UNKNOWN; #if !HAKMEM_BUILD_RELEASE diff --git a/core/box/front_gate_classifier.d b/core/box/front_gate_classifier.d index 0ce3de76..c0c2ffa5 100644 --- a/core/box/front_gate_classifier.d +++ b/core/box/front_gate_classifier.d @@ -11,7 +11,10 @@ core/box/front_gate_classifier.o: core/box/front_gate_classifier.c \ core/box/../tiny_debug_ring.h core/box/../tiny_remote.h \ core/box/../hakmem_tiny_superslab_constants.h \ core/box/../superslab/superslab_inline.h \ - core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_build_flags.h core/box/../hakmem_internal.h \ + core/box/../hakmem.h core/box/../hakmem_config.h \ + core/box/../hakmem_features.h core/box/../hakmem_sys.h \ + core/box/../hakmem_whale.h core/box/../hakmem_tiny_config.h \ core/box/../hakmem_super_registry.h core/box/../hakmem_tiny_superslab.h \ core/box/../pool_tls_registry.h core/box/front_gate_classifier.h: @@ -32,6 +35,12 @@ core/box/../tiny_remote.h: core/box/../hakmem_tiny_superslab_constants.h: core/box/../superslab/superslab_inline.h: core/box/../hakmem_build_flags.h: +core/box/../hakmem_internal.h: +core/box/../hakmem.h: +core/box/../hakmem_config.h: +core/box/../hakmem_features.h: +core/box/../hakmem_sys.h: +core/box/../hakmem_whale.h: core/box/../hakmem_tiny_config.h: core/box/../hakmem_super_registry.h: core/box/../hakmem_tiny_superslab.h: diff --git a/core/box/hak_wrappers.inc.h b/core/box/hak_wrappers.inc.h index 3990f42c..af40f9ef 100644 --- a/core/box/hak_wrappers.inc.h +++ b/core/box/hak_wrappers.inc.h @@ -151,6 +151,29 @@ void free(void* ptr) { #else (void)fg_libc_bypass_count; #endif + // Safety: If this is a HAKMEM-owned header allocation, free raw correctly + do { + void* raw = (char*)ptr - HEADER_SIZE; + int safe_same_page = (((uintptr_t)ptr & 0xFFFu) >= HEADER_SIZE); + if (!safe_same_page) { + if (!hak_is_memory_readable(raw)) break; + } + AllocHeader* hdr = (AllocHeader*)raw; + if (hdr->magic == HAKMEM_MAGIC) { + // Dispatch based on allocation method + if (hdr->method == ALLOC_METHOD_MALLOC) { + extern void __libc_free(void*); + ptr_trace_dump_now("wrap_libc_lockdepth_hak_hdr_malloc"); + __libc_free(raw); + return; + } else if (hdr->method == ALLOC_METHOD_MMAP) { + ptr_trace_dump_now("wrap_libc_lockdepth_hak_hdr_mmap"); + hkm_sys_munmap(raw, hdr->size); + return; + } + } + } while (0); + // Unknown pointer or non-HAKMEM: fall back to libc free(ptr) extern void __libc_free(void*); ptr_trace_dump_now("wrap_libc_lockdepth"); __libc_free(ptr); diff --git a/core/box/tls_sll_box.h b/core/box/tls_sll_box.h index b297792d..a794fd4d 100644 --- a/core/box/tls_sll_box.h +++ b/core/box/tls_sll_box.h @@ -28,6 +28,7 @@ #include "../ptr_trace.h" // Debug-only: pointer next read/write tracing #include "../hakmem_tiny_config.h" // For TINY_NUM_CLASSES #include "../hakmem_build_flags.h" +#include "../tiny_region_id.h" // HEADER_MAGIC / HEADER_CLASS_MASK // Debug guard: validate base pointer before SLL ops (Debug only) #if !HAKMEM_BUILD_RELEASE @@ -188,14 +189,25 @@ static inline uint32_t tls_sll_splice(int class_idx, void* chain_head, uint32_t // Limit splice size to available capacity uint32_t to_move = (count < available) ? count : available; - // Find chain tail (traverse to_move - 1 nodes) - // NOTE: Chain MUST be linked using base pointers (caller responsibility) - // Assume chain is linked with base pointers + // Determine how the chain is linked: base or user pointers. + // For C0-C6, header byte (0xA0|cls) resides at base. + // If chain_head points to base → *(uint8_t*)head has HEADER_MAGIC|cls + // If it points to user (base+1) → *(uint8_t*)head is user data (not magic) void* tail = chain_head; #if HAKMEM_TINY_HEADER_CLASSIDX - const size_t next_offset = 1; // Chain is built from header-safe links (C7 rejected) + size_t next_offset; + { + uint8_t hdr = *(uint8_t*)chain_head; + if ((hdr & 0xF0) == HEADER_MAGIC && (hdr & HEADER_CLASS_MASK) == (uint8_t)class_idx) { + // Chain nodes are base pointers; links live at base+1 + next_offset = 1; + } else { + // Chain nodes are user pointers; links live at user (base+1) → offset 0 from user + next_offset = 0; + } + } #else - const size_t next_offset = 0; + size_t next_offset = 0; #endif for (uint32_t i = 1; i < to_move; i++) { tls_sll_debug_guard(class_idx, tail, "splice_trav"); @@ -218,6 +230,12 @@ static inline uint32_t tls_sll_splice(int class_idx, void* chain_head, uint32_t PTR_NEXT_WRITE("tls_sp_link", class_idx, tail, next_offset, g_tls_sll_head[class_idx]); // CRITICAL: Normalize head before publishing to SLL (caller may pass user ptrs) void* head_norm = chain_head; +#if HAKMEM_TINY_HEADER_CLASSIDX + if (next_offset == 0) { + // Chain nodes were user pointers; convert head to base + head_norm = (uint8_t*)chain_head - 1; + } +#endif tls_sll_debug_guard(class_idx, head_norm, "splice_head"); #if !HAKMEM_BUILD_RELEASE fprintf(stderr, "[SPLICE_SET_HEAD] cls=%d head_norm=%p moved=%u\n", diff --git a/core/hakmem_build_flags.h b/core/hakmem_build_flags.h index bb856e9e..05e5edf7 100644 --- a/core/hakmem_build_flags.h +++ b/core/hakmem_build_flags.h @@ -37,7 +37,7 @@ // Batch refill P0 (can be toggled for A/B) #ifndef HAKMEM_TINY_P0_BATCH_REFILL -# define HAKMEM_TINY_P0_BATCH_REFILL 1 +# define HAKMEM_TINY_P0_BATCH_REFILL 0 #endif // Box refactor (Phase 6-1.7) — usually injected from build system @@ -52,7 +52,7 @@ // Default: OFF (enable after full validation in Task 5) // Build: make HEADER_CLASSIDX=1 or make phase7 #ifndef HAKMEM_TINY_HEADER_CLASSIDX -# define HAKMEM_TINY_HEADER_CLASSIDX 0 +# define HAKMEM_TINY_HEADER_CLASSIDX 1 #endif // Phase 7 Task 2: Aggressive inline TLS cache access diff --git a/core/hakmem_tiny.c b/core/hakmem_tiny.c index 6aab617f..4c03c64a 100644 --- a/core/hakmem_tiny.c +++ b/core/hakmem_tiny.c @@ -362,7 +362,7 @@ static int g_tiny_refill_max_hot = 192; // HAKMEM_TINY_REFILL_MAX_HOT for clas // hakmem_tiny_tls_list.h already included at top static __thread TinyTLSList g_tls_lists[TINY_NUM_CLASSES]; -static int g_tls_list_enable = 1; +static int g_tls_list_enable = 1; // Default ON (scope bug fixed 2025-11-11); disable via HAKMEM_TINY_TLS_LIST=0 static inline int tls_refill_from_tls_slab(int class_idx, TinyTLSList* tls, uint32_t want); static int g_fast_enable = 1; static uint16_t g_fast_cap[TINY_NUM_CLASSES]; @@ -1049,7 +1049,7 @@ static __attribute__((cold, noinline, unused)) void* tiny_slow_alloc_fast(int cl if (!tiny_fast_push(class_idx, extra)) { if (tls_enabled) { tiny_tls_list_guard_push(class_idx, tls, extra); - tls_list_push(tls, extra); + tls_list_push(tls, extra, class_idx); } } fast_need--; @@ -1062,7 +1062,7 @@ static __attribute__((cold, noinline, unused)) void* tiny_slow_alloc_fast(int cl slab->free_count--; void* extra = (void*)(base + ((size_t)extra_idx * block_size)); tiny_tls_list_guard_push(class_idx, tls, extra); - tls_list_push(tls, extra); + tls_list_push(tls, extra, class_idx); tls_need--; } diff --git a/core/hakmem_tiny.d b/core/hakmem_tiny.d index 8ca4e670..9eda0da7 100644 --- a/core/hakmem_tiny.d +++ b/core/hakmem_tiny.d @@ -25,6 +25,7 @@ core/hakmem_tiny.o: core/hakmem_tiny.c core/hakmem_tiny.h \ core/tiny_system.h core/hakmem_prof.h core/tiny_publish.h \ core/box/tls_sll_box.h core/box/../ptr_trace.h \ core/box/../hakmem_tiny_config.h core/box/../hakmem_build_flags.h \ + core/box/../tiny_region_id.h core/box/../hakmem_build_flags.h \ core/hakmem_tiny_hotmag.inc.h core/hakmem_tiny_hot_pop.inc.h \ core/hakmem_tiny_fastcache.inc.h core/hakmem_tiny_refill.inc.h \ core/tiny_box_geometry.h core/hakmem_tiny_refill_p0.inc.h \ @@ -103,6 +104,8 @@ core/box/tls_sll_box.h: core/box/../ptr_trace.h: core/box/../hakmem_tiny_config.h: core/box/../hakmem_build_flags.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_build_flags.h: core/hakmem_tiny_hotmag.inc.h: core/hakmem_tiny_hot_pop.inc.h: core/hakmem_tiny_fastcache.inc.h: diff --git a/core/hakmem_tiny_free.inc b/core/hakmem_tiny_free.inc index cfa60adf..8b90a569 100644 --- a/core/hakmem_tiny_free.inc +++ b/core/hakmem_tiny_free.inc @@ -216,7 +216,7 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { } } - if (g_tls_list_enable) { + if (g_tls_list_enable && class_idx != 7) { TinyTLSList* tls = &g_tls_lists[class_idx]; uint32_t seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed); if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) { @@ -234,7 +234,7 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { if (tls->count < tls->cap) { void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); - tls_list_push(tls, base); + tls_list_push(tls, base, class_idx); tiny_debug_ring_record(TINY_RING_EVENT_FREE_LOCAL, (uint16_t)class_idx, ptr, 0); HAK_STAT_FREE(class_idx); return; @@ -245,8 +245,8 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) { } { void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); - tiny_tls_list_guard_push(class_idx, tls, base); - tls_list_push(tls, base); + tiny_tls_list_guard_push(class_idx, tls, base); + tls_list_push(tls, base, class_idx); } if (tls_list_should_spill(tls)) { tls_list_spill_excess(class_idx, tls); diff --git a/core/hakmem_tiny_hotmag.inc.h b/core/hakmem_tiny_hotmag.inc.h index 7a5cbd44..4107585b 100644 --- a/core/hakmem_tiny_hotmag.inc.h +++ b/core/hakmem_tiny_hotmag.inc.h @@ -114,11 +114,17 @@ static inline int hotmag_try_refill(int class_idx, TinyHotMag* hm) { } void* chain_head = NULL; void* chain_tail = NULL; - uint32_t taken = tls_list_bulk_take(tls, target, &chain_head, &chain_tail); + uint32_t taken = tls_list_bulk_take(tls, target, &chain_head, &chain_tail, class_idx); if (taken > 0u) { void* node = chain_head; for (uint32_t i = 0; i < taken && node; i++) { - void* next = *(void**)node; + // Header-aware next from TLS list chain +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off_tls = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off_tls = 0; +#endif + void* next = *(void**)((uint8_t*)node + next_off_tls); hm->slots[hm->top++] = node; node = next; } diff --git a/core/hakmem_tiny_lifecycle.inc b/core/hakmem_tiny_lifecycle.inc index 2015162d..34a69f8b 100644 --- a/core/hakmem_tiny_lifecycle.inc +++ b/core/hakmem_tiny_lifecycle.inc @@ -144,8 +144,8 @@ void hak_tiny_trim(void) { static void tiny_tls_cache_drain(int class_idx) { TinyTLSList* tls = &g_tls_lists[class_idx]; - // Drain TLS SLL cache - void* sll = g_tls_sll_head[class_idx]; + // Drain TLS SLL cache (skip C7) + void* sll = (class_idx == 7) ? NULL : g_tls_sll_head[class_idx]; g_tls_sll_head[class_idx] = NULL; g_tls_sll_count[class_idx] = 0; while (sll) { @@ -156,12 +156,12 @@ static void tiny_tls_cache_drain(int class_idx) { #endif void* next = *(void**)((uint8_t*)sll + next_off_sll); tiny_tls_list_guard_push(class_idx, tls, sll); - tls_list_push(tls, sll); + tls_list_push(tls, sll, class_idx); sll = next; } - // Drain fast tier cache - void* fast = g_fast_head[class_idx]; + // Drain fast tier cache (skip C7) + void* fast = (class_idx == 7) ? NULL : g_fast_head[class_idx]; g_fast_head[class_idx] = NULL; g_fast_count[class_idx] = 0; while (fast) { @@ -172,7 +172,7 @@ static void tiny_tls_cache_drain(int class_idx) { #endif void* next = *(void**)((uint8_t*)fast + next_off_fast); tiny_tls_list_guard_push(class_idx, tls, fast); - tls_list_push(tls, fast); + tls_list_push(tls, fast, class_idx); fast = next; } @@ -180,11 +180,17 @@ static void tiny_tls_cache_drain(int class_idx) { void* head = NULL; void* tail = NULL; while (1) { - uint32_t taken = tls_list_bulk_take(tls, 0u, &head, &tail); + uint32_t taken = tls_list_bulk_take(tls, 0u, &head, &tail, class_idx); if (taken == 0u || head == NULL) break; void* cur = head; while (cur) { - void* next = *(void**)cur; + // Header-aware next pointer from TLS list chain +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off_tls = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off_tls = 0; +#endif + void* next = *(void**)((uint8_t*)cur + next_off_tls); SuperSlab* ss = hak_super_lookup(cur); if (ss && ss->magic == SUPERSLAB_MAGIC) { hak_tiny_free_superslab(cur, ss); diff --git a/core/hakmem_tiny_refill.inc.h b/core/hakmem_tiny_refill.inc.h index 660605e1..4e8f971c 100644 --- a/core/hakmem_tiny_refill.inc.h +++ b/core/hakmem_tiny_refill.inc.h @@ -157,23 +157,28 @@ static inline void* tiny_fast_refill_and_take(int class_idx, TinyTLSList* tls) { } void* batch_head = NULL; void* batch_tail = NULL; - uint32_t taken = tls_list_bulk_take(tls, need, &batch_head, &batch_tail); + uint32_t taken = tls_list_bulk_take(tls, need, &batch_head, &batch_tail, class_idx); if (taken == 0u || batch_head == NULL) { return NULL; } void* ret = batch_head; - void* node = *(void**)ret; +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off_tls = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off_tls = 0; +#endif + void* node = *(void**)((uint8_t*)ret + next_off_tls); uint32_t remaining = (taken > 0u) ? (taken - 1u) : 0u; while (node && remaining > 0u) { - void* next = *(void**)node; + void* next = *(void**)((uint8_t*)node + next_off_tls); if (tiny_fast_push(class_idx, node)) { node = next; remaining--; } else { // Push failed, return remaining to TLS - tls_list_bulk_put(tls, node, batch_tail, remaining); + tls_list_bulk_put(tls, node, batch_tail, remaining, class_idx); return ret; } } @@ -263,14 +268,14 @@ static inline int sll_refill_small_from_ss(int class_idx, int max_take) { if (!g_use_superslab || max_take <= 0) return 0; // ランタイムA/B: P0を有効化している場合はバッチrefillへ委譲 do { - // 既定: ON(HAKMEM_TINY_P0_ENABLE=0 で明示的にOFF) + // 既定: OFF(HAKMEM_TINY_P0_ENABLE=1 で有効化) static int g_p0_enable = -1; if (__builtin_expect(g_p0_enable == -1, 0)) { const char* e = getenv("HAKMEM_TINY_P0_ENABLE"); - // 環境変数が'0'のときだけ無効、それ以外(未設定含む)は有効 - g_p0_enable = (e && *e && *e == '0') ? 0 : 1; + // 環境変数が'1'のときだけ有効、それ以外(未設定含む)は無効 + g_p0_enable = (e && *e && *e == '1') ? 1 : 0; } - if (__builtin_expect(g_p0_enable, 1)) { + if (__builtin_expect(g_p0_enable, 0)) { return sll_refill_batch_from_ss(class_idx, max_take); } } while (0); diff --git a/core/hakmem_tiny_slow.inc b/core/hakmem_tiny_slow.inc index 7cbaa8b7..0f7c90d7 100644 --- a/core/hakmem_tiny_slow.inc +++ b/core/hakmem_tiny_slow.inc @@ -22,11 +22,16 @@ static void* __attribute__((cold, noinline)) hak_tiny_alloc_slow(size_t size, in if (ptr) { HAK_RET_ALLOC(class_idx, ptr); } } - // Try TLS list refill - if (g_tls_list_enable) { + // Try TLS list refill (C7 is headerless: skip TLS list entirely) + if (g_tls_list_enable && class_idx != 7) { TinyTLSList* tls = &g_tls_lists[class_idx]; + // Fail‑Fast: guard against poisoned head (remote sentinel) + if (__builtin_expect((uintptr_t)tls->head == TINY_REMOTE_SENTINEL, 0)) { + tls->head = NULL; + tls->count = 0; + } if (tls->count > 0) { - void* ptr = tls_list_pop(tls); + void* ptr = tls_list_pop(tls, class_idx); if (ptr) { HAK_RET_ALLOC(class_idx, ptr); } // ptr が NULL の場合でも、ここで終了せず後段の Superslab 経路へフォールバックする } @@ -34,7 +39,7 @@ static void* __attribute__((cold, noinline)) hak_tiny_alloc_slow(size_t size, in // Try refilling TLS list from slab uint32_t want = tls->refill_low > 0 ? tls->refill_low : 32; if (tls_refill_from_tls_slab(class_idx, tls, want) > 0) { - void* ptr = tls_list_pop(tls); + void* ptr = tls_list_pop(tls, class_idx); if (ptr) { HAK_RET_ALLOC(class_idx, ptr); } // ここでも NULL の場合は続行(後段へフォールバック) } diff --git a/core/hakmem_tiny_tls_list.h b/core/hakmem_tiny_tls_list.h index b7fcf676..a6007382 100644 --- a/core/hakmem_tiny_tls_list.h +++ b/core/hakmem_tiny_tls_list.h @@ -2,6 +2,7 @@ #define HAKMEM_TINY_TLS_LIST_H #include +#include "tiny_remote.h" // TINY_REMOTE_SENTINEL for head poisoning guard // Forward declarations typedef struct TinySlabMeta TinySlabMeta; @@ -47,17 +48,32 @@ static inline int tls_list_should_force_spill(const TinyTLSList* tls) { return tls->count > tls_list_spill_hard_threshold(tls); } -static inline void* tls_list_pop(TinyTLSList* tls) { +static inline void* tls_list_pop(TinyTLSList* tls, int class_idx) { void* head = tls->head; if (!head) return NULL; - tls->head = *(void**)head; + if (__builtin_expect((uintptr_t)head == TINY_REMOTE_SENTINEL, 0)) { + // Defensive: observed poisoned head. Reset list for safety. + tls->head = NULL; + tls->count = 0; + return NULL; + } + if (__builtin_expect(class_idx == 7, 0)) { + tls->head = *(void**)head; + } else { + tls->head = *(void**)((uint8_t*)head + 1); + } if (tls->count > 0) tls->count--; return head; } -static inline void tls_list_push(TinyTLSList* tls, void* node) { +static inline void tls_list_push(TinyTLSList* tls, void* node, int class_idx) { if (!node) return; - *(void**)node = tls->head; +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off = 0; +#endif + *(void**)((uint8_t*)node + next_off) = tls->head; tls->head = node; tls->count++; } @@ -65,10 +81,24 @@ static inline void tls_list_push(TinyTLSList* tls, void* node) { static inline uint32_t tls_list_bulk_take(TinyTLSList* tls, uint32_t want, void** out_head, - void** out_tail) { + void** out_tail, + int class_idx) { + // Define next_off at function scope to avoid scope violation +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off = 0; +#endif + if (out_head) *out_head = NULL; if (out_tail) *out_tail = NULL; if (tls->head == NULL || tls->count == 0) return 0; + // Guard poisoned head before traversal + if (__builtin_expect((uintptr_t)tls->head == TINY_REMOTE_SENTINEL, 0)) { + tls->head = NULL; + tls->count = 0; + return 0; + } if (want == 0 || want > tls->count) want = tls->count; @@ -76,14 +106,14 @@ static inline uint32_t tls_list_bulk_take(TinyTLSList* tls, void* cur = head; uint32_t taken = 1; while (taken < want) { - void* next = *(void**)cur; + void* next = *(void**)((uint8_t*)cur + next_off); if (!next) break; cur = next; taken++; } void* tail = cur; - void* rest = *(void**)tail; - *(void**)tail = NULL; + void* rest = *(void**)((uint8_t*)tail + next_off); + *(void**)((uint8_t*)tail + next_off) = NULL; tls->head = rest; tls->count -= taken; @@ -92,38 +122,46 @@ static inline uint32_t tls_list_bulk_take(TinyTLSList* tls, return taken; } -static inline uint32_t tls_list_count_chain(void* head) { +static inline uint32_t tls_list_count_chain(void* head, int class_idx) { uint32_t cnt = 0; - while (head) { - cnt++; - head = *(void**)head; - } + if (!head) return 0; +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off = 0; +#endif + while (head) { cnt++; head = *(void**)((uint8_t*)head + next_off); } return cnt; } static inline void tls_list_bulk_put(TinyTLSList* tls, void* head, void* tail, - uint32_t count) { + uint32_t count, + int class_idx) { + // Define next_off at function scope to avoid scope violation +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off = 0; +#endif + if (!head) return; if (!tail) { // Determine tail and count if not supplied tail = head; uint32_t computed = 1; - while (*(void**)tail) { - tail = *(void**)tail; - computed++; - } + while (*(void**)((uint8_t*)tail + next_off)) { tail = *(void**)((uint8_t*)tail + next_off); computed++; } if (count == 0) count = computed; } if (count == 0) { - count = tls_list_count_chain(head); + count = tls_list_count_chain(head, class_idx); // Move tail pointer to end if still NULL (just to be safe) void* cur = head; - while (*(void**)cur) cur = *(void**)cur; + while (*(void**)((uint8_t*)cur + next_off)) cur = *(void**)((uint8_t*)cur + next_off); tail = cur; } - *(void**)tail = tls->head; + *(void**)((uint8_t*)tail + next_off) = tls->head; tls->head = head; tls->count += count; } diff --git a/core/hakmem_tiny_tls_ops.h b/core/hakmem_tiny_tls_ops.h index 750820d4..6ef4e284 100644 --- a/core/hakmem_tiny_tls_ops.h +++ b/core/hakmem_tiny_tls_ops.h @@ -58,6 +58,12 @@ static inline int tls_refill_from_tls_slab(int class_idx, TinyTLSList* tls, uint if (want == 0u) return 0; size_t block_size = g_tiny_class_sizes[class_idx]; + // Header-aware TLS list next offset for chains we build here +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off_tls = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off_tls = 0; +#endif void* accum_head = NULL; void* accum_tail = NULL; uint32_t total = 0u; @@ -73,8 +79,8 @@ static inline int tls_refill_from_tls_slab(int class_idx, TinyTLSList* tls, uint uint32_t need = want - total; while (local < need && meta->freelist) { void* node = meta->freelist; - meta->freelist = *(void**)node; - *(void**)node = local_head; + meta->freelist = *(void**)node; // freelist is base-linked + *(void**)((uint8_t*)node + next_off_tls) = local_head; local_head = node; if (!local_tail) local_tail = node; local++; @@ -86,7 +92,7 @@ static inline int tls_refill_from_tls_slab(int class_idx, TinyTLSList* tls, uint accum_head = local_head; accum_tail = local_tail; } else { - *(void**)local_tail = accum_head; + *(void**)((uint8_t*)local_tail + next_off_tls) = accum_head; accum_head = local_head; } total += local; @@ -119,7 +125,7 @@ static inline int tls_refill_from_tls_slab(int class_idx, TinyTLSList* tls, uint uint8_t* cursor = base_cursor; for (uint32_t i = 1; i < need; ++i) { uint8_t* next = cursor + block_size; - *(void**)cursor = (void*)next; + *(void**)(cursor + next_off_tls) = (void*)next; cursor = next; } void* local_tail = (void*)cursor; @@ -130,14 +136,14 @@ static inline int tls_refill_from_tls_slab(int class_idx, TinyTLSList* tls, uint accum_head = local_head; accum_tail = local_tail; } else { - *(void**)local_tail = accum_head; + *(void**)((uint8_t*)local_tail + next_off_tls) = accum_head; accum_head = local_head; } total += need; } if (total > 0u && accum_head) { - tls_list_bulk_put(tls, accum_head, accum_tail, total); + tls_list_bulk_put(tls, accum_head, accum_tail, total, class_idx); return (int)total; } return 0; @@ -151,7 +157,7 @@ static inline void tls_list_spill_excess(int class_idx, TinyTLSList* tls) { uint32_t excess = tls->count - cap; void* head = NULL; void* tail = NULL; - uint32_t taken = tls_list_bulk_take(tls, excess, &head, &tail); + uint32_t taken = tls_list_bulk_take(tls, excess, &head, &tail, class_idx); if (taken == 0u || head == NULL) return; #if HAKMEM_PROF_STATIC && HAKMEM_BUILD_DEBUG @@ -174,8 +180,13 @@ static inline void tls_list_spill_excess(int class_idx, TinyTLSList* tls) { uint32_t self_tid = tiny_self_u32(); void* node = head; +#if HAKMEM_TINY_HEADER_CLASSIDX + const size_t next_off_tls = (class_idx == 7) ? 0 : 1; +#else + const size_t next_off_tls = 0; +#endif while (node) { - void* next = *(void**)node; + void* next = *(void**)((uint8_t*)node + next_off_tls); int handled = 0; // Phase 1: Try SuperSlab first (registry-based lookup, no false positives) @@ -189,7 +200,7 @@ static inline void tls_list_spill_excess(int class_idx, TinyTLSList* tls) { handled = 1; } else { void* prev = meta->freelist; - *(void**)node = prev; + *(void**)((uint8_t*)node + 0) = prev; // freelist within slab uses base link meta->freelist = node; tiny_failfast_log("tls_spill_ss", ss->size_class, ss, meta, node, prev); if (meta->used > 0) meta->used--; @@ -235,7 +246,7 @@ static inline void tls_list_spill_excess(int class_idx, TinyTLSList* tls) { } #endif if (!handled) { - *(void**)node = requeue_head; + *(void**)((uint8_t*)node + next_off_tls) = requeue_head; if (!requeue_head) requeue_tail = node; requeue_head = node; requeue_count++; @@ -248,7 +259,7 @@ static inline void tls_list_spill_excess(int class_idx, TinyTLSList* tls) { } if (requeue_head) { - tls_list_bulk_put(tls, requeue_head, requeue_tail, requeue_count); + tls_list_bulk_put(tls, requeue_head, requeue_tail, requeue_count, class_idx); } #if HAKMEM_PROF_STATIC && HAKMEM_BUILD_DEBUG diff --git a/core/tiny_free_magazine.inc.h b/core/tiny_free_magazine.inc.h index 4b4e8fc2..91ca6294 100644 --- a/core/tiny_free_magazine.inc.h +++ b/core/tiny_free_magazine.inc.h @@ -194,7 +194,7 @@ if (pthread_equal(slab->owner_tid, tiny_self_pt())) { int class_idx = slab->class_idx; - if (g_tls_list_enable) { + if (g_tls_list_enable && class_idx != 7) { TinyTLSList* tls = &g_tls_lists[class_idx]; uint32_t seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed); if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) { @@ -211,7 +211,7 @@ if (tls->count < tls->cap) { void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); - tls_list_push(tls, base); + tls_list_push(tls, base, class_idx); HAK_STAT_FREE(class_idx); return; } @@ -222,7 +222,7 @@ { void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1); tiny_tls_list_guard_push(class_idx, tls, base); - tls_list_push(tls, base); + tls_list_push(tls, base, class_idx); } if (tls_list_should_spill(tls)) { tls_list_spill_excess(class_idx, tls); diff --git a/core/tiny_superslab_alloc.inc.h b/core/tiny_superslab_alloc.inc.h index 0ced0ef9..c0fbfad9 100644 --- a/core/tiny_superslab_alloc.inc.h +++ b/core/tiny_superslab_alloc.inc.h @@ -743,7 +743,8 @@ static inline void* hak_tiny_alloc_superslab(int class_idx) { // } meta->used++; - // Debug: Log first C7 alloc for path verification + // Debug: Log first C7 alloc for path verification (debug-only) +#if HAKMEM_DEBUG_VERBOSE if (class_idx == 7) { static _Atomic int c7_alloc_count = 0; int count = atomic_fetch_add_explicit(&c7_alloc_count, 1, memory_order_relaxed); @@ -756,6 +757,7 @@ static inline void* hak_tiny_alloc_superslab(int class_idx) { fprintf(stderr, "[C7_FIRST_ALLOC] ptr=%p next=%p slab_idx=%d\n", block, next, slab_idx); } } +#endif // Track active blocks in SuperSlab for conservative reclamation ss_active_inc(ss); diff --git a/hakmem.d b/hakmem.d index 2dbe9cdb..a0f4aab5 100644 --- a/hakmem.d +++ b/hakmem.d @@ -24,7 +24,8 @@ hakmem.o: core/hakmem.c core/hakmem.h core/hakmem_build_flags.h \ core/box/../tiny_free_fast_v2.inc.h core/box/../tiny_region_id.h \ core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_config.h \ core/box/../box/tls_sll_box.h core/box/../box/../hakmem_tiny_config.h \ - core/box/../box/../hakmem_build_flags.h core/box/front_gate_classifier.h \ + core/box/../box/../hakmem_build_flags.h \ + core/box/../box/../tiny_region_id.h core/box/front_gate_classifier.h \ core/box/hak_wrappers.inc.h core/hakmem.h: core/hakmem_build_flags.h: @@ -86,5 +87,6 @@ core/box/../hakmem_tiny_config.h: core/box/../box/tls_sll_box.h: core/box/../box/../hakmem_tiny_config.h: core/box/../box/../hakmem_build_flags.h: +core/box/../box/../tiny_region_id.h: core/box/front_gate_classifier.h: core/box/hak_wrappers.inc.h: