Phase 7: header-aware TLS front caches and FG gating

- core/hakmem_tiny_fastcache.inc.h: make tiny_fast_pop/push read/write next at base+1 for C0–C6; clear C7 next on pop
- core/hakmem_tiny_hot_pop.inc.h: header-aware next reads for g_fast_head pops (classes 0–3)
- core/tiny_free_magazine.inc.h: header-aware chain linking for BG spill chain (base+1 for C0–C6)
- core/box/front_gate_classifier.c: registry fallback classifies headerless only for class 7; others as headered

Build OK; bench_fixed_size_hakmem still SIGBUS right after init. FREE_ROUTE trace shows invalid frees (ptr=0xa0, etc.). Next steps: instrument early frees and audit remaining header-aware writes in any front caches not yet patched.
This commit is contained in:
Moe Charm (CI)
2025-11-10 18:04:08 +09:00
parent d739ea7769
commit dde490f842
13 changed files with 166 additions and 37 deletions

View File

@ -41,11 +41,17 @@ static inline void refill_opt_dbg(const char* stage, int class_idx, uint32_t n)
#endif
}
static inline void trc_push_front(TinyRefillChain* c, void* node) {
// Phase 7 header-aware push_front: link using base+1 for C0-C6 (C7 not used here)
static inline void trc_push_front(TinyRefillChain* c, void* node, int class_idx) {
#if HAKMEM_TINY_HEADER_CLASSIDX
const size_t next_offset = (class_idx == 7) ? 0 : 1;
#else
const size_t next_offset = 0;
#endif
if (c->head == NULL) {
c->head = node; c->tail = node; *(void**)node = NULL; c->count = 1;
c->head = node; c->tail = node; *(void**)((uint8_t*)node + next_offset) = NULL; c->count = 1;
} else {
*(void**)node = c->head; c->head = node; c->count++;
*(void**)((uint8_t*)node + next_offset) = c->head; c->head = node; c->count++;
}
}
@ -167,7 +173,7 @@ static inline uint32_t trc_pop_from_freelist(struct TinySlabMeta* meta,
trc_failfast_abort("freelist_next", class_idx, ss_base, ss_limit, next);
}
meta->freelist = next;
trc_push_front(out, p);
trc_push_front(out, p, class_idx);
taken++;
}
// DEBUG REMOVED: refill_opt_dbg causes -26% regression (atomic CAS overhead)
@ -175,9 +181,12 @@ static inline uint32_t trc_pop_from_freelist(struct TinySlabMeta* meta,
}
// Carve a contiguous batch of size 'batch' from linear area, return as chain
// Phase 7 header-aware carve: link chain using header-safe next location
// class_idx is required to decide headerless (C7) vs headered (C0-C6)
static inline uint32_t trc_linear_carve(uint8_t* base, size_t bs,
struct TinySlabMeta* meta,
uint32_t batch,
int class_idx,
TinyRefillChain* out) {
if (!out || batch == 0) return 0;
trc_init(out);
@ -206,9 +215,18 @@ static inline uint32_t trc_linear_carve(uint8_t* base, size_t bs,
(void*)base, meta->carved, batch, (void*)cursor);
}
// CRITICAL FIX (Phase 7): header-aware next pointer placement
// For header classes (C0-C6), the first byte at base is the 1-byte header.
// Store the SLL next pointer at base+1 to avoid clobbering the header.
// For C7 (headerless), store at base.
#if HAKMEM_TINY_HEADER_CLASSIDX
const size_t next_offset = (class_idx == 7) ? 0 : 1;
#else
const size_t next_offset = 0;
#endif
for (uint32_t i = 1; i < batch; i++) {
uint8_t* next = cursor + stride;
*(void**)cursor = (void*)next;
*(void**)(cursor + next_offset) = (void*)next;
cursor = next;
}
void* tail = (void*)cursor;