Phase 7: header-aware TLS front caches and FG gating
- core/hakmem_tiny_fastcache.inc.h: make tiny_fast_pop/push read/write next at base+1 for C0–C6; clear C7 next on pop - core/hakmem_tiny_hot_pop.inc.h: header-aware next reads for g_fast_head pops (classes 0–3) - core/tiny_free_magazine.inc.h: header-aware chain linking for BG spill chain (base+1 for C0–C6) - core/box/front_gate_classifier.c: registry fallback classifies headerless only for class 7; others as headered Build OK; bench_fixed_size_hakmem still SIGBUS right after init. FREE_ROUTE trace shows invalid frees (ptr=0xa0, etc.). Next steps: instrument early frees and audit remaining header-aware writes in any front caches not yet patched.
This commit is contained in:
@ -41,11 +41,17 @@ static inline void refill_opt_dbg(const char* stage, int class_idx, uint32_t n)
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void trc_push_front(TinyRefillChain* c, void* node) {
|
||||
// Phase 7 header-aware push_front: link using base+1 for C0-C6 (C7 not used here)
|
||||
static inline void trc_push_front(TinyRefillChain* c, void* node, int class_idx) {
|
||||
#if HAKMEM_TINY_HEADER_CLASSIDX
|
||||
const size_t next_offset = (class_idx == 7) ? 0 : 1;
|
||||
#else
|
||||
const size_t next_offset = 0;
|
||||
#endif
|
||||
if (c->head == NULL) {
|
||||
c->head = node; c->tail = node; *(void**)node = NULL; c->count = 1;
|
||||
c->head = node; c->tail = node; *(void**)((uint8_t*)node + next_offset) = NULL; c->count = 1;
|
||||
} else {
|
||||
*(void**)node = c->head; c->head = node; c->count++;
|
||||
*(void**)((uint8_t*)node + next_offset) = c->head; c->head = node; c->count++;
|
||||
}
|
||||
}
|
||||
|
||||
@ -167,7 +173,7 @@ static inline uint32_t trc_pop_from_freelist(struct TinySlabMeta* meta,
|
||||
trc_failfast_abort("freelist_next", class_idx, ss_base, ss_limit, next);
|
||||
}
|
||||
meta->freelist = next;
|
||||
trc_push_front(out, p);
|
||||
trc_push_front(out, p, class_idx);
|
||||
taken++;
|
||||
}
|
||||
// DEBUG REMOVED: refill_opt_dbg causes -26% regression (atomic CAS overhead)
|
||||
@ -175,9 +181,12 @@ static inline uint32_t trc_pop_from_freelist(struct TinySlabMeta* meta,
|
||||
}
|
||||
|
||||
// Carve a contiguous batch of size 'batch' from linear area, return as chain
|
||||
// Phase 7 header-aware carve: link chain using header-safe next location
|
||||
// class_idx is required to decide headerless (C7) vs headered (C0-C6)
|
||||
static inline uint32_t trc_linear_carve(uint8_t* base, size_t bs,
|
||||
struct TinySlabMeta* meta,
|
||||
uint32_t batch,
|
||||
int class_idx,
|
||||
TinyRefillChain* out) {
|
||||
if (!out || batch == 0) return 0;
|
||||
trc_init(out);
|
||||
@ -206,9 +215,18 @@ static inline uint32_t trc_linear_carve(uint8_t* base, size_t bs,
|
||||
(void*)base, meta->carved, batch, (void*)cursor);
|
||||
}
|
||||
|
||||
// CRITICAL FIX (Phase 7): header-aware next pointer placement
|
||||
// For header classes (C0-C6), the first byte at base is the 1-byte header.
|
||||
// Store the SLL next pointer at base+1 to avoid clobbering the header.
|
||||
// For C7 (headerless), store at base.
|
||||
#if HAKMEM_TINY_HEADER_CLASSIDX
|
||||
const size_t next_offset = (class_idx == 7) ? 0 : 1;
|
||||
#else
|
||||
const size_t next_offset = 0;
|
||||
#endif
|
||||
for (uint32_t i = 1; i < batch; i++) {
|
||||
uint8_t* next = cursor + stride;
|
||||
*(void**)cursor = (void*)next;
|
||||
*(void**)(cursor + next_offset) = (void*)next;
|
||||
cursor = next;
|
||||
}
|
||||
void* tail = (void*)cursor;
|
||||
|
||||
Reference in New Issue
Block a user