tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Box TLS-SLL + free boundary hardening: normalize C0–C6 to base (ptr-1) at free boundary; route all caches/freelists via base; replace remaining g_tls_sll_head direct writes with Box API (tls_sll_push/splice) in refill/magazine/ultra; keep C7 excluded. Fixes rbp=0xa0 free crash by preventing header overwrite and centralizing TLS-SLL invariants.

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-11-10 16:48:20 +09:00
parent 1b6624dec4
commit b09ba4d40d
26 changed files with 1079 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
core/tiny_free_magazine.inc.h
View File

@ -19,7 +19,8 @@
if (g_quick_enable && class_idx <= 4) {
TinyQuickSlot* qs = &g_tls_quick[class_idx];
if (__builtin_expect(qs->top < QUICK_CAP, 1)) {
qs->items[qs->top++] = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
qs->items[qs->top++] = base;
HAK_STAT_FREE(class_idx);
return;
}
@ -28,17 +29,18 @@
// Fast path: TLS SLL push for hottest classes
if (!g_tls_list_enable && g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)cap)) {
*(void**)ptr = g_tls_sll_head[class_idx];
g_tls_sll_head[class_idx] = ptr;
g_tls_sll_count[class_idx]++;
// BUGFIX: Decrement used counter (was missing, causing Fail-Fast on next free)
meta->used--;
// Active → Inactive: count down immediately (TLS保管中は"使用中"ではない)
ss_active_dec_one(ss);
HAK_TP1(sll_push, class_idx);
tiny_debug_ring_record(TINY_RING_EVENT_FREE_LOCAL, (uint16_t)class_idx, ptr, 3);
HAK_STAT_FREE(class_idx);
return;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap);
if (tls_sll_push(class_idx, base, sll_cap)) {
// BUGFIX: Decrement used counter (was missing, causing Fail-Fast on next free)
meta->used--;
// Active → Inactive: count down immediately (TLS保管中は"使用中"ではない)
ss_active_dec_one(ss);
HAK_TP1(sll_push, class_idx);
tiny_debug_ring_record(TINY_RING_EVENT_FREE_LOCAL, (uint16_t)class_idx, ptr, 3);
HAK_STAT_FREE(class_idx);
return;
}
}
// Next: Magazine push（必要ならmag→SLLへバルク転送で空きを作る）
@ -47,7 +49,8 @@
(void)bulk_mag_to_sll_if_room(class_idx, mag, cap / 2);
}
if (mag->top < cap + g_spill_hyst) {
mag->items[mag->top].ptr = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = NULL; // SuperSlab owner not a TinySlab; leave NULL
#endif
@ -71,7 +74,7 @@
int limit = g_bg_spill_max_batch;
if (limit > cap/2) limit = cap/2;
if (limit > 32) limit = 32; // keep free-path bounded
void* head = ptr;
void* head = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
*(void**)head = NULL;
void* tail = head; // current tail
int taken = 1;
@ -138,7 +141,8 @@
// Finally, try FastCache push first (≤128B) — compile-out if HAKMEM_TINY_NO_FRONT_CACHE
#if !defined(HAKMEM_TINY_NO_FRONT_CACHE)
if (g_fastcache_enable && class_idx <= 4) {
if (fastcache_push(class_idx, ptr)) {
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (fastcache_push(class_idx, base)) {
HAK_TP1(front_push, class_idx);
HAK_STAT_FREE(class_idx);
return;
@ -147,11 +151,19 @@
#endif
// Then TLS SLL if room, else magazine
if (g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)mag->cap)) {
*(void**)ptr = g_tls_sll_head[class_idx];
g_tls_sll_head[class_idx] = ptr;
g_tls_sll_count[class_idx]++;
uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (!tls_sll_push(class_idx, base, sll_cap2)) {
// fallback to magazine
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
mag->top++;
}
} else {
mag->items[mag->top].ptr = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
@ -180,14 +192,16 @@
}
// TinyHotMag front push（8/16/32B, A/B）
if (__builtin_expect(g_hotmag_enable && class_idx <= 2, 1)) {
if (hotmag_push(class_idx, ptr)) {
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (hotmag_push(class_idx, base)) {
HAK_STAT_FREE(class_idx);
return;
}
}
if (tls->count < tls->cap) {
tiny_tls_list_guard_push(class_idx, tls, ptr);
tls_list_push(tls, ptr);
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
tiny_tls_list_guard_push(class_idx, tls, base);
tls_list_push(tls, base);
HAK_STAT_FREE(class_idx);
return;
}
@ -195,8 +209,11 @@
if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) {
tiny_tls_refresh_params(class_idx, tls);
}
tiny_tls_list_guard_push(class_idx, tls, ptr);
tls_list_push(tls, ptr);
{
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
tiny_tls_list_guard_push(class_idx, tls, base);
tls_list_push(tls, base);
}
if (tls_list_should_spill(tls)) {
tls_list_spill_excess(class_idx, tls);
}
@ -219,11 +236,11 @@
if (!g_tls_list_enable && g_tls_sll_enable && class_idx <= 5) {
uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap);
if (g_tls_sll_count[class_idx] < sll_cap) {
*(void**)ptr = g_tls_sll_head[class_idx];
g_tls_sll_head[class_idx] = ptr;
g_tls_sll_count[class_idx]++;
HAK_STAT_FREE(class_idx);
return;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (tls_sll_push(class_idx, base, sll_cap)) {
HAK_STAT_FREE(class_idx);
return;
}
}
}
// Next: if magazine has room, push immediately and return（満杯ならmag→SLLへバルク）
@ -232,11 +249,14 @@
}
// Remote-drain can be handled opportunistically on future calls.
if (mag->top < cap) {
mag->items[mag->top].ptr = ptr;
{
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
mag->items[mag->top].owner = slab;
#endif
mag->top++;
mag->top++;
}
#if HAKMEM_DEBUG_COUNTERS
g_magazine_push_count++; // Phase 7.6: Track pushes
@ -358,23 +378,33 @@
if (g_quick_enable && class_idx <= 4) {
TinyQuickSlot* qs = &g_tls_quick[class_idx];
if (__builtin_expect(qs->top < QUICK_CAP, 1)) {
qs->items[qs->top++] = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
qs->items[qs->top++] = base;
} else if (g_tls_sll_enable) {
uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
if (g_tls_sll_count[class_idx] < sll_cap2) {
*(void**)ptr = g_tls_sll_head[class_idx];
g_tls_sll_head[class_idx] = ptr;
g_tls_sll_count[class_idx]++;
} else if (!tiny_optional_push(class_idx, ptr)) {
mag->items[mag->top].ptr = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (!tls_sll_push(class_idx, base, sll_cap2)) {
if (!tiny_optional_push(class_idx, base)) {
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
mag->top++;
}
}
} else if (!tiny_optional_push(class_idx, (class_idx == 7 ? ptr : (void*)((uint8_t*)ptr - 1)))) {
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
mag->top++;
}
} else {
if (!tiny_optional_push(class_idx, ptr)) {
mag->items[mag->top].ptr = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (!tiny_optional_push(class_idx, base)) {
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
@ -387,19 +417,28 @@
if (g_tls_sll_enable && class_idx <= 5) {
uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
if (g_tls_sll_count[class_idx] < sll_cap2) {
*(void**)ptr = g_tls_sll_head[class_idx];
g_tls_sll_head[class_idx] = ptr;
g_tls_sll_count[class_idx]++;
} else if (!tiny_optional_push(class_idx, ptr)) {
mag->items[mag->top].ptr = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (!tls_sll_push(class_idx, base, sll_cap2)) {
if (!tiny_optional_push(class_idx, base)) {
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
mag->top++;
}
}
} else if (!tiny_optional_push(class_idx, (class_idx == 7 ? ptr : (void*)((uint8_t*)ptr - 1)))) {
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
mag->top++;
}
} else {
if (!tiny_optional_push(class_idx, ptr)) {
mag->items[mag->top].ptr = ptr;
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
if (!tiny_optional_push(class_idx, base)) {
mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
mag->items[mag->top].owner = slab;
#endif
@ -415,6 +454,7 @@
HAK_STAT_FREE(class_idx); // Phase 3
return;
} else {
tiny_remote_push(slab, ptr);
void* base = (slab && slab->class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
tiny_remote_push(slab, base);
}
}