tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix C7 warm/TLS Release path and unify debug instrumentation

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-05 23:41:01 +09:00
parent 96c2988381
commit d17ec46628
29 changed files with 1314 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
core/tiny_superslab_alloc.inc.h
View File

@ -7,6 +7,8 @@
#include "box/superslab_expansion_box.h" // Box E: Expansion with TLS state guarantee
#include "box/tiny_next_ptr_box.h" // Box API: Next pointer read/write
#include "box/tiny_tls_carve_one_block_box.h" // Box: Shared TLS carve helper
#include "box/c7_meta_used_counter_box.h" // Box: C7 meta->used telemetry
#include "hakmem_tiny_superslab_constants.h"
#include "tiny_box_geometry.h" // Box 3: Geometry & Capacity Calculator"
#include "tiny_debug_api.h" // Guard/failfast declarations
@ -33,6 +35,7 @@ static inline void* superslab_alloc_from_slab(SuperSlab* ss, int slab_idx) {
uint8_t* base = tiny_slab_base_for_geometry(ss, slab_idx);
void* block = tiny_block_at_index(base, meta->used, unit_sz);
meta->used++;
c7_meta_used_note(cls, C7_META_USED_SRC_FRONT);
ss_active_inc(ss);
HAK_RET_ALLOC(cls, block);
}
@ -105,6 +108,7 @@ static inline void* superslab_alloc_from_slab(SuperSlab* ss, int slab_idx) {
}
#endif
meta->used++;
c7_meta_used_note(meta->class_idx, C7_META_USED_SRC_FRONT);
void* user =
#if HAKMEM_TINY_HEADER_CLASSIDX
tiny_region_id_write_header(block_base, meta->class_idx);
@ -157,6 +161,7 @@ static inline void* superslab_alloc_from_slab(SuperSlab* ss, int slab_idx) {
meta->freelist = tiny_next_read(meta->class_idx, block);
meta->used++;
c7_meta_used_note(meta->class_idx, C7_META_USED_SRC_FRONT);
if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0) &&
__builtin_expect(meta->used > meta->capacity, 0)) {
@ -294,54 +299,33 @@ static inline void* hak_tiny_alloc_superslab(int class_idx) {
}
// Fast path: linear carve from current TLS slab
if (meta && meta->freelist == NULL && meta->used < meta->capacity && tls->slab_base) {
size_t block_size = tiny_stride_for_class(meta->class_idx);
uint8_t* base = tls->slab_base;
void* block = base + ((size_t)meta->used * block_size);
meta->used++;
if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) {
uintptr_t base_ss = (uintptr_t)tls->ss;
size_t ss_size = (size_t)1ULL << tls->ss->lg_size;
uintptr_t p = (uintptr_t)block;
int in_range = (p >= base_ss) && (p < base_ss + ss_size);
int aligned = ((p - (uintptr_t)base) % block_size) == 0;
int idx_ok = (tls->slab_idx >= 0) &&
(tls->slab_idx < ss_slabs_capacity(tls->ss));
if (!in_range || !aligned || !idx_ok || meta->used > meta->capacity) {
tiny_failfast_abort_ptr("alloc_ret_align",
tls->ss,
tls->slab_idx,
block,
"superslab_tls_invariant");
if (meta && tls->slab_base) {
TinyTLSCarveOneResult carve = tiny_tls_carve_one_block(tls, class_idx);
if (carve.block) {
#if !HAKMEM_BUILD_RELEASE
if (__builtin_expect(g_debug_remote_guard, 0)) {
const char* tag = (carve.path == TINY_TLS_CARVE_PATH_FREELIST)
? "freelist_alloc"
: "linear_alloc";
tiny_remote_track_on_alloc(tls->ss, slab_idx, carve.block, tag, 0);
tiny_remote_assert_not_remote(tls->ss, slab_idx, carve.block, tag, 0);
}
}
#endif
ss_active_inc(tls->ss);
ROUTE_MARK(11); ROUTE_COMMIT(class_idx, 0x60);
HAK_RET_ALLOC(class_idx, block);
}
// Freelist path from current TLS slab
if (meta && meta->freelist) {
void* block = meta->freelist;
if (__builtin_expect(g_tiny_safe_free, 0)) {
size_t blk = tiny_stride_for_class(meta->class_idx);
uint8_t* base = tiny_slab_base_for_geometry(tls->ss, tls->slab_idx);
uintptr_t delta = (uintptr_t)block - (uintptr_t)base;
int align_ok = ((delta % blk) == 0);
int range_ok = (delta / blk) < meta->capacity;
if (!align_ok || !range_ok) {
if (g_tiny_safe_free_strict) { raise(SIGUSR2); return NULL; }
return NULL;
#if HAKMEM_TINY_SS_TLS_HINT
{
void* ss_base = (void*)tls->ss;
size_t ss_size = (size_t)1ULL << tls->ss->lg_size;
tls_ss_hint_update(tls->ss, ss_base, ss_size);
}
#endif
if (carve.path == TINY_TLS_CARVE_PATH_LINEAR) {
ROUTE_MARK(11); ROUTE_COMMIT(class_idx, 0x60);
} else if (carve.path == TINY_TLS_CARVE_PATH_FREELIST) {
ROUTE_MARK(12); ROUTE_COMMIT(class_idx, 0x61);
}
HAK_RET_ALLOC(class_idx, carve.block);
}
void* next = tiny_next_read(class_idx, block);
meta->freelist = next;
meta->used++;
ss_active_inc(tls->ss);
ROUTE_MARK(12); ROUTE_COMMIT(class_idx, 0x61);
HAK_RET_ALLOC(class_idx, block);
}
// Slow path: acquire a new slab via shared pool
@ -363,6 +347,7 @@ static inline void* hak_tiny_alloc_superslab(int class_idx) {
size_t block_size = tiny_stride_for_class(meta->class_idx);
void* block = tiny_block_at_index(tls->slab_base, meta->used, block_size);
meta->used++;
c7_meta_used_note(meta->class_idx, C7_META_USED_SRC_FRONT);
ss_active_inc(ss);
HAK_RET_ALLOC(class_idx, block);
}