104 lines
4.5 KiB
C++
104 lines
4.5 KiB
C++
// hakmem_tiny_slow.inc
|
||
// Slow path allocation implementation
|
||
|
||
// Slow path allocation function
|
||
// Phase 6-1.7: Export for box refactor (Box 5 needs access from hakmem.c)
|
||
#ifdef HAKMEM_TINY_PHASE6_BOX_REFACTOR
|
||
void* __attribute__((cold, noinline)) hak_tiny_alloc_slow(size_t size, int class_idx) {
|
||
#else
|
||
static void* __attribute__((cold, noinline)) hak_tiny_alloc_slow(size_t size, int class_idx) {
|
||
#endif
|
||
(void)size; // size is already validated by caller
|
||
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) {
|
||
return NULL;
|
||
}
|
||
|
||
// Try refilling from HotMag
|
||
if (g_hotmag_enable && class_idx <= 3) {
|
||
TinyHotMag* hm = &g_tls_hot_mag[class_idx];
|
||
hotmag_try_refill(class_idx, hm);
|
||
void* ptr = hotmag_pop(class_idx);
|
||
if (ptr) { HAK_RET_ALLOC(class_idx, ptr); }
|
||
}
|
||
|
||
// Try TLS list refill (C7 is headerless: skip TLS list entirely)
|
||
if (g_tls_list_enable && class_idx != 7) {
|
||
TinyTLSList* tls = &g_tls_lists[class_idx];
|
||
// Fail‑Fast: guard against poisoned head (remote sentinel)
|
||
if (__builtin_expect((uintptr_t)tls->head == TINY_REMOTE_SENTINEL, 0)) {
|
||
tls->head = NULL;
|
||
tls->count = 0;
|
||
}
|
||
if (tls->count > 0) {
|
||
void* ptr = tls_list_pop(tls, class_idx);
|
||
if (ptr) { HAK_RET_ALLOC(class_idx, ptr); }
|
||
// ptr が NULL の場合でも、ここで終了せず後段の Superslab 経路へフォールバックする
|
||
}
|
||
|
||
// Try refilling TLS list from slab
|
||
uint32_t want = tls->refill_low > 0 ? tls->refill_low : 32;
|
||
if (tls_refill_from_tls_slab(class_idx, tls, want) > 0) {
|
||
void* ptr = tls_list_pop(tls, class_idx);
|
||
if (ptr) { HAK_RET_ALLOC(class_idx, ptr); }
|
||
// ここでも NULL の場合は続行（後段へフォールバック）
|
||
}
|
||
}
|
||
|
||
// Background coalescing/aggregation (ENV gated, very lightweight)
|
||
do {
|
||
// BG Remote Drain (coalescer)
|
||
static int bg_en = -1, bg_period = -1, bg_budget = -1;
|
||
static __thread uint32_t bg_tick[8];
|
||
if (__builtin_expect(bg_en == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_TINY_BG_REMOTE");
|
||
bg_en = (e && *e && *e != '0') ? 1 : 0;
|
||
const char* p = getenv("HAKMEM_TINY_BG_REMOTE_PERIOD");
|
||
bg_period = p ? atoi(p) : 1024;
|
||
if (bg_period <= 0) bg_period = 1024;
|
||
const char* b = getenv("HAKMEM_TINY_BG_REMOTE_BATCH");
|
||
bg_budget = b ? atoi(b) : 4;
|
||
if (bg_budget < 0) bg_budget = 0; if (bg_budget > 64) bg_budget = 64;
|
||
}
|
||
if (bg_en) {
|
||
if ((++bg_tick[class_idx] % (uint32_t)bg_period) == 0u) {
|
||
extern void tiny_remote_bg_drain_step(int class_idx, int budget);
|
||
tiny_remote_bg_drain_step(class_idx, bg_budget);
|
||
}
|
||
}
|
||
// Ready Aggregator (mailbox → ready push)
|
||
static int rdy_en = -1, rdy_period = -1, rdy_budget = -1;
|
||
static __thread uint32_t rdy_tick[8];
|
||
if (__builtin_expect(rdy_en == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_TINY_BG_READY");
|
||
rdy_en = (e && *e && *e != '0') ? 1 : 0;
|
||
const char* p = getenv("HAKMEM_TINY_BG_READY_PERIOD");
|
||
rdy_period = p ? atoi(p) : 1024;
|
||
if (rdy_period <= 0) rdy_period = 1024;
|
||
const char* b = getenv("HAKMEM_TINY_BG_READY_BUDGET");
|
||
rdy_budget = b ? atoi(b) : 1;
|
||
if (rdy_budget < 0) rdy_budget = 0; if (rdy_budget > 8) rdy_budget = 8;
|
||
}
|
||
if (rdy_en) {
|
||
if ((++rdy_tick[class_idx] % (uint32_t)rdy_period) == 0u) {
|
||
extern void tiny_ready_bg_aggregate_step(int class_idx, int mail_budget);
|
||
tiny_ready_bg_aggregate_step(class_idx, rdy_budget);
|
||
}
|
||
}
|
||
} while (0);
|
||
|
||
// Final fallback: allocate from superslab
|
||
void* ss_ptr = hak_tiny_alloc_superslab(class_idx);
|
||
if (ss_ptr) { HAK_RET_ALLOC(class_idx, ss_ptr); }
|
||
tiny_alloc_dump_tls_state(class_idx, "slow_fail", &g_tls_slabs[class_idx]);
|
||
// Optional one-shot debug when final slow path fails
|
||
static int g_alloc_dbg = -1; if (__builtin_expect(g_alloc_dbg == -1, 0)) { const char* e=getenv("HAKMEM_TINY_ALLOC_DEBUG"); g_alloc_dbg = (e && atoi(e)!=0)?1:0; }
|
||
if (g_alloc_dbg) {
|
||
static _Atomic int printed[8]; int exp=0;
|
||
if (atomic_compare_exchange_strong(&printed[class_idx], &exp, 1)) {
|
||
fprintf(stderr, "[ALLOC-SLOW] hak_tiny_alloc_superslab returned NULL class=%d size=%zu\n", class_idx, size);
|
||
}
|
||
}
|
||
return ss_ptr;
|
||
}
|