// hakmem_tiny_slow.inc // Slow path allocation implementation // Slow path allocation function // Phase 6-1.7: Export for box refactor (Box 5 needs access from hakmem.c) #ifdef HAKMEM_TINY_PHASE6_BOX_REFACTOR void* __attribute__((cold, noinline)) hak_tiny_alloc_slow(size_t size, int class_idx) { #else static void* __attribute__((cold, noinline)) hak_tiny_alloc_slow(size_t size, int class_idx) { #endif (void)size; // size is already validated by caller if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) { return NULL; } // Try refilling from HotMag if (g_hotmag_enable && class_idx <= 3) { TinyHotMag* hm = &g_tls_hot_mag[class_idx]; hotmag_try_refill(class_idx, hm); void* ptr = hotmag_pop(class_idx); if (ptr) { HAK_RET_ALLOC(class_idx, ptr); } } // Try TLS SLL via Box (Phase12 正式経路) // C7 は headerless: 既存仕様通り TLS/SLL をスキップ if (g_tls_sll_enable && class_idx != 7) { // Box: 単一APIで TLS SLL を扱う(内部で head/count/next を管理) void* ptr = NULL; if (tls_sll_pop(class_idx, &ptr)) { return ptr; } } // Try TLS list (legacy small-mag) via既存API(構造体直接触らない) if (g_tls_list_enable && class_idx != 7) { TinyTLSList* tls = &g_tls_lists[class_idx]; // Fail-Fast: guard against poisoned head (remote sentinel) if (__builtin_expect((uintptr_t)tls->head == TINY_REMOTE_SENTINEL, 0)) { tls->head = NULL; tls->count = 0; } if (tls->count > 0) { void* ptr = tls_list_pop(tls, class_idx); if (ptr) { HAK_RET_ALLOC(class_idx, ptr); } } // Try refilling TLS list from TLS-cached Superslab slab uint32_t want = tls->refill_low > 0 ? tls->refill_low : 32; if (tls_refill_from_tls_slab(class_idx, tls, want) > 0) { void* ptr = tls_list_pop(tls, class_idx); if (ptr) { HAK_RET_ALLOC(class_idx, ptr); } } } // Background coalescing/aggregation (ENV gated, very lightweight) do { // BG Remote Drain (coalescer) static int bg_en = -1, bg_period = -1, bg_budget = -1; static __thread uint32_t bg_tick[8]; if (__builtin_expect(bg_en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_BG_REMOTE"); bg_en = (e && *e && *e != '0') ? 1 : 0; const char* p = getenv("HAKMEM_TINY_BG_REMOTE_PERIOD"); bg_period = p ? atoi(p) : 1024; if (bg_period <= 0) bg_period = 1024; const char* b = getenv("HAKMEM_TINY_BG_REMOTE_BATCH"); bg_budget = b ? atoi(b) : 4; if (bg_budget < 0) bg_budget = 0; if (bg_budget > 64) bg_budget = 64; } if (bg_en) { if ((++bg_tick[class_idx] % (uint32_t)bg_period) == 0u) { extern void tiny_remote_bg_drain_step(int class_idx, int budget); tiny_remote_bg_drain_step(class_idx, bg_budget); } } // Ready Aggregator (mailbox → ready push) static int rdy_en = -1, rdy_period = -1, rdy_budget = -1; static __thread uint32_t rdy_tick[8]; if (__builtin_expect(rdy_en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_BG_READY"); rdy_en = (e && *e && *e != '0') ? 1 : 0; const char* p = getenv("HAKMEM_TINY_BG_READY_PERIOD"); rdy_period = p ? atoi(p) : 1024; if (rdy_period <= 0) rdy_period = 1024; const char* b = getenv("HAKMEM_TINY_BG_READY_BUDGET"); rdy_budget = b ? atoi(b) : 1; if (rdy_budget < 0) rdy_budget = 0; if (rdy_budget > 8) rdy_budget = 8; } if (rdy_en) { if ((++rdy_tick[class_idx] % (uint32_t)rdy_period) == 0u) { extern void tiny_ready_bg_aggregate_step(int class_idx, int mail_budget); tiny_ready_bg_aggregate_step(class_idx, rdy_budget); } } } while (0); // Final fallback: allocate from superslab via Box API wrapper (Stage A) // NOTE: // - hak_tiny_alloc_superslab_box() is a thin façade over the legacy // per-class SuperslabHead backend in Phase 12 Stage A. // - Callers (slow path) no longer depend on internal Superslab layout. void* ss_ptr = hak_tiny_alloc_superslab_box(class_idx); if (ss_ptr) { HAK_RET_ALLOC(class_idx, ss_ptr); } tiny_alloc_dump_tls_state(class_idx, "slow_fail", &g_tls_slabs[class_idx]); // Optional one-shot debug when final slow path fails static int g_alloc_dbg = -1; if (__builtin_expect(g_alloc_dbg == -1, 0)) { const char* e=getenv("HAKMEM_TINY_ALLOC_DEBUG"); g_alloc_dbg = (e && atoi(e)!=0)?1:0; } if (g_alloc_dbg) { static _Atomic int printed[8]; int exp=0; if (atomic_compare_exchange_strong(&printed[class_idx], &exp, 1)) { fprintf(stderr, "[ALLOC-SLOW] hak_tiny_alloc_superslab returned NULL class=%d size=%zu\n", class_idx, size); } } return ss_ptr; }