// tiny_superslab_alloc.inc.h - SuperSlab Allocation Layer (Box 4) // Purpose: Slab allocation, refill, and adoption logic (Phase 12 shared pool) // Public functions: // - superslab_alloc_from_slab(): Allocate from specific slab (linear or freelist) // - superslab_refill(): Refill TLS slab via shared pool // - hak_tiny_alloc_superslab(): Main SuperSlab allocation entry point #include "box/superslab_expansion_box.h" // Box E: Expansion with TLS state guarantee #include "box/tiny_next_ptr_box.h" // Box API: Next pointer read/write #include "box/tiny_tls_carve_one_block_box.h" // Box: Shared TLS carve helper #include "box/c7_meta_used_counter_box.h" // Box: C7 meta->used telemetry #include "hakmem_tiny_superslab_constants.h" #include "tiny_box_geometry.h" // Box 3: Geometry & Capacity Calculator" #include "tiny_debug_api.h" // Guard/failfast declarations #include "hakmem_env_cache.h" // Priority-2: ENV cache (eliminate syscalls) #if HAKMEM_TINY_SS_TLS_HINT #include "box/tls_ss_hint_box.h" // Phase 1: TLS SuperSlab Hint Cache for Headerless mode #endif // ============================================================================ // Phase 6.24: Allocate from SuperSlab slab (lazy freelist + linear allocation) // ============================================================================ static inline void* superslab_alloc_from_slab(SuperSlab* ss, int slab_idx) { TinySlabMeta* meta = &ss->slabs[slab_idx]; // Small hot classes (C0–C3): bump-only fast path if no remote/freelist do { uint8_t cls = meta->class_idx; if (__builtin_expect(cls <= 3, 1)) { if (atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_acquire) != 0) break; if (meta->freelist == NULL && meta->used < meta->capacity) { size_t unit_sz = tiny_stride_for_class(cls); uint8_t* base = tiny_slab_base_for_geometry(ss, slab_idx); void* block = tiny_block_at_index(base, meta->used, unit_sz); meta->used++; c7_meta_used_note(cls, C7_META_USED_SRC_FRONT); ss_active_inc(ss); HAK_RET_ALLOC(cls, block); } } } while (0); // Drain remote queue if needed before handing blocks back to TLS if (__builtin_expect(atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_acquire) != 0, 0)) { uint32_t self_tid = tiny_self_u32(); SlabHandle h = slab_try_acquire(ss, slab_idx, self_tid); if (slab_is_valid(&h)) { slab_drain_remote_full(&h); int pending = atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_acquire) != 0; if (__builtin_expect(pending, 0)) { if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_relaxed); tiny_remote_watch_note("alloc_pending_remote", ss, slab_idx, (void*)head, 0xA243u, self_tid, 0); } slab_release(&h); return NULL; } slab_release(&h); } else { if (__builtin_expect(g_debug_remote_guard, 0)) { tiny_remote_watch_note("alloc_acquire_fail", ss, slab_idx, meta, 0xA244u, self_tid, 0); } return NULL; } } if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head_pending = atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_acquire); if (head_pending != 0) { tiny_remote_watch_note("alloc_remote_pending", ss, slab_idx, (void*)head_pending, 0xA247u, tiny_self_u32(), 1); return NULL; } } // Linear allocation mode if (__builtin_expect(meta->freelist == NULL && meta->used < meta->capacity, 1)) { size_t unit_sz = tiny_stride_for_class(meta->class_idx); uint8_t* base = tiny_slab_base_for_geometry(ss, slab_idx); void* block_base = tiny_block_at_index(base, meta->used, unit_sz); #if !HAKMEM_BUILD_RELEASE if (__builtin_expect(!tiny_carve_guard(slab_idx, meta->used, unit_sz, 1), 0)) { size_t dbg_usable = tiny_usable_bytes_for_slab(slab_idx); uintptr_t dbg_off = (uintptr_t)((uint8_t*)block_base - base); fprintf(stderr, "[TINY_ALLOC_BOUNDS] cls=%u slab=%d used=%u cap=%u unit=%zu off=%lu usable=%zu\n", meta->class_idx, slab_idx, meta->used, meta->capacity, unit_sz, (unsigned long)dbg_off, dbg_usable); return NULL; } #endif meta->used++; c7_meta_used_note(meta->class_idx, C7_META_USED_SRC_FRONT); void* user = #if HAKMEM_TINY_HEADER_CLASSIDX tiny_region_id_write_header(block_base, meta->class_idx); #else block_base; #endif if (__builtin_expect(g_debug_remote_guard, 0)) { tiny_remote_track_on_alloc(ss, slab_idx, user, "linear_alloc", 0); tiny_remote_assert_not_remote(ss, slab_idx, user, "linear_alloc_ret", 0); } // Phase 1: Update TLS hint cache with this SuperSlab (fast free path optimization) #if HAKMEM_TINY_SS_TLS_HINT { void* ss_base = (void*)ss; size_t ss_size = (size_t)1ULL << ss->lg_size; tls_ss_hint_update(ss, ss_base, ss_size); } #endif return user; } // Freelist mode if (__builtin_expect(meta->freelist != NULL, 0)) { void* block = meta->freelist; if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) { size_t blk = g_tiny_class_sizes[meta->class_idx]; uint8_t* slab_base = tiny_slab_base_for(ss, slab_idx); uintptr_t block_addr = (uintptr_t)block; uintptr_t slab_addr = (uintptr_t)slab_base; uintptr_t offset = block_addr - slab_addr; fprintf(stderr, "[ALLOC_POP] cls=%u slab=%d block=%p offset=%zu (used=%u cap=%u)\n", meta->class_idx, slab_idx, block, offset, meta->used, meta->capacity); // Misaligned freelist entry → drop this slab's freelist to force new slab. if ((offset % blk) != 0) { fprintf(stderr, "[ALLOC_POP_MISALIGN] cls=%u slab=%d offset_mod=%zu blk=%zu base=%p ss=%p\n", meta->class_idx, slab_idx, (size_t)(offset % blk), blk, block, (void*)ss); meta->freelist = NULL; return NULL; } if (offset % blk != 0 || offset / blk >= meta->capacity) { fprintf(stderr, "[ALLOC_CORRUPT] Freelist head invalid\n"); tiny_failfast_abort_ptr("alloc_pop_invalid", ss, slab_idx, block, "freelist_head_corrupt"); } } meta->freelist = tiny_next_read(meta->class_idx, block); meta->used++; c7_meta_used_note(meta->class_idx, C7_META_USED_SRC_FRONT); if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0) && __builtin_expect(meta->used > meta->capacity, 0)) { fprintf(stderr, "[ALLOC_CORRUPT] meta->used overflow on freelist alloc\n"); tiny_failfast_abort_ptr("alloc_used_overflow", ss, slab_idx, block, "freelist_used_over_capacity"); } if (__builtin_expect(g_debug_remote_guard, 0)) { tiny_remote_track_on_alloc(ss, slab_idx, block, "freelist_alloc", 0); tiny_remote_assert_not_remote(ss, slab_idx, block, "freelist_alloc_ret", 0); } // CRITICAL FIX (Larson double-free): Write header for freelist allocations // Problem: Freelist path was returning BASE without writing header // Result: Stale headers from previous allocations → double-free on next free // Solution: Always write header before returning (same as linear carve path) void* user = #if HAKMEM_TINY_HEADER_CLASSIDX tiny_region_id_write_header(block, meta->class_idx); #else block; #endif // Phase 1: Update TLS hint cache with this SuperSlab (fast free path optimization) #if HAKMEM_TINY_SS_TLS_HINT { void* ss_base = (void*)ss; size_t ss_size = (size_t)1ULL << ss->lg_size; tls_ss_hint_update(ss, ss_base, ss_size); } #endif return user; } return NULL; } // ============================================================================ /* * Phase 12: Shared SuperSlab Pool based superslab_refill * * ポリシー: * - superslab_refill(int class_idx) は shared pool を経由して * 「class_idx 用の slab を1枚 TLS にバインドする」単一のエントリポイントとする。 * - 呼び出し側は、この関数が: * * 成功時: TinyTLSSlab (g_tls_slabs[class_idx]) が有効な ss/meta/slab_base を指す * * 失敗時: NULL を返し、TLS は変更しない or クリーンに巻き戻される * ことだけを前提にすればよい。 * - shared_pool_acquire_slab() の戻り値は 0=成功 / 非0=失敗 とみなし、 * 成功時に (*ss_out, *slab_idx_out) が設定される想定とする。 * - superslab_init_slab() / tiny_tls_bind_slab() は再帰的に superslab_refill() を * 呼ばない設計前提(自己呼び出し禁止)。ここで安全側に防御チェックを行う。 */ // TLS Bind Box Logic moved to core/box/ss_tls_bind_box.h #include "box/ss_tls_bind_box.h" SuperSlab* superslab_refill(int class_idx) { #if HAKMEM_DEBUG_COUNTERS g_superslab_refill_calls_dbg[class_idx]++; #endif // Bounds check (defensive, should be enforced by callers too) if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) { return NULL; } TinyTLSSlab* tls = &g_tls_slabs[class_idx]; // Shared pool API: // 0 == success, (*ss_out, *slab_idx_out) に有効値が入る。 // !=0 == failure, 出力は未定義とみなす。 extern int shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out); SuperSlab* ss = NULL; int slab_idx = -1; if (shared_pool_acquire_slab(class_idx, &ss, &slab_idx) != 0 || !ss || slab_idx < 0) { return NULL; } if (!ss_tls_bind_one(class_idx, tls, ss, slab_idx, tiny_self_u32())) { return NULL; } return ss; } // ============================================================================ // Phase 6.24: SuperSlab-based allocation using TLS slab // ============================================================================ static inline void* hak_tiny_alloc_superslab(int class_idx) { // MidTC fast path do { void* mp = midtc_pop(class_idx); if (mp) { HAK_RET_ALLOC(class_idx, mp); } } while (0); TinyTLSSlab* tls = &g_tls_slabs[class_idx]; TinySlabMeta* meta = tls->meta; int slab_idx = tls->slab_idx; if (meta && slab_idx >= 0 && tls->ss) { // Ensure TLS metadata matches class and slab base if (tls->meta->class_idx != (uint8_t)class_idx) { tls->ss = NULL; tls->meta = NULL; tls->slab_idx = -1; tls->slab_base = NULL; meta = NULL; } else { uint8_t* canonical = tiny_slab_base_for(tls->ss, slab_idx); if (tls->slab_base != canonical) { tls->slab_base = canonical; } } // Drain remote if needed (ownership-checked elsewhere) // Priority-2: Use cached ENV (eliminate lazy-init syscall overhead) if (meta) { uintptr_t pending = atomic_load_explicit( &tls->ss->remote_heads[slab_idx], HAK_ENV_TINY_ALLOC_REMOTE_RELAX() ? memory_order_relaxed : memory_order_acquire); if (__builtin_expect(pending != 0, 0)) { uint32_t self_tid = tiny_self_u32(); if (ss_owner_try_acquire(meta, self_tid)) { _ss_remote_drain_to_freelist_unsafe(tls->ss, slab_idx, meta); } } } } // Fast path: linear carve from current TLS slab if (meta && tls->slab_base) { TinyTLSCarveOneResult carve = tiny_tls_carve_one_block(tls, class_idx); if (carve.block) { #if !HAKMEM_BUILD_RELEASE if (__builtin_expect(g_debug_remote_guard, 0)) { const char* tag = (carve.path == TINY_TLS_CARVE_PATH_FREELIST) ? "freelist_alloc" : "linear_alloc"; tiny_remote_track_on_alloc(tls->ss, slab_idx, carve.block, tag, 0); tiny_remote_assert_not_remote(tls->ss, slab_idx, carve.block, tag, 0); } #endif #if HAKMEM_TINY_SS_TLS_HINT { void* ss_base = (void*)tls->ss; size_t ss_size = (size_t)1ULL << tls->ss->lg_size; tls_ss_hint_update(tls->ss, ss_base, ss_size); } #endif if (carve.path == TINY_TLS_CARVE_PATH_LINEAR) { ROUTE_MARK(11); ROUTE_COMMIT(class_idx, 0x60); } else if (carve.path == TINY_TLS_CARVE_PATH_FREELIST) { ROUTE_MARK(12); ROUTE_COMMIT(class_idx, 0x61); } HAK_RET_ALLOC(class_idx, carve.block); } } // Slow path: acquire a new slab via shared pool SuperSlab* ss = superslab_refill(class_idx); if (!ss) { static int log_oom = 0; if (log_oom < 2) { fprintf(stderr, "[DEBUG] superslab_refill returned NULL (OOM)\n"); log_oom++; } return NULL; } // Retry after refill tls = &g_tls_slabs[class_idx]; meta = tls->meta; if (meta && meta->freelist == NULL && meta->used < meta->capacity && tls->slab_base) { size_t block_size = tiny_stride_for_class(meta->class_idx); void* block = tiny_block_at_index(tls->slab_base, meta->used, block_size); meta->used++; c7_meta_used_note(meta->class_idx, C7_META_USED_SRC_FRONT); ss_active_inc(ss); HAK_RET_ALLOC(class_idx, block); } return NULL; }