hakmem/core/box/tiny_heap_box.h

// tiny_heap_box.h - TinyHeap front (mimalloc 風) の共通 Box
// 役割:
//   - クラスごとの TinyHeap コンテキスト（current/partial/full + freelist）を TLS で保持。
//   - ホットパス alloc/free は TinyHeap 内で完結させ、ページ枯渇/全 free の境界でのみ
//     Superslab/Tier/Warm などの下層 Box に触れる。
//   - C7HotBox はこの TinyHeapBox 上のクラス7ラッパとして実装する。
#pragma once

#include <stdint.h>
#include <stddef.h>
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>

#include "../hakmem_build_flags.h"
#include "../hakmem_tiny_superslab.h"    // SuperSlab, TinySlabMeta, superslab_refill
#include "../hakmem_super_registry.h"    // hak_super_lookup
#include "../superslab/superslab_inline.h"  // slab_index_for
#include "../tiny_tls.h"                 // TinyTLSSlab
#include "../tiny_box_geometry.h"        // tiny_stride_for_class
#include "../tiny_region_id.h"           // tiny_region_id_write_header
#include "tiny_layout_box.h"             // tiny_user_offset
#include "tiny_next_ptr_box.h"           // tiny_next_read/write
#include "tiny_heap_env_box.h"           // profile gates

// Forward decls for SuperSlab active counters (definitions in hakmem_tiny_ss_active_box.inc)
void ss_active_add(SuperSlab* ss, uint32_t n);

#ifndef TINY_HEAP_MAX_PAGES_PER_CLASS
#define TINY_HEAP_MAX_PAGES_PER_CLASS 4
#endif

typedef struct tiny_heap_page_t {
    void* free_list;            // ページ内 free list 先頭 (BASE)
    uint16_t used;              // 現在の使用数（active count）
    uint16_t capacity;          // ページ内の総ブロック数
    uint16_t slab_idx;          // 対応する slab index
    uint16_t _pad;
    uint8_t* base;              // slab 先頭（データ領域先頭）
    TinySlabMeta* meta;         // Superslab メタ（owner/Tier 判定用）
    SuperSlab* ss;              // 所有する Superslab
    struct tiny_heap_page_t* next;
    int32_t active_delta;    // meta-light 用: total_active_blocks の差分（主に C6/C7）
    int32_t used_delta;      // meta-light 用: meta->used の差分（主に C6/C7）
    int32_t stats_active_pending; // Cold Stats Box 用 pending（C7 SAFE）
    int32_t stats_used_pending;   // Cold Stats Box 用 pending（C7 SAFE）
    int32_t last_delta_site; // debug: 直近で delta を触ったサイト（C6/C7 meta-light 用）
} tiny_heap_page_t;

typedef struct tiny_heap_class_t {
    tiny_heap_page_t* current_page;
    tiny_heap_page_t* partial_pages;
    tiny_heap_page_t* full_pages;
    tiny_heap_page_t  nodes[TINY_HEAP_MAX_PAGES_PER_CLASS];
    uint8_t           node_in_use[TINY_HEAP_MAX_PAGES_PER_CLASS];
    uint16_t          stride;   // cached block stride for this class
    uint16_t          _pad;
} tiny_heap_class_t;

typedef struct tiny_heap_ctx_t {
    tiny_heap_class_t cls[TINY_NUM_CLASSES];
    uint8_t initialized;
} tiny_heap_ctx_t;

typedef struct {
    TinySlabMeta* meta;
    SuperSlab*    ss;
    uint8_t*      base;
    void*         freelist;
    uint16_t      capacity;
    uint16_t      slab_idx;
    tiny_heap_page_t* page;  // v1 TinyHeap の page 構造体（lease 元）
} TinyHeapPageLease;

// Stats Box（Cold 側への集計フック）
#include "tiny_stats_box.h"

// TLS state (定義は core/hakmem_tiny.c)
extern __thread tiny_heap_ctx_t g_tiny_heap_ctx;
extern __thread int g_tiny_heap_ctx_init;
extern __thread TinyTLSSlab g_tls_slabs[TINY_NUM_CLASSES];

static inline int tiny_heap_stats_enabled(void) {
    static int g = -1;
    if (__builtin_expect(g == -1, 0)) {
        const char* eh = getenv("HAKMEM_TINY_HEAP_STATS");
        const char* e = getenv("HAKMEM_TINY_C7_HEAP_STATS");
        g = ((eh && *eh && *eh != '0') || (e && *e && *e != '0')) ? 1 : 0;
    }
    return g;
}

static inline int tiny_c7_heap_stats_enabled(void) {
    return tiny_heap_stats_enabled();
}

// meta mode: 0=OFF, 1=SAFE meta-light (delta + flush), 2=ULTRA (bench only, meta/active をほぼ触らない)
// meta mode: 0=OFF, 1=SAFE meta-light (delta + flush), 2=ULTRA (bench only, meta/active をほぼ触らない)
static inline int tiny_c7_meta_mode(void);
static inline int tiny_heap_meta_mode_for_class(int class_idx) {
    if (class_idx == 7) {
        return tiny_c7_meta_mode();
    }
    if (class_idx == 6) {
        static int g_c6_mode = -1;
        if (__builtin_expect(g_c6_mode == -1, 0)) {
            const char* e = getenv("HAKMEM_TINY_C6_META_MODE");
            if (e && *e && *e != '0') {
                g_c6_mode = 1;
            } else {
                g_c6_mode = 0;
            }
        }
        // NOTE: mode 1 for class 6 is experimental (bench-only, may crash). See CURRENT_TASK for guidance.
        return g_c6_mode;
    }
    return 0;
}

static inline int tiny_c7_meta_mode(void) {
    static int g_mode = -1;
    if (__builtin_expect(g_mode == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_C7_META_MODE");
        if (!e || !*e) {
            const char* l = getenv("HAKMEM_TINY_C7_META_LIGHT");
            if (l && *l && *l != '0') {
                g_mode = 1;
            } else {
                g_mode = tiny_heap_profile_default_c7_meta_mode(tiny_heap_profile_mode());
            }
        } else {
            int v = atoi(e);
            if (v < 0) v = 0;
            if (v > 2) v = 2;
            g_mode = v;
        }
    }
    return g_mode;
}

static inline int tiny_heap_behavior_mode_for_class(int class_idx) {
    return tiny_heap_meta_mode_for_class(class_idx);
}

static inline int tiny_heap_meta_light_enabled_for_class(int class_idx) {
    return tiny_heap_behavior_mode_for_class(class_idx) >= 1;
}

static inline int tiny_heap_meta_ultra_enabled_for_class(int class_idx) {
    return (class_idx == 7) && tiny_heap_meta_mode_for_class(class_idx) == 2;
}

static inline int tiny_heap_meta_mode_effective(int class_idx) {
    return tiny_heap_meta_mode_for_class(class_idx);
}

static inline int tiny_c7_meta_light_enabled(void) {
    return tiny_heap_meta_light_enabled_for_class(7);
}

static inline int tiny_c7_meta_ultra_enabled(void) {
    return tiny_heap_meta_ultra_enabled_for_class(7);
}

static inline int tiny_c7_delta_debug_enabled(void) {
    static int g = -1;
    if (__builtin_expect(g == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_C7_DELTA_DEBUG");
        g = (e && *e && *e != '0') ? 1 : 0;
    }
    return g;
}

static inline int tiny_c6_delta_debug_enabled(void) {
    static int g = -1;
    if (__builtin_expect(g == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_C6_DELTA_DEBUG");
        g = (e && *e && *e != '0') ? 1 : 0;
    }
    return g;
}

static inline int tiny_c6_delta_trace_enabled(void) {
    static int g = -1;
    if (__builtin_expect(g == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_C6_DELTA_TRACE");
        g = (e && *e && *e != '0') ? 1 : 0;
    }
    return g;
}

static inline int tiny_c6_debug_pop_enabled(void) {
    static int g = -1;
    if (__builtin_expect(g == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_C6_DEBUG_POP");
        g = (e && *e && *e != '0') ? 1 : 0;
    }
    return g;
}

typedef struct {
    _Atomic uint64_t alloc_fast_current;
    _Atomic uint64_t alloc_slow_prepare;
    _Atomic uint64_t free_fast_local;
    _Atomic uint64_t free_slow_fallback;
    _Atomic uint64_t alloc_prepare_fail;
    _Atomic uint64_t alloc_fail;
} TinyHeapClassStats;

extern TinyHeapClassStats g_tiny_heap_stats[TINY_NUM_CLASSES];

typedef struct {
    _Atomic uint64_t prepare_calls;
    _Atomic uint64_t prepare_with_current_null;
    _Atomic uint64_t prepare_from_partial;
    _Atomic uint64_t current_set_from_free;
    _Atomic uint64_t current_dropped_to_partial;
} TinyC7PageStats;

extern TinyC7PageStats g_c7_page_stats;

typedef enum {
    C6_DELTA_NONE = 0,
    C6_DELTA_ALLOC = 1,
    C6_DELTA_FREE = 2,
    C6_DELTA_EMPTY = 3,
    C6_DELTA_ATTACH = 4,
    C6_DELTA_THRESHOLD = 5,
} tiny_c6_delta_site_t;

static inline int tiny_c7_page_stats_enabled(void) {
    return tiny_heap_stats_enabled();
}

static inline TinyHeapClassStats* tiny_heap_stats_for_class(int class_idx) {
    if (!tiny_heap_stats_enabled()) return NULL;
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL;
    return &g_tiny_heap_stats[class_idx];
}

static inline int tiny_heap_cold_drain_and_free(int class_idx, void* base) {
    (void)class_idx;
    (void)base;
    return 0;
}

static inline tiny_heap_ctx_t* tiny_heap_ctx_for_thread(void) {
    tiny_heap_ctx_t* ctx = &g_tiny_heap_ctx;
    if (!g_tiny_heap_ctx_init) {
        memset(ctx, 0, sizeof(*ctx));
        g_tiny_heap_ctx_init = 1;
        ctx->initialized = 1;
        for (int c = 0; c < TINY_NUM_CLASSES; c++) {
            tiny_heap_class_t* hcls = &ctx->cls[c];
            hcls->stride = (uint16_t)tiny_stride_for_class(c);
        }
    }
    return ctx;
}

static inline tiny_heap_class_t* tiny_heap_class(tiny_heap_ctx_t* ctx, int class_idx) {
    if (!ctx || class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL;
    return &ctx->cls[class_idx];
}

static inline int tiny_heap_tls_try_resolve(int class_idx,
                                            void* base,
                                            SuperSlab** out_ss,
                                            int* out_slab_idx,
                                            TinySlabMeta** out_meta) {
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return 0;
    TinyTLSSlab* tls = &g_tls_slabs[class_idx];
    if (!tls->ss || !tls->slab_base || !tls->meta) return 0;

    const size_t stride = (size_t)tiny_stride_for_class(class_idx);
    const size_t cap = (size_t)tls->meta->capacity;
    if (stride == 0 || cap == 0) return 0;

    uint8_t* low = tls->slab_base;
    uint8_t* high = low + stride * cap;
    if ((uint8_t*)base < low || (uint8_t*)base >= high) {
        return 0;
    }

    if (out_ss) {
        *out_ss = tls->ss;
    }
    if (out_slab_idx) {
        *out_slab_idx = (int)tls->slab_idx;
    }
    if (out_meta) {
        *out_meta = tls->meta;
    }
    return 1;
}

static inline int tiny_heap_page_is_valid(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
    if (!hcls || !page) return 0;
    return (page >= hcls->nodes) && (page < (hcls->nodes + TINY_HEAP_MAX_PAGES_PER_CLASS));
}

static inline int tiny_heap_ptr_in_page_range(tiny_heap_page_t* page, void* ptr) {
    if (!page || !ptr || page->capacity == 0 || !page->base) return 0;
    uintptr_t p = (uintptr_t)ptr;
    uintptr_t low = (uintptr_t)page->base;
    size_t stride = 0;
    // stride は caller が hcls->stride を更新済みの前提。0 の場合は range 判定を諦める。
    // (debug 用なので false positive でも安全側に倒す)
    if (page->meta) {
        stride = tiny_stride_for_class((int)page->meta->class_idx);
    }
    if (stride == 0) {
        return 0;
    }
    uintptr_t high = low + (uintptr_t)(stride * (size_t)page->capacity);
    return (p >= low) && (p < high);
}

static inline size_t tiny_heap_block_stride(int class_idx) {
    return tiny_stride_for_class(class_idx);
}

static inline void tiny_heap_page_clear(tiny_heap_page_t* page) {
    if (!page) return;
    page->free_list = NULL;
    page->used = 0;
    page->capacity = 0;
    page->slab_idx = 0;
    page->base = NULL;
    page->meta = NULL;
    page->ss = NULL;
    page->next = NULL;
    page->active_delta = 0;
    page->used_delta = 0;
    page->stats_active_pending = 0;
    page->stats_used_pending = 0;
    page->last_delta_site = 0;
}

static inline void tiny_c6_mark_delta_site(tiny_heap_page_t* page, tiny_c6_delta_site_t site) {
    if (!page) return;
    if (!tiny_c6_delta_trace_enabled()) return;
    page->last_delta_site = (int32_t)site;
}

static inline void tiny_heap_meta_flush_page(int class_idx, tiny_heap_page_t* page) {
    if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
        return;
    }
    if (!page || !tiny_heap_meta_light_enabled_for_class(class_idx)) return;
    if (!page->meta || !page->ss) return;

    int32_t active_delta = page->active_delta;
    int32_t used_delta = page->used_delta;
    if (active_delta == 0 && used_delta == 0) {
        if (class_idx == 7 && tiny_stats_box_enabled() && tiny_stats_batch_enabled()) {
            tiny_stats_maybe_flush_for_page(class_idx, page, page->used == 0);
        }
        return;
    }

    page->active_delta = 0;
    page->used_delta = 0;

    if (class_idx == 7 && tiny_stats_box_enabled()) {
        tiny_stats_flush_for_page(class_idx, page, used_delta, active_delta);
        return;
    }

    if (used_delta != 0) {
        atomic_fetch_add_explicit(&page->meta->used, used_delta, memory_order_relaxed);
    }
    if (active_delta != 0) {
        if (active_delta > 0) {
            ss_active_add(page->ss, (uint32_t)active_delta);
        } else {
            int32_t n = -active_delta;
            for (int32_t i = 0; i < n; i++) {
                ss_active_dec_one(page->ss);
            }
        }
    }
}

static inline int tiny_heap_delta_should_flush(int class_idx, tiny_heap_page_t* page) {
    if (!page) return 0;
    if (!tiny_heap_meta_light_enabled_for_class(class_idx)) return 0;
    if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) return 0;

    int32_t ud = page->used_delta;
    int32_t ad = page->active_delta;
    int32_t abs_ud = (ud >= 0) ? ud : -ud;
    int32_t abs_ad = (ad >= 0) ? ad : -ad;
    int32_t abs_max = (abs_ud > abs_ad) ? abs_ud : abs_ad;
    if (abs_max == 0) {
        return 0;
    }

    uint16_t cap = page->capacity;
    int32_t base_th = (cap > 0) ? ((int32_t)cap * 16) : 256;
    if (base_th < 256) {
        base_th = 256;
    }
    return abs_max >= base_th;
}

static __attribute__((noinline, unused)) void tiny_heap_debug_dump_deltas_for_class(int class_idx, const char* tag) {
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return;
    if (!tiny_heap_meta_light_enabled_for_class(class_idx)) return;
    if (class_idx == 7 && !tiny_c7_delta_debug_enabled()) return;
    if (class_idx == 6 && !tiny_c6_delta_debug_enabled()) return;

    tiny_heap_ctx_t* ctx = tiny_heap_ctx_for_thread();
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!hcls) return;

    uint64_t nonzero_pages = 0;
    int64_t used_delta_sum = 0;
    int64_t active_delta_sum = 0;

    for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) {
        tiny_heap_page_t* p = &hcls->nodes[i];
        if (p->used_delta != 0 || p->active_delta != 0) {
            nonzero_pages++;
            used_delta_sum += (int64_t)p->used_delta;
            active_delta_sum += (int64_t)p->active_delta;
            fprintf(stderr,
                    "[%s_DELTA_PAGE] idx=%d used_delta=%d active_delta=%d used=%u cap=%u ss=%p slab=%u\n",
                    tag ? tag : "C?_DELTA",
                    i,
                    p->used_delta,
                    p->active_delta,
                    (unsigned)p->used,
                    (unsigned)p->capacity,
                    (void*)p->ss,
                    (unsigned)p->slab_idx);
        }
    }

    fprintf(stderr,
            "[%s_DELTA_SUMMARY] nonzero_pages=%llu used_delta_sum=%lld active_delta_sum=%lld\n",
            tag ? tag : "C?_DELTA",
            (unsigned long long)nonzero_pages,
            (long long)used_delta_sum,
            (long long)active_delta_sum);
}

static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(void) {
    if (!tiny_c7_meta_light_enabled()) {
        return;
    }
    tiny_heap_debug_dump_deltas_for_class(7, "C7");
}

static __attribute__((noinline, unused)) void tiny_c6_heap_debug_dump_deltas(void) {
    if (!tiny_heap_meta_light_enabled_for_class(6)) {
        return;
    }
    tiny_heap_debug_dump_deltas_for_class(6, "C6");
}

static inline tiny_heap_page_t* tiny_heap_class_acquire_node(tiny_heap_class_t* hcls) {
    if (!hcls) return NULL;
    for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) {
        if (hcls->node_in_use[i] == 0) {
            hcls->node_in_use[i] = 1;
            tiny_heap_page_t* p = &hcls->nodes[i];
            tiny_heap_page_clear(p);
            return p;
        }
    }
    return NULL;
}

static inline void tiny_heap_class_release_node(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
    if (!hcls || !page) return;
    int class_idx = (page->meta) ? page->meta->class_idx : -1;
    if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
        page->active_delta = 0;
        page->used_delta = 0;
    } else if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
        if (class_idx == 6) {
            tiny_c6_mark_delta_site(page, C6_DELTA_EMPTY);
        }
        tiny_heap_meta_flush_page(class_idx, page);
    }
    intptr_t idx = page - hcls->nodes;
    if (idx >= 0 && idx < TINY_HEAP_MAX_PAGES_PER_CLASS) {
        hcls->node_in_use[idx] = 0;
        tiny_heap_page_clear(page);
    }
}

static inline tiny_heap_page_t* tiny_heap_class_find_page(tiny_heap_class_t* hcls, SuperSlab* ss, int slab_idx) {
    if (!hcls || !ss || slab_idx < 0) return NULL;
    tiny_heap_page_t* lists[] = {hcls->current_page, hcls->partial_pages, hcls->full_pages};
    for (size_t li = 0; li < sizeof(lists) / sizeof(lists[0]); li++) {
        tiny_heap_page_t* p = lists[li];
        while (p) {
            if (p->ss == ss && (int)p->slab_idx == slab_idx) {
                return p;
            }
            p = p->next;
        }
    }
    return NULL;
}

static inline tiny_heap_page_t* tiny_heap_attach_page(tiny_heap_ctx_t* ctx,
                                                      int class_idx,
                                                      SuperSlab* ss,
                                                      int slab_idx) {
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!hcls || !ss || slab_idx < 0) return NULL;
    TinySlabMeta* meta = &ss->slabs[slab_idx];
    if (meta->class_idx != class_idx) {
        return NULL;
    }

    tiny_heap_page_t* page = tiny_heap_class_find_page(hcls, ss, slab_idx);
    if (!page) {
        page = tiny_heap_class_acquire_node(hcls);
        if (!page) return NULL;
        page->ss = ss;
        page->slab_idx = (uint16_t)slab_idx;
        page->meta = meta;
        page->base = tiny_slab_base_for(ss, slab_idx);
        page->capacity = meta->capacity;
        page->used = atomic_load_explicit(&meta->used, memory_order_relaxed);
        page->free_list = atomic_load_explicit(&meta->freelist, memory_order_acquire);
        page->next = hcls->partial_pages;
        hcls->partial_pages = page;
    } else {
        page->capacity = meta->capacity;
        page->used = atomic_load_explicit(&meta->used, memory_order_relaxed);
        page->free_list = atomic_load_explicit(&meta->freelist, memory_order_acquire);
        if (!page->base) {
            page->base = tiny_slab_base_for(ss, slab_idx);
        }
    }
    if (class_idx == 6 && tiny_heap_meta_mode_effective(class_idx) == 1) {
        // Fail-Fast: meta->freelist が範囲外なら破棄しておく（古い slab の残骸を拾わない）
        if (page->free_list && !tiny_heap_ptr_in_page_range(page, page->free_list)) {
            if (tiny_c6_debug_pop_enabled()) {
                fprintf(stderr,
                        "[C6_ATTACH_OOB] freelist=%p base=%p cap=%u ss=%p slab=%d\n",
                        page->free_list,
                        (void*)page->base,
                        (unsigned)page->capacity,
                        (void*)page->ss,
                        slab_idx);
            }
            page->free_list = NULL;
            atomic_store_explicit(&meta->freelist, NULL, memory_order_release);
        }
    }
    if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
        if (!tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
            if (page->capacity > 0 && page->used > page->capacity) {
                page->used = page->capacity;
            }
        }
        page->used_delta = 0;
        page->active_delta = 0;
        if (class_idx == 6) {
            tiny_c6_mark_delta_site(page, C6_DELTA_ATTACH);
        }
    }
    return page;
}

static inline tiny_heap_page_t* tiny_heap_take_current(tiny_heap_class_t* hcls) {
    if (!hcls) return NULL;
    if (hcls->current_page) {
        return hcls->current_page;
    }
    if (hcls->partial_pages) {
        tiny_heap_page_t* page = hcls->partial_pages;
        hcls->partial_pages = page->next;
        page->next = NULL;
        hcls->current_page = page;
        return page;
    }
    return NULL;
}

static inline void tiny_heap_class_unlink(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
    if (!hcls || !page) return;
    if (hcls->current_page == page) {
        hcls->current_page = NULL;
    }
    tiny_heap_page_t** lists[] = {&hcls->partial_pages, &hcls->full_pages};
    for (size_t i = 0; i < sizeof(lists) / sizeof(lists[0]); i++) {
        tiny_heap_page_t** head = lists[i];
        tiny_heap_page_t* prev = NULL;
        tiny_heap_page_t* cur = head ? *head : NULL;
        while (cur) {
            if (cur == page) {
                if (prev) {
                    prev->next = cur->next;
                } else {
                    *head = cur->next;
                }
                break;
            }
            prev = cur;
            cur = cur->next;
        }
    }
}

static inline tiny_heap_page_t* tiny_heap_page_of(tiny_heap_ctx_t* ctx, int class_idx, void* base_ptr) {
    if (!ctx || !base_ptr || class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL;
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!hcls) return NULL;

    // TLS fast path: TLS slab の範囲内なら lookup を避ける
    TinyTLSSlab* tls = &g_tls_slabs[class_idx];
    if (tls->ss && tls->meta && tls->slab_base) {
        size_t stride = hcls->stride;
        if (stride == 0) {
            stride = tiny_heap_block_stride(class_idx);
            hcls->stride = (uint16_t)stride;
        }
        uint16_t cap = tls->meta->capacity;
        if (cap > 0) {
            uint8_t* slab_base = tls->slab_base;
            size_t slab_size = stride * (size_t)cap;
            uint8_t* base_u8 = (uint8_t*)base_ptr;
            if (base_u8 >= slab_base && base_u8 < (slab_base + slab_size)) {
                return tiny_heap_attach_page(ctx, class_idx, tls->ss, tls->slab_idx);
            }
        }
    }

    // fallback: Superslab lookup
    SuperSlab* ss = hak_super_lookup(base_ptr);
    if (!ss || ss->magic != SUPERSLAB_MAGIC) {
        return NULL;
    }
    int slab_idx = slab_index_for(ss, base_ptr);
    if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
        return NULL;
    }
    return tiny_heap_attach_page(ctx, class_idx, ss, slab_idx);
}

static inline void tiny_heap_page_push_to_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
    if (!hcls || !page) return;
    page->next = hcls->full_pages;
    hcls->full_pages = page;
}

static inline void tiny_heap_page_push_to_partial(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
    if (!hcls || !page) return;
    page->next = hcls->partial_pages;
    hcls->partial_pages = page;
}

static inline void tiny_heap_page_becomes_empty(tiny_heap_ctx_t* ctx, int class_idx, tiny_heap_page_t* page) {
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!hcls || !page) return;
    const int mode = tiny_heap_meta_mode_effective(class_idx);

    if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
        // ULTRA: C7 は 1 ページ前提で保持し続ける。publish/unlink/release を避ける。
        hcls->current_page = page;
        return;
    }
    if (mode == 1) {
        // SAFE: delta を反映（C6/C7）
        if (class_idx == 6) {
            tiny_c6_mark_delta_site(page, C6_DELTA_EMPTY);
        }
        tiny_heap_meta_flush_page(class_idx, page);
        if (class_idx == 7) {
            // C7 SAFE: current を保持し、頻繁な detach を避ける
            tiny_heap_class_unlink(hcls, page);
            hcls->current_page = page;
            page->next = NULL;
            return;
        }
        if (class_idx == 6) {
            // C6 SAFE: freelist の残骸を次回 attach へ渡さない
            atomic_store_explicit(&page->meta->freelist, NULL, memory_order_release);
            if (tiny_c6_debug_pop_enabled()) {
                page->free_list = (void*)0xDEAD6EED;
            } else {
                page->free_list = NULL;
            }
        }
    }
    if (page->meta && page->ss) {
        ss_partial_publish(class_idx, page->ss);
    }
    tiny_heap_class_unlink(hcls, page);
    tiny_heap_class_release_node(hcls, page);
}

static inline void tiny_heap_page_mark_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
    if (!hcls || !page) return;
    int class_idx = -1;
    if (page->meta) {
        class_idx = page->meta->class_idx;
    }
    if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
        // ULTRA: full 判定で current を追い出さない
        return;
    }
    if (tiny_heap_meta_light_enabled_for_class(class_idx) && hcls->current_page == page) {
        // SAFE: current page は可能な限り保持し、頻繁な unlink を避ける（C6/C7）
        return;
    }
    tiny_heap_class_unlink(hcls, page);
    tiny_heap_page_push_to_full(hcls, page);
}

static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, tiny_heap_page_t* page) {
    if (!tiny_heap_page_is_valid(hcls, page)) return NULL;

    const int mode = tiny_heap_meta_mode_effective(class_idx);
    if (class_idx == 7 && __builtin_expect(mode == 1, 1)) {
        if (!page->meta || !page->ss || !page->base) return NULL;
        void* block = NULL;
        if (page->free_list) {
            block = page->free_list;
            void* next = tiny_next_read(class_idx, block);
            page->free_list = next;
            atomic_store_explicit(&page->meta->freelist, next, memory_order_release);
        } else {
            if (page->used >= page->capacity) {
                return NULL;
            }
            size_t stride = hcls->stride;
            if (stride == 0) {
                stride = tiny_heap_block_stride(class_idx);
                hcls->stride = (uint16_t)stride;
            }
            block = (void*)(page->base + ((size_t)page->used * stride));
            if (page->meta->carved < page->capacity) {
                page->meta->carved++;
            }
        }
        page->used++;
        page->used_delta++;
        page->active_delta++;
        if (tiny_heap_delta_should_flush(class_idx, page)) {
            tiny_heap_meta_flush_page(class_idx, page);
        }
        return tiny_region_id_write_header(block, class_idx);
    }

    const int c6_pop_dbg = (class_idx == 6) && tiny_c6_debug_pop_enabled();
    if (!page->meta || !page->ss || !page->base) return NULL;
    if (c6_pop_dbg && mode == 1) {
        int fail = 0;
        const char* reason = NULL;
        SuperSlab* ss_chk = hak_super_lookup(page->base);
        if (!page->meta || page->meta->class_idx != class_idx) {
            fail = 1; reason = "meta_cls";
        } else if (page->capacity == 0) {
            fail = 1; reason = "cap0";
        } else if (!page->free_list && page->used >= page->capacity) {
            fail = 1; reason = "exhausted";
        } else if (!ss_chk) {
            fail = 1; reason = "ss_lookup_null";
        } else if (page->ss && ss_chk != page->ss) {
            fail = 1; reason = "ss_mismatch";
        } else if (page->free_list && !tiny_heap_ptr_in_page_range(page, page->free_list)) {
            fail = 1; reason = "freelist_oob";
        }
        if (fail) {
            fprintf(stderr,
                    "[C6_POP_FAIL] reason=%s page=%p base=%p freelist=%p used=%u cap=%u ss=%p ss_chk=%p meta=%p last_site=%d\n",
                    reason ? reason : "unknown",
                    (void*)page,
                    (void*)page->base,
                    page->free_list,
                    (unsigned)page->used,
                    (unsigned)page->capacity,
                    (void*)page->ss,
                    (void*)ss_chk,
                    (void*)page->meta,
                    page->last_delta_site);
            abort();
        }
    }
    void* block = NULL;
    if (class_idx == 6 && __builtin_expect(mode == 1, 0) && c6_pop_dbg) {
        static _Atomic uint32_t g_c6_pop_dbg = 0;
        uint32_t n = atomic_fetch_add_explicit(&g_c6_pop_dbg, 1, memory_order_relaxed);
        if (n < 512) {
            fprintf(stderr,
                    "[C6_POP_DEBUG] page=%p used=%u cap=%u free_list=%p meta=%p ss=%p base=%p last_site=%d\n",
                    (void*)page,
                    (unsigned)page->used,
                    (unsigned)page->capacity,
                    page->free_list,
                    (void*)page->meta,
                    (void*)page->ss,
                    (void*)page->base,
                    page->last_delta_site);
        }
    }
    if (page->free_list) {
        block = page->free_list;
        void* next = tiny_next_read(class_idx, block);
        page->free_list = next;
        if (__builtin_expect(mode != 2, 1)) {
            atomic_store_explicit(&page->meta->freelist, next, memory_order_release);
        }
    } else if (page->used < page->capacity) {
        size_t stride = hcls->stride;
        if (stride == 0) {
            stride = tiny_heap_block_stride(class_idx);
            hcls->stride = (uint16_t)stride;
        }
        block = (void*)(page->base + ((size_t)page->used * stride));
        if (__builtin_expect(mode != 2, 1) && page->meta->carved < page->capacity) {
            page->meta->carved++;
        }
    } else {
        return NULL;
    }

    page->used++;
    if (__builtin_expect(mode == 2, 0)) {
        return tiny_region_id_write_header(block, class_idx);
    }
    if (__builtin_expect(mode == 1, 0)) {
        if (class_idx == 6) {
            tiny_c6_mark_delta_site(page, C6_DELTA_ALLOC);
        }
        page->used_delta++;
        page->active_delta++;
        if (tiny_heap_delta_should_flush(class_idx, page)) {
            if (class_idx == 6) {
                tiny_c6_mark_delta_site(page, C6_DELTA_THRESHOLD);
            }
            tiny_heap_meta_flush_page(class_idx, page);
        }
        return tiny_region_id_write_header(block, class_idx);
    }
    atomic_fetch_add_explicit(&page->meta->used, 1, memory_order_relaxed);
    ss_active_add(page->ss, 1u);

    return tiny_region_id_write_header(block, class_idx);
}

static inline void tiny_heap_page_push_free(int class_idx, tiny_heap_page_t* page, void* base_ptr) {
    if (!page || !base_ptr || !page->meta) return;
    const int ultra = tiny_heap_meta_ultra_enabled_for_class(class_idx);
    tiny_next_write(class_idx, base_ptr, page->free_list);
    page->free_list = base_ptr;
    if (!__builtin_expect(ultra, 0)) {
        atomic_store_explicit(&page->meta->freelist, base_ptr, memory_order_release);
    }
}

static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx,
                                             int class_idx,
                                             tiny_heap_page_t* page,
                                             void* base_ptr) {
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!hcls || !page || !base_ptr) return;
    if (!tiny_heap_page_is_valid(hcls, page)) return;
    TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
    if (__builtin_expect(stats != NULL, 0)) {
        atomic_fetch_add_explicit(&stats->free_fast_local, 1, memory_order_relaxed);
    }
    const int behavior_mode = tiny_heap_behavior_mode_for_class(class_idx);
    const int mode = tiny_heap_meta_mode_effective(class_idx);
    const int page_stats_on = (class_idx == 7 && tiny_c7_page_stats_enabled());
    if (class_idx == 6 && __builtin_expect(mode == 1, 0) && tiny_c6_debug_pop_enabled()) {
        SuperSlab* ss_chk = hak_super_lookup(base_ptr);
        if (!ss_chk || ss_chk != page->ss || !page->meta || page->meta->class_idx != class_idx) {
            fprintf(stderr,
                    "[C6_FREE_FAIL] ss_chk=%p page_ss=%p meta=%p meta_cls=%d base=%p page=%p used=%u cap=%u last_site=%d\n",
                    (void*)ss_chk,
                    (void*)page->ss,
                    (void*)page->meta,
                    page->meta ? page->meta->class_idx : -1,
                    base_ptr,
                    (void*)page,
                    (unsigned)page->used,
                    (unsigned)page->capacity,
                    page->last_delta_site);
            abort();
        }
    }
    tiny_heap_page_push_free(class_idx, page, base_ptr);
    if (page->used > 0) {
        page->used--;
        if (__builtin_expect(mode == 2, 0)) {
            // ULTRA: meta/active は触らない
        } else if (!__builtin_expect(mode == 1, 0)) {
            atomic_fetch_sub_explicit(&page->meta->used, 1, memory_order_relaxed);
            ss_active_dec_one(page->ss);
        } else {
            if (class_idx == 6) {
                tiny_c6_mark_delta_site(page, C6_DELTA_FREE);
            }
            page->used_delta--;
            page->active_delta--;
            if (tiny_heap_delta_should_flush(class_idx, page)) {
                if (class_idx == 6) {
                    tiny_c6_mark_delta_site(page, C6_DELTA_THRESHOLD);
                }
                tiny_heap_meta_flush_page(class_idx, page);
            }
        }
    }
    if (class_idx == 6 && __builtin_expect(mode == 1, 0) && tiny_c6_debug_pop_enabled()) {
        static _Atomic uint32_t g_c6_free_dbg = 0;
        uint32_t n = atomic_fetch_add_explicit(&g_c6_free_dbg, 1, memory_order_relaxed);
        if (n < 512) {
            fprintf(stderr,
                    "[C6_FREE_DEBUG] page=%p used=%u cap=%u free_list=%p meta=%p ss=%p base=%p used_delta=%d active_delta=%d\n",
                    (void*)page,
                    (unsigned)page->used,
                    (unsigned)page->capacity,
                    page->free_list,
                    (void*)page->meta,
                    (void*)page->ss,
                    (void*)page->base,
                    page->used_delta,
                    page->active_delta);
        }
    }

    if (class_idx == 7 && __builtin_expect(behavior_mode == 2, 0)) {
        tiny_heap_page_t* old_cur = hcls->current_page;
        if (__builtin_expect(page_stats_on, 0)) {
            if (!old_cur) {
                atomic_fetch_add_explicit(&g_c7_page_stats.current_set_from_free, 1, memory_order_relaxed);
            } else if (old_cur != page) {
                atomic_fetch_add_explicit(&g_c7_page_stats.current_dropped_to_partial, 1, memory_order_relaxed);
            }
        }
        // ULTRA: 1ページで回す前提。current を強制保持し、partial/full には追い出さない。
        hcls->current_page = page;
        if (old_cur && old_cur != page) {
            tiny_heap_page_push_to_partial(hcls, old_cur);
        }
        // partial list に page が重複して残っている場合を避ける
        tiny_heap_page_t** pp = &hcls->partial_pages;
        while (*pp) {
            if (*pp == page) {
                *pp = (*pp)->next;
                break;
            }
            pp = &(*pp)->next;
        }
        return;
    }

    if (__builtin_expect(behavior_mode == 1, 0)) {
        if (page->used == 0) {
            tiny_heap_page_becomes_empty(ctx, class_idx, page);
            return;
        }
        if (page->used < page->capacity && page->free_list) {
            tiny_heap_page_t* old_cur = hcls->current_page;
            if (class_idx == 7 && __builtin_expect(page_stats_on, 0)) {
                if (!old_cur) {
                    atomic_fetch_add_explicit(&g_c7_page_stats.current_set_from_free, 1, memory_order_relaxed);
                } else if (old_cur != page) {
                    atomic_fetch_add_explicit(&g_c7_page_stats.current_dropped_to_partial, 1, memory_order_relaxed);
                }
            }
            tiny_heap_class_unlink(hcls, page);
            page->next = NULL;
            if (class_idx == 7) {
                hcls->current_page = page;
                if (old_cur && old_cur != page) {
                    tiny_heap_page_push_to_partial(hcls, old_cur);
                }
            } else if (class_idx == 6) {
                hcls->current_page = page;
                if (old_cur && old_cur != page) {
                    tiny_heap_page_push_to_partial(hcls, old_cur);
                }
            } else if (!hcls->current_page) {
                hcls->current_page = page;
            } else {
                tiny_heap_page_push_to_partial(hcls, page);
            }
        }
        return;
    }

    if (page->used == 0) {
        tiny_heap_page_becomes_empty(ctx, class_idx, page);
    } else if (page->used < page->capacity && page->free_list) {
        tiny_heap_page_t* old_cur = hcls->current_page;
        tiny_heap_class_unlink(hcls, page);
        page->next = NULL;
        if (class_idx == 7) {
            if (__builtin_expect(page_stats_on, 0)) {
                if (!old_cur) {
                    atomic_fetch_add_explicit(&g_c7_page_stats.current_set_from_free, 1, memory_order_relaxed);
                } else if (old_cur != page) {
                    atomic_fetch_add_explicit(&g_c7_page_stats.current_dropped_to_partial, 1, memory_order_relaxed);
                }
            }
            hcls->current_page = page;
            if (old_cur && old_cur != page) {
                tiny_heap_page_push_to_partial(hcls, old_cur);
            }
        } else {
            if (!hcls->current_page) {
                hcls->current_page = page;
            } else {
                tiny_heap_page_push_to_partial(hcls, page);
            }
        }
    }
}

static inline tiny_heap_page_t* tiny_heap_prepare_page(tiny_heap_ctx_t* ctx, int class_idx) {
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
    const int page_stats_on = (class_idx == 7 && tiny_c7_page_stats_enabled());
    if (!hcls) {
        if (__builtin_expect(stats != NULL, 0)) {
            atomic_fetch_add_explicit(&stats->alloc_prepare_fail, 1, memory_order_relaxed);
        }
        return NULL;
    }

    if (__builtin_expect(page_stats_on, 0)) {
        atomic_fetch_add_explicit(&g_c7_page_stats.prepare_calls, 1, memory_order_relaxed);
        if (!hcls->current_page) {
            atomic_fetch_add_explicit(&g_c7_page_stats.prepare_with_current_null, 1, memory_order_relaxed);
            if (hcls->partial_pages) {
                atomic_fetch_add_explicit(&g_c7_page_stats.prepare_from_partial, 1, memory_order_relaxed);
            }
        }
    }

    if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
        if (hcls->current_page) {
            return hcls->current_page;
        }
    }
    if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
        if (hcls->current_page &&
            (hcls->current_page->free_list || hcls->current_page->used < hcls->current_page->capacity)) {
            return hcls->current_page;
        }
    }

    tiny_heap_page_t* page = tiny_heap_take_current(hcls);
    if (page) return page;

    TinyTLSSlab* tls = &g_tls_slabs[class_idx];
    if (!tls->ss) {
        if (superslab_refill(class_idx) == NULL) {
            if (__builtin_expect(stats != NULL, 0)) {
                atomic_fetch_add_explicit(&stats->alloc_prepare_fail, 1, memory_order_relaxed);
            }
            return NULL;
        }
    }
    tls = &g_tls_slabs[class_idx];  // superslab_refill で更新されるため再取得
    if (!tls->ss || !tls->meta) {
        if (__builtin_expect(stats != NULL, 0)) {
            atomic_fetch_add_explicit(&stats->alloc_prepare_fail, 1, memory_order_relaxed);
        }
        return NULL;
    }
    page = tiny_heap_attach_page(ctx, class_idx, tls->ss, tls->slab_idx);
    if (page) {
        tiny_heap_class_unlink(hcls, page);
        page->next = NULL;
        hcls->current_page = page;
    }
    return page;
}

// =============================================================================
// C7 HotHeap v2 page lease helpers (Phase 32)
// =============================================================================
static inline TinyHeapPageLease tiny_heap_page_lease_nil(void) {
    TinyHeapPageLease lease;
    memset(&lease, 0, sizeof(lease));
    return lease;
}

static inline TinyHeapPageLease tiny_heap_c7_lease_page_for_v2(void) {
    TinyHeapPageLease lease = tiny_heap_page_lease_nil();
    tiny_heap_ctx_t* ctx = tiny_heap_ctx_for_thread();
    if (!ctx) return lease;
    tiny_heap_page_t* page = tiny_heap_prepare_page(ctx, 7);
    if (!page) return lease;
    lease.page = page;
    lease.meta = page->meta;
    lease.ss = page->ss;
    lease.base = page->base;
    lease.capacity = page->capacity;
    lease.slab_idx = page->slab_idx;
    lease.freelist = page->free_list;
    return lease;
}

static inline void tiny_heap_c7_return_page_from_v2(TinyHeapPageLease* lease) {
    (void)lease;
    // Phase32: C7 v2 は 1 枚使い切り前提。返却処理はまだ持たない。
}

// class_idx 固定での alloc ホットパス
static inline void* tiny_heap_alloc_slow_from_class(tiny_heap_ctx_t* ctx, int class_idx) {
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!hcls) return NULL;

    TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
    tiny_heap_page_t* page = tiny_heap_prepare_page(ctx, class_idx);
    if (!page) {
        if (__builtin_expect(stats != NULL, 0)) {
            atomic_fetch_add_explicit(&stats->alloc_fail, 1, memory_order_relaxed);
        }
        return NULL;
    }

    void* user = tiny_heap_page_pop(hcls, class_idx, page);
    if (user && page->used >= page->capacity && page->free_list == NULL) {
        tiny_heap_page_mark_full(hcls, page);
    }
    return user;
}

// class_idx 固定での alloc ホットパス
__attribute__((always_inline)) static inline void* tiny_heap_alloc_class_fast(tiny_heap_ctx_t* ctx,
                                                                               int class_idx,
                                                                               size_t size) {
    (void)size;
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    tiny_heap_page_t* page = hcls ? hcls->current_page : NULL;
    TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);

    if (page) {
        if (page->free_list || page->used < page->capacity) {
            void* user = tiny_heap_page_pop(hcls, class_idx, page);
            if (user) {
                if (page->used >= page->capacity && page->free_list == NULL) {
                    tiny_heap_page_mark_full(hcls, page);
                }
                if (__builtin_expect(stats != NULL, 0)) {
                    atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
                }
                return user;
            }
        }
    }

    if (__builtin_expect(stats != NULL, 0)) {
        atomic_fetch_add_explicit(&stats->alloc_slow_prepare, 1, memory_order_relaxed);
    }
    return tiny_heap_alloc_slow_from_class(ctx, class_idx);
}

// Superslab/Slab メタが既に分かっている場合の free（Gate から渡されるホットパス用）
static inline void tiny_heap_free_class_fast_with_meta(tiny_heap_ctx_t* ctx,
                                                       int class_idx,
                                                       SuperSlab* ss,
                                                       int slab_idx,
                                                       void* base) {
    tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
    if (!base || !ss || slab_idx < 0 || !hcls) return;
    TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);

    tiny_heap_page_t* page = tiny_heap_attach_page(ctx, class_idx, ss, slab_idx);
    if (!page) {
        if (__builtin_expect(stats != NULL, 0)) {
            atomic_fetch_add_explicit(&stats->free_slow_fallback, 1, memory_order_relaxed);
        }
        tiny_heap_cold_drain_and_free(class_idx, base);
        return;
    }

    tiny_heap_page_free_local(ctx, class_idx, page, base);
}

// class_idx 固定での free ホットパス（ptr は USER ポインタ）
static inline void tiny_heap_free_class_fast(tiny_heap_ctx_t* ctx, int class_idx, void* ptr) {
    if (!ptr) return;

#if HAKMEM_TINY_HEADERLESS
    void* base = ptr;
#else
    void* base = (void*)((uint8_t*)ptr - 1);
#endif
    SuperSlab* ss = NULL;
    int slab_idx = -1;
    if (class_idx == 7 || class_idx == 6) {
        if (!tiny_heap_tls_try_resolve(class_idx, base, &ss, &slab_idx, NULL)) {
            ss = NULL;
        }
    }
    if (!ss) {
        ss = hak_super_lookup(base);
        if (!ss || ss->magic != SUPERSLAB_MAGIC) {
            TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
            if (__builtin_expect(stats != NULL, 0)) {
                atomic_fetch_add_explicit(&stats->free_slow_fallback, 1, memory_order_relaxed);
            }
            tiny_heap_cold_drain_and_free(class_idx, base);
            return;
        }
    }
    if (slab_idx < 0) {
        slab_idx = slab_index_for(ss, base);
    }
    if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
        TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
        if (__builtin_expect(stats != NULL, 0)) {
            atomic_fetch_add_explicit(&stats->free_slow_fallback, 1, memory_order_relaxed);
        }
        tiny_heap_cold_drain_and_free(class_idx, base);
        return;
    }

    tiny_heap_free_class_fast_with_meta(ctx, class_idx, ss, slab_idx, base);
}