hakmem/core/hakmem_tiny_superslab.c

// hakmem_tiny_superslab.c - SuperSlab allocator implementation (Phase 6.22)
// Purpose: 2MB aligned slab allocation with fast pointer→slab lookup
// License: MIT
// Date: 2025-10-24

#include "hakmem_tiny_superslab.h"
#include "box/ss_hot_cold_box.h"  // Phase 3d-C: Hot/Cold Split
#include "hakmem_super_registry.h"  // Phase 1: Registry integration
#include "hakmem_tiny.h"  // For tiny_self_u32
#include "hakmem_tiny_config.h"  // For extern g_tiny_class_sizes
#include "hakmem_shared_pool.h"  // Phase 12: Shared SuperSlab pool backend (skeleton)
#include <sys/mman.h>
#include <sys/resource.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>  // getenv, atoi
#include <pthread.h>
#include <unistd.h>
#include <sys/resource.h>  // getrlimit for OOM diagnostics
#include <sys/mman.h>
#include "hakmem_internal.h"  // HAKMEM_LOG for release-silent logging
#include "tiny_region_id.h"   // For HEADER_MAGIC / HEADER_CLASS_MASK (restore header on remote-drain)
#include "hakmem_tiny_integrity.h"  // HAK_CHECK_CLASS_IDX
#include "box/tiny_next_ptr_box.h" // For tiny_next_write
#include "box/slab_freelist_atomic.h" // Phase 1: Atomic freelist accessor

static int g_ss_force_lg = -1;
static _Atomic int g_ss_populate_once = 0;

// Forward: decide next SuperSlab lg for a class (ACE-aware, clamped)
static inline uint8_t hak_tiny_superslab_next_lg(int class_idx)
{
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
        return SUPERSLAB_LG_DEFAULT;
    }
    // Prefer ACE target if within allowed range
    uint8_t t = atomic_load_explicit((_Atomic uint8_t*)&g_ss_ace[class_idx].target_lg,
                                     memory_order_relaxed);
    if (t < SUPERSLAB_LG_MIN || t > SUPERSLAB_LG_MAX) {
        return SUPERSLAB_LG_DEFAULT;
    }
    return t;
}

// ============================================================================
// Global Statistics
// ============================================================================

static pthread_mutex_t g_superslab_lock = PTHREAD_MUTEX_INITIALIZER;
uint64_t g_superslabs_allocated = 0;  // Non-static for debugging
uint64_t g_superslabs_freed = 0;      // Phase 7.6: Non-static for test access
uint64_t g_bytes_allocated = 0;  // Non-static for debugging

// ============================================================================
// Phase 2a: Dynamic Expansion - Global per-class SuperSlabHeads
// ============================================================================

SuperSlabHead* g_superslab_heads[TINY_NUM_CLASSES_SS] = {NULL};

// Debug counters
_Atomic uint64_t g_ss_active_dec_calls = 0;
_Atomic uint64_t g_hak_tiny_free_calls = 0;
_Atomic uint64_t g_ss_remote_push_calls = 0;
// Free path instrumentation (lightweight, for OOM/route diagnosis)
_Atomic uint64_t g_free_ss_enter = 0;          // hak_tiny_free_superslab() entries
_Atomic uint64_t g_free_local_box_calls = 0;   // same-thread freelist pushes
_Atomic uint64_t g_free_remote_box_calls = 0;  // cross-thread remote pushes
// Per-class counters for gating/metrics (Tiny classes = 8)
uint64_t g_ss_alloc_by_class[8] = {0};
uint64_t g_ss_freed_by_class[8] = {0};

typedef struct SuperslabCacheEntry {
    struct SuperslabCacheEntry* next;
} SuperslabCacheEntry;

static SuperslabCacheEntry* g_ss_cache_head[8] = {0};
static size_t g_ss_cache_count[8] = {0};
static size_t g_ss_cache_cap[8] = {0};
static size_t g_ss_precharge_target[8] = {0};
static _Atomic int g_ss_precharge_done[8] = {0};
static int g_ss_cache_enabled = 0;

static pthread_once_t g_ss_cache_once = PTHREAD_ONCE_INIT;
static pthread_mutex_t g_ss_cache_lock[8];

uint64_t g_ss_cache_hits[8] = {0};
uint64_t g_ss_cache_misses[8] = {0};
uint64_t g_ss_cache_puts[8] = {0};
uint64_t g_ss_cache_drops[8] = {0};
uint64_t g_ss_cache_precharged[8] = {0};

uint64_t g_superslabs_reused = 0;
uint64_t g_superslabs_cached = 0;

static void ss_cache_global_init(void) {
    for (int i = 0; i < 8; i++) {
        pthread_mutex_init(&g_ss_cache_lock[i], NULL);
    }
}

static inline void ss_cache_ensure_init(void) {
    pthread_once(&g_ss_cache_once, ss_cache_global_init);
}

static void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int populate);
static void ss_cache_precharge(uint8_t size_class, size_t ss_size, uintptr_t ss_mask);
static SuperslabCacheEntry* ss_cache_pop(uint8_t size_class);
static int ss_cache_push(uint8_t size_class, SuperSlab* ss);

// Drain remote MPSC stack into freelist (ownership already verified by caller)
void _ss_remote_drain_to_freelist_unsafe(SuperSlab* ss, int slab_idx, TinySlabMeta* meta)
{
    if (!ss || slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss) || !meta) return;

    static _Atomic uint32_t g_remote_drain_diag_once = 0;
    static int g_remote_drain_diag_en = -1;

    // Atomically take the whole remote list
    uintptr_t head = atomic_exchange_explicit(&ss->remote_heads[slab_idx], 0,
                                              memory_order_acq_rel);
    if (head == 0) return;

    // Convert remote stack (offset 0 next) into freelist encoding via Box API
    // and splice in front of current freelist preserving relative order.
    void* prev = meta->freelist;
    int cls = (int)meta->class_idx;
    HAK_CHECK_CLASS_IDX(cls, "_ss_remote_drain_to_freelist_unsafe");
    if (__builtin_expect(cls < 0 || cls >= TINY_NUM_CLASSES, 0)) {
        static _Atomic int g_remote_drain_cls_oob = 0;
        if (atomic_fetch_add_explicit(&g_remote_drain_cls_oob, 1, memory_order_relaxed) == 0) {
            fprintf(stderr,
                    "[REMOTE_DRAIN_CLASS_OOB] ss=%p slab_idx=%d meta=%p cls=%d head=%#lx\n",
                    (void*)ss, slab_idx, (void*)meta, cls, (unsigned long)head);
        }
        return;
    }
    uintptr_t cur = head;
    while (cur != 0) {
        uintptr_t next = *(uintptr_t*)cur;  // remote-next stored at offset 0
        if (__builtin_expect(g_remote_drain_diag_en == -1, 0)) {
            const char* e = getenv("HAKMEM_TINY_SLL_DIAG");
            g_remote_drain_diag_en = (e && *e && *e != '0') ? 1 : 0;
        }
        if (__builtin_expect(g_remote_drain_diag_en, 0)) {
            uintptr_t addr = (uintptr_t)next;
            if (addr != 0 && (addr < 4096 || addr > 0x00007fffffffffffULL)) {
                uint32_t shot = atomic_fetch_add_explicit(&g_remote_drain_diag_once, 1, memory_order_relaxed);
                if (shot < 8) {
                    fprintf(stderr,
                            "[REMOTE_DRAIN_NEXT_INVALID] cls=%d slab=%d cur=%p next=%p head=%#lx prev=%p count=%u\n",
                            cls,
                            slab_idx,
                            (void*)cur,
                            (void*)next,
                            (unsigned long)head,
                            prev,
                            (unsigned)meta->used);
                }
            }
#if HAKMEM_TINY_HEADER_CLASSIDX
            int hdr_cls = tiny_region_id_read_header((uint8_t*)cur + 1);
            if (hdr_cls >= 0 && hdr_cls != cls) {
                uint32_t shot = atomic_fetch_add_explicit(&g_remote_drain_diag_once, 1, memory_order_relaxed);
                if (shot < 8) {
                    fprintf(stderr,
                            "[REMOTE_DRAIN_HDR_MISMATCH] cls=%d slab=%d cur=%p hdr_cls=%d meta_cls=%d head=%#lx\n",
                            cls, slab_idx, (void*)cur, hdr_cls, (int)meta->class_idx, (unsigned long)head);
                }
            }
#endif
        }
#if HAKMEM_TINY_HEADER_CLASSIDX
        // Cross-check header vs meta before writing next (even if diag is off)
        {
            int hdr_cls_pre = tiny_region_id_read_header((uint8_t*)cur + 1);
            if (hdr_cls_pre >= 0 && hdr_cls_pre != cls) {
                static _Atomic uint32_t g_hdr_meta_mismatch_rd = 0;
                uint32_t n = atomic_fetch_add_explicit(&g_hdr_meta_mismatch_rd, 1, memory_order_relaxed);
                if (n < 16) {
                    fprintf(stderr,
                            "[REMOTE_DRAIN_HDR_META_MISMATCH] cls=%d slab=%d cur=%p hdr_cls=%d meta_cls=%d\n",
                            cls, slab_idx, (void*)cur, hdr_cls_pre, (int)meta->class_idx);
                }
            }
        }
#endif
        // Restore header for header-classes (class 1-6) which were clobbered by remote push
#if HAKMEM_TINY_HEADER_CLASSIDX
        if (cls != 0) {
            uint8_t expected = (uint8_t)(HEADER_MAGIC | (cls & HEADER_CLASS_MASK));
            *(uint8_t*)(uintptr_t)cur = expected;
        }
#endif
        // Rewrite next pointer to Box representation for this class
        tiny_next_write(cls, (void*)cur, prev);
        prev = (void*)cur;
        cur = next;
    }
    meta->freelist = prev;
    // Reset remote count after full drain
    atomic_store_explicit(&ss->remote_counts[slab_idx], 0, memory_order_release);

    // Update freelist/nonempty visibility bits
    uint32_t bit = (1u << slab_idx);
    atomic_fetch_or_explicit(&ss->freelist_mask, bit, memory_order_release);
    atomic_fetch_or_explicit(&ss->nonempty_mask, bit, memory_order_release);
}

static inline void ss_stats_os_alloc(uint8_t size_class, size_t ss_size) {
    pthread_mutex_lock(&g_superslab_lock);
    g_superslabs_allocated++;
    if (size_class < 8) {
        g_ss_alloc_by_class[size_class]++;
    }
    g_bytes_allocated += ss_size;
    pthread_mutex_unlock(&g_superslab_lock);
}

static inline void ss_stats_cache_reuse(void) {
    pthread_mutex_lock(&g_superslab_lock);
    g_superslabs_reused++;
    pthread_mutex_unlock(&g_superslab_lock);
}

static inline void ss_stats_cache_store(void) {
    pthread_mutex_lock(&g_superslab_lock);
    g_superslabs_cached++;
    pthread_mutex_unlock(&g_superslab_lock);
}

// ============================================================================
// Phase 8.3: ACE (Adaptive Cache Engine) State
// ============================================================================

SuperSlabACEState g_ss_ace[TINY_NUM_CLASSES_SS] = {{0}};

// Phase 8.3: hak_now_ns() is now defined in hakmem_tiny_superslab.h as static inline

// ============================================================================ 
// Diagnostics
// ============================================================================ 

static void log_superslab_oom_once(size_t ss_size, size_t alloc_size, int err) {
    static int logged = 0;
    if (logged) return;
    logged = 1;

    // CRITICAL FIX: Increment lock depth FIRST before any LIBC calls
    // fopen/fclose/getrlimit/fprintf all may call malloc internally
    // Must bypass HAKMEM wrapper to avoid header mismatch crash
    extern __thread int g_hakmem_lock_depth;
    g_hakmem_lock_depth++;  // Force wrapper to use __libc_malloc

    struct rlimit rl = {0};
    if (getrlimit(RLIMIT_AS, &rl) != 0) {
        rl.rlim_cur = RLIM_INFINITY;
        rl.rlim_max = RLIM_INFINITY;
    }

    unsigned long vm_size_kb = 0;
    unsigned long vm_rss_kb = 0;
    FILE* status = fopen("/proc/self/status", "r");
    if (status) {
        char line[256];
        while (fgets(line, sizeof(line), status)) {
            if (strncmp(line, "VmSize:", 7) == 0) {
                (void)sscanf(line + 7, "%lu", &vm_size_kb);
            } else if (strncmp(line, "VmRSS:", 6) == 0) {
                (void)sscanf(line + 6, "%lu", &vm_rss_kb);
            }
        }
        fclose(status);
    }
    // CRITICAL FIX: Do NOT decrement lock_depth yet!
    // fprintf() below may call malloc for buffering

    char rl_cur_buf[32];
    char rl_max_buf[32];
    if (rl.rlim_cur == RLIM_INFINITY) {
        strcpy(rl_cur_buf, "inf");
    } else {
        snprintf(rl_cur_buf, sizeof(rl_cur_buf), "%llu", (unsigned long long)rl.rlim_cur);
    }
    if (rl.rlim_max == RLIM_INFINITY) {
        strcpy(rl_max_buf, "inf");
    } else {
        snprintf(rl_max_buf, sizeof(rl_max_buf), "%llu", (unsigned long long)rl.rlim_max);
    }

#if !HAKMEM_BUILD_RELEASE
    fprintf(stderr,
            "[SS OOM] mmap failed: err=%d ss_size=%zu alloc_size=%zu "
            "alloc=%llu freed=%llu bytes=%llu "
            "RLIMIT_AS(cur=%s max=%s) VmSize=%lu kB VmRSS=%lu kB\n",
            err,
            ss_size,
            alloc_size,
            (unsigned long long)g_superslabs_allocated,
            (unsigned long long)g_superslabs_freed,
            (unsigned long long)g_bytes_allocated,
            rl_cur_buf,
            rl_max_buf,
            vm_size_kb,
            vm_rss_kb);
#endif

    g_hakmem_lock_depth--;  // Now safe to restore (all libc calls complete)
}

// Global counters for debugging (non-static for external access)
_Atomic uint64_t g_ss_mmap_count = 0;
_Atomic uint64_t g_final_fallback_mmap_count = 0;

static void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int populate) {
    void* ptr = NULL;
    static int log_count = 0;

#ifdef MAP_ALIGNED_SUPER
    int map_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER;
#ifdef MAP_POPULATE
    if (populate) {
        map_flags |= MAP_POPULATE;
    }
#endif
    ptr = mmap(NULL, ss_size,
               PROT_READ | PROT_WRITE,
               map_flags,
               -1, 0);
    if (ptr != MAP_FAILED) {
        atomic_fetch_add(&g_ss_mmap_count, 1);
        if (((uintptr_t)ptr & ss_mask) == 0) {
            ss_stats_os_alloc(size_class, ss_size);
            return ptr;
        }
        munmap(ptr, ss_size);
        ptr = NULL;
    } else {
        log_superslab_oom_once(ss_size, ss_size, errno);
    }
#endif

    size_t alloc_size = ss_size * 2;
    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
#ifdef MAP_POPULATE
    if (populate) {
        flags |= MAP_POPULATE;
    }
#endif
    void* raw = mmap(NULL, alloc_size,
                     PROT_READ | PROT_WRITE,
                     flags,
                     -1, 0);
    if (raw != MAP_FAILED) {
        uint64_t count = atomic_fetch_add(&g_ss_mmap_count, 1) + 1;
        #if !HAKMEM_BUILD_RELEASE
        if (log_count < 10) {
            fprintf(stderr, "[SUPERSLAB_MMAP] #%lu: class=%d size=%zu (total SuperSlab mmaps so far)\n",
                    (unsigned long)count, size_class, ss_size);
            log_count++;
        }
        #endif
    }
    if (raw == MAP_FAILED) {
        log_superslab_oom_once(ss_size, alloc_size, errno);
        return NULL;
    }

    uintptr_t raw_addr = (uintptr_t)raw;
    uintptr_t aligned_addr = (raw_addr + ss_mask) & ~ss_mask;
    ptr = (void*)aligned_addr;

    size_t prefix_size = aligned_addr - raw_addr;
    if (prefix_size > 0) {
        munmap(raw, prefix_size);
    }
    size_t suffix_size = alloc_size - prefix_size - ss_size;
    if (suffix_size > 0) {
        if (populate) {
#ifdef MADV_DONTNEED
            madvise((char*)ptr + ss_size, suffix_size, MADV_DONTNEED);
#endif
        } else {
            munmap((char*)ptr + ss_size, suffix_size);
        }
    }

    ss_stats_os_alloc(size_class, ss_size);
    return ptr;
}

static void ss_cache_precharge(uint8_t size_class, size_t ss_size, uintptr_t ss_mask) {
    if (!g_ss_cache_enabled) return;
    if (size_class >= 8) return;
    if (g_ss_precharge_target[size_class] == 0) return;
    if (atomic_load_explicit(&g_ss_precharge_done[size_class], memory_order_acquire)) return;

    ss_cache_ensure_init();
    pthread_mutex_lock(&g_ss_cache_lock[size_class]);
    size_t target = g_ss_precharge_target[size_class];
    size_t cap = g_ss_cache_cap[size_class];
    size_t desired = target;
    if (cap != 0 && desired > cap) {
        desired = cap;
    }
    while (g_ss_cache_count[size_class] < desired) {
        void* raw = ss_os_acquire(size_class, ss_size, ss_mask, 1);
        if (!raw) {
            break;
        }
        SuperslabCacheEntry* entry = (SuperslabCacheEntry*)raw;
        entry->next = g_ss_cache_head[size_class];
        g_ss_cache_head[size_class] = entry;
        g_ss_cache_count[size_class]++;
        g_ss_cache_precharged[size_class]++;
    }
    atomic_store_explicit(&g_ss_precharge_done[size_class], 1, memory_order_release);
    pthread_mutex_unlock(&g_ss_cache_lock[size_class]);
}

static SuperslabCacheEntry* ss_cache_pop(uint8_t size_class) {
    if (!g_ss_cache_enabled) return NULL;
    if (size_class >= 8) return NULL;

    ss_cache_ensure_init();

    pthread_mutex_lock(&g_ss_cache_lock[size_class]);
    SuperslabCacheEntry* entry = g_ss_cache_head[size_class];
    if (entry) {
        g_ss_cache_head[size_class] = entry->next;
        if (g_ss_cache_count[size_class] > 0) {
            g_ss_cache_count[size_class]--;
        }
        entry->next = NULL;
        g_ss_cache_hits[size_class]++;
    } else {
        g_ss_cache_misses[size_class]++;
    }
    pthread_mutex_unlock(&g_ss_cache_lock[size_class]);
    return entry;
}

static int ss_cache_push(uint8_t size_class, SuperSlab* ss) {
    if (!g_ss_cache_enabled) return 0;
    if (size_class >= 8) return 0;

    ss_cache_ensure_init();
    pthread_mutex_lock(&g_ss_cache_lock[size_class]);
    size_t cap = g_ss_cache_cap[size_class];
    if (cap != 0 && g_ss_cache_count[size_class] >= cap) {
        g_ss_cache_drops[size_class]++;
        pthread_mutex_unlock(&g_ss_cache_lock[size_class]);
        return 0;
    }
    SuperslabCacheEntry* entry = (SuperslabCacheEntry*)ss;
    entry->next = g_ss_cache_head[size_class];
    g_ss_cache_head[size_class] = entry;
    g_ss_cache_count[size_class]++;
    g_ss_cache_puts[size_class]++;
    pthread_mutex_unlock(&g_ss_cache_lock[size_class]);
    return 1;
}

/*
 * Legacy backend for hak_tiny_alloc_superslab_box().
 *
 * Phase 12 Stage A/B:
 *  - Uses per-class SuperSlabHead (g_superslab_heads) as the implementation.
 *  - Callers MUST use hak_tiny_alloc_superslab_box() and never touch this directly.
 *  - Later Stage C: this function will be replaced by a shared_pool backend.
 */
static SuperSlabHead* init_superslab_head(int class_idx);
static int expand_superslab_head(SuperSlabHead* head);

static void* hak_tiny_alloc_superslab_backend_legacy(int class_idx)
{
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
        return NULL;
    }

    SuperSlabHead* head = g_superslab_heads[class_idx];
    if (!head) {
        head = init_superslab_head(class_idx);
        if (!head) {
            return NULL;
        }
        g_superslab_heads[class_idx] = head;
    }

    SuperSlab* chunk = head->current_chunk ? head->current_chunk : head->first_chunk;

    while (chunk) {
        int cap = ss_slabs_capacity(chunk);
        for (int slab_idx = 0; slab_idx < cap; slab_idx++) {
            TinySlabMeta* meta = &chunk->slabs[slab_idx];

            // Skip slabs that belong to a different class (or are uninitialized).
            if (meta->class_idx != (uint8_t)class_idx) {
                continue;
            }

            if (meta->capacity == 0) {
                continue;
            }

            if (meta->used < meta->capacity) {
                size_t stride = tiny_block_stride_for_class(class_idx);
                size_t offset = (size_t)meta->used * stride;
                uint8_t* base = (uint8_t*)chunk
                              + SUPERSLAB_SLAB0_DATA_OFFSET
                              + (size_t)slab_idx * SUPERSLAB_SLAB_USABLE_SIZE
                              + offset;

                meta->used++;
                atomic_fetch_add_explicit(&chunk->total_active_blocks, 1, memory_order_relaxed);
                return (void*)base;
            }
        }
        chunk = chunk->next_chunk;
    }

    if (expand_superslab_head(head) < 0) {
        return NULL;
    }

    SuperSlab* new_chunk = head->current_chunk;
    if (!new_chunk) {
        return NULL;
    }

    int cap2 = ss_slabs_capacity(new_chunk);
    for (int slab_idx = 0; slab_idx < cap2; slab_idx++) {
        TinySlabMeta* meta = &new_chunk->slabs[slab_idx];
        if (meta->capacity == 0) continue;
        if (meta->used < meta->capacity) {
            size_t stride = tiny_block_stride_for_class(class_idx);
            size_t offset = (size_t)meta->used * stride;
            uint8_t* base = (uint8_t*)new_chunk
                          + SUPERSLAB_SLAB0_DATA_OFFSET
                          + (size_t)slab_idx * SUPERSLAB_SLAB_USABLE_SIZE
                          + offset;

            meta->used++;
            atomic_fetch_add_explicit(&new_chunk->total_active_blocks, 1, memory_order_relaxed);
            return (void*)base;
        }
    }

    return NULL;
}

/*
 * Shared pool backend for hak_tiny_alloc_superslab_box().
 *
 * Phase 12-2:
 *  - Uses SharedSuperSlabPool (g_shared_pool) to obtain a SuperSlab/slab
 *    for the requested class_idx.
 *  - This backend EXPRESSLY owns only:
 *      - choosing (ss, slab_idx) via shared_pool_acquire_slab()
 *      - initializing that slab's TinySlabMeta via superslab_init_slab()
 *    and nothing else; all callers must go through hak_tiny_alloc_superslab_box().
 *
 *  - For now this is a minimal, conservative implementation:
 *      - One linear bump-run is carved from the acquired slab using tiny_block_stride_for_class().
 *      - No complex per-slab freelist or refill policy yet (Phase 12-3+).
 *      - If shared_pool_acquire_slab() fails, we fall back to legacy backend.
 */
static void* hak_tiny_alloc_superslab_backend_shared(int class_idx)
{
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
        return NULL;
    }

    SuperSlab* ss = NULL;
    int slab_idx = -1;

    if (shared_pool_acquire_slab(class_idx, &ss, &slab_idx) != 0 || !ss) {
        // Shared pool could not provide a slab; caller may choose to fall back.
        return NULL;
    }

    TinySlabMeta* meta = &ss->slabs[slab_idx];

    // Defensive: shared_pool must either hand us an UNASSIGNED slab or one
    // already bound to this class. Anything else is a hard bug.
    if (meta->class_idx != 255 && meta->class_idx != (uint8_t)class_idx) {
#if !HAKMEM_BUILD_RELEASE
        fprintf(stderr,
                "[HAKMEM][SS_SHARED] BUG: acquire_slab mismatch: cls=%d meta->class_idx=%u slab_idx=%d ss=%p\n",
                class_idx, (unsigned)meta->class_idx, slab_idx, (void*)ss);
#endif
        return NULL;
    }

    // Initialize slab geometry once for this class.
    if (meta->capacity == 0) {
        size_t block_size = g_tiny_class_sizes[class_idx];
        // owner_tid_low is advisory; we can use 0 in this backend.
        superslab_init_slab(ss, slab_idx, block_size, 0);
        meta = &ss->slabs[slab_idx];

        // CRITICAL FIX: Always set class_idx after init to avoid C0/C7 confusion.
        // New SuperSlabs start with meta->class_idx=0 (mmap zero-init).
        // Must explicitly set to requested class, not just when class_idx==255.
        meta->class_idx = (uint8_t)class_idx;
    }

    // Final contract check before computing addresses.
    if (meta->class_idx != (uint8_t)class_idx ||
        meta->capacity == 0 ||
        meta->used > meta->capacity) {
#if !HAKMEM_BUILD_RELEASE
        fprintf(stderr,
                "[HAKMEM][SS_SHARED] BUG: invalid slab meta before alloc: "
                "cls=%d slab_idx=%d meta_cls=%u used=%u cap=%u ss=%p\n",
                class_idx, slab_idx,
                (unsigned)meta->class_idx,
                (unsigned)meta->used,
                (unsigned)meta->capacity,
                (void*)ss);
#endif
        return NULL;
    }

    // Simple bump allocation within this slab.
    if (meta->used >= meta->capacity) {
        // Slab exhausted: in minimal Phase12-2 backend we do not loop;
        // caller or future logic must acquire another slab.
        return NULL;
    }

    size_t stride = tiny_block_stride_for_class(class_idx);
    size_t offset = (size_t)meta->used * stride;

    // Phase 12-2 minimal geometry:
    //  - slab 0 data offset via SUPERSLAB_SLAB0_DATA_OFFSET
    //  - subsequent slabs at fixed SUPERSLAB_SLAB_USABLE_SIZE strides.
    size_t slab_base_off = SUPERSLAB_SLAB0_DATA_OFFSET
                         + (size_t)slab_idx * SUPERSLAB_SLAB_USABLE_SIZE;
    uint8_t* base = (uint8_t*)ss + slab_base_off + offset;

    meta->used++;
    atomic_fetch_add_explicit(&ss->total_active_blocks, 1, memory_order_relaxed);

    return (void*)base;
}

/*
 * Box API entry:
 *  - Single front-door for tiny-side Superslab allocations.
 *
 * Phase 12 policy:
 *  - HAKMEM_TINY_SS_SHARED=0 → legacy backendのみ（回帰確認用）
 *  - HAKMEM_TINY_SS_SHARED=1 → shared backendを優先し、失敗時のみ legacy にフォールバック
 */
void* hak_tiny_alloc_superslab_box(int class_idx)
{
    static int g_ss_shared_mode = -1;
    static _Atomic uint32_t g_ss_backend_log = 0;
    if (__builtin_expect(g_ss_shared_mode == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_SS_SHARED");
        if (!e || !*e) {
            g_ss_shared_mode = 1; // デフォルト: shared 有効
        } else {
            int v = atoi(e);
            g_ss_shared_mode = (v != 0) ? 1 : 0;
        }
    }

    if (g_ss_shared_mode == 1) {
        void* p = hak_tiny_alloc_superslab_backend_shared(class_idx);
        if (p != NULL) {
            uint32_t n = atomic_fetch_add_explicit(&g_ss_backend_log, 1, memory_order_relaxed);
            if (n < 4) {
                fprintf(stderr, "[SS_BACKEND] shared cls=%d ptr=%p\n", class_idx, p);
            }
            return p;
        }
        // shared backend が失敗した場合は安全側で legacy にフォールバック
        uint32_t n = atomic_fetch_add_explicit(&g_ss_backend_log, 1, memory_order_relaxed);
        if (n < 4) {
            fprintf(stderr, "[SS_BACKEND] shared_fail→legacy cls=%d\n", class_idx);
        }
        return hak_tiny_alloc_superslab_backend_legacy(class_idx);
    }

    // shared OFF 時は legacy のみ
    uint32_t n = atomic_fetch_add_explicit(&g_ss_backend_log, 1, memory_order_relaxed);
    if (n < 4) {
        fprintf(stderr, "[SS_BACKEND] legacy cls=%d\n", class_idx);
    }
    return hak_tiny_alloc_superslab_backend_legacy(class_idx);
}

// ============================================================================
// SuperSlab Allocation (2MB aligned)
// ============================================================================

SuperSlab* superslab_allocate(uint8_t size_class) {
    // Optional fault injection for testing: HAKMEM_TINY_SS_FAULT_RATE=N → 1/N で失敗
    static int fault_rate = -1;  // -1=unparsed, 0=disabled, >0=rate
    static __thread unsigned long fault_tick = 0;
    if (__builtin_expect(fault_rate == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_SS_FAULT_RATE");
        if (e && *e) {
            int v = atoi(e); if (v < 0) v = 0; fault_rate = v;
        } else {
            fault_rate = 0;
        }
    }
    if (fault_rate > 0) {
        unsigned long t = ++fault_tick;
        if ((t % (unsigned long)fault_rate) == 0ul) {
            return NULL;  // simulate OOM
        }
    }
    // Optional env clamp for SuperSlab size
    static int env_parsed = 0;
    static uint8_t g_ss_min_lg_env = SUPERSLAB_LG_DEFAULT;  // Start with default (2MB)
    static uint8_t g_ss_max_lg_env = SUPERSLAB_LG_MAX;
    if (!env_parsed) {
        char* maxmb = getenv("HAKMEM_TINY_SS_MAX_MB");
        if (maxmb) {
            int m = atoi(maxmb); if (m == 1) g_ss_max_lg_env = 20; else if (m == 2) g_ss_max_lg_env = 21;
        }
        char* minmb = getenv("HAKMEM_TINY_SS_MIN_MB");
        if (minmb) {
            int m = atoi(minmb); if (m == 1) g_ss_min_lg_env = 20; else if (m == 2) g_ss_min_lg_env = 21;
        }
        if (g_ss_min_lg_env > g_ss_max_lg_env) g_ss_min_lg_env = g_ss_max_lg_env;
        const char* force_lg_env = getenv("HAKMEM_TINY_SS_FORCE_LG");
        if (force_lg_env && *force_lg_env) {
            int v = atoi(force_lg_env);
            if (v >= SUPERSLAB_LG_MIN && v <= SUPERSLAB_LG_MAX) {
                g_ss_force_lg = v;
                g_ss_min_lg_env = g_ss_max_lg_env = v;
            }
        }
        size_t precharge_default = 0;
        const char* precharge_env = getenv("HAKMEM_TINY_SS_PRECHARGE");
        if (precharge_env && *precharge_env) {
            long v = atol(precharge_env);
            if (v < 0) v = 0;
            precharge_default = (size_t)v;
            if (v > 0) {
                atomic_store_explicit(&g_ss_populate_once, 1, memory_order_relaxed);
            }
        }
        size_t cache_default = 0;
        const char* cache_env = getenv("HAKMEM_TINY_SS_CACHE");
        if (cache_env && *cache_env) {
            long v = atol(cache_env);
            if (v < 0) v = 0;
            cache_default = (size_t)v;
        }
        for (int i = 0; i < 8; i++) {
            g_ss_cache_cap[i] = cache_default;
            g_ss_precharge_target[i] = precharge_default;
        }
        for (int i = 0; i < 8; i++) {
            char name[64];
            snprintf(name, sizeof(name), "HAKMEM_TINY_SS_CACHE_C%d", i);
            char* cap_env = getenv(name);
            if (cap_env && *cap_env) {
                long v = atol(cap_env);
                if (v < 0) v = 0;
                g_ss_cache_cap[i] = (size_t)v;
            }
            snprintf(name, sizeof(name), "HAKMEM_TINY_SS_PRECHARGE_C%d", i);
            char* pre_env = getenv(name);
            if (pre_env && *pre_env) {
                long v = atol(pre_env);
                if (v < 0) v = 0;
                g_ss_precharge_target[i] = (size_t)v;
                if (v > 0) {
                    atomic_store_explicit(&g_ss_populate_once, 1, memory_order_relaxed);
                }
            }
            if (g_ss_cache_cap[i] > 0 || g_ss_precharge_target[i] > 0) {
                g_ss_cache_enabled = 1;
            }
        }
        const char* populate_env = getenv("HAKMEM_TINY_SS_POPULATE_ONCE");
        if (populate_env && atoi(populate_env) != 0) {
            atomic_store_explicit(&g_ss_populate_once, 1, memory_order_relaxed);
        }
        env_parsed = 1;
    }

    uint8_t lg = (g_ss_force_lg >= 0) ? (uint8_t)g_ss_force_lg : hak_tiny_superslab_next_lg(size_class);
    if (lg < g_ss_min_lg_env) lg = g_ss_min_lg_env;
    if (lg > g_ss_max_lg_env) lg = g_ss_max_lg_env;
    size_t ss_size = (size_t)1 << lg;  // 2^20 = 1MB, 2^21 = 2MB
    uintptr_t ss_mask = ss_size - 1;
    int from_cache = 0;
    void* ptr = NULL;

    // Debug logging flag (lazy init)
    static __thread int dbg = -1;
    if (__builtin_expect(dbg == -1, 0)) {
        const char* e = getenv("HAKMEM_SS_PREWARM_DEBUG");
        dbg = (e && *e && *e != '0') ? 1 : 0;
    }

    // Phase 9: Try LRU cache first (lazy deallocation)
    SuperSlab* cached_ss = hak_ss_lru_pop(size_class);
    if (cached_ss) {
        ptr = (void*)cached_ss;
        from_cache = 1;
        // Debug logging for REFILL from LRU
        if (dbg == 1) {
            fprintf(stderr, "[REFILL] class=%d from_lru=1 ss=%p\n",
                    size_class, (void*)cached_ss);
        }
        // Skip old cache path - LRU cache takes priority
    } else if (g_ss_cache_enabled && size_class < 8) {
        // Fallback to old cache (will be deprecated)
        ss_cache_precharge(size_class, ss_size, ss_mask);
        SuperslabCacheEntry* old_cached = ss_cache_pop(size_class);
        if (old_cached) {
            ptr = (void*)old_cached;
            from_cache = 1;
            // Debug logging for REFILL from prewarm (old cache is essentially prewarm)
            if (dbg == 1) {
                fprintf(stderr, "[REFILL] class=%d from_prewarm=1 ss=%p\n",
                        size_class, (void*)old_cached);
            }
        }
    }

    if (!ptr) {
        int populate = atomic_exchange_explicit(&g_ss_populate_once, 0, memory_order_acq_rel);
        ptr = ss_os_acquire(size_class, ss_size, ss_mask, populate);
        if (!ptr) {
            return NULL;
        }
        // Debug logging for REFILL with new allocation
        if (dbg == 1) {
            fprintf(stderr, "[REFILL] class=%d new_alloc=1 ss=%p\n",
                    size_class, (void*)ptr);
        }
    }

    // Initialize SuperSlab header (Phase 12: no global size_class field)
    SuperSlab* ss = (SuperSlab*)ptr;
    ss->magic = SUPERSLAB_MAGIC;
    ss->active_slabs = 0;
    ss->lg_size = lg;  // Phase 8.3: Use ACE-determined lg_size (20=1MB, 21=2MB)
    ss->slab_bitmap = 0;
    ss->nonempty_mask = 0;  // Phase 6-2.1: ChatGPT Pro P0 - init nonempty mask
    ss->partial_epoch = 0;
    ss->publish_hint = 0xFF;

    // Initialize atomics explicitly
    atomic_store_explicit(&ss->total_active_blocks, 0, memory_order_relaxed);
    atomic_store_explicit(&ss->refcount, 0, memory_order_relaxed);
    atomic_store_explicit(&ss->listed, 0, memory_order_relaxed);
    ss->partial_next = NULL;

    // Phase 9: Initialize LRU fields
    ss->last_used_ns = 0;
    ss->generation = 0;
    ss->lru_prev = NULL;
    ss->lru_next = NULL;

    // Phase 3d-C: Initialize Hot/Cold Split fields
    ss->hot_count = 0;
    ss->cold_count = 0;
    for (int i = 0; i < 16; i++) {
        ss->hot_indices[i] = 0;
        ss->cold_indices[i] = 0;
    }

    // Initialize all slab metadata (only up to max slabs for this size)
    int max_slabs = (int)(ss_size / SLAB_SIZE);

    // DEFENSIVE FIX: Zero all slab metadata arrays to prevent ANY uninitialized pointers
    // This catches the 0xa2a2a2a2a2a2a2a2 pattern bug (ASan/debug fill pattern)
    // Even though mmap should return zeroed pages, sanitizers may fill with debug patterns
    memset(ss->slabs, 0, max_slabs * sizeof(TinySlabMeta));
    memset(ss->remote_heads, 0, max_slabs * sizeof(uintptr_t));
    memset(ss->remote_counts, 0, max_slabs * sizeof(uint32_t));
    memset(ss->slab_listed, 0, max_slabs * sizeof(uint32_t));

    for (int i = 0; i < max_slabs; i++) {
        // Phase 1: Atomic initialization (freelist + used are now _Atomic)
        slab_freelist_store_relaxed(&ss->slabs[i], NULL);  // Explicit NULL (redundant after memset, but clear intent)
        atomic_store_explicit(&ss->slabs[i].used, 0, memory_order_relaxed);
        ss->slabs[i].capacity = 0;
        ss->slabs[i].owner_tid_low = 0;

        // Initialize remote queue atomics (memset already zeroed, but use proper atomic init)
        atomic_store_explicit(&ss->remote_heads[i], 0, memory_order_relaxed);
        atomic_store_explicit(&ss->remote_counts[i], 0, memory_order_relaxed);
        atomic_store_explicit(&ss->slab_listed[i], 0, memory_order_relaxed);
    }

    if (from_cache) {
        ss_stats_cache_reuse();
    }

    // Phase 8.3: Update ACE current_lg to match allocated size
    g_ss_ace[size_class].current_lg = lg;

    // Phase 1: Register SuperSlab in global registry for fast lookup
    // CRITICAL: Register AFTER full initialization (ss structure is ready)
    uintptr_t base = (uintptr_t)ss;
    if (!hak_super_register(base, ss)) {
        // Registry full - this is a fatal error
        fprintf(stderr, "HAKMEM FATAL: SuperSlab registry full, cannot register %p\n", ss);
        // Still return ss to avoid memory leak, but lookups may fail
    }

    return ss;
}

// ============================================================================
// Phase 2a: Dynamic Expansion - Chunk Management Functions
// ============================================================================

// Initialize SuperSlabHead for a class
SuperSlabHead* init_superslab_head(int class_idx) {
    if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
        return NULL;
    }

    // Allocate SuperSlabHead structure
    SuperSlabHead* head = (SuperSlabHead*)calloc(1, sizeof(SuperSlabHead));
    if (!head) {
        extern __thread int g_hakmem_lock_depth;
        g_hakmem_lock_depth++;
        fprintf(stderr, "[HAKMEM] CRITICAL: Failed to allocate SuperSlabHead for class %d\n", class_idx);
        g_hakmem_lock_depth--;
        return NULL;
    }

    head->class_idx = (uint8_t)class_idx;
    atomic_store_explicit(&head->total_chunks, 0, memory_order_relaxed);
    head->first_chunk = NULL;
    head->current_chunk = NULL;
    pthread_mutex_init(&head->expansion_lock, NULL);

    // Allocate initial chunk(s)
    // Hot classes (1, 4, 6) get 2 initial chunks to reduce contention
    int initial_chunks = 1;

    // Phase 2a: Start with 1 chunk for all classes (expansion will handle growth)
    // This reduces startup memory overhead while still allowing unlimited growth
    initial_chunks = 1;

    for (int i = 0; i < initial_chunks; i++) {
        if (expand_superslab_head(head) < 0) {
            extern __thread int g_hakmem_lock_depth;
            g_hakmem_lock_depth++;
            fprintf(stderr, "[HAKMEM] CRITICAL: Failed to allocate initial chunk %d for class %d\n",
                    i, class_idx);
            g_hakmem_lock_depth--;

            // Cleanup on failure
            SuperSlab* chunk = head->first_chunk;
            while (chunk) {
                SuperSlab* next = chunk->next_chunk;
                superslab_free(chunk);
                chunk = next;
            }
            pthread_mutex_destroy(&head->expansion_lock);
            free(head);
            return NULL;
        }
    }

    extern __thread int g_hakmem_lock_depth;
    g_hakmem_lock_depth++;
#if !HAKMEM_BUILD_RELEASE
    fprintf(stderr, "[HAKMEM] Initialized SuperSlabHead for class %d: %zu initial chunks\n",
            class_idx, atomic_load_explicit(&head->total_chunks, memory_order_relaxed));
#endif
    g_hakmem_lock_depth--;

    return head;
}

// Expand SuperSlabHead by allocating and linking a new chunk
int expand_superslab_head(SuperSlabHead* head) {
    if (!head) {
        return -1;
    }

    // Allocate new chunk via existing superslab_allocate
    SuperSlab* new_chunk = superslab_allocate(head->class_idx);
    if (!new_chunk) {
#if !defined(NDEBUG) || defined(HAKMEM_SUPERSLAB_VERBOSE)
        extern __thread int g_hakmem_lock_depth;
        g_hakmem_lock_depth++;
        fprintf(stderr, "[HAKMEM] CRITICAL: Failed to allocate new chunk for class %d (system OOM)\n",
                head->class_idx);
        g_hakmem_lock_depth--;
#endif
        return -1;  // True OOM (system out of memory)
    }

    // CRITICAL FIX: Initialize slab 0 so bitmap != 0x00000000
    // Phase 2a chunks must have at least one usable slab after allocation
    size_t block_size = g_tiny_class_sizes[head->class_idx];
    // Use pthread_self() directly since tiny_self_u32() is static inline in hakmem_tiny.c
    uint32_t owner_tid = (uint32_t)(uintptr_t)pthread_self();

    superslab_init_slab(new_chunk, 0, block_size, owner_tid);

    // Initialize the next_chunk link to NULL
    new_chunk->next_chunk = NULL;

    // Thread-safe linking
    pthread_mutex_lock(&head->expansion_lock);

    if (head->current_chunk) {
        // Find the tail of the list (optimization: could cache tail pointer)
        SuperSlab* tail = head->current_chunk;
        while (tail->next_chunk) {
            tail = tail->next_chunk;
        }
        tail->next_chunk = new_chunk;
    } else {
        // First chunk
        head->first_chunk = new_chunk;
    }

    // Update current chunk to new chunk (for fast allocation)
    head->current_chunk = new_chunk;

    // Increment total chunks atomically
    size_t old_count = atomic_fetch_add_explicit(&head->total_chunks, 1, memory_order_relaxed);
    size_t new_count = old_count + 1;

    pthread_mutex_unlock(&head->expansion_lock);

#if !defined(NDEBUG) || defined(HAKMEM_SUPERSLAB_VERBOSE)
    extern __thread int g_hakmem_lock_depth;
    g_hakmem_lock_depth++;
    fprintf(stderr, "[HAKMEM] Expanded SuperSlabHead for class %d: %zu chunks now (bitmap=0x%08x)\n",
            head->class_idx, new_count, new_chunk->slab_bitmap);
    g_hakmem_lock_depth--;
#endif

    return 0;
}

// Find which chunk a pointer belongs to
SuperSlab* find_chunk_for_ptr(void* ptr, int class_idx) {
    if (!ptr || class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
        return NULL;
    }

    SuperSlabHead* head = g_superslab_heads[class_idx];
    if (!head) {
        return NULL;
    }

    uintptr_t ptr_addr = (uintptr_t)ptr;

    // Walk the chunk list
    SuperSlab* chunk = head->first_chunk;
    while (chunk) {
        // Check if ptr is within this chunk's memory range
        // Each chunk is aligned to SUPERSLAB_SIZE (1MB or 2MB)
        uintptr_t chunk_start = (uintptr_t)chunk;
        size_t chunk_size = (size_t)1 << chunk->lg_size;  // Use actual chunk size
        uintptr_t chunk_end = chunk_start + chunk_size;

        if (ptr_addr >= chunk_start && ptr_addr < chunk_end) {
            // Found the chunk
            return chunk;
        }

        chunk = chunk->next_chunk;
    }

    return NULL;  // Not found in any chunk
}

// ============================================================================
// SuperSlab Deallocation
// ============================================================================

void superslab_free(SuperSlab* ss) {
    if (!ss || ss->magic != SUPERSLAB_MAGIC) {
        return;  // Invalid SuperSlab
    }

    // ADD DEBUG LOGGING
    static __thread int dbg = -1;
    if (__builtin_expect(dbg == -1, 0)) {
        const char* e = getenv("HAKMEM_SS_FREE_DEBUG");
        dbg = (e && *e && *e != '0') ? 1 : 0;
    }
    if (dbg == 1) {
        fprintf(stderr, "[SS_FREE] CALLED: ss=%p lg_size=%d active_slabs=%u\n",
                (void*)ss, ss->lg_size, ss->active_slabs);
    }

    // Phase 9: Lazy Deallocation - try to cache in LRU instead of munmap
    size_t ss_size = (size_t)1 << ss->lg_size;

    // Phase 1: Unregister SuperSlab from registry FIRST
    // CRITICAL: Must unregister BEFORE adding to LRU cache
    // Reason: Cached SuperSlabs should NOT be found by lookups
    uintptr_t base = (uintptr_t)ss;
    hak_super_unregister(base);

    // Memory fence to ensure unregister is visible
    atomic_thread_fence(memory_order_release);

    // Phase 9: Try LRU cache first (lazy deallocation)
    // NOTE: LRU cache keeps magic=SUPERSLAB_MAGIC for validation
    // Magic will be cleared on eviction or reuse
    int lru_cached = hak_ss_lru_push(ss);
    if (dbg == 1) {
        fprintf(stderr, "[SS_FREE] hak_ss_lru_push() returned %d\n", lru_cached);
    }
    if (lru_cached) {
        // Successfully cached in LRU - defer munmap
        return;
    }

    // LRU cache full or disabled - try old cache using head class_idx (if known)
    int old_cached = ss_cache_push(0, ss);
    if (old_cached) {
        ss_stats_cache_store();
        return;
    }

    // Both caches full - immediately free to OS (eager deallocation)
    // Clear magic to prevent use-after-free
    ss->magic = 0;

#if !HAKMEM_BUILD_RELEASE
    fprintf(stderr, "[DEBUG ss_os_release] Freeing SuperSlab ss=%p size=%zu active=%u (LRU full)\n",
            (void*)ss, ss_size,
            atomic_load_explicit(&ss->total_active_blocks, memory_order_relaxed));
#endif

    munmap(ss, ss_size);

    // Update statistics for actual release to OS
    pthread_mutex_lock(&g_superslab_lock);
    g_superslabs_freed++;
    // Phase 12: we no longer track per-SS size_class on header; skip g_ss_freed_by_class here
    g_bytes_allocated -= ss_size;
    pthread_mutex_unlock(&g_superslab_lock);

#if !HAKMEM_BUILD_RELEASE
    fprintf(stderr, "[DEBUG ss_os_release] g_superslabs_freed now = %llu\n",
            (unsigned long long)g_superslabs_freed);
#endif
}

// ============================================================================
// Slab Initialization within SuperSlab
// ============================================================================

void superslab_init_slab(SuperSlab* ss, int slab_idx, size_t block_size, uint32_t owner_tid)
{
    if (!ss || slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
        return;
    }

    // Phase E1-CORRECT unified geometry:
    // - block_size is the TOTAL stride for this class (g_tiny_class_sizes[cls])
    // - usable bytes are determined by slab index (slab0 vs others)
    // - capacity = usable / stride for ALL classes (including former C7)
    size_t usable_size = (slab_idx == 0)
                           ? SUPERSLAB_SLAB0_USABLE_SIZE
                           : SUPERSLAB_SLAB_USABLE_SIZE;
    size_t stride = block_size;
    uint16_t capacity = (uint16_t)(usable_size / stride);

#if !HAKMEM_BUILD_RELEASE
    if (slab_idx == 0) {
        fprintf(stderr,
                "[SUPERSLAB_INIT] slab 0: usable_size=%zu stride=%zu capacity=%u\n",
                usable_size, stride, (unsigned)capacity);
    }
#endif

    TinySlabMeta* meta = &ss->slabs[slab_idx];
    meta->freelist = NULL;          // NULL = linear allocation mode
    meta->used = 0;
    meta->capacity = capacity;
    meta->carved = 0;
    meta->owner_tid_low = (uint8_t)(owner_tid & 0xFFu);
    // Fail-safe: stamp class_idx from geometry (stride → class).
    // This ensures legacy/shared/legacy-refill paths all end with a correct class.
    for (int i = 0; i < TINY_NUM_CLASSES; i++) {
        if (g_tiny_class_sizes[i] == stride) {
            meta->class_idx = (uint8_t)i;
            break;
        }
    }

    superslab_activate_slab(ss, slab_idx);
}

// ============================================================================
// Slab Bitmap Management
// ============================================================================

void superslab_activate_slab(SuperSlab* ss, int slab_idx) {
    if (!ss || slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
        return;
    }
    uint32_t mask = 1u << slab_idx;
    if ((ss->slab_bitmap & mask) == 0) {
        ss->slab_bitmap |= mask;
        ss->active_slabs++;

        // Phase 3d-C: Update hot/cold indices after activating new slab
        ss_update_hot_cold_indices(ss);
    }
}

void superslab_deactivate_slab(SuperSlab* ss, int slab_idx) {
    if (!ss || slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
        return;
    }
    uint32_t mask = 1u << slab_idx;
    if (ss->slab_bitmap & mask) {
        ss->slab_bitmap &= ~mask;
        ss->active_slabs--;
    }
}

int superslab_find_free_slab(SuperSlab* ss) {
    if (!ss) return -1;
    if ((int)ss->active_slabs >= ss_slabs_capacity(ss)) {
        return -1;  // No free slabs
    }
    // Find first 0 bit in bitmap
    int cap = ss_slabs_capacity(ss);
    for (int i = 0; i < cap; i++) {
        if ((ss->slab_bitmap & (1u << i)) == 0) {
            return i;
        }
    }
    return -1;
}

// ============================================================================
// Statistics / Debugging
// ============================================================================

void superslab_print_stats(SuperSlab* ss) {
    if (!ss || ss->magic != SUPERSLAB_MAGIC) {
        printf("Invalid SuperSlab\n");
        return;
    }

    printf("=== SuperSlab Stats ===\n");
    printf("Address: %p\n", (void*)ss);
    // Phase 12: per-SS size_class removed; classes are per-slab via meta->class_idx.
    printf("Active slabs: %u / %d\n", ss->active_slabs, ss_slabs_capacity(ss));
    printf("Bitmap: 0x%08X\n", ss->slab_bitmap);
    printf("\nPer-slab details:\n");
    for (int i = 0; i < ss_slabs_capacity(ss); i++) {
        if (ss->slab_bitmap & (1u << i)) {
            TinySlabMeta* meta = &ss->slabs[i];
            printf("  Slab %2d: used=%u/%u freelist=%p class=%u owner_tid_low=%u\n",
                   i, meta->used, meta->capacity, meta->freelist,
                   (unsigned)meta->class_idx, (unsigned)meta->owner_tid_low);
        }
    }
    printf("\n");
}

// Global statistics
void superslab_print_global_stats(void) {
    pthread_mutex_lock(&g_superslab_lock);
    printf("=== Global SuperSlab Stats ===\n");
    printf("SuperSlabs allocated: %lu\n", g_superslabs_allocated);
    printf("SuperSlabs freed: %lu\n", g_superslabs_freed);
    printf("SuperSlabs active: %lu\n", g_superslabs_allocated - g_superslabs_freed);
    printf("Total bytes allocated: %lu MB\n", g_bytes_allocated / (1024 * 1024));
    pthread_mutex_unlock(&g_superslab_lock);
}

// ============================================================================
// Phase 8.3: ACE Statistics / Debugging
// ============================================================================

void superslab_ace_print_stats(void) {
    printf("=== ACE (Adaptive Cache Engine) Stats ===\n");
    const char* class_names[8] = {"8B", "16B", "24B", "32B", "40B", "48B", "56B", "64B"};

    printf("Class   Curr  Targ  Hot  Allocs  Refills  Spills  LiveBlks\n");
    printf("--------------------------------------------------------------\n");

    for (int i = 0; i < TINY_NUM_CLASSES_SS; i++) {
        SuperSlabACEState* c = &g_ss_ace[i];
        printf("%-6s  %2uMB  %2uMB  %4u  %7u  %8u  %7u  %9u\n",
               class_names[i],
               (1u << c->current_lg) / (1024 * 1024),
               (1u << c->target_lg) / (1024 * 1024),
               c->hot_score,
               c->alloc_count,
               c->refill_count,
               c->spill_count,
               c->live_blocks);
    }
    printf("\n");
}

// ============================================================================
// Phase 8.3: ACE Tick Function (Promotion/Demotion Logic)
// ============================================================================

#define ACE_TICK_NS        (150ULL * 1000 * 1000)  // 150ms tick interval
#define ACE_COOLDOWN_NS    (800ULL * 1000 * 1000)  // 0.8s cooldown (anti-oscillation)

// Simplified thresholds for refill activity
#define HI_REFILL(k)       (g_ss_ace[k].refill_count > 64)   // High refill rate
#define MID_REFILL(k)      (g_ss_ace[k].refill_count > 16)   // Medium refill rate

// Object sizes per class (for capacity calculation)
// Must match TINY size classes: 8, 16, 24, 32, 40, 48, 56, 64 bytes
static const int g_tiny_obj_sizes[TINY_NUM_CLASSES_SS] = {8, 16, 24, 32, 40, 48, 56, 64};

void hak_tiny_superslab_ace_tick(int k, uint64_t now) {
    if (k < 0 || k >= TINY_NUM_CLASSES_SS) return;

    SuperSlabACEState* c = &g_ss_ace[k];

    // Rate limiting: only tick every ACE_TICK_NS (~150ms)
    if (now - c->last_tick_ns < ACE_TICK_NS) return;

    // Calculate capacity for 1MB and 2MB SuperSlabs
    int obj_size = g_tiny_obj_sizes[k];
    double cap1MB = (double)((1U << 20) / obj_size);  // 1MB capacity
    double cap2MB = (double)((1U << 21) / obj_size);  // 2MB capacity

    // Calculate hotness score (weighted: 60% live blocks, 40% refill rate)
    double hot = 0.6 * (double)c->live_blocks + 0.4 * (double)c->refill_count;
    if (hot < 0) hot = 0;
    if (hot > 1000) hot = 1000;
    c->hot_score = (uint16_t)hot;

    // Cooldown mechanism: prevent size changes within 0.8s of last change
    static uint64_t last_switch_ns[TINY_NUM_CLASSES_SS] = {0};

    if (now - last_switch_ns[k] >= ACE_COOLDOWN_NS) {
        if (c->current_lg <= 20) {
            // Promotion condition: 1MB → 2MB
            // High demand (live > 75% capacity) AND high refill rate
            if (c->live_blocks > 0.75 * cap1MB && HI_REFILL(k)) {
                c->target_lg = 21;  // Promote to 2MB
                last_switch_ns[k] = now;
            }
        } else {
            // Demotion condition: 2MB → 1MB
            // Low demand (live < 35% capacity) AND low refill rate
            if (c->live_blocks < 0.35 * cap2MB && !MID_REFILL(k)) {
                c->target_lg = 20;  // Demote to 1MB
                last_switch_ns[k] = now;
            }
        }
    }

    // EMA-style decay for counters (reduce by 75% each tick)
    c->alloc_count  = c->alloc_count  / 4;
    c->refill_count = c->refill_count / 4;
    c->spill_count  = c->spill_count  / 4;
    // live_blocks is updated incrementally by alloc/free, not decayed here

    c->last_tick_ns = now;
}

// ============================================================================
// Phase 8.4: ACE Observer (Registry-based, zero hot-path overhead)
// ============================================================================

// Global debug flag (set once at initialization)
static int g_ace_debug = 0;

// Registry-based observation: scan all SuperSlabs for usage stats
static void ace_observe_and_decide(int k) {
    if (k < 0 || k >= TINY_NUM_CLASSES_SS) return;

    SuperSlabACEState* c = &g_ss_ace[k];

    // Scan Registry to count SuperSlabs and total live blocks
    int ss_count = 0;
    uint32_t total_live = 0;

    for (int i = 0; i < SUPER_REG_SIZE; i++) {
        SuperRegEntry* e = &g_super_reg[i];

        // Atomic read (thread-safe)
        uintptr_t base = atomic_load_explicit(
            (_Atomic uintptr_t*)&e->base,
            memory_order_acquire);

        if (base == 0) continue;  // Empty slot

        // Phase 8.4: Safety check - skip if ss pointer is invalid
        if (!e->ss) continue;
        // Phase 12: per-SS size_class removed; registry entries are per-class by construction.

        ss_count++;
        // Phase 8.4: Scan all slabs to count used blocks (zero hot-path overhead)
        uint32_t ss_live = 0;
        int cap_scan = ss_slabs_capacity(e->ss);
        for (int slab_idx = 0; slab_idx < cap_scan; slab_idx++) {
            TinySlabMeta* meta = &e->ss->slabs[slab_idx];
            // Relaxed read is OK (stats only, no hot-path impact)
            ss_live += meta->used;
        }
        total_live += ss_live;
    }

    // Calculate utilization
    int obj_size = g_tiny_obj_sizes[k];
    uint8_t current_lg = atomic_load_explicit(
        (_Atomic uint8_t*)&c->current_lg,
        memory_order_relaxed);

    uint32_t capacity = (ss_count > 0) ? ss_count * ((1U << current_lg) / obj_size) : 1;
    double util = (double)total_live / capacity;

    // Update hot_score (for debugging/visualization)
    c->hot_score = (uint16_t)(util * 1000);
    if (c->hot_score > 1000) c->hot_score = 1000;

    // Promotion/Demotion decision
    uint8_t new_target = current_lg;

    if (current_lg <= 20) {
        // Promotion: 1MB → 2MB
        if (util > 0.75) {
            new_target = 21;
        }
    } else {
        // Demotion: 2MB → 1MB
        if (util < 0.35) {
            new_target = 20;
        }
    }

    // Debug output (if enabled)
    if (g_ace_debug && ss_count > 0) {
        fprintf(stderr, "[ACE] Class %d (%dB): ss=%d live=%u cap=%u util=%.2f%% lg=%d->%d hot=%d\n",
                k, obj_size, ss_count, total_live, capacity, util * 100.0,
                current_lg, new_target, c->hot_score);
    }

    // Atomic write (thread-safe)
    if (new_target != current_lg) {
        atomic_store_explicit(
            (_Atomic uint8_t*)&c->target_lg,
            new_target,
            memory_order_release);
        if (g_ace_debug) {
            fprintf(stderr, "[ACE] *** Class %d: SIZE CHANGE %dMB -> %dMB (util=%.2f%%)\n",
                    k, 1 << (current_lg - 20), 1 << (new_target - 20), util * 100.0);
        }
    }
}

// Called from Learner thread (background observation)
void hak_tiny_superslab_ace_observe_all(void) {
    // Initialize debug flag once
    static int initialized = 0;
    if (!initialized) {
        const char* ace_debug = getenv("HAKMEM_ACE_DEBUG");
        g_ace_debug = (ace_debug && atoi(ace_debug) != 0) ? 1 : 0;
        initialized = 1;
    }

    for (int k = 0; k < TINY_NUM_CLASSES_SS; k++) {
        ace_observe_and_decide(k);
    }
}