hakmem/core/tiny_free_magazine.inc.h

// tiny_free_magazine.inc.h - Magazine Layer for hak_tiny_free_with_slab()
// Purpose: TLS caching (TinyQuickSlot, TLS SLL, Magazine) and spill logic
// Extracted from: hakmem_tiny_free.inc lines 208-620
// Box Theory: Box 5 (TLS Cache) integration
//
// Context: This file is #included within hak_tiny_free_with_slab() function body
// Prerequisites: ss, meta, class_idx, ptr variables must be defined in calling scope

#if !HAKMEM_BUILD_RELEASE
        // SuperSlab uses Magazine for TLS caching (same as TinySlab)
        tiny_small_mags_init_once();
        if (class_idx > 3) tiny_mag_init_if_needed(class_idx);
        TinyTLSMag* mag = &g_tls_mags[class_idx];
        int cap = mag->cap;

        // 32/64B: SLL優先（mag優先は無効化）
        // Prefer TinyQuickSlot (compile-out if HAKMEM_TINY_NO_QUICK)
#if !defined(HAKMEM_TINY_NO_QUICK)
        if (g_quick_enable && class_idx <= 4) {
            TinyQuickSlot* qs = &g_tls_quick[class_idx];
            if (__builtin_expect(qs->top < QUICK_CAP, 1)) {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                qs->items[qs->top++] = base;
                HAK_STAT_FREE(class_idx);
                return;
            }
        }
#endif

        // Fast path: TLS SLL push for hottest classes
        if (!g_tls_list_enable && g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)cap)) {
            // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
            void* base = (void*)((uint8_t*)ptr - 1);
            uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap);
            if (tls_sll_push(class_idx, base, sll_cap)) {
                // BUGFIX: Decrement used counter (was missing, causing Fail-Fast on next free)
                meta->used--;
                // Active → Inactive: count down immediately (TLS保管中は"使用中"ではない)
                ss_active_dec_one(ss);
                HAK_TP1(sll_push, class_idx);
                tiny_debug_ring_record(TINY_RING_EVENT_FREE_LOCAL, (uint16_t)class_idx, ptr, 3);
                HAK_STAT_FREE(class_idx);
                return;
            }
        }

        // Next: Magazine push（必要ならmag→SLLへバルク転送で空きを作る）
        // Hysteresis: allow slight overfill before deciding to spill under lock
        if (mag->top >= cap && g_spill_hyst > 0) {
            (void)bulk_mag_to_sll_if_room(class_idx, mag, cap / 2);
        }
        if (mag->top < cap + g_spill_hyst) {
            // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
            void* base = (void*)((uint8_t*)ptr - 1);
            mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
            mag->items[mag->top].owner = NULL; // SuperSlab owner not a TinySlab; leave NULL
#endif
            mag->top++;
#if HAKMEM_DEBUG_COUNTERS
            g_magazine_push_count++;  // Phase 7.6: Track pushes
#endif
            // Active → Inactive: decrement now（アプリ解放時に非アクティブ扱い）
            ss_active_dec_one(ss);
            HAK_TP1(mag_push, class_idx);
            tiny_debug_ring_record(TINY_RING_EVENT_FREE_RETURN_MAG, (uint16_t)class_idx, ptr, 2);
            HAK_STAT_FREE(class_idx);
            return;
        }

        // Background spill: queue to BG thread instead of locking (when enabled)
        if (g_bg_spill_enable) {
            uint32_t qlen = atomic_load_explicit(&g_bg_spill_len[class_idx], memory_order_relaxed);
            if ((int)qlen < g_bg_spill_target) {
                // Build a small chain: include current ptr and pop from mag up to limit
                int limit = g_bg_spill_max_batch;
                if (limit > cap/2) limit = cap/2;
                if (limit > 32) limit = 32; // keep free-path bounded
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* head = (void*)((uint8_t*)ptr - 1);
#if HAKMEM_TINY_HEADER_CLASSIDX
                const size_t next_off = 1;  // Phase E1-CORRECT: Always 1
#else
                const size_t next_off = 0;
#endif
                #include "box/tiny_next_ptr_box.h"
                tiny_next_write(head, NULL);
                void* tail = head; // current tail
                int taken = 1;
                while (taken < limit && mag->top > 0) {
                    void* p2 = mag->items[--mag->top].ptr;
#if HAKMEM_TINY_HEADER_CLASSIDX
                    const size_t next_off2 = 1;  // Phase E1-CORRECT: Always 1
#else
                    const size_t next_off2 = 0;
#endif
                    tiny_next_write(p2, head);
                    head = p2;
                    taken++;
                }
                // Push chain to spill queue (single CAS)
                bg_spill_push_chain(class_idx, head, tail, taken);
                tiny_debug_ring_record(TINY_RING_EVENT_FREE_RETURN_MAG, (uint16_t)class_idx, ptr, 3);
                HAK_STAT_FREE(class_idx);
                return;
            }
        }

        // Spill half (SuperSlab version - simpler than TinySlab)
                pthread_mutex_t* lock = &g_tiny_class_locks[class_idx].m;
        hkm_prof_begin(NULL);
        pthread_mutex_lock(lock);
        // Batch spill: reduce lock frequency and work per call
        int spill = cap / 2;
        int over = mag->top - (cap + g_spill_hyst);
        if (over > 0 && over < spill) spill = over;

        for (int i = 0; i < spill && mag->top > 0; i++) {
            TinyMagItem it = mag->items[--mag->top];

            // Phase 7.6: SuperSlab spill - return to freelist
            SuperSlab* owner_ss = hak_super_lookup(it.ptr);
            if (owner_ss && owner_ss->magic == SUPERSLAB_MAGIC) {
                // Direct freelist push (same as old hak_tiny_free_superslab)
                // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation
                void* base = (void*)((uint8_t*)it.ptr - 1);
                int slab_idx = slab_index_for(owner_ss, base);
                // BUGFIX: Validate slab_idx before array access (prevents OOB)
                if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(owner_ss)) {
                    continue;  // Skip invalid index
                }
                TinySlabMeta* meta = &owner_ss->slabs[slab_idx];
                tiny_next_write(owner_ss->size_class, it.ptr, meta->freelist);
                meta->freelist = it.ptr;
                meta->used--;
                // Decrement SuperSlab active counter (spill returns blocks to SS)
                ss_active_dec_one(owner_ss);

                // Phase 8.4: Empty SuperSlab detection (will use meta->used scan)
                // TODO: Implement scan-based empty detection
                // Empty SuperSlab detection/munmapは別途フラッシュAPIで実施（ホットパスから除外）
            }
        }

        pthread_mutex_unlock(lock);
        hkm_prof_end(ss_time, HKP_TINY_SPILL, &tss);

        // Adaptive increase of cap after spill
        int max_cap = tiny_cap_max_for_class(class_idx);
                if (mag->cap < max_cap) {
            int new_cap = mag->cap + (mag->cap / 2);
            if (new_cap > max_cap) new_cap = max_cap;
            if (new_cap > TINY_TLS_MAG_CAP) new_cap = TINY_TLS_MAG_CAP;
            mag->cap = new_cap;
        }

        // Finally, try FastCache push first (≤128B) — compile-out if HAKMEM_TINY_NO_FRONT_CACHE
#if !defined(HAKMEM_TINY_NO_FRONT_CACHE)
        if (g_fastcache_enable && class_idx <= 4) {
            // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
            void* base = (void*)((uint8_t*)ptr - 1);
            if (fastcache_push(class_idx, base)) {
                HAK_TP1(front_push, class_idx);
                HAK_STAT_FREE(class_idx);
                return;
            }
        }
#endif
        // Then TLS SLL if room, else magazine
        if (g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)mag->cap)) {
            uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
            // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
            void* base = (void*)((uint8_t*)ptr - 1);
            if (!tls_sll_push(class_idx, base, sll_cap2)) {
                // fallback to magazine
                mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                mag->items[mag->top].owner = slab;
#endif
                mag->top++;
            }
        } else {
            // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
            void* base = (void*)((uint8_t*)ptr - 1);
            mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
            mag->items[mag->top].owner = slab;
#endif
            mag->top++;
        }

#if HAKMEM_DEBUG_COUNTERS
        g_magazine_push_count++;  // Phase 7.6: Track pushes
#endif
        HAK_STAT_FREE(class_idx);
        return;
#endif  // HAKMEM_BUILD_RELEASE
    }

    // Phase 7.6: TinySlab path (original)
    //g_tiny_free_with_slab_count++;  // Phase 7.6: Track calls - DISABLED due to segfault
    // Same-thread → TLS magazine; remote-thread → MPSC stack
    if (pthread_equal(slab->owner_tid, tiny_self_pt())) {
        int class_idx = slab->class_idx;

        // Phase E1-CORRECT: C7 now has headers, can use TLS list like other classes
        if (g_tls_list_enable) {
            TinyTLSList* tls = &g_tls_lists[class_idx];
            uint32_t seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed);
            if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) {
                tiny_tls_refresh_params(class_idx, tls);
            }
            // TinyHotMag front push（8/16/32B, A/B）
            if (__builtin_expect(g_hotmag_enable && class_idx <= 2, 1)) {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                if (hotmag_push(class_idx, base)) {
                    HAK_STAT_FREE(class_idx);
                    return;
                }
            }
            if (tls->count < tls->cap) {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                tiny_tls_list_guard_push(class_idx, tls, base);
                tls_list_push_fast(tls, base, class_idx);
                HAK_STAT_FREE(class_idx);
                return;
            }
            seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed);
            if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) {
                tiny_tls_refresh_params(class_idx, tls);
            }
            {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                tiny_tls_list_guard_push(class_idx, tls, base);
                tls_list_push_fast(tls, base, class_idx);
            }
            if (tls_list_should_spill(tls)) {
                tls_list_spill_excess(class_idx, tls);
            }
            HAK_STAT_FREE(class_idx);
            return;
        }

        tiny_mag_init_if_needed(class_idx);
        TinyTLSMag* mag = &g_tls_mags[class_idx];
        int cap = mag->cap;
        // 32/64B: SLL優先（mag優先は無効化）
        // Fast path: FastCache push (preferred for ≤128B), then TLS SLL
        if (g_fastcache_enable && class_idx <= 4) {
            if (fastcache_push(class_idx, ptr)) {
                HAK_STAT_FREE(class_idx);
                return;
            }
        }
        // Fast path: TLS SLL push (preferred)
        if (!g_tls_list_enable && g_tls_sll_enable && class_idx <= 5) {
            uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap);
            if (g_tls_sll_count[class_idx] < sll_cap) {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                if (tls_sll_push(class_idx, base, sll_cap)) {
                    HAK_STAT_FREE(class_idx);
                    return;
                }
            }
        }
        // Next: if magazine has room, push immediately and return（満杯ならmag→SLLへバルク）
        if (mag->top >= cap) {
            (void)bulk_mag_to_sll_if_room(class_idx, mag, cap / 2);
        }
        // Remote-drain can be handled opportunistically on future calls.
        if (mag->top < cap) {
            {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                mag->items[mag->top].owner = slab;
#endif
                mag->top++;
            }

#if HAKMEM_DEBUG_COUNTERS
            g_magazine_push_count++;  // Phase 7.6: Track pushes
#endif
            // Note: SuperSlab uses separate path (slab == NULL branch above)
            HAK_STAT_FREE(class_idx);  // Phase 3
            return;
        }
        // Magazine full: before spilling, opportunistically drain remotes once under lock.
        if (atomic_load_explicit(&slab->remote_count, memory_order_relaxed) >= (unsigned)g_remote_drain_thresh_per_class[class_idx] || atomic_load_explicit(&slab->remote_head, memory_order_acquire)) {
            pthread_mutex_t* lock = &g_tiny_class_locks[class_idx].m;
            pthread_mutex_lock(lock);
            HAK_TP1(remote_drain, class_idx);
            tiny_remote_drain_locked(slab);
            pthread_mutex_unlock(lock);
        }
        // Spill half under class lock
        pthread_mutex_t* lock = &g_tiny_class_locks[class_idx].m;
        pthread_mutex_lock(lock);
        int spill = cap / 2;

        // Phase 4.2: High-water threshold for gating Phase 4 logic
        int high_water = (cap * 3) / 4;  // 75% of capacity

        for (int i = 0; i < spill && mag->top > 0; i++) {
            TinyMagItem it = mag->items[--mag->top];

            // Phase 7.6: Check for SuperSlab first (mixed Magazine support)
            SuperSlab* ss_owner = hak_super_lookup(it.ptr);
            if (ss_owner && ss_owner->magic == SUPERSLAB_MAGIC) {
                // SuperSlab spill - return to freelist
                // ✅ FIX: Phase E1-CORRECT - Convert USER → BASE before slab index calculation
                void* base = (void*)((uint8_t*)it.ptr - 1);
                int slab_idx = slab_index_for(ss_owner, base);
                // BUGFIX: Validate slab_idx before array access (prevents OOB)
                if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss_owner)) {
                    HAK_STAT_FREE(class_idx);
                    continue;  // Skip invalid index
                }
                TinySlabMeta* meta = &ss_owner->slabs[slab_idx];
                tiny_next_write(ss_owner->size_class, it.ptr, meta->freelist);
                meta->freelist = it.ptr;
                meta->used--;
                // 空SuperSlab処理はフラッシュ/バックグラウンドで対応（ホットパス除外）
                HAK_STAT_FREE(class_idx);
                continue;  // Skip TinySlab processing
            }

            TinySlab* owner =
#if HAKMEM_TINY_MAG_OWNER
                it.owner;
#else
                NULL;
#endif
            if (!owner) {
                owner = tls_active_owner_for_ptr(class_idx, it.ptr);
            }
            if (!owner) {
                owner = hak_tiny_owner_slab(it.ptr);
            }
            if (!owner) continue;

            // Phase 4.2: Adaptive gating - skip Phase 4 when TLS Magazine is high-water
            // Rationale: When mag->top >= 75%, next alloc will come from TLS anyway
            //            so pushing to mini-mag is wasted work
            int is_high_water = (mag->top >= high_water);

            if (!is_high_water) {
                // Low-water: Phase 4.1 logic (try mini-magazine first)
                uint8_t cidx = owner->class_idx;  // Option A: 1回だけ読む
                TinySlab* tls_a = g_tls_active_slab_a[cidx];
                TinySlab* tls_b = g_tls_active_slab_b[cidx];

                // Option B: Branch prediction hint (spill → TLS-active への戻りが likely)
                if (__builtin_expect((owner == tls_a || owner == tls_b) &&
                                     !mini_mag_is_full(&owner->mini_mag), 1)) {
                    // Fast path: mini-magazineに戻す（bitmap触らない）
                    mini_mag_push(&owner->mini_mag, it.ptr);
                    HAK_TP1(spill_tiny, cidx);
                    HAK_STAT_FREE(cidx);
                    continue;  // bitmap操作スキップ
                }
            }
            // High-water or Phase 4.1 mini-mag full: fall through to bitmap

            // Slow path: bitmap直接書き込み（既存ロジック）
            size_t bs = g_tiny_class_sizes[owner->class_idx];
            int idx = ((uintptr_t)it.ptr - (uintptr_t)owner->base) / bs;
            if (hak_tiny_is_used(owner, idx)) {
                hak_tiny_set_free(owner, idx);
                int was_full = (owner->free_count == 0);
                owner->free_count++;
                if (was_full) move_to_free_list(owner->class_idx, owner);
                if (owner->free_count == owner->total_count) {
                    // If this slab is TLS-active for this thread, clear the pointer before releasing
                    if (g_tls_active_slab_a[owner->class_idx] == owner) g_tls_active_slab_a[owner->class_idx] = NULL;
                    if (g_tls_active_slab_b[owner->class_idx] == owner) g_tls_active_slab_b[owner->class_idx] = NULL;
                    TinySlab** headp = &g_tiny_pool.free_slabs[owner->class_idx];
                    TinySlab* prev = NULL;
                    for (TinySlab* s = *headp; s; prev = s, s = s->next) {
                        if (s == owner) { if (prev) prev->next = s->next; else *headp = s->next; break; }
                    }
                    release_slab(owner);
                }
                HAK_TP1(spill_tiny, owner->class_idx);
                HAK_STAT_FREE(owner->class_idx);
            }
        }
        pthread_mutex_unlock(lock);
        hkm_prof_end(ss, HKP_TINY_SPILL, &tss);
        // Adaptive increase of cap after spill
        int max_cap = tiny_cap_max_for_class(class_idx);
        if (mag->cap < max_cap) {
            int new_cap = mag->cap + (mag->cap / 2);
            if (new_cap > max_cap) new_cap = max_cap;
            if (new_cap > TINY_TLS_MAG_CAP) new_cap = TINY_TLS_MAG_CAP;
            mag->cap = new_cap;
        }
        // Finally: prefer TinyQuickSlot → SLL → UltraFront → HotMag → Magazine（順序で局所性を確保）
#if !HAKMEM_BUILD_RELEASE && !defined(HAKMEM_TINY_NO_QUICK)
        if (g_quick_enable && class_idx <= 4) {
            TinyQuickSlot* qs = &g_tls_quick[class_idx];
            if (__builtin_expect(qs->top < QUICK_CAP, 1)) {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                qs->items[qs->top++] = base;
            } else if (g_tls_sll_enable) {
                uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
                if (g_tls_sll_count[class_idx] < sll_cap2) {
                    // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                    void* base = (void*)((uint8_t*)ptr - 1);
                    if (!tls_sll_push(class_idx, base, sll_cap2)) {
                        if (!tiny_optional_push(class_idx, base)) {
                            mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                            mag->items[mag->top].owner = slab;
#endif
                            mag->top++;
                        }
                    }
                } else if (!tiny_optional_push(class_idx, (void*)((uint8_t*)ptr - 1))) {  // Phase E1-CORRECT
                    // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                    void* base = (void*)((uint8_t*)ptr - 1);
                    mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                    mag->items[mag->top].owner = slab;
#endif
                    mag->top++;
                }
            } else {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                if (!tiny_optional_push(class_idx, base)) {
                    mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                    mag->items[mag->top].owner = slab;
#endif
                    mag->top++;
                }
            }
        } else
#endif
        {
            if (g_tls_sll_enable && class_idx <= 5) {
                uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
                if (g_tls_sll_count[class_idx] < sll_cap2) {
                    // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                    void* base = (void*)((uint8_t*)ptr - 1);
                    if (!tls_sll_push(class_idx, base, sll_cap2)) {
                        if (!tiny_optional_push(class_idx, base)) {
                            mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                            mag->items[mag->top].owner = slab;
#endif
                            mag->top++;
                        }
                    }
                } else if (!tiny_optional_push(class_idx, (void*)((uint8_t*)ptr - 1))) {  // Phase E1-CORRECT
                    // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                    void* base = (void*)((uint8_t*)ptr - 1);
                    mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                    mag->items[mag->top].owner = slab;
#endif
                    mag->top++;
                }
            } else {
                // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
                void* base = (void*)((uint8_t*)ptr - 1);
                if (!tiny_optional_push(class_idx, base)) {
                    mag->items[mag->top].ptr = base;
#if HAKMEM_TINY_MAG_OWNER
                    mag->items[mag->top].owner = slab;
#endif
                    mag->top++;
                }
            }
        }

#if HAKMEM_DEBUG_COUNTERS
        g_magazine_push_count++;  // Phase 7.6: Track pushes
#endif
        // Note: SuperSlab uses separate path (slab == NULL branch above)
        HAK_STAT_FREE(class_idx);  // Phase 3
        return;
    } else {
        // Phase E1-CORRECT: ALL classes (C0-C7) have 1-byte header
        void* base = (void*)((uint8_t*)ptr - 1);
        tiny_remote_push(slab, base);
    }
}