hakmem/core/tiny_refill.h

// tiny_refill.h - Refill Boundary box (inline helpers)
#pragma once
#include <stdatomic.h>
#include "hakmem_tiny_superslab.h"
#include "slab_handle.h"
#include "tiny_sticky.h"
#include "tiny_mailbox.h"
#include <stdio.h>
#include <stdlib.h>

// External helpers from main TU
static inline uint32_t tiny_self_u32(void);
static inline void tiny_tls_bind_slab(TinyTLSSlab* tls, SuperSlab* ss, int slab_idx);

// Forward decls in main TU
static inline uintptr_t hot_slot_pop(int class_idx);
static inline uintptr_t bench_pub_pop(int class_idx);
static inline SuperSlab* slab_entry_ss(uintptr_t ent);
static inline int slab_entry_idx(uintptr_t ent);

// Registry scan window (ENV: HAKMEM_TINY_REG_SCAN_MAX, default 256)
static inline int tiny_reg_scan_max(void) {
    static int v = -1;
    if (__builtin_expect(v == -1, 0)) {
        const char* s = getenv("HAKMEM_TINY_REG_SCAN_MAX");
        int defv = 256;  // conservative default
        if (s && *s) {
            int parsed = atoi(s);
            v = (parsed > 0) ? parsed : defv;
        } else {
            v = defv;
        }
    }
    return v;
}

// Mid-size simple refill (ENV: HAKMEM_TINY_MID_REFILL_SIMPLE)
static inline int tiny_mid_refill_simple_enabled(void) {
    static int v = -1;
    if (__builtin_expect(v == -1, 0)) {
        const char* s = getenv("HAKMEM_TINY_MID_REFILL_SIMPLE");
        v = (s && *s && *s != '0') ? 1 : 0;
    }
    return v;
}

// Try a quick adopt from sticky/hot/bench/mailbox (single pass)
static inline SuperSlab* tiny_refill_try_fast(int class_idx, TinyTLSSlab* tls) {
    // One-shot entry trace (env: HAKMEM_TINY_RF_TRACE)
    do {
        static int en = -1; static _Atomic int printed[8];
        if (__builtin_expect(en == -1, 0)) {
            const char* e = getenv("HAKMEM_TINY_RF_TRACE");
            en = (e && atoi(e) != 0) ? 1 : 0;
        }
        if (en) {
            int expected = 0;
            (void)atomic_compare_exchange_strong(&printed[class_idx], &expected, 1);
            if (expected == 0) {
                fprintf(stderr, "[RFTRACE] fast-refill enter class=%d\n", class_idx);
            }
        }
    } while (0);
    // For hot tiny classes (0..3), try mailbox first to avoid deeper scans
    if (class_idx <= 3) {
        uint32_t self_tid = tiny_self_u32();
        uintptr_t mail = tiny_mailbox_fetch(class_idx);
        if (mail) {
            SuperSlab* mss = slab_entry_ss(mail);
            int midx = slab_entry_idx(mail);
            SlabHandle h = slab_try_acquire(mss, midx, self_tid);
            if (slab_is_valid(&h)) {
                if (slab_remote_pending(&h)) {
                    slab_drain_remote_full(&h);
                    slab_release(&h);
                } else if (slab_freelist(&h)) {
                    tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
                    tiny_sticky_save(class_idx, h.ss, h.slab_idx);
                    return h.ss;
                } else {
                    slab_release(&h);
                }
            }
        }
    }

    // Sticky ring (Box: SlabHandle)
    uint32_t self_tid = tiny_self_u32();
    for (int r = 0; r < TINY_STICKY_RING; r++) {
        SuperSlab* last_ss = g_tls_sticky_ss[class_idx][r];
        if (!(last_ss && last_ss->magic == SUPERSLAB_MAGIC)) { tiny_sticky_clear(class_idx, r); continue; }
        int li = g_tls_sticky_idx[class_idx][r];
        int cap = ss_slabs_capacity(last_ss);
        if (li < 0 || li >= cap) { tiny_sticky_clear(class_idx, r); continue; }

        // Box: Try to acquire ownership
        SlabHandle h = slab_try_acquire(last_ss, li, self_tid);
        if (slab_is_valid(&h)) {
            if (slab_remote_pending(&h)) {
                slab_drain_remote_full(&h);
                if (__builtin_expect(g_debug_remote_guard, 0)) {
                    uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
                    tiny_remote_watch_note("sticky_remote_pending",
                                           h.ss,
                                           h.slab_idx,
                                           (void*)head,
                                           0xA250u,
                                           self_tid,
                                           0);
                }
                slab_release(&h);
            } else if (slab_freelist(&h)) {
                tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
                return h.ss;
            } else {
                slab_release(&h);
            }
        }

        int has_remote = (atomic_load_explicit(&last_ss->remote_heads[li], memory_order_acquire) != 0);
        if (!has_remote) tiny_sticky_clear(class_idx, r);
    }
    // Hot slot
    {
        uintptr_t hs = hot_slot_pop(class_idx);
        if (hs) {
            SuperSlab* hss = slab_entry_ss(hs);
            int hidx = slab_entry_idx(hs);

            // Box: Try to acquire ownership
            SlabHandle h = slab_try_acquire(hss, hidx, self_tid);
        if (slab_is_valid(&h)) {
            if (slab_remote_pending(&h)) {
                slab_drain_remote_full(&h);
                if (__builtin_expect(g_debug_remote_guard, 0)) {
                    uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
                    tiny_remote_watch_note("hot_remote_pending",
                                           h.ss,
                                           h.slab_idx,
                                           (void*)head,
                                           0xA251u,
                                           self_tid,
                                           0);
                }
                slab_release(&h);
            } else if (slab_freelist(&h)) {
                tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
                tiny_sticky_save(class_idx, h.ss, h.slab_idx);
                return h.ss;
            } else {
                slab_release(&h);
            }
        }
        }
    }
    // Bench
    {
        uintptr_t entb = bench_pub_pop(class_idx);
        if (entb) {
            SuperSlab* bss = slab_entry_ss(entb);
            int bidx = slab_entry_idx(entb);

            // Box: Try to acquire ownership
            SlabHandle h = slab_try_acquire(bss, bidx, self_tid);
        if (slab_is_valid(&h)) {
            if (slab_remote_pending(&h)) {
                slab_drain_remote_full(&h);
                if (__builtin_expect(g_debug_remote_guard, 0)) {
                    uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
                    tiny_remote_watch_note("bench_remote_pending",
                                           h.ss,
                                           h.slab_idx,
                                           (void*)head,
                                           0xA252u,
                                           self_tid,
                                           0);
                }
                slab_release(&h);
            } else if (slab_freelist(&h)) {
                tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
                tiny_sticky_save(class_idx, h.ss, h.slab_idx);
                return h.ss;
            } else {
                slab_release(&h);
            }
        }
        }
    }
    // Mailbox (for non-hot classes)
    if (class_idx > 3) {
        uintptr_t mail = tiny_mailbox_fetch(class_idx);
        if (mail) {
            SuperSlab* mss = slab_entry_ss(mail);
            int midx = slab_entry_idx(mail);

            // Box: Try to acquire ownership
            SlabHandle h = slab_try_acquire(mss, midx, self_tid);
        if (slab_is_valid(&h)) {
            if (slab_remote_pending(&h)) {
                slab_drain_remote_full(&h);
                if (__builtin_expect(g_debug_remote_guard, 0)) {
                    uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
                    tiny_remote_watch_note("mailbox_remote_pending",
                                           h.ss,
                                           h.slab_idx,
                                           (void*)head,
                                           0xA253u,
                                           self_tid,
                                           0);
                }
                slab_release(&h);
            } else if (slab_freelist(&h)) {
                tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
                tiny_sticky_save(class_idx, h.ss, h.slab_idx);
                return h.ss;
            } else {
                slab_release(&h);
            }
        }
        }
    }
    return NULL;
}