// tiny_refill.h - Refill Boundary box (inline helpers) #pragma once #include #include "hakmem_tiny_superslab.h" #include "slab_handle.h" #include "tiny_sticky.h" #include "tiny_mailbox.h" #include #include // External helpers from main TU static inline uint32_t tiny_self_u32(void); static inline void tiny_tls_bind_slab(TinyTLSSlab* tls, SuperSlab* ss, int slab_idx); // Forward decls in main TU static inline uintptr_t hot_slot_pop(int class_idx); static inline uintptr_t bench_pub_pop(int class_idx); static inline SuperSlab* slab_entry_ss(uintptr_t ent); static inline int slab_entry_idx(uintptr_t ent); // Registry scan window (ENV: HAKMEM_TINY_REG_SCAN_MAX, default 256) static inline int tiny_reg_scan_max(void) { static int v = -1; if (__builtin_expect(v == -1, 0)) { const char* s = getenv("HAKMEM_TINY_REG_SCAN_MAX"); int defv = 256; // conservative default if (s && *s) { int parsed = atoi(s); v = (parsed > 0) ? parsed : defv; } else { v = defv; } } return v; } // Mid-size simple refill (ENV: HAKMEM_TINY_MID_REFILL_SIMPLE) static inline int tiny_mid_refill_simple_enabled(void) { static int v = -1; if (__builtin_expect(v == -1, 0)) { const char* s = getenv("HAKMEM_TINY_MID_REFILL_SIMPLE"); v = (s && *s && *s != '0') ? 1 : 0; } return v; } // Try a quick adopt from sticky/hot/bench/mailbox (single pass) static inline SuperSlab* tiny_refill_try_fast(int class_idx, TinyTLSSlab* tls) { // One-shot entry trace (env: HAKMEM_TINY_RF_TRACE) do { static int en = -1; static _Atomic int printed[8]; if (__builtin_expect(en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_RF_TRACE"); en = (e && atoi(e) != 0) ? 1 : 0; } if (en) { int expected = 0; (void)atomic_compare_exchange_strong(&printed[class_idx], &expected, 1); if (expected == 0) { fprintf(stderr, "[RFTRACE] fast-refill enter class=%d\n", class_idx); } } } while (0); // For hot tiny classes (0..3), try mailbox first to avoid deeper scans if (class_idx <= 3) { uint32_t self_tid = tiny_self_u32(); uintptr_t mail = tiny_mailbox_fetch(class_idx); if (mail) { SuperSlab* mss = slab_entry_ss(mail); int midx = slab_entry_idx(mail); SlabHandle h = slab_try_acquire(mss, midx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); slab_release(&h); } else if (slab_freelist(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); return h.ss; } else { slab_release(&h); } } } } // Sticky ring (Box: SlabHandle) uint32_t self_tid = tiny_self_u32(); for (int r = 0; r < TINY_STICKY_RING; r++) { SuperSlab* last_ss = g_tls_sticky_ss[class_idx][r]; if (!(last_ss && last_ss->magic == SUPERSLAB_MAGIC)) { tiny_sticky_clear(class_idx, r); continue; } int li = g_tls_sticky_idx[class_idx][r]; int cap = ss_slabs_capacity(last_ss); if (li < 0 || li >= cap) { tiny_sticky_clear(class_idx, r); continue; } // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(last_ss, li, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("sticky_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA250u, self_tid, 0); } slab_release(&h); } else if (slab_freelist(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); return h.ss; } else { slab_release(&h); } } int has_remote = (atomic_load_explicit(&last_ss->remote_heads[li], memory_order_acquire) != 0); if (!has_remote) tiny_sticky_clear(class_idx, r); } // Hot slot { uintptr_t hs = hot_slot_pop(class_idx); if (hs) { SuperSlab* hss = slab_entry_ss(hs); int hidx = slab_entry_idx(hs); // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(hss, hidx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("hot_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA251u, self_tid, 0); } slab_release(&h); } else if (slab_freelist(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); return h.ss; } else { slab_release(&h); } } } } // Bench { uintptr_t entb = bench_pub_pop(class_idx); if (entb) { SuperSlab* bss = slab_entry_ss(entb); int bidx = slab_entry_idx(entb); // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(bss, bidx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("bench_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA252u, self_tid, 0); } slab_release(&h); } else if (slab_freelist(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); return h.ss; } else { slab_release(&h); } } } } // Mailbox (for non-hot classes) if (class_idx > 3) { uintptr_t mail = tiny_mailbox_fetch(class_idx); if (mail) { SuperSlab* mss = slab_entry_ss(mail); int midx = slab_entry_idx(mail); // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(mss, midx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("mailbox_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA253u, self_tid, 0); } slab_release(&h); } else if (slab_freelist(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); return h.ss; } else { slab_release(&h); } } } } return NULL; }