// tiny_refill.h - Refill Boundary box (inline helpers) #pragma once #include #include "hakmem_tiny_superslab.h" #include "slab_handle.h" #include "tiny_sticky.h" #include "tiny_ready.h" #include "box/mailbox_box.h" #include "tiny_remote_bg.h" // Background remote-drain step (best-effort) #include "tiny_ready_bg.h" // Ready aggregator (mailbox→ready hint) #include "tiny_route.h" // Route Fingerprint (Box boundary tracing) #include #include // External helpers from main TU static inline uint32_t tiny_self_u32(void); static inline void tiny_tls_bind_slab(TinyTLSSlab* tls, SuperSlab* ss, int slab_idx); // Forward decls in main TU static inline uintptr_t hot_slot_pop(int class_idx); static inline uintptr_t bench_pub_pop(int class_idx); static inline SuperSlab* slab_entry_ss(uintptr_t ent); static inline int slab_entry_idx(uintptr_t ent); // Registry scan window (ENV: HAKMEM_TINY_REG_SCAN_MAX, default 256) static inline int tiny_reg_scan_max(void) { static int v = -1; if (__builtin_expect(v == -1, 0)) { const char* s = getenv("HAKMEM_TINY_REG_SCAN_MAX"); int defv = 256; // conservative default if (s && *s) { int parsed = atoi(s); v = (parsed > 0) ? parsed : defv; } else { v = defv; } } return v; } // Opportunistic background remote-drain knobs (ENV parsed lazily) static inline int tiny_bg_remote_tryrate(void) { static int v = -1; if (__builtin_expect(v == -1, 0)) { const char* s = getenv("HAKMEM_TINY_BG_REMOTE_TRYRATE"); int defv = 16; if (s && *s) { int t = atoi(s); v = (t > 0) ? t : defv; } else { v = defv; } } return v; } static inline int tiny_bg_remote_budget_default(void) { static int b = -1; if (__builtin_expect(b == -1, 0)) { const char* s = getenv("HAKMEM_TINY_BG_REMOTE_BUDGET"); int defb = 2; if (s && *s) { int t = atoi(s); if (t <= 0) t = defb; if (t > 64) t = 64; b = t; } else { b = defb; } } return b; } // Mid-size simple refill (ENV: HAKMEM_TINY_MID_REFILL_SIMPLE) static inline int tiny_mid_refill_simple_enabled(void) { static int v = -1; if (__builtin_expect(v == -1, 0)) { const char* s = getenv("HAKMEM_TINY_MID_REFILL_SIMPLE"); v = (s && *s && *s != '0') ? 1 : 0; } return v; } // Try a quick adopt from sticky/hot/bench/mailbox (single pass) static inline SuperSlab* tiny_refill_try_fast(int class_idx, TinyTLSSlab* tls) { ROUTE_BEGIN(class_idx); ROUTE_MARK(0); // Ready list (Box: Ready) — O(1) candidates published by free/publish { // ENV: HAKMEM_TINY_READY_BUDGET (default 1) static int rb = -1; if (__builtin_expect(rb == -1, 0)) { const char* s = getenv("HAKMEM_TINY_READY_BUDGET"); int defv = 1; if (s && *s) { int v = atoi(s); rb = (v > 0 && v <= 8) ? v : defv; } else rb = defv; } for (int attempt = 0; attempt < rb; attempt++) { ROUTE_MARK(1); // ready_try uintptr_t ent = tiny_ready_pop(class_idx); if (!ent) break; SuperSlab* rss = slab_entry_ss(ent); int ridx = slab_entry_idx(ent); uint32_t self_tid = tiny_self_u32(); SlabHandle h = slab_try_acquire(rss, ridx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); slab_release(&h); } else if (slab_is_safe_to_bind(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); extern unsigned long long g_rf_hit_ready[]; g_rf_hit_ready[class_idx]++; ROUTE_MARK(2); ROUTE_COMMIT(class_idx, 0x01); return h.ss; } else { slab_release(&h); } } } } // One-shot entry trace (env: HAKMEM_TINY_RF_TRACE) do { static int en = -1; static _Atomic int printed[8]; if (__builtin_expect(en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_RF_TRACE"); en = (e && atoi(e) != 0) ? 1 : 0; } if (en) { int expected = 0; (void)atomic_compare_exchange_strong(&printed[class_idx], &expected, 1); if (expected == 0) { fprintf(stderr, "[RFTRACE] fast-refill enter class=%d\n", class_idx); } } } while (0); // For hot tiny classes (0..3), try mailbox first to avoid deeper scans if (class_idx <= 3) { uint32_t self_tid = tiny_self_u32(); ROUTE_MARK(3); // mail_try uintptr_t mail = mailbox_box_fetch(class_idx); if (mail) { SuperSlab* mss = slab_entry_ss(mail); int midx = slab_entry_idx(mail); SlabHandle h = slab_try_acquire(mss, midx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); slab_release(&h); } else if (slab_is_safe_to_bind(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); ROUTE_MARK(4); ROUTE_COMMIT(class_idx, 0x02); return h.ss; } else { slab_release(&h); } } } } // Sticky ring (Box: SlabHandle) uint32_t self_tid = tiny_self_u32(); for (int r = 0; r < TINY_STICKY_RING; r++) { ROUTE_MARK(5); // sticky_try SuperSlab* last_ss = g_tls_sticky_ss[class_idx][r]; if (!(last_ss && last_ss->magic == SUPERSLAB_MAGIC)) { tiny_sticky_clear(class_idx, r); continue; } int li = g_tls_sticky_idx[class_idx][r]; int cap = ss_slabs_capacity(last_ss); if (li < 0 || li >= cap) { tiny_sticky_clear(class_idx, r); continue; } // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(last_ss, li, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("sticky_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA250u, self_tid, 0); } slab_release(&h); } else if (slab_is_safe_to_bind(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); ROUTE_MARK(6); ROUTE_COMMIT(class_idx, 0x03); return h.ss; } else { slab_release(&h); } } int has_remote = (atomic_load_explicit(&last_ss->remote_heads[li], memory_order_acquire) != 0); if (!has_remote) tiny_sticky_clear(class_idx, r); } // Hot slot { ROUTE_MARK(7); // hot_try uintptr_t hs = hot_slot_pop(class_idx); if (hs) { SuperSlab* hss = slab_entry_ss(hs); int hidx = slab_entry_idx(hs); // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(hss, hidx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("hot_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA251u, self_tid, 0); } slab_release(&h); } else if (slab_is_safe_to_bind(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); ROUTE_MARK(8); ROUTE_COMMIT(class_idx, 0x04); return h.ss; } else { slab_release(&h); } } } } // Bench { ROUTE_MARK(9); // bench_try uintptr_t entb = bench_pub_pop(class_idx); if (entb) { SuperSlab* bss = slab_entry_ss(entb); int bidx = slab_entry_idx(entb); // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(bss, bidx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("bench_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA252u, self_tid, 0); } slab_release(&h); } else if (slab_is_safe_to_bind(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); ROUTE_MARK(10); ROUTE_COMMIT(class_idx, 0x05); return h.ss; } else { slab_release(&h); } } } } // Mailbox (for non-hot classes) if (class_idx > 3) { ROUTE_MARK(3); // mail_try (non-hot) uintptr_t mail = mailbox_box_fetch(class_idx); if (mail) { SuperSlab* mss = slab_entry_ss(mail); int midx = slab_entry_idx(mail); // Box: Try to acquire ownership SlabHandle h = slab_try_acquire(mss, midx, self_tid); if (slab_is_valid(&h)) { if (slab_remote_pending(&h)) { slab_drain_remote_full(&h); if (__builtin_expect(g_debug_remote_guard, 0)) { uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed); tiny_remote_watch_note("mailbox_remote_pending", h.ss, h.slab_idx, (void*)head, 0xA253u, self_tid, 0); } slab_release(&h); } else if (slab_is_safe_to_bind(&h)) { tiny_tls_bind_slab(tls, h.ss, h.slab_idx); tiny_sticky_save(class_idx, h.ss, h.slab_idx); ROUTE_MARK(4); ROUTE_COMMIT(class_idx, 0x02); return h.ss; } else { slab_release(&h); } } } } // Opportunistic background remote-drain (Box: Remote Drain Coalescer) // Every N misses, coalesce a few remote queues into freelists under ownership do { // ENV gate: HAKMEM_TINY_BG_REMOTE=1 enables this light step extern int g_bg_remote_enable; // from hakmem_tiny_remote_target.c if (__builtin_expect(!g_bg_remote_enable, 1)) break; // TLS miss tick per class static __thread unsigned miss_tick[8]; unsigned t = ++miss_tick[class_idx]; int period = tiny_bg_remote_tryrate(); if (__builtin_expect(period <= 1 || (t % (unsigned)period) == 0, 0)) { int budget = tiny_bg_remote_budget_default(); tiny_remote_bg_drain_step(class_idx, budget); // Quick second chance from Ready after drain uintptr_t ent2 = tiny_ready_pop(class_idx); if (ent2) { SuperSlab* ss2 = slab_entry_ss(ent2); int idx2 = slab_entry_idx(ent2); uint32_t self_tid = tiny_self_u32(); SlabHandle h2 = slab_try_acquire(ss2, idx2, self_tid); if (slab_is_valid(&h2)) { if (slab_is_safe_to_bind(&h2)) { tiny_tls_bind_slab(tls, h2.ss, h2.slab_idx); tiny_sticky_save(class_idx, h2.ss, h2.slab_idx); extern unsigned long long g_rf_hit_ready[]; g_rf_hit_ready[class_idx]++; slab_release(&h2); return h2.ss; } slab_release(&h2); } } // Ready Aggregator: peek mailbox and surface one hint into Ready do { static int agg_en = -1; // ENV: HAKMEM_TINY_READY_AGG=1 if (__builtin_expect(agg_en == -1, 0)) { const char* e = getenv("HAKMEM_TINY_READY_AGG"); agg_en = (e && *e && *e != '0') ? 1 : 0; } if (agg_en) { // Budget: ENV HAKMEM_TINY_READY_AGG_MAIL_BUDGET (default 1) static int mb = -1; if (__builtin_expect(mb == -1, 0)) { const char* s = getenv("HAKMEM_TINY_READY_AGG_MAIL_BUDGET"); int defb = 1; if (s && *s) { int v = atoi(s); mb = (v>0 && v<=4)?v:defb; } else mb = defb; } tiny_ready_bg_aggregate_step(class_idx, mb); // Try Ready once more after aggregation uintptr_t ent3 = tiny_ready_pop(class_idx); if (ent3) { SuperSlab* ss3 = slab_entry_ss(ent3); int idx3 = slab_entry_idx(ent3); uint32_t self_tid = tiny_self_u32(); SlabHandle h3 = slab_try_acquire(ss3, idx3, self_tid); if (slab_is_valid(&h3)) { if (slab_is_safe_to_bind(&h3)) { tiny_tls_bind_slab(tls, h3.ss, h3.slab_idx); tiny_sticky_save(class_idx, h3.ss, h3.slab_idx); extern unsigned long long g_rf_hit_ready[]; g_rf_hit_ready[class_idx]++; slab_release(&h3); return h3.ss; } slab_release(&h3); } } } } while (0); } } while (0); ROUTE_COMMIT(class_idx, 0xFF); // no candidate hit; fall back to slab/slow return NULL; }