// hakmem_tiny_assist.inc.h // Tiny: helper routines to assist targeted remote-drain // Keep inline to avoid call overhead on hot slow-paths. #ifndef HAKMEM_TINY_ASSIST_INC_H #define HAKMEM_TINY_ASSIST_INC_H #include #include "hakmem_tiny_superslab.h" #include "hakmem_tiny_ss_target.h" #include "hakmem_tiny_drain_ema.inc.h" #include "box/tiny_next_ptr_box.h" // Box API: Next pointer read/write static inline uint16_t tiny_assist_drain_owned(int class_idx, int max_items) { int drained_sets = 0; while (drained_sets < max_items) { SuperSlab* t = ss_target_pop(class_idx); if (!t) break; uint32_t mytid = tiny_self_u32(); for (int i = 0; i < SLABS_PER_SUPERSLAB; i++) { TinySlabMeta* m = &t->slabs[i]; if (m->owner_tid != mytid) continue; TinySlabPrefix* pref = tiny_slab_prefix(t, i); _Atomic(uintptr_t)* rhead = (_Atomic(uintptr_t)*)&pref->reserved[0]; _Atomic(uint32_t)* rcount = (_Atomic(uint32_t)*)&pref->reserved[1]; uint32_t pending = atomic_load_explicit(rcount, memory_order_relaxed); if (pending == 0) continue; uintptr_t chain = atomic_exchange_explicit(rhead, 0, memory_order_acquire); uint32_t cnt = atomic_exchange_explicit(rcount, 0, memory_order_relaxed); while (chain && cnt > 0) { void* node = (void*)chain; uintptr_t next = (uintptr_t)tiny_next_read(class_idx, node); tiny_next_write(class_idx, node, m->freelist); m->freelist = node; if (m->used > 0) m->used--; ss_active_dec_one(t); chain = next; cnt--; } } drained_sets++; } return (uint16_t)drained_sets; } // Auto-sized assist based on EMA static inline void tiny_assist_drain_auto(int class_idx) { uint16_t want = tiny_drain_target_from_ema(class_idx); uint16_t got = tiny_assist_drain_owned(class_idx, want); tiny_drain_ema_update(class_idx, got); } // Periodic assist from free-path (very lightweight) static __thread uint32_t g_tls_free_tick[TINY_NUM_CLASSES]; static inline void tiny_assist_maybe_drain_on_free(int class_idx) { uint32_t k = ++g_tls_free_tick[class_idx]; if ((k & 0x3Fu) == 0u) { // every 64 frees uint16_t want = tiny_drain_target_from_ema(class_idx); if (want > 8u) want = 8u; // keep it small on free path uint16_t got = tiny_assist_drain_owned(class_idx, want); tiny_drain_ema_update(class_idx, got); } } #endif // HAKMEM_TINY_ASSIST_INC_H