hakmem/core/hakmem_tiny_assist.inc.h

// hakmem_tiny_assist.inc.h
// Tiny: helper routines to assist targeted remote-drain
// Keep inline to avoid call overhead on hot slow-paths.

#ifndef HAKMEM_TINY_ASSIST_INC_H
#define HAKMEM_TINY_ASSIST_INC_H

#include <stdatomic.h>
#include "hakmem_tiny_superslab.h"
#include "hakmem_tiny_ss_target.h"
#include "hakmem_tiny_drain_ema.inc.h"
#include "box/tiny_next_ptr_box.h"  // Box API: Next pointer read/write

static inline uint16_t tiny_assist_drain_owned(int class_idx, int max_items) {
    int drained_sets = 0;
    while (drained_sets < max_items) {
        SuperSlab* t = ss_target_pop(class_idx);
        if (!t) break;
        uint32_t mytid = tiny_self_u32();
        for (int i = 0; i < SLABS_PER_SUPERSLAB; i++) {
            TinySlabMeta* m = &t->slabs[i];
            if (m->owner_tid != mytid) continue;
            TinySlabPrefix* pref = tiny_slab_prefix(t, i);
            _Atomic(uintptr_t)* rhead = (_Atomic(uintptr_t)*)&pref->reserved[0];
            _Atomic(uint32_t)* rcount = (_Atomic(uint32_t)*)&pref->reserved[1];
            uint32_t pending = atomic_load_explicit(rcount, memory_order_relaxed);
            if (pending == 0) continue;
            uintptr_t chain = atomic_exchange_explicit(rhead, 0, memory_order_acquire);
            uint32_t cnt = atomic_exchange_explicit(rcount, 0, memory_order_relaxed);
            while (chain && cnt > 0) {
                void* node = (void*)chain;
                uintptr_t next = (uintptr_t)tiny_next_read(class_idx, node);
                tiny_next_write(class_idx, node, m->freelist);
                m->freelist = node;
                if (m->used > 0) m->used--;
                ss_active_dec_one(t);
                chain = next;
                cnt--;
            }
        }
        drained_sets++;
    }
    return (uint16_t)drained_sets;
}

// Auto-sized assist based on EMA
static inline void tiny_assist_drain_auto(int class_idx) {
    uint16_t want = tiny_drain_target_from_ema(class_idx);
    uint16_t got = tiny_assist_drain_owned(class_idx, want);
    tiny_drain_ema_update(class_idx, got);
}

// Periodic assist from free-path (very lightweight)
static __thread uint32_t g_tls_free_tick[TINY_NUM_CLASSES];
static inline void tiny_assist_maybe_drain_on_free(int class_idx) {
    uint32_t k = ++g_tls_free_tick[class_idx];
    if ((k & 0x3Fu) == 0u) { // every 64 frees
        uint16_t want = tiny_drain_target_from_ema(class_idx);
        if (want > 8u) want = 8u; // keep it small on free path
        uint16_t got = tiny_assist_drain_owned(class_idx, want);
        tiny_drain_ema_update(class_idx, got);
    }
}

#endif // HAKMEM_TINY_ASSIST_INC_H