hakmem/core/hakmem_tiny_hotmag.inc.h

// Inline helpers for Tiny HotMag (small TLS magazine for hot classes)
// This header is textually included from hakmem_tiny.c after the following
// symbols are defined:
//   - static int g_hotmag_enable;
//   - static __thread TinyHotMag g_tls_hot_mag[TINY_NUM_CLASSES];
//   - tiny_mag_init_if_needed(int)
//   - g_tls_sll_head[], g_tls_sll_count[], g_tls_mags[]

static inline int hkm_is_hot_class(int class_idx) {
    return class_idx >= 0 && class_idx <= 3 && g_hotmag_class_en[class_idx];
}

static inline uint16_t hotmag_effective_cap(int class_idx) {
    if (!hkm_is_hot_class(class_idx)) return 0;
    int cap = g_hotmag_cap_current[class_idx];
    if (!g_hotmag_cap_locked[class_idx] || cap <= 0) {
        cap = g_hotmag_cap_default;
    }
    if (cap < 16) cap = 16;
    if (cap > 1024) cap = 1024;
    return (uint16_t)cap;
}

static inline uint16_t hotmag_refill_target(int class_idx) {
    if (!hkm_is_hot_class(class_idx)) return 0;
    int refill = g_hotmag_refill_current[class_idx];
    if (!g_hotmag_refill_locked[class_idx] && refill <= 0) {
        refill = g_hotmag_refill_default;
    }
    if (refill < 0) refill = 0;
    uint16_t cap = hotmag_effective_cap(class_idx);
    if ((uint16_t)refill > cap) refill = cap;
    return (uint16_t)refill;
}

static inline void hotmag_init_if_needed(int class_idx) {
    if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return;
    TinyHotMag* hm = &g_tls_hot_mag[class_idx];
    uint16_t desired = hotmag_effective_cap(class_idx);
    if (hm->cap != desired) {
        hm->cap = desired;
        if (hm->top > hm->cap) hm->top = hm->cap;
    }
}

static inline void* hotmag_pop(int class_idx) {
    if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return NULL;
    TinyHotMag* hm = &g_tls_hot_mag[class_idx];
    if (__builtin_expect(hm->top > 0, 1)) {
        return hm->slots[--hm->top];
    }
    return NULL;
}

static inline int hotmag_push(int class_idx, void* ptr) {
    if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return 0;
    TinyHotMag* hm = &g_tls_hot_mag[class_idx];
    hotmag_init_if_needed(class_idx);
    if (__builtin_expect(hm->top < hm->cap, 1)) {
        hm->slots[hm->top++] = ptr;
        return 1;
    }
    return 0;
}

static inline int hotmag_refill_from_sll(int class_idx, int max_take) {
    if (!g_hotmag_enable || !hkm_is_hot_class(class_idx) || max_take <= 0) return 0;
    TinyHotMag* hm = &g_tls_hot_mag[class_idx];
    int room = (int)hm->cap - (int)hm->top;
    if (room <= 0) return 0;
    if (max_take > room) max_take = room;
    int taken = 0;
    while (taken < max_take) {
        void* head = g_tls_sll_head[class_idx];
        if (!head) break;
        g_tls_sll_head[class_idx] = *(void**)head;
        if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
        hm->slots[hm->top++] = head;
        taken++;
    }
    return taken;
}

static inline int hotmag_refill_from_mag(int class_idx, int max_take) {
    if (!g_hotmag_enable || !hkm_is_hot_class(class_idx) || max_take <= 0) return 0;
    tiny_mag_init_if_needed(class_idx);
    TinyTLSMag* mag = &g_tls_mags[class_idx];
    TinyHotMag* hm = &g_tls_hot_mag[class_idx];
    int room = (int)hm->cap - (int)hm->top;
    if (room <= 0 || mag->top <= 0) return 0;
    if (max_take > room) max_take = room;
    int take = mag->top < max_take ? mag->top : max_take;
    for (int i = 0; i < take; i++) {
        hm->slots[hm->top++] = mag->items[--mag->top].ptr;
    }
    return take;
}

static inline int hotmag_try_refill(int class_idx, TinyHotMag* hm) {
    if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return 0;
    uint16_t target = hotmag_refill_target(class_idx);
    if (target == 0) return 0;

    int room = (int)hm->cap - (int)hm->top;
    if (room <= 0) return 0;
    if (target > (uint16_t)room) target = (uint16_t)room;

    int filled = 0;

    if (g_tls_list_enable && target > 0) {
        TinyTLSList* tls = &g_tls_lists[class_idx];
        uint32_t seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed);
        if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) {
            tiny_tls_refresh_params(class_idx, tls);
        }
        void* chain_head = NULL;
        void* chain_tail = NULL;
        uint32_t taken = tls_list_bulk_take(tls, target, &chain_head, &chain_tail);
        if (taken > 0u) {
            void* node = chain_head;
            for (uint32_t i = 0; i < taken && node; i++) {
                void* next = *(void**)node;
                hm->slots[hm->top++] = node;
                node = next;
            }
            filled += (int)taken;
            room = (int)hm->cap - (int)hm->top;
            target -= (uint16_t)taken;
        }
    }

    if (target > 0 && g_tls_sll_enable && room > 0) {
        int from_sll = hotmag_refill_from_sll(class_idx, target);
        filled += from_sll;
        room -= from_sll;
        if (room < 0) room = 0;
        if (target > from_sll) target -= (uint16_t)((from_sll < 0) ? 0 : from_sll);
        else target = 0;
    }

    if (target > 0 && room > 0) {
        int from_mag = hotmag_refill_from_mag(class_idx, target);
        filled += from_mag;
    }

    return filled;
}