hakmem/core/hakmem_tiny_bg_spill.c

#include "hakmem_tiny_bg_spill.h"
#include "hakmem_tiny_superslab.h"  // For SuperSlab, TinySlabMeta, ss_active_dec_one
#include "hakmem_super_registry.h"  // For hak_super_lookup
#include "tiny_remote.h"
#include "hakmem_tiny.h"
#include <pthread.h>

static inline uint32_t tiny_self_u32_guard(void) {
    return (uint32_t)(uintptr_t)pthread_self();
}
#include <stdlib.h>  // For getenv, atoi

// Global variables
int g_bg_spill_enable = 0;           // HAKMEM_TINY_BG_SPILL=1
int g_bg_spill_target = 128;         // HAKMEM_TINY_BG_TARGET (per class)
int g_bg_spill_max_batch = 128;      // HAKMEM_TINY_BG_MAX_BATCH
_Atomic uintptr_t g_bg_spill_head[TINY_NUM_CLASSES];
_Atomic uint32_t g_bg_spill_len[TINY_NUM_CLASSES];

void bg_spill_init(void) {
    // Parse environment variables
    char* bs = getenv("HAKMEM_TINY_BG_SPILL");
    if (bs) g_bg_spill_enable = (atoi(bs) != 0) ? 1 : 0;
    char* bt2 = getenv("HAKMEM_TINY_BG_TARGET");
    if (bt2) { int v = atoi(bt2); if (v > 0 && v <= 8192) g_bg_spill_target = v; }
    char* mb = getenv("HAKMEM_TINY_BG_MAX_BATCH");
    if (mb) { int v = atoi(mb); if (v > 0 && v <= 4096) g_bg_spill_max_batch = v; }

    // Initialize atomic queues
    for (int k = 0; k < TINY_NUM_CLASSES; k++) {
        atomic_store_explicit(&g_bg_spill_head[k], (uintptr_t)0, memory_order_relaxed);
        atomic_store_explicit(&g_bg_spill_len[k], 0u, memory_order_relaxed);
    }
}

void bg_spill_drain_class(int class_idx, pthread_mutex_t* lock) {
    uint32_t approx = atomic_load_explicit(&g_bg_spill_len[class_idx], memory_order_relaxed);
    if (approx == 0) return;

    uintptr_t chain = atomic_exchange_explicit(&g_bg_spill_head[class_idx], (uintptr_t)0, memory_order_acq_rel);
    if (chain == 0) return;

    // Split chain up to max_batch
    int processed = 0;
    void* rest = NULL;
    void* cur = (void*)chain;
    void* prev = NULL;
    // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_off = (class_idx == 7) ? 0 : 1;
#else
    const size_t next_off = 0;
#endif
    while (cur && processed < g_bg_spill_max_batch) {
        prev = cur;
        cur = *(void**)((uint8_t*)cur + next_off);
        processed++;
    }
    if (cur != NULL) { rest = cur; *(void**)((uint8_t*)prev + next_off) = NULL; }

    // Return processed nodes to SS freelists
    pthread_mutex_lock(lock);
    uint32_t self_tid = tiny_self_u32_guard();
    void* node = (void*)chain;
    while (node) {
        void* next = *(void**)((uint8_t*)node + next_off);
        SuperSlab* owner_ss = hak_super_lookup(node);
        if (owner_ss && owner_ss->magic == SUPERSLAB_MAGIC) {
            int slab_idx = slab_index_for(owner_ss, node);
            TinySlabMeta* meta = &owner_ss->slabs[slab_idx];
            if (!tiny_remote_guard_allow_local_push(owner_ss, slab_idx, meta, node, "bg_spill", self_tid)) {
                (void)ss_remote_push(owner_ss, slab_idx, node);
                if (meta->used > 0) meta->used--;
                node = next;
                continue;
            }
            void* prev = meta->freelist;
            // SuperSlab freelist uses base offset (no header while free)
            *(void**)node = prev;
            meta->freelist = node;
            tiny_failfast_log("bg_spill", owner_ss->size_class, owner_ss, meta, node, prev);
            meta->used--;
            // Active was decremented at free time
        }
        node = next;
    }
    pthread_mutex_unlock(lock);

    if (processed > 0) {
        atomic_fetch_sub_explicit(&g_bg_spill_len[class_idx], (uint32_t)processed, memory_order_relaxed);
    }

    if (rest) {
        // Prepend remainder back to head
        uintptr_t old_head;
        void* tail = rest;
        while (*(void**)((uint8_t*)tail + next_off)) tail = *(void**)((uint8_t*)tail + next_off);
        do {
            old_head = atomic_load_explicit(&g_bg_spill_head[class_idx], memory_order_acquire);
            *(void**)((uint8_t*)tail + next_off) = (void*)old_head;
        } while (!atomic_compare_exchange_weak_explicit(&g_bg_spill_head[class_idx], &old_head,
                                                        (uintptr_t)rest,
                                                        memory_order_release, memory_order_relaxed));
    }
}