hakmem/core/hakmem_tiny_fastcache.inc.h

// hakmem_tiny_fastcache.inc.h
// Phase 2D-1: Hot-path inline functions - Fast cache and quick slot operations
//
// This file contains fast cache and quick slot inline functions.
// These functions are extracted from hakmem_tiny.c to improve maintainability and
// reduce the main file size by approximately 53 lines.
//
// Functions handle:
// - tiny_fast_pop/push: Fast TLS cache operations (lines 377-404)
// - fastcache_pop/push: Frontend fast cache (lines 873-888)
// - quick_pop: Quick slot pop operation (line 892-896)

#ifndef HAKMEM_TINY_FASTCACHE_INC_H
#define HAKMEM_TINY_FASTCACHE_INC_H

#include "hakmem_tiny.h"
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdatomic.h>

// External TLS variables
extern int g_fast_enable;
extern uint16_t g_fast_cap[TINY_NUM_CLASSES];
extern __thread void* g_fast_head[TINY_NUM_CLASSES];
extern __thread uint16_t g_fast_count[TINY_NUM_CLASSES];

// Fast cache capacity
#define TINY_FASTCACHE_CAP 128

// Quick slot capacity
#define QUICK_CAP 6

// External variable declarations
// Note: TinyFastCache and TinyQuickSlot types must be defined before including this file
extern int g_fastcache_enable;
extern __thread TinyFastCache g_fast_cache[TINY_NUM_CLASSES];
extern int g_quick_enable;
extern __thread TinyQuickSlot g_tls_quick[TINY_NUM_CLASSES];
extern unsigned long long g_free_via_fastcache[];
extern unsigned long long g_fast_push_hits[];
extern unsigned long long g_fast_push_full[];
extern unsigned long long g_fast_push_disabled[];
extern unsigned long long g_fast_push_zero_cap[];

static int g_fast_debug_mode = -1;
static int g_fast_debug_limit = 8;
static _Atomic int g_fast_debug_seen[TINY_NUM_CLASSES];

static inline void tiny_fast_debug_log(int class_idx, const char* event, uint16_t count, uint16_t cap) {
    if (__builtin_expect(g_fast_debug_mode == -1, 0)) {
        const char* e = getenv("HAKMEM_TINY_FAST_DEBUG");
        g_fast_debug_mode = (e && atoi(e) != 0) ? 1 : 0;
        const char* limit_env = getenv("HAKMEM_TINY_FAST_DEBUG_MAX");
        if (limit_env && *limit_env) {
            int v = atoi(limit_env);
            if (v > 0) g_fast_debug_limit = v;
        }
    }
    if (!g_fast_debug_mode) return;
    int limit = g_fast_debug_limit;
    if (limit <= 0) limit = 8;
    int seen = atomic_fetch_add_explicit(&g_fast_debug_seen[class_idx], 1, memory_order_relaxed);
    if (seen < limit) {
        fprintf(stderr, "[FASTDBG] class=%d event=%s count=%u cap=%u\n",
                class_idx, event, (unsigned)count, (unsigned)cap);
    }
}

// Tracepoint macros (no-op if not defined)
#ifndef HAK_TP1
#define HAK_TP1(name, idx) do { (void)(idx); } while(0)
#endif

// Basic fast cache operations
static inline __attribute__((always_inline)) void* tiny_fast_pop(int class_idx) {
    if (!g_fast_enable) return NULL;
    uint16_t cap = g_fast_cap[class_idx];
    if (cap == 0) return NULL;
    void* head = g_fast_head[class_idx];
    if (!head) return NULL;
    // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_offset = (class_idx == 7) ? 0 : 1;
#else
    const size_t next_offset = 0;
#endif
    // Use safe unaligned load for "next" to avoid UB when offset==1
    void* next = NULL;
    {
        #include "tiny_nextptr.h"
        next = tiny_next_load(head, class_idx);
    }
    g_fast_head[class_idx] = next;
    uint16_t count = g_fast_count[class_idx];
    if (count > 0) {
        g_fast_count[class_idx] = (uint16_t)(count - 1);
    } else {
        g_fast_count[class_idx] = 0;
    }
    // CRITICAL FIX: Convert base -> user pointer for classes 0-6
    // Headerless class (1KB): clear embedded next pointer before returning to user
    if (__builtin_expect(class_idx == 7, 0)) {
        *(void**)head = NULL;
        return head;  // C7: return base (headerless)
    }
    // C0-C6: return user pointer (base+1)
    return (void*)((uint8_t*)head + 1);
}

static inline __attribute__((always_inline)) int tiny_fast_push(int class_idx, void* ptr) {
    if (!g_fast_enable) {
        g_fast_push_disabled[class_idx]++;
        tiny_fast_debug_log(class_idx, "disabled", 0, 0);
        return 0;
    }
    uint16_t cap = g_fast_cap[class_idx];
    if (cap == 0) {
        g_fast_push_zero_cap[class_idx]++;
        tiny_fast_debug_log(class_idx, "zero_cap", g_fast_count[class_idx], cap);
        return 0;
    }
    uint16_t count = g_fast_count[class_idx];
    if (count >= cap) {
        g_fast_push_full[class_idx]++;
        tiny_fast_debug_log(class_idx, "full", count, cap);
        return 0;
    }
    // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_offset2 = (class_idx == 7) ? 0 : 1;
#else
    const size_t next_offset2 = 0;
#endif
    {
        #include "tiny_nextptr.h"
        tiny_next_store(ptr, class_idx, g_fast_head[class_idx]);
    }
    g_fast_head[class_idx] = ptr;
    g_fast_count[class_idx] = (uint16_t)(count + 1);
    g_fast_push_hits[class_idx]++;
    tiny_fast_debug_log(class_idx, "hit", (uint16_t)(count + 1), cap);
    return 1;
}

// Frontend fast cache operations
static inline void* fastcache_pop(int class_idx) {
    TinyFastCache* fc = &g_fast_cache[class_idx];
    if (__builtin_expect(fc->top > 0, 1)) {
        void* base = fc->items[--fc->top];
        // CRITICAL FIX: Convert base -> user pointer for classes 0-6
        // FastCache stores base pointers, user needs base+1
        if (class_idx == 7) {
            return base;  // C7: headerless, return base
        }
        return (void*)((uint8_t*)base + 1);  // C0-C6: return user pointer
    }
    return NULL;
}

static inline int fastcache_push(int class_idx, void* ptr) {
    TinyFastCache* fc = &g_fast_cache[class_idx];
    if (__builtin_expect(fc->top < TINY_FASTCACHE_CAP, 1)) {
        fc->items[fc->top++] = ptr;
        g_free_via_fastcache[class_idx]++;
        return 1;
    }
    return 0;
}

// Quick slot pop operation
static inline void* quick_pop(int class_idx) {
    TinyQuickSlot* qs = &g_tls_quick[class_idx];
    if (__builtin_expect(qs->top > 0, 1)) {
        void* p = qs->items[--qs->top];
        HAK_TP1(quick_pop, class_idx);
        return p;
    }
    return NULL;
}

#endif // HAKMEM_TINY_FASTCACHE_INC_H