hakmem/core/box/tls_sll_box.h

// tls_sll_box.h - Box TLS-SLL: Single-Linked List API (C7-safe)
//
// Purpose: Centralized TLS SLL management with C7 protection
// Design: Zero-overhead static inline API, C7 always rejected
//
// Key Rules:
//   1. C7 (1KB headerless) is ALWAYS rejected (returns false/0)
//   2. All SLL direct writes MUST go through this API
//   3. Pop returns with first 8 bytes cleared for C7 (safety)
//   4. Capacity checks prevent overflow
//
// Architecture:
//   - Box TLS-SLL (this): Push/Pop/Splice authority
//   - Caller: Provides capacity limits, handles fallback on failure
//
// Performance:
//   - Static inline → zero function call overhead
//   - C7 check: 1 comparison + predict-not-taken (< 1 cycle)
//   - Same performance as direct SLL access for C0-C6

#ifndef TLS_SLL_BOX_H
#define TLS_SLL_BOX_H

#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>     // For fprintf in debug
#include <stdlib.h>    // For abort in debug
#include "../ptr_trace.h"              // Debug-only: pointer next read/write tracing
#include "../hakmem_tiny_config.h"     // For TINY_NUM_CLASSES
#include "../hakmem_build_flags.h"
#include "../tiny_region_id.h"        // HEADER_MAGIC / HEADER_CLASS_MASK

// Debug guard: validate base pointer before SLL ops (Debug only)
#if !HAKMEM_BUILD_RELEASE
extern const size_t g_tiny_class_sizes[];
static inline void tls_sll_debug_guard(int class_idx, void* base, const char* where) {
    (void)g_tiny_class_sizes;
    // Only a minimal guard: tiny integers are always invalid
    if ((uintptr_t)base < 4096) {
        fprintf(stderr, "[TLS_SLL_GUARD] %s: small ptr=%p cls=%d (likely corruption)\n", where, base, class_idx);
        abort();
    }
    // NOTE: Do NOT check alignment vs class size here.
    // Blocks are stride-aligned (size+header) from slab base; modulo class size is not 0.
}
#else
static inline void tls_sll_debug_guard(int class_idx, void* base, const char* where) { (void)class_idx; (void)base; (void)where; }
#endif

// Normalize a possibly user-pointer (base+1) to base (header classes)
static inline void* tls_sll_normalize_base(int class_idx, void* node) {
    (void)class_idx;
    // Caller must pass base pointers; do not heuristically adjust.
    return node;
}

// External TLS SLL state (defined elsewhere)
extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES];
extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES];

// ========== Push ==========

// Push pointer to TLS SLL
// Returns: true on success, false if C7 or capacity exceeded
//
// CRITICAL Phase 7 Header Design:
//   - C0-C6 (header classes): [1B header][user data]
//                             ^base      ^ptr (caller passes this)
//   - SLL stores "base" (ptr-1) to avoid overwriting header
//   - C7 (headerless): ptr == base (no offset)
//
// Safety:
//   - C7 always rejected (headerless, first 8 bytes = user data)
//   - Capacity check prevents overflow
//   - Header protection: stores base (ptr-1) for C0-C6
//
// Performance: 3-4 cycles (C0-C6), < 1 cycle (C7 fast rejection)
static inline bool tls_sll_push(int class_idx, void* ptr, uint32_t capacity) {
    // CRITICAL: C7 (1KB) is headerless - MUST NOT use TLS SLL
    // Reason: SLL stores next pointer in first 8 bytes (user data for C7)
    if (__builtin_expect(class_idx == 7, 0)) {
        return false;  // C7 rejected
    }

    // Capacity check
    if (g_tls_sll_count[class_idx] >= capacity) {
        return false;  // SLL full
    }

    // CRITICAL: Caller must pass "base" pointer (NOT user ptr)
    // Phase 7 carve operations return base (stride includes header)
    // SLL stores base to avoid overwriting header with next pointer

    // Phase 7: Store next pointer at header-safe offset (base+1 for C0-C6)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_offset = 1;  // C7 is rejected above; always skip header
#else
    const size_t next_offset = 0;
#endif
    tls_sll_debug_guard(class_idx, ptr, "push");
    PTR_NEXT_WRITE("tls_push", class_idx, ptr, next_offset, g_tls_sll_head[class_idx]);
    g_tls_sll_head[class_idx] = ptr;
    g_tls_sll_count[class_idx]++;

    return true;
}

// ========== Pop ==========

// Pop pointer from TLS SLL
// Returns: true on success (writes user ptr to *out), false if empty
//
// CRITICAL Phase 7 Header Design:
//   - SLL stores "base" (ptr-1) for C0-C6
//   - Must return "ptr" (base+1) to user
//   - C7: base == ptr (no offset)
//
// Safety:
//   - C7 protection: clears first 8 bytes on pop (prevents next pointer leak)
//   - Header protection: returns ptr (base+1) for C0-C6
//   - NULL check before deref
//
// Performance: 4-5 cycles
static inline bool tls_sll_pop(int class_idx, void** out) {
    void* base = g_tls_sll_head[class_idx];
    if (!base) {
        return false;  // SLL empty
    }

    // Pop from SLL (reads next from base)
    // Phase 7: Read next pointer at header-safe offset
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_offset = (class_idx == 7) ? 0 : 1;
#else
    const size_t next_offset = 0;
#endif
    tls_sll_debug_guard(class_idx, base, "pop");
    void* next; PTR_NEXT_READ("tls_pop", class_idx, base, next_offset, next);
    g_tls_sll_head[class_idx] = next;
    if (g_tls_sll_count[class_idx] > 0) {
        g_tls_sll_count[class_idx]--;
    }

    // CRITICAL: C7 (1KB) returns with first 8 bytes cleared
    // Reason: C7 is headerless, first 8 bytes are user data area
    // Without this: user sees stale SLL next pointer → corruption
    // Cost: 1 store instruction (~1 cycle), only for C7 (~1% of allocations)
    //
    // Note: C0-C6 have 1-byte header, so first 8 bytes are safe (header hides next)
    // Caller responsibility: Convert base → ptr (base+1) for C0-C6 before returning to user
    if (__builtin_expect(class_idx == 7, 0)) {
        *(void**)base = NULL;
    }

    *out = base;  // Return base (caller converts to ptr if needed)
    return true;
}

// ========== Splice ==========

// Splice chain of pointers to TLS SLL (batch push)
// Returns: actual count moved (0 for C7 or if capacity exceeded)
//
// CRITICAL Phase 7 Header Design:
//   - Caller MUST pass chain of "base" pointers (ptr-1 for C0-C6)
//   - Chain links are stored at base (*(void**)base = next_base)
//   - SLL head stores base pointers
//
// Safety:
//   - C7 always returns 0 (no splice)
//   - Capacity check limits splice size
//   - Chain traversal with safety (breaks on NULL)
//   - Assumes chain is already linked using base pointers
//
// Performance: ~5 cycles + O(count) for chain traversal
static inline uint32_t tls_sll_splice(int class_idx, void* chain_head, uint32_t count, uint32_t capacity) {
    // CRITICAL: C7 (1KB) is headerless - MUST NOT splice to TLS SLL
    if (__builtin_expect(class_idx == 7, 0)) {
        return 0;  // C7 rejected
    }

    // Calculate available capacity
    uint32_t available = (capacity > g_tls_sll_count[class_idx])
                         ? (capacity - g_tls_sll_count[class_idx]) : 0;
    if (available == 0 || count == 0 || !chain_head) {
        return 0;  // No space or empty chain
    }

    // Limit splice size to available capacity
    uint32_t to_move = (count < available) ? count : available;

    // Determine how the chain is linked: base or user pointers.
    // For C0-C6, header byte (0xA0|cls) resides at base.
    // If chain_head points to base → *(uint8_t*)head has HEADER_MAGIC|cls
    // If it points to user (base+1) → *(uint8_t*)head is user data (not magic)
    void* tail = chain_head;
#if HAKMEM_TINY_HEADER_CLASSIDX
    size_t next_offset;
    {
        uint8_t hdr = *(uint8_t*)chain_head;
        if ((hdr & 0xF0) == HEADER_MAGIC && (hdr & HEADER_CLASS_MASK) == (uint8_t)class_idx) {
            // Chain nodes are base pointers; links live at base+1
            next_offset = 1;
        } else {
            // Chain nodes are user pointers; links live at user (base+1) → offset 0 from user
            next_offset = 0;
        }
    }
#else
    size_t next_offset = 0;
#endif
    for (uint32_t i = 1; i < to_move; i++) {
        tls_sll_debug_guard(class_idx, tail, "splice_trav");
        void* next; PTR_NEXT_READ("tls_sp_trav", class_idx, tail, next_offset, next);
        if (!next) {
            // Chain shorter than expected, adjust to_move
            to_move = i;
            break;
        }
        tail = next;
    }

    // Splice chain to SLL head
    // tail is a base pointer by construction
    tls_sll_debug_guard(class_idx, tail, "splice_link");
#if !HAKMEM_BUILD_RELEASE
    fprintf(stderr, "[SPLICE_LINK] cls=%d tail=%p off=%zu old_head=%p\n",
            class_idx, tail, (size_t)next_offset, g_tls_sll_head[class_idx]);
#endif
    PTR_NEXT_WRITE("tls_sp_link", class_idx, tail, next_offset, g_tls_sll_head[class_idx]);
    // CRITICAL: Normalize head before publishing to SLL (caller may pass user ptrs)
    void* head_norm = chain_head;
#if HAKMEM_TINY_HEADER_CLASSIDX
    if (next_offset == 0) {
        // Chain nodes were user pointers; convert head to base
        head_norm = (uint8_t*)chain_head - 1;
    }
#endif
    tls_sll_debug_guard(class_idx, head_norm, "splice_head");
#if !HAKMEM_BUILD_RELEASE
    fprintf(stderr, "[SPLICE_SET_HEAD] cls=%d head_norm=%p moved=%u\n",
            class_idx, head_norm, (unsigned)to_move);
#endif
    g_tls_sll_head[class_idx] = head_norm;
    g_tls_sll_count[class_idx] += to_move;

    return to_move;
}

// ========== Debug/Stats (optional) ==========

#if !HAKMEM_BUILD_RELEASE
// Verify C7 is not in SLL (debug only, call at safe points)
static inline void tls_sll_verify_no_c7(void) {
    void* head = g_tls_sll_head[7];
    if (head != NULL) {
        fprintf(stderr, "[TLS_SLL_BUG] C7 found in TLS SLL! head=%p count=%u\n",
                head, g_tls_sll_count[7]);
        fprintf(stderr, "[TLS_SLL_BUG] This should NEVER happen - C7 is headerless!\n");
        abort();
    }
}
#endif

#endif // TLS_SLL_BOX_H