// tls_sll_box.h - Box TLS-SLL: Single-Linked List API (C7-safe) // // Purpose: Centralized TLS SLL management with C7 protection // Design: Zero-overhead static inline API, C7 always rejected // // Key Rules: // 1. C7 (1KB headerless) is ALWAYS rejected (returns false/0) // 2. All SLL direct writes MUST go through this API // 3. Pop returns with first 8 bytes cleared for C7 (safety) // 4. Capacity checks prevent overflow // // Architecture: // - Box TLS-SLL (this): Push/Pop/Splice authority // - Caller: Provides capacity limits, handles fallback on failure // // Performance: // - Static inline → zero function call overhead // - C7 check: 1 comparison + predict-not-taken (< 1 cycle) // - Same performance as direct SLL access for C0-C6 #ifndef TLS_SLL_BOX_H #define TLS_SLL_BOX_H #include #include #include // For fprintf in debug #include // For abort in debug #include "../ptr_trace.h" // Debug-only: pointer next read/write tracing #include "../hakmem_tiny_config.h" // For TINY_NUM_CLASSES #include "../hakmem_build_flags.h" // Debug guard: validate base pointer before SLL ops (Debug only) #if !HAKMEM_BUILD_RELEASE extern const size_t g_tiny_class_sizes[]; static inline void tls_sll_debug_guard(int class_idx, void* base, const char* where) { (void)g_tiny_class_sizes; // Only a minimal guard: tiny integers are always invalid if ((uintptr_t)base < 4096) { fprintf(stderr, "[TLS_SLL_GUARD] %s: small ptr=%p cls=%d (likely corruption)\n", where, base, class_idx); abort(); } // NOTE: Do NOT check alignment vs class size here. // Blocks are stride-aligned (size+header) from slab base; modulo class size is not 0. } #else static inline void tls_sll_debug_guard(int class_idx, void* base, const char* where) { (void)class_idx; (void)base; (void)where; } #endif // Normalize a possibly user-pointer (base+1) to base (header classes) static inline void* tls_sll_normalize_base(int class_idx, void* node) { (void)class_idx; // Caller must pass base pointers; do not heuristically adjust. return node; } // External TLS SLL state (defined elsewhere) extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES]; extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES]; // ========== Push ========== // Push pointer to TLS SLL // Returns: true on success, false if C7 or capacity exceeded // // CRITICAL Phase 7 Header Design: // - C0-C6 (header classes): [1B header][user data] // ^base ^ptr (caller passes this) // - SLL stores "base" (ptr-1) to avoid overwriting header // - C7 (headerless): ptr == base (no offset) // // Safety: // - C7 always rejected (headerless, first 8 bytes = user data) // - Capacity check prevents overflow // - Header protection: stores base (ptr-1) for C0-C6 // // Performance: 3-4 cycles (C0-C6), < 1 cycle (C7 fast rejection) static inline bool tls_sll_push(int class_idx, void* ptr, uint32_t capacity) { // CRITICAL: C7 (1KB) is headerless - MUST NOT use TLS SLL // Reason: SLL stores next pointer in first 8 bytes (user data for C7) if (__builtin_expect(class_idx == 7, 0)) { return false; // C7 rejected } // Capacity check if (g_tls_sll_count[class_idx] >= capacity) { return false; // SLL full } // CRITICAL: Caller must pass "base" pointer (NOT user ptr) // Phase 7 carve operations return base (stride includes header) // SLL stores base to avoid overwriting header with next pointer // Phase 7: Store next pointer at header-safe offset (base+1 for C0-C6) #if HAKMEM_TINY_HEADER_CLASSIDX const size_t next_offset = 1; // C7 is rejected above; always skip header #else const size_t next_offset = 0; #endif tls_sll_debug_guard(class_idx, ptr, "push"); PTR_NEXT_WRITE("tls_push", class_idx, ptr, next_offset, g_tls_sll_head[class_idx]); g_tls_sll_head[class_idx] = ptr; g_tls_sll_count[class_idx]++; return true; } // ========== Pop ========== // Pop pointer from TLS SLL // Returns: true on success (writes user ptr to *out), false if empty // // CRITICAL Phase 7 Header Design: // - SLL stores "base" (ptr-1) for C0-C6 // - Must return "ptr" (base+1) to user // - C7: base == ptr (no offset) // // Safety: // - C7 protection: clears first 8 bytes on pop (prevents next pointer leak) // - Header protection: returns ptr (base+1) for C0-C6 // - NULL check before deref // // Performance: 4-5 cycles static inline bool tls_sll_pop(int class_idx, void** out) { void* base = g_tls_sll_head[class_idx]; if (!base) { return false; // SLL empty } // Pop from SLL (reads next from base) // Phase 7: Read next pointer at header-safe offset #if HAKMEM_TINY_HEADER_CLASSIDX const size_t next_offset = (class_idx == 7) ? 0 : 1; #else const size_t next_offset = 0; #endif tls_sll_debug_guard(class_idx, base, "pop"); void* next; PTR_NEXT_READ("tls_pop", class_idx, base, next_offset, next); g_tls_sll_head[class_idx] = next; if (g_tls_sll_count[class_idx] > 0) { g_tls_sll_count[class_idx]--; } // CRITICAL: C7 (1KB) returns with first 8 bytes cleared // Reason: C7 is headerless, first 8 bytes are user data area // Without this: user sees stale SLL next pointer → corruption // Cost: 1 store instruction (~1 cycle), only for C7 (~1% of allocations) // // Note: C0-C6 have 1-byte header, so first 8 bytes are safe (header hides next) // Caller responsibility: Convert base → ptr (base+1) for C0-C6 before returning to user if (__builtin_expect(class_idx == 7, 0)) { *(void**)base = NULL; } *out = base; // Return base (caller converts to ptr if needed) return true; } // ========== Splice ========== // Splice chain of pointers to TLS SLL (batch push) // Returns: actual count moved (0 for C7 or if capacity exceeded) // // CRITICAL Phase 7 Header Design: // - Caller MUST pass chain of "base" pointers (ptr-1 for C0-C6) // - Chain links are stored at base (*(void**)base = next_base) // - SLL head stores base pointers // // Safety: // - C7 always returns 0 (no splice) // - Capacity check limits splice size // - Chain traversal with safety (breaks on NULL) // - Assumes chain is already linked using base pointers // // Performance: ~5 cycles + O(count) for chain traversal static inline uint32_t tls_sll_splice(int class_idx, void* chain_head, uint32_t count, uint32_t capacity) { // CRITICAL: C7 (1KB) is headerless - MUST NOT splice to TLS SLL if (__builtin_expect(class_idx == 7, 0)) { return 0; // C7 rejected } // Calculate available capacity uint32_t available = (capacity > g_tls_sll_count[class_idx]) ? (capacity - g_tls_sll_count[class_idx]) : 0; if (available == 0 || count == 0 || !chain_head) { return 0; // No space or empty chain } // Limit splice size to available capacity uint32_t to_move = (count < available) ? count : available; // Find chain tail (traverse to_move - 1 nodes) // NOTE: Chain MUST be linked using base pointers (caller responsibility) // Assume chain is linked with base pointers void* tail = chain_head; #if HAKMEM_TINY_HEADER_CLASSIDX const size_t next_offset = 1; // Chain is built from header-safe links (C7 rejected) #else const size_t next_offset = 0; #endif for (uint32_t i = 1; i < to_move; i++) { tls_sll_debug_guard(class_idx, tail, "splice_trav"); void* next; PTR_NEXT_READ("tls_sp_trav", class_idx, tail, next_offset, next); if (!next) { // Chain shorter than expected, adjust to_move to_move = i; break; } tail = next; } // Splice chain to SLL head // tail is a base pointer by construction tls_sll_debug_guard(class_idx, tail, "splice_link"); #if !HAKMEM_BUILD_RELEASE fprintf(stderr, "[SPLICE_LINK] cls=%d tail=%p off=%zu old_head=%p\n", class_idx, tail, (size_t)next_offset, g_tls_sll_head[class_idx]); #endif PTR_NEXT_WRITE("tls_sp_link", class_idx, tail, next_offset, g_tls_sll_head[class_idx]); // CRITICAL: Normalize head before publishing to SLL (caller may pass user ptrs) void* head_norm = chain_head; tls_sll_debug_guard(class_idx, head_norm, "splice_head"); #if !HAKMEM_BUILD_RELEASE fprintf(stderr, "[SPLICE_SET_HEAD] cls=%d head_norm=%p moved=%u\n", class_idx, head_norm, (unsigned)to_move); #endif g_tls_sll_head[class_idx] = head_norm; g_tls_sll_count[class_idx] += to_move; return to_move; } // ========== Debug/Stats (optional) ========== #if !HAKMEM_BUILD_RELEASE // Verify C7 is not in SLL (debug only, call at safe points) static inline void tls_sll_verify_no_c7(void) { void* head = g_tls_sll_head[7]; if (head != NULL) { fprintf(stderr, "[TLS_SLL_BUG] C7 found in TLS SLL! head=%p count=%u\n", head, g_tls_sll_count[7]); fprintf(stderr, "[TLS_SLL_BUG] This should NEVER happen - C7 is headerless!\n"); abort(); } } #endif #endif // TLS_SLL_BOX_H