// hakmem_tiny_fastcache.inc.h // Phase 2D-1: Hot-path inline functions - Fast cache and quick slot operations // // This file contains fast cache and quick slot inline functions. // These functions are extracted from hakmem_tiny.c to improve maintainability and // reduce the main file size by approximately 53 lines. // // Functions handle: // - tiny_fast_pop/push: Fast TLS cache operations (lines 377-404) // - fastcache_pop/push: Frontend fast cache (lines 873-888) // - quick_pop: Quick slot pop operation (line 892-896) #ifndef HAKMEM_TINY_FASTCACHE_INC_H #define HAKMEM_TINY_FASTCACHE_INC_H #include "hakmem_tiny.h" #include #include #include #include #include "tiny_remote.h" // For TINY_REMOTE_SENTINEL detection #include "box/tiny_next_ptr_box.h" // For tiny_next_read(class_idx, ) // External TLS variables extern int g_fast_enable; extern uint16_t g_fast_cap[TINY_NUM_CLASSES]; extern __thread void* g_fast_head[TINY_NUM_CLASSES]; extern __thread uint16_t g_fast_count[TINY_NUM_CLASSES]; // Fast cache capacity #define TINY_FASTCACHE_CAP 128 // Quick slot capacity #define QUICK_CAP 6 // External variable declarations // Note: TinyFastCache and TinyQuickSlot types must be defined before including this file extern int g_fastcache_enable; extern __thread TinyFastCache g_fast_cache[TINY_NUM_CLASSES]; extern int g_quick_enable; extern __thread TinyQuickSlot g_tls_quick[TINY_NUM_CLASSES]; extern unsigned long long g_free_via_fastcache[]; extern unsigned long long g_fast_push_hits[]; extern unsigned long long g_fast_push_full[]; extern unsigned long long g_fast_push_disabled[]; extern unsigned long long g_fast_push_zero_cap[]; static int g_fast_debug_mode = -1; static int g_fast_debug_limit = 8; static _Atomic int g_fast_debug_seen[TINY_NUM_CLASSES]; static inline void tiny_fast_debug_log(int class_idx, const char* event, uint16_t count, uint16_t cap) { if (__builtin_expect(g_fast_debug_mode == -1, 0)) { const char* e = getenv("HAKMEM_TINY_FAST_DEBUG"); g_fast_debug_mode = (e && atoi(e) != 0) ? 1 : 0; const char* limit_env = getenv("HAKMEM_TINY_FAST_DEBUG_MAX"); if (limit_env && *limit_env) { int v = atoi(limit_env); if (v > 0) g_fast_debug_limit = v; } } if (!g_fast_debug_mode) return; int limit = g_fast_debug_limit; if (limit <= 0) limit = 8; int seen = atomic_fetch_add_explicit(&g_fast_debug_seen[class_idx], 1, memory_order_relaxed); if (seen < limit) { fprintf(stderr, "[FASTDBG] class=%d event=%s count=%u cap=%u\n", class_idx, event, (unsigned)count, (unsigned)cap); } } // Tracepoint macros (no-op if not defined) #ifndef HAK_TP1 #define HAK_TP1(name, idx) do { (void)(idx); } while(0) #endif // Basic fast cache operations static inline __attribute__((always_inline)) void* tiny_fast_pop(int class_idx) { if (!g_fast_enable) return NULL; uint16_t cap = g_fast_cap[class_idx]; if (cap == 0) return NULL; void* head = g_fast_head[class_idx]; if (!head) return NULL; // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base) #if HAKMEM_TINY_HEADER_CLASSIDX // Phase E1-CORRECT: ALL classes have 1-byte header, next ptr at offset 1 const size_t next_offset = 1; #else const size_t next_offset = 0; #endif // Phase E1-CORRECT: Use Box API for next pointer read (ALL classes: base+1) #include "box/tiny_next_ptr_box.h" void* next = tiny_next_read(class_idx, head); g_fast_head[class_idx] = next; uint16_t count = g_fast_count[class_idx]; if (count > 0) { g_fast_count[class_idx] = (uint16_t)(count - 1); } else { g_fast_count[class_idx] = 0; } // Phase E1-CORRECT: All classes return user pointer (base+1) return (void*)((uint8_t*)head + 1); } static inline __attribute__((always_inline)) int tiny_fast_push(int class_idx, void* ptr) { // ✅ CRITICAL FIX: Prevent sentinel-poisoned nodes from entering fast cache // Remote free operations can write SENTINEL to node->next, which eventually // propagates through freelist → TLS list → fast cache. If we push such a node, // the next pop will try to dereference the sentinel → SEGV! if (__builtin_expect((uintptr_t)ptr == TINY_REMOTE_SENTINEL, 0)) { static __thread int sentinel_ptr_logged = 0; if (sentinel_ptr_logged < 5) { fprintf(stderr, "[FAST_PUSH_SENTINEL] cls=%d ptr=%p BLOCKED (ptr is sentinel)!\n", class_idx, ptr); sentinel_ptr_logged++; } return 0; // Reject push } // ✅ CRITICAL FIX #2: Also check if node's NEXT pointer is sentinel (defense-in-depth) // This catches nodes that have sentinel in their next field (from remote free) void* next_check = tiny_next_read(class_idx, ptr); if (__builtin_expect((uintptr_t)next_check == TINY_REMOTE_SENTINEL, 0)) { static __thread int sentinel_next_logged = 0; if (sentinel_next_logged < 5) { fprintf(stderr, "[FAST_PUSH_NEXT_SENTINEL] cls=%d ptr=%p next=%p BLOCKED (next is sentinel)!\n", class_idx, ptr, next_check); sentinel_next_logged++; } return 0; // Reject push } if (!g_fast_enable) { g_fast_push_disabled[class_idx]++; tiny_fast_debug_log(class_idx, "disabled", 0, 0); return 0; } uint16_t cap = g_fast_cap[class_idx]; if (cap == 0) { g_fast_push_zero_cap[class_idx]++; tiny_fast_debug_log(class_idx, "zero_cap", g_fast_count[class_idx], cap); return 0; } uint16_t count = g_fast_count[class_idx]; if (count >= cap) { g_fast_push_full[class_idx]++; tiny_fast_debug_log(class_idx, "full", count, cap); return 0; } // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base) #if HAKMEM_TINY_HEADER_CLASSIDX // Phase E1-CORRECT: ALL classes have 1-byte header, next ptr at offset 1 const size_t next_offset2 = 1; #else const size_t next_offset2 = 0; #endif // Phase E1-CORRECT: Use Box API for next pointer write (ALL classes: base+1) #include "box/tiny_next_ptr_box.h" tiny_next_write(class_idx, ptr, g_fast_head[class_idx]); g_fast_head[class_idx] = ptr; g_fast_count[class_idx] = (uint16_t)(count + 1); g_fast_push_hits[class_idx]++; tiny_fast_debug_log(class_idx, "hit", (uint16_t)(count + 1), cap); return 1; } // Frontend fast cache operations static inline void* fastcache_pop(int class_idx) { TinyFastCache* fc = &g_fast_cache[class_idx]; if (__builtin_expect(fc->top > 0, 1)) { void* base = fc->items[--fc->top]; // ✅ FIX #16: Return BASE pointer (not USER) // FastCache stores base pointers. Caller will apply HAK_RET_ALLOC // which does BASE → USER conversion via tiny_region_id_write_header return base; } return NULL; } static inline int fastcache_push(int class_idx, void* ptr) { TinyFastCache* fc = &g_fast_cache[class_idx]; if (__builtin_expect(fc->top < TINY_FASTCACHE_CAP, 1)) { fc->items[fc->top++] = ptr; g_free_via_fastcache[class_idx]++; return 1; } return 0; } // Quick slot pop operation static inline void* quick_pop(int class_idx) { TinyQuickSlot* qs = &g_tls_quick[class_idx]; if (__builtin_expect(qs->top > 0, 1)) { void* p = qs->items[--qs->top]; HAK_TP1(quick_pop, class_idx); return p; } return NULL; } #endif // HAKMEM_TINY_FASTCACHE_INC_H