// hakmem_tiny_superslab_internal.h - Internal declarations for superslab refactor // Purpose: Shared declarations between superslab implementation files // License: MIT // Date: 2025-11-28 #ifndef HAKMEM_TINY_SUPERSLAB_INTERNAL_H #define HAKMEM_TINY_SUPERSLAB_INTERNAL_H #include "hakmem_tiny_superslab.h" #include "box/ss_hot_cold_box.h" #include "box/ss_allocation_box.h" // CRITICAL: For superslab_allocate() declaration (fixes implicit int assumption) #include "hakmem_super_registry.h" #include "hakmem_debug_master.h" // For unified debug level control #include "hakmem_tiny.h" #include "hakmem_tiny_config.h" #include "hakmem_shared_pool.h" #include "hakmem_internal.h" #include "tiny_region_id.h" #include "hakmem_tiny_integrity.h" #include "box/tiny_next_ptr_box.h" #include "box/slab_freelist_atomic.h" #include #include #include #include #include #include #include #include // ============================================================================ // Allocation Path Tracking // ============================================================================ // Allocation path identifiers for debugging #define ALLOC_PATH_UNKNOWN 0 #define ALLOC_PATH_BACKEND 1 // SuperSlab backend (legacy/shared) #define ALLOC_PATH_TLS_POP 2 // TLS SLL pop #define ALLOC_PATH_CARVE 3 // Linear carve from slab #define ALLOC_PATH_FREELIST 4 // Freelist pop #define ALLOC_PATH_HOTMAG 5 // Hot magazine #define ALLOC_PATH_FASTCACHE 6 // Fast cache #define ALLOC_PATH_BUMP 7 // Bump allocator #define ALLOC_PATH_REFILL 8 // Refill/adoption // ============================================================================ // Unified Return Macros // ============================================================================ // HAK_RET_ALLOC_BLOCK - Single exit point for SuperSlab allocations // Purpose: Ensures consistent header writing across all SuperSlab allocation paths // Usage: HAK_RET_ALLOC_BLOCK(class_idx, base_ptr); // Note: Must be used in function context (macro contains return statement) #if HAKMEM_TINY_HEADER_CLASSIDX #define HAK_RET_ALLOC_BLOCK(cls, base_ptr) \ return tiny_region_id_write_header((base_ptr), (cls)) #else #define HAK_RET_ALLOC_BLOCK(cls, base_ptr) \ return (void*)(base_ptr) #endif // HAK_RET_ALLOC_BLOCK_TRACED - Same as above but with path tracking // Enabled via HAKMEM_DEBUG_LEVEL >= 3 (INFO level) // Legacy: HAKMEM_ALLOC_PATH_TRACE=1 still works for compatibility #if HAKMEM_TINY_HEADER_CLASSIDX #define HAK_RET_ALLOC_BLOCK_TRACED(cls, base_ptr, path_id) do { \ static int g_trace_enabled = -1; \ if (__builtin_expect(g_trace_enabled == -1, 0)) { \ g_trace_enabled = hak_debug_check_level("HAKMEM_ALLOC_PATH_TRACE", 3); \ } \ if (__builtin_expect(g_trace_enabled, 0)) { \ static _Atomic uint32_t g_trace_count = 0; \ uint32_t n = atomic_fetch_add(&g_trace_count, 1); \ if (n < 20) { \ fprintf(stderr, "[ALLOC_PATH] cls=%d path=%d base=%p\n", \ (cls), (path_id), (void*)(base_ptr)); \ } \ } \ return tiny_region_id_write_header((base_ptr), (cls)); \ } while(0) #else #define HAK_RET_ALLOC_BLOCK_TRACED(cls, base_ptr, path_id) \ return (void*)(base_ptr) #endif // ============================================================================ // Global Variables (defined in superslab_stats.c) // ============================================================================ extern pthread_mutex_t g_superslab_lock; extern uint64_t g_superslabs_allocated; extern uint64_t g_superslabs_freed; extern uint64_t g_bytes_allocated; extern _Atomic uint64_t g_ss_active_dec_calls; extern _Atomic uint64_t g_hak_tiny_free_calls; extern _Atomic uint64_t g_ss_remote_push_calls; extern _Atomic uint64_t g_free_ss_enter; extern _Atomic uint64_t g_free_local_box_calls; extern _Atomic uint64_t g_free_remote_box_calls; extern uint64_t g_ss_alloc_by_class[8]; extern uint64_t g_ss_freed_by_class[8]; extern _Atomic uint64_t g_ss_mmap_count; extern _Atomic uint64_t g_final_fallback_mmap_count; // ============================================================================ // SuperSlabHead Management (defined in superslab_head.c) // ============================================================================ extern SuperSlabHead* g_superslab_heads[TINY_NUM_CLASSES_SS]; // ============================================================================ // Cache System (defined in superslab_cache.c) // ============================================================================ typedef struct SuperslabCacheEntry { struct SuperslabCacheEntry* next; } SuperslabCacheEntry; extern SuperslabCacheEntry* g_ss_cache_head[8]; extern size_t g_ss_cache_count[8]; extern size_t g_ss_cache_cap[8]; extern size_t g_ss_precharge_target[8]; extern _Atomic int g_ss_precharge_done[8]; extern int g_ss_cache_enabled; extern pthread_once_t g_ss_cache_once; extern pthread_mutex_t g_ss_cache_lock[8]; extern uint64_t g_ss_cache_hits[8]; extern uint64_t g_ss_cache_misses[8]; extern uint64_t g_ss_cache_puts[8]; extern uint64_t g_ss_cache_drops[8]; extern uint64_t g_ss_cache_precharged[8]; extern uint64_t g_superslabs_reused; extern uint64_t g_superslabs_cached; // Cache functions (defined in superslab_cache.c) void ss_cache_global_init(void); void ss_cache_ensure_init(void); void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int populate); void ss_cache_precharge(uint8_t size_class, size_t ss_size, uintptr_t ss_mask); SuperslabCacheEntry* ss_cache_pop(uint8_t size_class); int ss_cache_push(uint8_t size_class, SuperSlab* ss); // ============================================================================ // ACE (Adaptive Cache Engine) - defined in superslab_ace.c // ============================================================================ extern SuperSlabACEState g_ss_ace[TINY_NUM_CLASSES_SS]; extern int g_ss_force_lg; extern _Atomic int g_ss_populate_once; uint8_t hak_tiny_superslab_next_lg(int class_idx); void ace_observe_and_decide(int k); // ============================================================================ // Statistics (defined in superslab_stats.c) // ============================================================================ void ss_stats_os_alloc(uint8_t size_class, size_t ss_size); void ss_stats_cache_reuse(void); void ss_stats_cache_store(void); void log_superslab_oom_once(size_t ss_size, size_t alloc_size, int err); // ============================================================================ // Slab Management (defined in superslab_slab.c) // ============================================================================ // Drain remote MPSC stack into freelist (ownership already verified by caller) void _ss_remote_drain_to_freelist_unsafe(SuperSlab* ss, int slab_idx, TinySlabMeta* meta); // ============================================================================ // Backend Allocation (defined in superslab_backend.c) // ============================================================================ void* hak_tiny_alloc_superslab_backend_legacy(int class_idx); void* hak_tiny_alloc_superslab_backend_shared(int class_idx); // ============================================================================ // SuperSlabHead Management (defined in superslab_head.c) // ============================================================================ SuperSlabHead* init_superslab_head(int class_idx); int expand_superslab_head(SuperSlabHead* head); SuperSlab* find_chunk_for_ptr(void* ptr, int class_idx); #endif // HAKMEM_TINY_SUPERSLAB_INTERNAL_H