// hak_free_api.inc.h — Box: hak_free_at() implementation #ifndef HAK_FREE_API_INC_H #define HAK_FREE_API_INC_H #include "hakmem_tiny_superslab.h" // For SUPERSLAB_MAGIC, SuperSlab #include "../tiny_free_fast_v2.inc.h" // Phase 7: Header-based ultra-fast free #ifdef HAKMEM_POOL_TLS_PHASE1 #include "../pool_tls.h" #endif // Optional route trace: print first N classification lines when enabled by env static inline int hak_free_route_trace_on(void) { static int g_trace = -1; if (__builtin_expect(g_trace == -1, 0)) { const char* e = getenv("HAKMEM_FREE_ROUTE_TRACE"); g_trace = (e && *e && *e != '0') ? 1 : 0; } return g_trace; } static inline int* hak_free_route_budget_ptr(void) { static int g_budget = 32; // first 32 frees only return &g_budget; } static inline void hak_free_route_log(const char* tag, void* p) { if (!hak_free_route_trace_on()) return; int* budget = hak_free_route_budget_ptr(); if (*budget <= 0) return; (*budget)--; fprintf(stderr, "[FREE_ROUTE] %s ptr=%p\n", tag, p); } // Optional: request-trace for invalid-magic cases (first N hits) static inline int hak_super_reg_reqtrace_on(void) { static int g_on = -1; if (__builtin_expect(g_on == -1, 0)) { const char* e = getenv("HAKMEM_SUPER_REG_REQTRACE"); g_on = (e && *e && *e != '0') ? 1 : 0; } return g_on; } static inline int* hak_super_reg_reqtrace_budget_ptr(void) { static int g_budget = 16; // trace first 16 occurrences return &g_budget; } static inline void hak_super_reg_reqtrace_dump(void* ptr) { if (!hak_super_reg_reqtrace_on()) return; int* b = hak_super_reg_reqtrace_budget_ptr(); if (*b <= 0) return; (*b)--; uintptr_t p = (uintptr_t)ptr; uintptr_t m20 = ((uintptr_t)1 << 20) - 1; uintptr_t m21 = ((uintptr_t)1 << 21) - 1; SuperSlab* s20 = (SuperSlab*)(p & ~m20); SuperSlab* s21 = (SuperSlab*)(p & ~m21); unsigned long long mg20 = 0, mg21 = 0; // Best-effort reads (may be unmapped; wrap in volatile access) mg20 = (unsigned long long)(s20 ? s20->magic : 0); mg21 = (unsigned long long)(s21 ? s21->magic : 0); fprintf(stderr, "[SUPER_REG_REQTRACE] ptr=%p base1M=%p magic1M=0x%llx base2M=%p magic2M=0x%llx\n", ptr, (void*)s20, mg20, (void*)s21, mg21); } #ifndef HAKMEM_TINY_PHASE6_BOX_REFACTOR __attribute__((always_inline)) inline #endif void hak_free_at(void* ptr, size_t size, hak_callsite_t site) { #if HAKMEM_DEBUG_TIMING HKM_TIME_START(t0); #endif (void)site; (void)size; if (!ptr) { #if HAKMEM_DEBUG_TIMING HKM_TIME_END(HKM_CAT_HAK_FREE, t0); #endif return; } #ifdef HAKMEM_POOL_TLS_PHASE1 // Phase 1: Try Pool TLS free FIRST for 8KB-52KB range // CRITICAL: Must come before Phase 7 Tiny to avoid magic mismatch SEGV // Pool TLS uses magic 0xb0, Tiny uses magic 0xa0 - must distinguish! { void* header_addr = (char*)ptr - 1; // Safety vs performance trade-off: // - If HAKMEM_TINY_SAFE_FREE=1 (strict), validate with mincore() always // - Else (default), only validate on page-boundary risk to avoid syscall cost #if HAKMEM_TINY_SAFE_FREE if (!hak_is_memory_readable(header_addr)) { goto skip_pool_tls; } #else uintptr_t off = (uintptr_t)header_addr & 0xFFF; if (__builtin_expect(off == 0, 0)) { if (!hak_is_memory_readable(header_addr)) { goto skip_pool_tls; } } #endif uint8_t header = *(uint8_t*)header_addr; if ((header & 0xF0) == POOL_MAGIC) { pool_free(ptr); hak_free_route_log("pool_tls", ptr); goto done; } // Not Pool TLS - fall through to other paths } skip_pool_tls: #endif #if HAKMEM_TINY_HEADER_CLASSIDX // Phase 7: Dual-header dispatch (1-byte Tiny header OR 16-byte malloc/mmap header) // // Step 1: Try 1-byte Tiny header (fast path: 5-10 cycles) if (__builtin_expect(hak_tiny_free_fast_v2(ptr), 1)) { hak_free_route_log("header_fast", ptr); #if !HAKMEM_BUILD_RELEASE hak_free_v2_track_fast(); // Track hit rate in debug #endif goto done; // Success - done in 5-10 cycles! NO SuperSlab lookup! } // Step 2: Try 16-byte AllocHeader (malloc/mmap allocations) // CRITICAL: Must check this BEFORE calling hak_tiny_free() to avoid silent failures! { void* raw = (char*)ptr - HEADER_SIZE; // SAFETY: Check if raw header is accessible before dereferencing // This prevents SEGV when malloc metadata is unmapped // // OPTIMIZATION: raw = ptr - HEADER_SIZE (16 bytes) // Page boundary case: if ptr is in first 16 bytes of page, raw crosses page boundary // Check: (ptr & 0xFFF) < HEADER_SIZE → raw might be on previous (unmapped) page uintptr_t offset_in_page = (uintptr_t)ptr & 0xFFF; if (__builtin_expect(offset_in_page < HEADER_SIZE, 0)) { // Potential page boundary crossing - do safety check if (!hak_is_memory_readable(raw)) { goto slow_path_after_step2; } } // Normal case (99.6%): raw is on same page as ptr (no mincore call!) // Safe to dereference now AllocHeader* hdr = (AllocHeader*)raw; if (hdr->magic == HAKMEM_MAGIC) { // Valid 16-byte header found (malloc/mmap allocation) hak_free_route_log("header_16byte", ptr); if (hdr->method == ALLOC_METHOD_MALLOC) { // CRITICAL: raw was allocated with __libc_malloc, so free with __libc_free extern void __libc_free(void*); __libc_free(raw); goto done; } // Handle other methods (mmap, etc) - continue to slow path below } slow_path_after_step2:; } // Fallback: Invalid header (non-tiny) or TLS cache full #if !HAKMEM_BUILD_RELEASE hak_free_v2_track_slow(); #endif #endif // SS-first free(既定ON) #if !HAKMEM_TINY_HEADER_CLASSIDX // Only run SS-first if Phase 7 header-based free is not enabled // (Phase 7 already does the SS lookup and handles SS allocations) { static int s_free_to_ss = -2; if (s_free_to_ss == -2) { const char* e = getenv("HAKMEM_TINY_FREE_TO_SS"); s_free_to_ss = (e && *e) ? ((*e!='0')?1:0) : 1; } if (s_free_to_ss) { extern int g_use_superslab; if (__builtin_expect(g_use_superslab != 0, 1)) { SuperSlab* ss = hak_super_lookup(ptr); if (ss && ss->magic == SUPERSLAB_MAGIC) { int sidx = slab_index_for(ss, ptr); int cap = ss_slabs_capacity(ss); if (__builtin_expect(sidx >= 0 && sidx < cap, 1)) { hak_free_route_log("ss_hit", ptr); hak_tiny_free(ptr); goto done; } } // FIX: Removed dangerous "guess loop" (lines 92-95) // The loop dereferenced unmapped memory causing SEGV // If registry lookup fails, allocation is not from SuperSlab } } } #endif // Mid/L25 headerless経路 { extern int hak_pool_mid_lookup(void* ptr, size_t* out_size); extern void hak_pool_free_fast(void* ptr, uintptr_t site_id); size_t mid_sz = 0; if (hak_pool_mid_lookup(ptr, &mid_sz)) { hak_free_route_log("mid_hit", ptr); hak_pool_free_fast(ptr, (uintptr_t)site); goto done; } } { extern int hak_l25_lookup(void* ptr, size_t* out_size); extern void hak_l25_pool_free_fast(void* ptr, uintptr_t site_id); size_t l25_sz = 0; if (hak_l25_lookup(ptr, &l25_sz)) { hak_free_route_log("l25_hit", ptr); hkm_ace_stat_large_free(); hak_l25_pool_free_fast(ptr, (uintptr_t)site); goto done; } } // Raw header dispatch(mmap/malloc/BigCacheなど) { void* raw = (char*)ptr - HEADER_SIZE; // CRITICAL FIX (2025-11-07): Check if memory is accessible before dereferencing // This prevents SEGV when ptr has no header (Tiny alloc where SS lookup failed, or libc alloc) if (!hak_is_memory_readable(raw)) { // Memory not accessible, ptr likely has no header hak_free_route_log("unmapped_header_fallback", ptr); // In direct-link mode, try tiny_free (handles headerless Tiny allocs) if (!g_ldpreload_mode && g_invalid_free_mode) { hak_tiny_free(ptr); goto done; } // LD_PRELOAD mode: route to libc (might be libc allocation) extern void __libc_free(void*); __libc_free(ptr); goto done; } // Safe to dereference header now AllocHeader* hdr = (AllocHeader*)raw; if (hdr->magic != HAKMEM_MAGIC) { // CRITICAL FIX (2025-11-07): Invalid magic could mean: // 1. Tiny allocation where SuperSlab lookup failed (NO header exists) // 2. Libc allocation from mixed environment // 3. Double-free or corrupted pointer if (g_invalid_free_log) fprintf(stderr, "[hakmem] ERROR: Invalid magic 0x%X (expected 0x%X)\n", hdr->magic, HAKMEM_MAGIC); // One-shot request-trace to help diagnose SS registry lookups hak_super_reg_reqtrace_dump(ptr); // In direct-link mode, try routing to tiny free as best-effort recovery // This handles case #1 where SuperSlab lookup failed but allocation is valid if (!g_ldpreload_mode && g_invalid_free_mode) { // Attempt tiny free (will validate internally and handle gracefully if invalid) hak_free_route_log("invalid_magic_tiny_recovery", ptr); hak_tiny_free(ptr); goto done; } // LD_PRELOAD mode or fallback mode: route to libc // IMPORTANT: Use ptr (not raw), as NO header exists if (g_invalid_free_mode) { // Skip mode: leak memory (original behavior, but logged) static int leak_warn = 0; if (!leak_warn) { fprintf(stderr, "[hakmem] WARNING: Skipping free of invalid pointer %p (may leak memory)\n", ptr); leak_warn = 1; } goto done; } else { // Fallback mode: route to libc extern void __libc_free(void*); __libc_free(ptr); // Use ptr, not raw! goto done; } } if (HAK_ENABLED_CACHE(HAKMEM_FEATURE_BIGCACHE) && hdr->class_bytes >= 2097152) { if (hak_bigcache_put(ptr, hdr->size, hdr->alloc_site)) goto done; } { static int g_bc_l25_en_free = -1; if (g_bc_l25_en_free == -1) { const char* e = getenv("HAKMEM_BIGCACHE_L25"); g_bc_l25_en_free = (e && atoi(e) != 0) ? 1 : 0; } if (g_bc_l25_en_free && HAK_ENABLED_CACHE(HAKMEM_FEATURE_BIGCACHE) && hdr->size >= 524288 && hdr->size < 2097152) { if (hak_bigcache_put(ptr, hdr->size, hdr->alloc_site)) goto done; } } switch (hdr->method) { case ALLOC_METHOD_POOL: if (HAK_ENABLED_ALLOC(HAKMEM_FEATURE_POOL)) { hkm_ace_stat_mid_free(); hak_pool_free(ptr, hdr->size, hdr->alloc_site); goto done; } break; case ALLOC_METHOD_L25_POOL: hkm_ace_stat_large_free(); hak_l25_pool_free(ptr, hdr->size, hdr->alloc_site); goto done; case ALLOC_METHOD_MALLOC: // CRITICAL FIX: raw was allocated with __libc_malloc, so free with __libc_free // Using free(raw) would go through wrapper → infinite recursion hak_free_route_log("malloc_hdr", ptr); extern void __libc_free(void*); __libc_free(raw); break; case ALLOC_METHOD_MMAP: #ifdef __linux__ if (HAK_ENABLED_MEMORY(HAKMEM_FEATURE_BATCH_MADVISE) && hdr->size >= BATCH_MIN_SIZE) { hak_batch_add(raw, hdr->size); goto done; } if (hkm_whale_put(raw, hdr->size) != 0) { hkm_sys_munmap(raw, hdr->size); } #else // CRITICAL FIX: Same as ALLOC_METHOD_MALLOC extern void __libc_free(void*); __libc_free(raw); #endif break; default: fprintf(stderr, "[hakmem] ERROR: Unknown allocation method: %d\n", hdr->method); break; } } done: #if HAKMEM_DEBUG_TIMING HKM_TIME_END(HKM_CAT_HAK_FREE, t0); #endif return; } #endif // HAK_FREE_API_INC_H