d378ee11a0
- Implement Box BenchMeta pattern in bench_random_mixed.c (BENCH_META_CALLOC/FREE) - Add enhanced debug logging to external_guard_box.h (caller tracking, FG classification) - Document investigation in PHASE15_BUG_ANALYSIS.md Issue: Page-aligned MIDCAND pointer not in SuperSlab registry → ExternalGuard → crash Hypothesis: May be pre-existing SuperSlab bug (not Phase 15-specific) Next: Test in Phase 14-C to verify
258 lines
10 KiB
C
258 lines
10 KiB
C
// tiny_free_fast_v2.inc.h - Phase 7: Ultra-Fast Free Path (Header-based)
|
||
// Purpose: Eliminate SuperSlab lookup bottleneck (52.63% CPU → <5%)
|
||
// Design: Read class_idx from inline header (O(1), 2-3 cycles)
|
||
// Performance: 1.2M → 40-60M ops/s (30-50x improvement)
|
||
//
|
||
// Key Innovation: Smart Headers
|
||
// - 1-byte header before each block stores class_idx
|
||
// - Slab[0]: 0% overhead (reuses 960B wasted padding)
|
||
// - Other slabs: ~1.5% overhead (1 byte per block)
|
||
// - Total: <2% memory overhead for 30-50x speed gain
|
||
//
|
||
// Flow (3-5 instructions, 5-10 cycles):
|
||
// 1. Read class_idx from header (ptr-1) [1 instruction, 2-3 cycles]
|
||
// 2. Push to TLS freelist [2-3 instructions, 3-5 cycles]
|
||
// 3. Done! (No lookup, no validation, no atomic)
|
||
|
||
#pragma once
|
||
#include "tiny_region_id.h"
|
||
#include "hakmem_build_flags.h"
|
||
#include "hakmem_tiny_config.h" // For TINY_TLS_MAG_CAP, TINY_NUM_CLASSES
|
||
#include "box/tls_sll_box.h" // Box TLS-SLL API
|
||
#include "box/tls_sll_drain_box.h" // Box TLS-SLL Drain (Option B)
|
||
#include "hakmem_tiny_integrity.h" // PRIORITY 1-4: Corruption detection
|
||
#include "front/tiny_heap_v2.h" // Phase 13-B: TinyHeapV2 magazine supply
|
||
#include "front/tiny_ultra_hot.h" // Phase 14: TinyUltraHot C1/C2 ultra-fast path
|
||
|
||
// Phase 7: Header-based ultra-fast free
|
||
#if HAKMEM_TINY_HEADER_CLASSIDX
|
||
|
||
// External TLS variables (defined in hakmem_tiny.c)
|
||
extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES];
|
||
extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES];
|
||
extern int g_tls_sll_enable; // Honored for fast free: when 0, fall back to slow path
|
||
|
||
// External functions
|
||
extern void hak_tiny_free(void* ptr); // Fallback for non-header allocations
|
||
|
||
// ========== Ultra-Fast Free (Header-based) ==========
|
||
|
||
// Ultra-fast free for header-based allocations
|
||
// Returns: 1 if handled, 0 if needs slow path
|
||
//
|
||
// Performance: 3-5 instructions, 5-10 cycles
|
||
// vs Current: 330+ lines, 500+ cycles (100x faster!)
|
||
//
|
||
// Assembly (x86-64, release build):
|
||
// movzbl -0x1(%rdi),%eax # Read header (class_idx)
|
||
// mov g_tls_sll_head(,%rax,8),%rdx # Load head
|
||
// mov %rdx,(%rdi) # ptr->next = head
|
||
// mov %rdi,g_tls_sll_head(,%rax,8) # head = ptr
|
||
// addl $0x1,g_tls_sll_count(,%rax,4) # count++
|
||
// ret
|
||
//
|
||
// Expected: 3-5 instructions, 5-10 cycles (L1 hit)
|
||
static inline int hak_tiny_free_fast_v2(void* ptr) {
|
||
if (__builtin_expect(!ptr, 0)) return 0;
|
||
|
||
// Respect global SLL toggle: when disabled, do not use TLS SLL fast path.
|
||
if (__builtin_expect(!g_tls_sll_enable, 0)) {
|
||
return 0; // Force slow path
|
||
}
|
||
|
||
// Phase E3-1: Remove registry lookup (50-100 cycles overhead)
|
||
// Reason: Phase E1 added headers to C7, making this check redundant
|
||
// Header magic validation (2-3 cycles) is now sufficient for all classes
|
||
// Expected: 9M → 30-50M ops/s recovery (+226-443%)
|
||
|
||
// CRITICAL: Check if header is accessible before reading
|
||
void* header_addr = (char*)ptr - 1;
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
// Debug: Validate header accessibility (metadata-based check)
|
||
// Phase 9: mincore() REMOVED - no syscall overhead (0 cycles)
|
||
// Strategy: Trust internal metadata (registry ensures memory is valid)
|
||
// Benefit: Catch invalid pointers via header magic validation below
|
||
extern int hak_is_memory_readable(void* addr);
|
||
if (!hak_is_memory_readable(header_addr)) {
|
||
return 0; // Header not accessible - not a Tiny allocation
|
||
}
|
||
#else
|
||
// Release: Phase 9 optimization - mincore() completely removed
|
||
// OLD: Page boundary check + mincore() syscall (~634 cycles)
|
||
// NEW: No check needed - trust internal metadata (0 cycles)
|
||
// Safety: Header magic validation below catches invalid pointers
|
||
// Performance: 841 syscalls → 0 (100% elimination)
|
||
// (Page boundary check removed - adds 1-2 cycles without benefit)
|
||
#endif
|
||
|
||
// 1. Read class_idx from header (2-3 cycles, L1 hit)
|
||
// Note: In release mode, tiny_region_id_read_header() skips magic validation (saves 2-3 cycles)
|
||
#if HAKMEM_DEBUG_VERBOSE
|
||
static _Atomic int debug_calls = 0;
|
||
if (atomic_fetch_add(&debug_calls, 1) < 5) {
|
||
fprintf(stderr, "[TINY_FREE_V2] Before read_header, ptr=%p\n", ptr);
|
||
}
|
||
#endif
|
||
int class_idx = tiny_region_id_read_header(ptr);
|
||
#if HAKMEM_DEBUG_VERBOSE
|
||
if (atomic_load(&debug_calls) <= 5) {
|
||
fprintf(stderr, "[TINY_FREE_V2] After read_header, class_idx=%d\n", class_idx);
|
||
}
|
||
#endif
|
||
|
||
// Check if header read failed (invalid magic in debug, or out-of-bounds class_idx)
|
||
if (__builtin_expect(class_idx < 0, 0)) {
|
||
// Invalid header - route to slow path (non-header allocation or corrupted header)
|
||
return 0;
|
||
}
|
||
|
||
// PRIORITY 1: Bounds check on class_idx from header
|
||
if (__builtin_expect(class_idx >= TINY_NUM_CLASSES, 0)) {
|
||
fprintf(stderr, "[TINY_FREE_V2] FATAL: class_idx=%d out of bounds (from header at %p)\n",
|
||
class_idx, ptr);
|
||
fflush(stderr);
|
||
assert(0 && "class_idx from header out of bounds");
|
||
return 0;
|
||
}
|
||
atomic_fetch_add(&g_integrity_check_class_bounds, 1);
|
||
|
||
// 2. Check TLS freelist capacity (defense in depth - ALWAYS ENABLED)
|
||
// CRITICAL: Enable in both debug and release to prevent corruption accumulation
|
||
// Reason: If C7 slips through magic validation, capacity limit prevents unbounded growth
|
||
// Cost: 1 comparison (~1 cycle, predict-not-taken)
|
||
// Benefit: Fail-safe against TLS SLL pollution from false positives
|
||
uint32_t cap = (uint32_t)TINY_TLS_MAG_CAP;
|
||
if (__builtin_expect(g_tls_sll_count[class_idx] >= cap, 0)) {
|
||
return 0; // Route to slow path for spill (Front Gate will catch corruption)
|
||
}
|
||
|
||
// 3. Push base to TLS freelist (4 instructions, 5-7 cycles)
|
||
// Must push base (block start) not user pointer!
|
||
// Phase E1: ALL classes (C0-C7) have 1-byte header → base = ptr-1
|
||
void* base = (char*)ptr - 1;
|
||
|
||
// Phase 14-C: UltraHot は free 時に横取りしない（Borrowing 設計）
|
||
// → 正史（TLS SLL）の在庫を正しく保つ
|
||
// → UltraHot refill は alloc 側で TLS SLL から借りる
|
||
|
||
// Phase 13-B: TinyHeapV2 magazine supply (C0-C3 only)
|
||
// Two supply modes (controlled by HAKMEM_TINY_HEAP_V2_LEFTOVER_MODE):
|
||
// Mode 0 (default): L0 gets blocks first ("stealing" design)
|
||
// Mode 1: L1 primary owner, L0 gets leftovers (ChatGPT recommended design)
|
||
if (class_idx <= 3 && tiny_heap_v2_enabled() && !tiny_heap_v2_leftover_mode()) {
|
||
// Mode 0: Try to supply to magazine first (L0 cache, faster than TLS SLL)
|
||
// Falls back to TLS SLL if magazine is full
|
||
if (tiny_heap_v2_try_push(class_idx, base)) {
|
||
// Successfully supplied to magazine
|
||
return 1;
|
||
}
|
||
// Magazine full → fall through to TLS SLL
|
||
}
|
||
|
||
// REVERT E3-2: Use Box TLS-SLL for all builds (testing hypothesis)
|
||
// Hypothesis: Box TLS-SLL acts as verification layer, masking underlying bugs
|
||
if (!tls_sll_push(class_idx, base, UINT32_MAX)) {
|
||
// C7 rejected or capacity exceeded - route to slow path
|
||
return 0;
|
||
}
|
||
|
||
// Phase 13-B: Leftover mode - L0 gets leftovers from L1
|
||
// Mode 1: L1 (TLS SLL) is primary owner, L0 (magazine) gets leftovers
|
||
// Only refill L0 if it's empty (don't reduce L1 capacity)
|
||
if (class_idx <= 3 && tiny_heap_v2_enabled() && tiny_heap_v2_leftover_mode()) {
|
||
TinyHeapV2Mag* mag = &g_tiny_heap_v2_mag[class_idx];
|
||
if (mag->top == 0) { // Only refill if magazine is empty
|
||
void* leftover;
|
||
if (tls_sll_pop(class_idx, &leftover)) {
|
||
mag->items[mag->top++] = leftover;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Option B: Periodic TLS SLL Drain (restore slab accounting consistency)
|
||
// Purpose: Every N frees (default: 1024), drain TLS SLL → slab freelist
|
||
// Impact: Enables empty detection → SuperSlabs freed → LRU cache functional
|
||
// Cost: 2-3 cycles (counter increment + comparison, predict-not-taken)
|
||
// Benefit: +1,300-1,700% throughput (563K → 8-10M ops/s expected)
|
||
tiny_tls_sll_try_drain(class_idx);
|
||
|
||
return 1; // Success - handled in fast path
|
||
}
|
||
|
||
// ========== Free Entry Point ==========
|
||
|
||
// Entry point for free() - tries fast path first, falls back to slow path
|
||
//
|
||
// Flow:
|
||
// 1. Try ultra-fast free (header-based) → 95-99% hit rate
|
||
// 2. Miss → Fallback to slow path → 1-5% (non-header, cache full)
|
||
//
|
||
// Performance:
|
||
// - Fast path: 5-10 cycles (header read + TLS push)
|
||
// - Slow path: 500+ cycles (SuperSlab lookup + validation)
|
||
// - Weighted average: ~10-30 cycles (vs 500+ current)
|
||
static inline void hak_free_fast_v2_entry(void* ptr) {
|
||
// Try ultra-fast free (header-based)
|
||
if (__builtin_expect(hak_tiny_free_fast_v2(ptr), 1)) {
|
||
return; // Success - done in 5-10 cycles!
|
||
}
|
||
|
||
// Slow path: Non-header allocation or TLS cache full
|
||
hak_tiny_free(ptr);
|
||
}
|
||
|
||
// ========== Performance Counters (Debug) ==========
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
// Performance counters (TLS, lightweight)
|
||
static __thread uint64_t g_free_v2_fast_hits = 0;
|
||
static __thread uint64_t g_free_v2_slow_hits = 0;
|
||
|
||
// Track fast path hit rate
|
||
static inline void hak_free_v2_track_fast(void) {
|
||
g_free_v2_fast_hits++;
|
||
}
|
||
|
||
static inline void hak_free_v2_track_slow(void) {
|
||
g_free_v2_slow_hits++;
|
||
}
|
||
|
||
// Print stats at exit
|
||
static void hak_free_v2_print_stats(void) __attribute__((destructor));
|
||
static void hak_free_v2_print_stats(void) {
|
||
uint64_t total = g_free_v2_fast_hits + g_free_v2_slow_hits;
|
||
if (total == 0) return;
|
||
|
||
double hit_rate = (double)g_free_v2_fast_hits / total * 100.0;
|
||
fprintf(stderr, "[FREE_V2] Fast hits: %lu, Slow hits: %lu, Hit rate: %.2f%%\n",
|
||
g_free_v2_fast_hits, g_free_v2_slow_hits, hit_rate);
|
||
}
|
||
#else
|
||
// Release: No tracking overhead
|
||
static inline void hak_free_v2_track_fast(void) {}
|
||
static inline void hak_free_v2_track_slow(void) {}
|
||
#endif
|
||
|
||
// ========== Benchmark Comparison ==========
|
||
//
|
||
// Current (hak_tiny_free_superslab):
|
||
// - 2x SuperSlab lookup: 200+ cycles
|
||
// - Safety checks (O(n) duplicate scan): 100+ cycles
|
||
// - Validation, atomics, diagnostics: 200+ cycles
|
||
// - Total: 500+ cycles
|
||
// - Throughput: 1.2M ops/s
|
||
//
|
||
// Phase 7 (hak_tiny_free_fast_v2):
|
||
// - Header read: 2-3 cycles
|
||
// - TLS push: 3-5 cycles
|
||
// - Total: 5-10 cycles (100x faster!)
|
||
// - Throughput: 40-60M ops/s (30-50x improvement)
|
||
//
|
||
// vs System malloc tcache:
|
||
// - System: 10-15 cycles (3-4 instructions)
|
||
// - HAKMEM: 5-10 cycles (3-5 instructions)
|
||
// - Result: 70-110% of System speed (互角〜勝ち!)
|
||
|
||
#endif // HAKMEM_TINY_HEADER_CLASSIDX
|