7975e243ee
MAJOR SUCCESS: HAKMEM now achieves 85-92% of System malloc on tiny allocations (128-512B) and BEATS System at 146% on 1024B allocations! Performance Results: - Random Mixed 128B: 21M → 59M ops/s (+181%) 🚀 - Random Mixed 256B: 19M → 70M ops/s (+268%) 🚀 - Random Mixed 512B: 21M → 68M ops/s (+224%) 🚀 - Random Mixed 1024B: 21M → 65M ops/s (+210%, 146% of System!) 🏆 - Larson 1T: 2.68M ops/s (stable, no regression) Implementation: 1. Task 3a: Remove profiling overhead in release builds - Wrapped RDTSC calls in #if !HAKMEM_BUILD_RELEASE - Compiler can eliminate profiling code completely - Effect: +2% (2.68M → 2.73M Larson) 2. Task 3b: Simplify refill logic - Use constants from hakmem_build_flags.h - TLS cache already optimal - Effect: No regression 3. Task 3c: Pre-warm TLS cache (GAME CHANGER!) - Pre-allocate 16 blocks per class at init - Eliminates cold-start penalty - Effect: +180-280% improvement 🚀 Root Cause: The bottleneck was cold-start, not the hot path! First allocation in each class triggered a SuperSlab refill (100+ cycles). Pre-warming eliminated this penalty, revealing Phase 7's true potential. Files Modified: - core/hakmem_tiny.c: Pre-warm function implementation - core/box/hak_core_init.inc.h: Pre-warm initialization call - core/tiny_alloc_fast.inc.h: Profiling overhead removal - core/hakmem_phase7_config.h: Task 3 constants (NEW) - core/hakmem_build_flags.h: Phase 7 feature flags - Makefile: PREWARM_TLS flag, phase7 targets - CLAUDE.md: Phase 7 success summary - PHASE7_TASK3_RESULTS.md: Comprehensive results report (NEW) Build: make HEADER_CLASSIDX=1 AGGRESSIVE_INLINE=1 PREWARM_TLS=1 phase7-bench 🎉 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
100 lines
3.4 KiB
C
100 lines
3.4 KiB
C
// tiny_alloc_fast_inline.h - Phase 7 Task 2: Aggressive inline TLS cache access
|
|
// Purpose: Eliminate function call overhead (5-10 cycles) in hot path
|
|
// Design: Macro-based inline expansion of TLS freelist operations
|
|
// Performance: Expected +10-15% (22M → 24-25M ops/s)
|
|
|
|
#ifndef TINY_ALLOC_FAST_INLINE_H
|
|
#define TINY_ALLOC_FAST_INLINE_H
|
|
|
|
#include <stddef.h>
|
|
#include "hakmem_build_flags.h"
|
|
|
|
// External TLS variables (defined in hakmem_tiny.c)
|
|
extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES];
|
|
extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES];
|
|
|
|
#ifndef TINY_NUM_CLASSES
|
|
#define TINY_NUM_CLASSES 8
|
|
#endif
|
|
|
|
// ========== Inline Macro: TLS Freelist Pop ==========
|
|
//
|
|
// Aggressive inline expansion of tiny_alloc_fast_pop()
|
|
// Saves: 5-10 cycles (function call overhead + register spilling)
|
|
//
|
|
// Assembly comparison (x86-64):
|
|
// Function call:
|
|
// push %rbx ; Save registers
|
|
// mov %edi, %ebx ; class_idx to %ebx
|
|
// call tiny_alloc_fast_pop ; Call (5-10 cycles overhead)
|
|
// pop %rbx ; Restore registers
|
|
// test %rax, %rax ; Check result
|
|
//
|
|
// Inline macro:
|
|
// mov g_tls_sll_head(%rdi), %rax ; Direct access (3-4 cycles)
|
|
// test %rax, %rax
|
|
// je .miss
|
|
// mov (%rax), %rdx
|
|
// mov %rdx, g_tls_sll_head(%rdi)
|
|
//
|
|
// Result: 5-10 fewer instructions, better register allocation
|
|
//
|
|
#define TINY_ALLOC_FAST_POP_INLINE(class_idx, ptr_out) do { \
|
|
void* _head = g_tls_sll_head[(class_idx)]; \
|
|
if (__builtin_expect(_head != NULL, 1)) { \
|
|
void* _next = *(void**)_head; \
|
|
g_tls_sll_head[(class_idx)] = _next; \
|
|
if (g_tls_sll_count[(class_idx)] > 0) { \
|
|
g_tls_sll_count[(class_idx)]--; \
|
|
} \
|
|
(ptr_out) = _head; \
|
|
} else { \
|
|
(ptr_out) = NULL; \
|
|
} \
|
|
} while(0)
|
|
|
|
// ========== Inline Macro: TLS Freelist Push ==========
|
|
//
|
|
// Aggressive inline expansion of tiny_alloc_fast_push()
|
|
// Saves: 5-10 cycles (function call overhead)
|
|
//
|
|
// Assembly comparison:
|
|
// Function call:
|
|
// mov %rdi, %rsi ; ptr to %rsi
|
|
// mov %ebx, %edi ; class_idx to %edi
|
|
// call tiny_alloc_fast_push ; Call (5-10 cycles)
|
|
//
|
|
// Inline macro:
|
|
// mov g_tls_sll_head(%rdi), %rax ; Direct inline (2-3 cycles)
|
|
// mov %rax, (%rsi)
|
|
// mov %rsi, g_tls_sll_head(%rdi)
|
|
//
|
|
#define TINY_ALLOC_FAST_PUSH_INLINE(class_idx, ptr) do { \
|
|
*(void**)(ptr) = g_tls_sll_head[(class_idx)]; \
|
|
g_tls_sll_head[(class_idx)] = (ptr); \
|
|
g_tls_sll_count[(class_idx)]++; \
|
|
} while(0)
|
|
|
|
// ========== Performance Notes ==========
|
|
//
|
|
// Benchmark results (expected):
|
|
// - Random Mixed 128B: 21M → 23M ops/s (+10%)
|
|
// - Random Mixed 256B: 19M → 22M ops/s (+15%)
|
|
// - Larson 1T: 2.7M → 3.0M ops/s (+11%)
|
|
//
|
|
// Key optimizations:
|
|
// 1. No function call overhead (save 5-10 cycles)
|
|
// 2. Better register allocation (inline knows full context)
|
|
// 3. No stack frame setup/teardown
|
|
// 4. Compiler can optimize across macro boundaries
|
|
//
|
|
// Trade-offs:
|
|
// 1. Code size: +100-200 bytes (each call site expanded)
|
|
// 2. Debug visibility: Macros harder to step through
|
|
// 3. Maintenance: Changes must be kept in sync with function version
|
|
//
|
|
// Recommendation: Use inline macros for CRITICAL hot paths only
|
|
// (alloc/free fast path), keep functions for diagnostics/debugging
|
|
|
|
#endif // TINY_ALLOC_FAST_INLINE_H
|