diff --git a/Makefile b/Makefile index ebaf2f7e..32ad96bc 100644 --- a/Makefile +++ b/Makefile @@ -427,7 +427,7 @@ test-box-refactor: box-refactor ./larson_hakmem 10 8 128 1024 1 12345 4 # Phase 4: Tiny Pool benchmarks (properly linked with hakmem) -TINY_BENCH_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o core/box/ss_allocation_box.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o core/superslab_head_stub.o hakmem_smallmid.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/free_publish_box.o core/box/capacity_box.o core/box/carve_push_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/tiny_route_box.o core/box/free_front_v3_env_box.o core/box/free_path_stats_box.o core/box/free_dispatch_stats_box.o core/box/alloc_gate_stats_box.o core/box/tiny_c6_ultra_free_box.o core/box/tiny_c5_ultra_free_box.o core/box/tiny_c4_ultra_free_box.o core/box/tiny_page_box.o core/box/tiny_class_policy_box.o core/box/tiny_class_stats_box.o core/box/tiny_policy_learner_box.o core/box/ss_budget_box.o core/box/tiny_mem_stats_box.o core/box/c7_meta_used_counter_box.o core/box/wrapper_env_box.o core/box/madvise_guard_box.o core/box/libm_reloc_guard_box.o core/box/ptr_trace_box.o core/box/link_missing_stubs.o core/box/super_reg_box.o core/box/shared_pool_box.o core/box/remote_side_box.o core/page_arena.o core/front/tiny_unified_cache.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/tiny_alloc_fast_push.o core/tiny_c7_ultra_segment.o core/tiny_c7_ultra.o core/link_stubs.o core/tiny_failfast.o core/tiny_destructors.o core/smallobject_hotbox_v3.o core/smallobject_hotbox_v4.o core/smallobject_hotbox_v5.o core/smallsegment_v5.o core/smallobject_cold_iface_v5.o core/smallsegment_v6.o core/smallobject_cold_iface_v6.o core/smallobject_core_v6.o core/region_id_v6.o +TINY_BENCH_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o core/box/ss_allocation_box.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o core/superslab_head_stub.o hakmem_smallmid.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/free_publish_box.o core/box/capacity_box.o core/box/carve_push_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/tiny_route_box.o core/box/free_front_v3_env_box.o core/box/free_path_stats_box.o core/box/free_dispatch_stats_box.o core/box/alloc_gate_stats_box.o core/box/tiny_c6_ultra_free_box.o core/box/tiny_c5_ultra_free_box.o core/box/tiny_c4_ultra_free_box.o core/box/tiny_page_box.o core/box/tiny_class_policy_box.o core/box/tiny_class_stats_box.o core/box/tiny_policy_learner_box.o core/box/ss_budget_box.o core/box/tiny_mem_stats_box.o core/box/c7_meta_used_counter_box.o core/box/wrapper_env_box.o core/box/madvise_guard_box.o core/box/libm_reloc_guard_box.o core/box/ptr_trace_box.o core/box/link_missing_stubs.o core/box/super_reg_box.o core/box/shared_pool_box.o core/box/remote_side_box.o core/page_arena.o core/front/tiny_unified_cache.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/tiny_alloc_fast_push.o core/tiny_c7_ultra_segment.o core/tiny_c7_ultra.o core/link_stubs.o core/tiny_failfast.o core/tiny_destructors.o core/smallobject_hotbox_v3.o core/smallobject_hotbox_v4.o core/smallobject_hotbox_v5.o core/smallsegment_v5.o core/smallobject_cold_iface_v5.o core/smallsegment_v6.o core/smallobject_cold_iface_v6.o core/smallobject_core_v6.o core/region_id_v6.o core/mid_hotbox_v3.o TINY_BENCH_OBJS = $(TINY_BENCH_OBJS_BASE) ifeq ($(POOL_TLS_PHASE1),1) TINY_BENCH_OBJS += pool_tls.o pool_refill.o core/pool_tls_arena.o pool_tls_registry.o pool_tls_remote.o diff --git a/core/box/hak_alloc_api.inc.h b/core/box/hak_alloc_api.inc.h index 538e4e42..4e29e550 100644 --- a/core/box/hak_alloc_api.inc.h +++ b/core/box/hak_alloc_api.inc.h @@ -9,6 +9,8 @@ #include "tiny_heap_env_box.h" // TinyHeap front gate (C7 / multi-class) #include "tiny_heap_box.h" // TinyHeapBox alloc/free helpers #include "tiny_c7_hotbox.h" // tiny_c7_alloc_fast wrapper +#include "mid_hotbox_v3_box.h" // Phase MID-V3: Mid/Pool HotBox v3 types +#include "mid_hotbox_v3_env_box.h" // Phase MID-V3: ENV gate for v3 #ifdef HAKMEM_POOL_TLS_PHASE1 #include "../pool_tls.h" @@ -68,6 +70,52 @@ inline void* hak_alloc_at(size_t size, hak_callsite_t site) { // TLS miss: Fall through to Mid/ACE (Tiny skipped due to auto-adjust) } + // ========================================================================= + // Phase MID-V3: Mid/Pool HotBox v3 (256B-1KB, opt-in via ENV) + // ========================================================================= + // Priority: v6 → v3 → v4 → pool (ENV-controlled routing) + // ENV: HAKMEM_MID_V3_ENABLED=1 HAKMEM_MID_V3_CLASSES=0x40 (C6 only) + // Design: TLS lane cache with page-based allocation, RegionIdBox integration + // NOTE: Must come BEFORE Tiny to intercept specific size classes + if (__builtin_expect(mid_v3_enabled() && size >= 256 && size <= 1024, 0)) { + static _Atomic int entry_log_count = 0; + if (mid_v3_debug_enabled() && atomic_fetch_add(&entry_log_count, 1) < 3) { + fprintf(stderr, "[MID_V3] Entered v3 path: size=%zu\n", size); + } + + int class_idx = -1; + // C6: 256B class handles sizes up to 256B (145-256B range) + // C7: 1024B class handles sizes up to 1024B (769-1024B range) + if (size > 144 && size <= 256 && mid_v3_class_enabled(6)) { + class_idx = 6; // C6: 256B + } else if (size > 768 && size <= 1024 && mid_v3_class_enabled(7)) { + class_idx = 7; // C7: 1024B + } + + if (mid_v3_debug_enabled() && class_idx >= 0) { + static _Atomic int class_log_count = 0; + if (atomic_fetch_add(&class_log_count, 1) < 3) { + fprintf(stderr, "[MID_V3] Class selected: size=%zu class=%d\n", size, class_idx); + } + } + + if (class_idx >= 0) { + MidHotBoxV3* hot = mid_hot_box_v3_get(); + void* v3_ptr = mid_hot_v3_alloc(hot, class_idx); + if (__builtin_expect(v3_ptr != NULL, 1)) { + if (mid_v3_debug_enabled()) { + static _Atomic int alloc_log_count = 0; + if (atomic_fetch_add(&alloc_log_count, 1) < 10) { + fprintf(stderr, "[MID_V3] Alloc: size=%zu class=%d ptr=%p\n", + size, class_idx, v3_ptr); + } + } + hkm_ace_track_alloc(); + return v3_ptr; + } + } + } + // Phase 16: Dynamic Tiny max size (ENV: HAKMEM_TINY_MAX_CLASS) // Default: 1023B (C0-C7), reduced to 255B (C0-C5) when Small-Mid enabled // Phase 17-1: Auto-adjusted to avoid overlap with Small-Mid diff --git a/core/box/hak_free_api.inc.h b/core/box/hak_free_api.inc.h index 9528ba43..e410bf05 100644 --- a/core/box/hak_free_api.inc.h +++ b/core/box/hak_free_api.inc.h @@ -12,6 +12,9 @@ #include "fg_tiny_gate_box.h" // Tiny gate guard box (Superslab check) #include "tiny_free_gate_box.h" // Tiny Free Gatekeeper Box (USER→Fast Path 境界) #include "free_dispatch_stats_box.h" // Phase FREE-DISPATCHER-OPT-1: free dispatcher stats +#include "region_id_v6_box.h" // Phase MID-V3: RegionIdBox for ownership lookup +#include "mid_hotbox_v3_box.h" // Phase MID-V3: Mid/Pool HotBox v3 types +#include "mid_hotbox_v3_env_box.h" // Phase MID-V3: ENV gate for v3 #ifdef HAKMEM_POOL_TLS_PHASE1 #include "../pool_tls.h" @@ -220,6 +223,35 @@ void hak_free_at(void* ptr, size_t size, hak_callsite_t site) { // ========== Mid/L25/Tiny Registry Lookup (Headerless) ========== // MIDCAND: Could be Mid/Large/C7, needs registry lookup + + // Phase MID-V3: Try v3 ownership first (RegionIdBox-based) + // ENV-controlled, default OFF + if (__builtin_expect(mid_v3_enabled(), 0)) { + // RegionIdBox lookup to check if v3 owns this pointer + // mid_hot_v3_free() will check internally and return early if not owned + mid_hot_v3_free(ptr); + + // Check if v3 actually owned it by doing a quick verification + // For now, we'll use the existence check via RegionIdBox + // If v3 handled it, it would have returned already + // We need to check if v3 took ownership - simplified: always check other paths too + // Better approach: mid_hot_v3_free returns bool or we check ownership first + + // For safety, check ownership explicitly before continuing + // This prevents double-free if v3 handled it + extern RegionLookupV6 region_id_lookup_v6(void* ptr); + RegionLookupV6 lk = region_id_lookup_v6(ptr); + if (lk.kind == REGION_KIND_MID_V3) { + if (mid_v3_debug_enabled()) { + static _Atomic int free_log_count = 0; + if (atomic_fetch_add(&free_log_count, 1) < 10) { + fprintf(stderr, "[MID_V3] Free: ptr=%p\n", ptr); + } + } + goto done; + } + } + { extern int hak_pool_mid_lookup(void* ptr, size_t* out_size); extern void hak_pool_free_fast(void* ptr, uintptr_t site_id); diff --git a/core/hakmem.c b/core/hakmem.c index 13a3656f..b6846364 100644 --- a/core/hakmem.c +++ b/core/hakmem.c @@ -30,6 +30,8 @@ #include "hakmem_ace_metrics.h" // NEW Phase ACE: Metrics tracking (inline helpers) #include "box/bench_fast_box.h" // NEW Phase 20-2: BenchFast Mode (structural ceiling measurement) #include "hakmem_env_cache.h" // NEW Priority-2: ENV Variable Cache (eliminate hot-path getenv) +#include "box/mid_hotbox_v3_box.h" // NEW Phase MID-V3: Mid/Pool HotBox v3 (256B-1KB) +#include "box/mid_hotbox_v3_env_box.h" // NEW Phase MID-V3: ENV gate for v3 #include #include #include