diff --git a/Makefile b/Makefile index fb69259b..d709852d 100644 --- a/Makefile +++ b/Makefile @@ -219,12 +219,12 @@ LDFLAGS += $(EXTRA_LDFLAGS) # Targets TARGET = test_hakmem -OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/capacity_box.o core/box/carve_push_box.o core/box/unified_batch_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o test_hakmem.o +OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/capacity_box.o core/box/carve_push_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o test_hakmem.o OBJS = $(OBJS_BASE) # Shared library SHARED_LIB = libhakmem.so -SHARED_OBJS = hakmem_shared.o hakmem_config_shared.o hakmem_tiny_config_shared.o hakmem_ucb1_shared.o hakmem_bigcache_shared.o hakmem_pool_shared.o hakmem_l25_pool_shared.o hakmem_site_rules_shared.o hakmem_tiny_shared.o superslab_allocate_shared.o superslab_stats_shared.o superslab_cache_shared.o superslab_ace_shared.o superslab_slab_shared.o superslab_backend_shared.o superslab_head_shared.o hakmem_smallmid_shared.o hakmem_smallmid_superslab_shared.o core/box/superslab_expansion_box_shared.o core/box/integrity_box_shared.o core/box/mailbox_box_shared.o core/box/front_gate_box_shared.o core/box/front_gate_classifier_shared.o core/box/free_publish_box_shared.o core/box/capacity_box_shared.o core/box/carve_push_box_shared.o core/box/unified_batch_box_shared.o core/box/prewarm_box_shared.o core/box/ss_hot_prewarm_box_shared.o core/box/front_metrics_box_shared.o core/box/bench_fast_box_shared.o core/box/ss_addr_map_box_shared.o core/box/slab_recycling_box_shared.o core/box/pagefault_telemetry_box_shared.o core/box/tiny_sizeclass_hist_box_shared.o core/box/tiny_env_box_shared.o core/box/wrapper_env_box_shared.o core/page_arena_shared.o core/front/tiny_unified_cache_shared.o core/tiny_alloc_fast_push_shared.o core/link_stubs_shared.o core/tiny_failfast_shared.o tiny_sticky_shared.o tiny_remote_shared.o tiny_publish_shared.o tiny_debug_ring_shared.o hakmem_tiny_magazine_shared.o hakmem_tiny_stats_shared.o hakmem_tiny_sfc_shared.o hakmem_tiny_query_shared.o hakmem_tiny_rss_shared.o hakmem_tiny_registry_shared.o hakmem_tiny_remote_target_shared.o hakmem_tiny_bg_spill_shared.o tiny_adaptive_sizing_shared.o hakmem_super_registry_shared.o hakmem_shared_pool_shared.o hakmem_shared_pool_acquire_shared.o hakmem_shared_pool_release_shared.o hakmem_elo_shared.o hakmem_batch_shared.o hakmem_p2_shared.o hakmem_sizeclass_dist_shared.o hakmem_evo_shared.o hakmem_debug_shared.o hakmem_sys_shared.o hakmem_whale_shared.o hakmem_policy_shared.o hakmem_ace_shared.o hakmem_ace_stats_shared.o hakmem_ace_controller_shared.o hakmem_ace_metrics_shared.o hakmem_ace_ucb1_shared.o hakmem_prof_shared.o hakmem_learner_shared.o hakmem_size_hist_shared.o hakmem_learn_log_shared.o hakmem_syscall_shared.o tiny_fastcache_shared.o +SHARED_OBJS = hakmem_shared.o hakmem_config_shared.o hakmem_tiny_config_shared.o hakmem_ucb1_shared.o hakmem_bigcache_shared.o hakmem_pool_shared.o hakmem_l25_pool_shared.o hakmem_site_rules_shared.o hakmem_tiny_shared.o superslab_allocate_shared.o superslab_stats_shared.o superslab_cache_shared.o superslab_ace_shared.o superslab_slab_shared.o superslab_backend_shared.o superslab_head_shared.o hakmem_smallmid_shared.o hakmem_smallmid_superslab_shared.o core/box/superslab_expansion_box_shared.o core/box/integrity_box_shared.o core/box/mailbox_box_shared.o core/box/front_gate_box_shared.o core/box/front_gate_classifier_shared.o core/box/free_publish_box_shared.o core/box/capacity_box_shared.o core/box/carve_push_box_shared.o core/box/prewarm_box_shared.o core/box/ss_hot_prewarm_box_shared.o core/box/front_metrics_box_shared.o core/box/bench_fast_box_shared.o core/box/ss_addr_map_box_shared.o core/box/slab_recycling_box_shared.o core/box/pagefault_telemetry_box_shared.o core/box/tiny_sizeclass_hist_box_shared.o core/box/tiny_env_box_shared.o core/box/wrapper_env_box_shared.o core/page_arena_shared.o core/front/tiny_unified_cache_shared.o core/tiny_alloc_fast_push_shared.o core/link_stubs_shared.o core/tiny_failfast_shared.o tiny_sticky_shared.o tiny_remote_shared.o tiny_publish_shared.o tiny_debug_ring_shared.o hakmem_tiny_magazine_shared.o hakmem_tiny_stats_shared.o hakmem_tiny_sfc_shared.o hakmem_tiny_query_shared.o hakmem_tiny_rss_shared.o hakmem_tiny_registry_shared.o hakmem_tiny_remote_target_shared.o hakmem_tiny_bg_spill_shared.o tiny_adaptive_sizing_shared.o hakmem_super_registry_shared.o hakmem_shared_pool_shared.o hakmem_shared_pool_acquire_shared.o hakmem_shared_pool_release_shared.o hakmem_elo_shared.o hakmem_batch_shared.o hakmem_p2_shared.o hakmem_sizeclass_dist_shared.o hakmem_evo_shared.o hakmem_debug_shared.o hakmem_sys_shared.o hakmem_whale_shared.o hakmem_policy_shared.o hakmem_ace_shared.o hakmem_ace_stats_shared.o hakmem_ace_controller_shared.o hakmem_ace_metrics_shared.o hakmem_ace_ucb1_shared.o hakmem_prof_shared.o hakmem_learner_shared.o hakmem_size_hist_shared.o hakmem_learn_log_shared.o hakmem_syscall_shared.o tiny_fastcache_shared.o # Pool TLS Phase 1 (enable with POOL_TLS_PHASE1=1) ifeq ($(POOL_TLS_PHASE1),1) @@ -251,7 +251,7 @@ endif # Benchmark targets BENCH_HAKMEM = bench_allocators_hakmem BENCH_SYSTEM = bench_allocators_system -BENCH_HAKMEM_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/capacity_box.o core/box/carve_push_box.o core/box/unified_batch_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o bench_allocators_hakmem.o +BENCH_HAKMEM_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/capacity_box.o core/box/carve_push_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o bench_allocators_hakmem.o BENCH_HAKMEM_OBJS = $(BENCH_HAKMEM_OBJS_BASE) ifeq ($(POOL_TLS_PHASE1),1) BENCH_HAKMEM_OBJS += pool_tls.o pool_refill.o pool_tls_arena.o pool_tls_registry.o pool_tls_remote.o @@ -428,7 +428,7 @@ test-box-refactor: box-refactor ./larson_hakmem 10 8 128 1024 1 12345 4 # Phase 4: Tiny Pool benchmarks (properly linked with hakmem) -TINY_BENCH_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/free_publish_box.o core/box/capacity_box.o core/box/carve_push_box.o core/box/unified_batch_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/tiny_sizeclass_hist_box.o core/box/pagefault_telemetry_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o +TINY_BENCH_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/free_publish_box.o core/box/capacity_box.o core/box/carve_push_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/slab_recycling_box.o core/box/tiny_sizeclass_hist_box.o core/box/pagefault_telemetry_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o TINY_BENCH_OBJS = $(TINY_BENCH_OBJS_BASE) ifeq ($(POOL_TLS_PHASE1),1) TINY_BENCH_OBJS += pool_tls.o pool_refill.o core/pool_tls_arena.o pool_tls_registry.o pool_tls_remote.o diff --git a/core/box/carve_push_box.d b/core/box/carve_push_box.d index e6b6c644..80581244 100644 --- a/core/box/carve_push_box.d +++ b/core/box/carve_push_box.d @@ -29,11 +29,17 @@ core/box/carve_push_box.o: core/box/carve_push_box.c \ core/tiny_region_id.h core/superslab/superslab_inline.h \ core/box/tiny_layout_box.h core/box/tiny_header_box.h \ core/box/tiny_layout_box.h core/box/../tiny_debug_ring.h \ - core/box/../superslab/superslab_inline.h core/box/tiny_header_box.h \ - core/box/../tiny_refill_opt.h core/box/../tiny_region_id.h \ - core/box/../box/slab_freelist_atomic.h \ + core/box/ss_addr_map_box.h core/box/../superslab/superslab_inline.h \ + core/box/tiny_ptr_bridge_box.h \ + core/box/../hakmem_tiny_superslab_internal.h \ + core/box/../box/ss_hot_cold_box.h \ core/box/../box/../superslab/superslab_types.h \ - core/box/../hakmem_debug_master.h core/box/../box/tls_sll_box.h \ + core/box/../box/ss_allocation_box.h core/hakmem_tiny_superslab.h \ + core/box/../hakmem_debug_master.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_shared_pool.h core/box/../hakmem_internal.h \ + core/box/../tiny_region_id.h core/box/../hakmem_tiny_integrity.h \ + core/box/../box/slab_freelist_atomic.h core/box/tiny_header_box.h \ + core/box/../tiny_refill_opt.h core/box/../box/tls_sll_box.h \ core/box/../tiny_box_geometry.h core/box/../hakmem_tiny.h: core/box/../hakmem_build_flags.h: @@ -88,12 +94,22 @@ core/box/tiny_layout_box.h: core/box/tiny_header_box.h: core/box/tiny_layout_box.h: core/box/../tiny_debug_ring.h: +core/box/ss_addr_map_box.h: core/box/../superslab/superslab_inline.h: +core/box/tiny_ptr_bridge_box.h: +core/box/../hakmem_tiny_superslab_internal.h: +core/box/../box/ss_hot_cold_box.h: +core/box/../box/../superslab/superslab_types.h: +core/box/../box/ss_allocation_box.h: +core/hakmem_tiny_superslab.h: +core/box/../hakmem_debug_master.h: +core/box/../hakmem_tiny_config.h: +core/box/../hakmem_shared_pool.h: +core/box/../hakmem_internal.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_tiny_integrity.h: +core/box/../box/slab_freelist_atomic.h: core/box/tiny_header_box.h: core/box/../tiny_refill_opt.h: -core/box/../tiny_region_id.h: -core/box/../box/slab_freelist_atomic.h: -core/box/../box/../superslab/superslab_types.h: -core/box/../hakmem_debug_master.h: core/box/../box/tls_sll_box.h: core/box/../tiny_box_geometry.h: diff --git a/core/box/front_gate_box.d b/core/box/front_gate_box.d index 312850eb..d82b6114 100644 --- a/core/box/front_gate_box.d +++ b/core/box/front_gate_box.d @@ -22,9 +22,18 @@ core/box/front_gate_box.o: core/box/front_gate_box.c \ core/box/../hakmem_debug_master.h core/box/../tiny_remote.h \ core/box/../hakmem_tiny_integrity.h core/box/../hakmem_tiny.h \ core/box/../ptr_track.h core/box/../ptr_trace.h \ - core/box/../tiny_debug_ring.h core/box/../superslab/superslab_inline.h \ - core/box/tiny_header_box.h core/box/ptr_conversion_box.h \ - core/box/ptr_type_box.h + core/box/../tiny_debug_ring.h core/box/ss_addr_map_box.h \ + core/box/../superslab/superslab_inline.h core/box/tiny_ptr_bridge_box.h \ + core/box/../hakmem_tiny_superslab_internal.h \ + core/box/../hakmem_tiny_superslab.h core/box/../box/ss_hot_cold_box.h \ + core/box/../box/../superslab/superslab_types.h \ + core/box/../box/ss_allocation_box.h core/hakmem_tiny_superslab.h \ + core/box/../hakmem_debug_master.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_shared_pool.h core/box/../superslab/superslab_types.h \ + core/box/../hakmem_internal.h core/box/../tiny_region_id.h \ + core/box/../hakmem_tiny_integrity.h \ + core/box/../box/slab_freelist_atomic.h core/box/tiny_header_box.h \ + core/box/ptr_conversion_box.h core/box/ptr_type_box.h core/box/front_gate_box.h: core/hakmem_tiny.h: core/hakmem_build_flags.h: @@ -75,7 +84,23 @@ core/box/../hakmem_tiny.h: core/box/../ptr_track.h: core/box/../ptr_trace.h: core/box/../tiny_debug_ring.h: +core/box/ss_addr_map_box.h: core/box/../superslab/superslab_inline.h: +core/box/tiny_ptr_bridge_box.h: +core/box/../hakmem_tiny_superslab_internal.h: +core/box/../hakmem_tiny_superslab.h: +core/box/../box/ss_hot_cold_box.h: +core/box/../box/../superslab/superslab_types.h: +core/box/../box/ss_allocation_box.h: +core/hakmem_tiny_superslab.h: +core/box/../hakmem_debug_master.h: +core/box/../hakmem_tiny_config.h: +core/box/../hakmem_shared_pool.h: +core/box/../superslab/superslab_types.h: +core/box/../hakmem_internal.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_tiny_integrity.h: +core/box/../box/slab_freelist_atomic.h: core/box/tiny_header_box.h: core/box/ptr_conversion_box.h: core/box/ptr_type_box.h: diff --git a/core/box/front_gate_classifier.d b/core/box/front_gate_classifier.d index a5d92954..d249c6bf 100644 --- a/core/box/front_gate_classifier.d +++ b/core/box/front_gate_classifier.d @@ -19,7 +19,8 @@ core/box/front_gate_classifier.o: core/box/front_gate_classifier.c \ core/box/../hakmem_build_flags.h core/box/../hakmem_internal.h \ core/box/../hakmem.h core/box/../hakmem_config.h \ core/box/../hakmem_features.h core/box/../hakmem_sys.h \ - core/box/../hakmem_whale.h core/box/../hakmem_tiny_config.h + core/box/../hakmem_whale.h core/box/../hakmem_tiny_config.h \ + core/box/../pool_tls_registry.h core/box/front_gate_classifier.h: core/box/../tiny_region_id.h: core/box/../hakmem_build_flags.h: @@ -54,3 +55,4 @@ core/box/../hakmem_features.h: core/box/../hakmem_sys.h: core/box/../hakmem_whale.h: core/box/../hakmem_tiny_config.h: +core/box/../pool_tls_registry.h: diff --git a/core/box/unified_batch_box.c b/core/box/unified_batch_box.c deleted file mode 100644 index 0fe27fec..00000000 --- a/core/box/unified_batch_box.c +++ /dev/null @@ -1,26 +0,0 @@ -// unified_batch_box.c - Box U2: Batch Alloc Connector Implementation -#include "unified_batch_box.h" -#include "carve_push_box.h" -#include "../box/tls_sll_box.h" -#include - -// Batch allocate blocks from SuperSlab -// Returns: Actual count allocated (0 = failed) -int superslab_batch_alloc(int class_idx, void** blocks, int max_count) { - if (!blocks || max_count <= 0) return 0; - - // Step 1: Carve N blocks from SuperSlab and push to TLS SLL - // (uses existing Box C1 carve_push logic) - uint32_t carved = box_carve_and_push_with_freelist(class_idx, (uint32_t)max_count); - if (carved == 0) return 0; - - // Step 2: Pop carved blocks from TLS SLL into output array - int got = 0; - for (uint32_t i = 0; i < carved; i++) { - void* base; - if (!tls_sll_pop(class_idx, &base)) break; // Should not happen - blocks[got++] = base; - } - - return got; -} diff --git a/core/box/unified_batch_box.d b/core/box/unified_batch_box.d index 2ae78582..a49fe6b0 100644 --- a/core/box/unified_batch_box.d +++ b/core/box/unified_batch_box.d @@ -34,7 +34,18 @@ core/box/unified_batch_box.o: core/box/unified_batch_box.c \ core/box/../hakmem_tiny_config.h core/box/tiny_header_box.h \ core/box/../hakmem_build_flags.h core/box/tiny_layout_box.h \ core/box/../tiny_region_id.h core/box/../box/../tiny_debug_ring.h \ + core/box/../box/ss_addr_map_box.h \ core/box/../box/../superslab/superslab_inline.h \ + core/box/../box/tiny_ptr_bridge_box.h \ + core/box/../box/../hakmem_tiny_superslab_internal.h \ + core/box/../box/../box/ss_hot_cold_box.h \ + core/box/../box/../box/../superslab/superslab_types.h \ + core/box/../box/../box/ss_allocation_box.h core/hakmem_tiny_superslab.h \ + core/box/../box/../hakmem_debug_master.h \ + core/box/../box/../hakmem_shared_pool.h \ + core/box/../box/../hakmem_internal.h core/box/../box/../tiny_region_id.h \ + core/box/../box/../hakmem_tiny_integrity.h \ + core/box/../box/../box/slab_freelist_atomic.h \ core/box/../box/tiny_header_box.h core/box/unified_batch_box.h: core/box/carve_push_box.h: @@ -88,5 +99,18 @@ core/box/../hakmem_build_flags.h: core/box/tiny_layout_box.h: core/box/../tiny_region_id.h: core/box/../box/../tiny_debug_ring.h: +core/box/../box/ss_addr_map_box.h: core/box/../box/../superslab/superslab_inline.h: +core/box/../box/tiny_ptr_bridge_box.h: +core/box/../box/../hakmem_tiny_superslab_internal.h: +core/box/../box/../box/ss_hot_cold_box.h: +core/box/../box/../box/../superslab/superslab_types.h: +core/box/../box/../box/ss_allocation_box.h: +core/hakmem_tiny_superslab.h: +core/box/../box/../hakmem_debug_master.h: +core/box/../box/../hakmem_shared_pool.h: +core/box/../box/../hakmem_internal.h: +core/box/../box/../tiny_region_id.h: +core/box/../box/../hakmem_tiny_integrity.h: +core/box/../box/../box/slab_freelist_atomic.h: core/box/../box/tiny_header_box.h: diff --git a/core/box/unified_batch_box.h b/core/box/unified_batch_box.h deleted file mode 100644 index c8736d89..00000000 --- a/core/box/unified_batch_box.h +++ /dev/null @@ -1,29 +0,0 @@ -// unified_batch_box.h - Box U2: Batch Alloc Connector for Unified Cache -// -// Purpose: Provide batch allocation API for Unified Frontend Cache (Box U1) -// Design: Thin wrapper over existing Box flow (Carve/Push Box C1) -// -// API: -// int superslab_batch_alloc(int class_idx, void** blocks, int max_count) -// - Allocates up to max_count blocks from SuperSlab -// - Returns actual count allocated -// - blocks[] receives BASE pointers (caller converts to USER) -// -// Box Theory: -// - Box U2 (this) = Connector layer (no state, pure function) -// - Box U1 (Unified Cache) calls this for batch refill -// - This delegates to Box C1 (Carve/Push) for actual allocation -// -// ENV: None (controlled by caller Box U1) - -#ifndef HAK_BOX_UNIFIED_BATCH_BOX_H -#define HAK_BOX_UNIFIED_BATCH_BOX_H - -#include - -// Batch allocate blocks from SuperSlab (for Unified Cache refill) -// Returns: Actual count allocated (0 = failed) -// Note: blocks[] contains BASE pointers (not USER pointers) -int superslab_batch_alloc(int class_idx, void** blocks, int max_count); - -#endif // HAK_BOX_UNIFIED_BATCH_BOX_H diff --git a/core/front/tiny_unified_cache.c b/core/front/tiny_unified_cache.c index 2cb79ad3..9f7b6daa 100644 --- a/core/front/tiny_unified_cache.c +++ b/core/front/tiny_unified_cache.c @@ -1,6 +1,5 @@ // tiny_unified_cache.c - Phase 23: Unified Frontend Cache Implementation #include "tiny_unified_cache.h" -#include "../box/unified_batch_box.h" // Phase 23-D: Box U2 batch alloc (deprecated in 23-E) #include "../tiny_tls.h" // Phase 23-E: TinyTLSSlab, TinySlabMeta #include "../tiny_box_geometry.h" // Phase 23-E: tiny_stride_for_class, tiny_slab_base_for_geometry #include "../box/tiny_next_ptr_box.h" // Phase 23-E: tiny_next_read (freelist traversal) diff --git a/core/front/tiny_unified_cache.d b/core/front/tiny_unified_cache.d index 8281258b..566c58f9 100644 --- a/core/front/tiny_unified_cache.d +++ b/core/front/tiny_unified_cache.d @@ -1,9 +1,8 @@ core/front/tiny_unified_cache.o: core/front/tiny_unified_cache.c \ core/front/tiny_unified_cache.h core/front/../hakmem_build_flags.h \ - core/front/../hakmem_tiny_config.h \ + core/front/../hakmem_tiny_config.h core/front/../box/ptr_type_box.h \ core/front/../box/tiny_front_config_box.h \ - core/front/../box/../hakmem_build_flags.h \ - core/front/../box/unified_batch_box.h core/front/../tiny_tls.h \ + core/front/../box/../hakmem_build_flags.h core/front/../tiny_tls.h \ core/front/../hakmem_tiny_superslab.h \ core/front/../superslab/superslab_types.h \ core/hakmem_tiny_superslab_constants.h \ @@ -32,9 +31,9 @@ core/front/tiny_unified_cache.o: core/front/tiny_unified_cache.c \ core/front/tiny_unified_cache.h: core/front/../hakmem_build_flags.h: core/front/../hakmem_tiny_config.h: +core/front/../box/ptr_type_box.h: core/front/../box/tiny_front_config_box.h: core/front/../box/../hakmem_build_flags.h: -core/front/../box/unified_batch_box.h: core/front/../tiny_tls.h: core/front/../hakmem_tiny_superslab.h: core/front/../superslab/superslab_types.h: diff --git a/core/hakmem_tiny_phase6_wrappers_box.inc b/core/hakmem_tiny_phase6_wrappers_box.inc index e13dd391..4bed7e85 100644 --- a/core/hakmem_tiny_phase6_wrappers_box.inc +++ b/core/hakmem_tiny_phase6_wrappers_box.inc @@ -10,9 +10,6 @@ // Box 5: Allocation Fast Path (Layer 1 - 3-4 instructions) #include "tiny_alloc_fast.inc.h" - // Box 6: Free Fast Path (Layer 2 - 2-3 instructions) - #include "tiny_free_fast.inc.h" - // ---------------- Refill count (Front) global config ---------------- // Parsed once at init; hot path reads plain ints (no getenv). int g_refill_count_global = 0; // HAKMEM_TINY_REFILL_COUNT @@ -82,19 +79,15 @@ } void hak_tiny_free_fast_wrapper(void* ptr) { - // Phase E5: Ultra fast path (6-8 instruction free) - #if HAKMEM_ULTRA_FAST_PATH - tiny_free_fast_ultra(ptr); - return; - #endif - static _Atomic uint64_t free_call_count = 0; uint64_t call_num = atomic_fetch_add(&free_call_count, 1); if (call_num > 14135 && call_num < 14145) { fprintf(stderr, "[HAK_TINY_FREE_FAST_WRAPPER] call=%lu ptr=%p\n", call_num, ptr); fflush(stderr); } - tiny_free_fast(ptr); + // Box 6 v1 (tiny_free_fast) は v2 に置き換え済み。 + // Wrapper からは Box 経路の slow/fast 判定に委ねる。 + hak_tiny_free(ptr); if (call_num > 14135 && call_num < 14145) { fprintf(stderr, "[HAK_TINY_FREE_FAST_WRAPPER] call=%lu completed\n", call_num); fflush(stderr); diff --git a/core/tiny_alloc_fast_push.d b/core/tiny_alloc_fast_push.d index 0b516fa9..d50a955b 100644 --- a/core/tiny_alloc_fast_push.d +++ b/core/tiny_alloc_fast_push.d @@ -26,7 +26,16 @@ core/tiny_alloc_fast_push.o: core/tiny_alloc_fast_push.c \ core/tiny_region_id.h core/superslab/superslab_inline.h \ core/box/tiny_layout_box.h core/box/tiny_header_box.h \ core/box/tiny_layout_box.h core/box/../tiny_debug_ring.h \ - core/box/../superslab/superslab_inline.h core/box/tiny_header_box.h \ + core/box/ss_addr_map_box.h core/box/../superslab/superslab_inline.h \ + core/box/tiny_ptr_bridge_box.h \ + core/box/../hakmem_tiny_superslab_internal.h \ + core/box/../box/ss_hot_cold_box.h \ + core/box/../box/../superslab/superslab_types.h \ + core/box/../box/ss_allocation_box.h core/hakmem_tiny_superslab.h \ + core/box/../hakmem_debug_master.h core/box/../hakmem_shared_pool.h \ + core/box/../hakmem_internal.h core/box/../tiny_region_id.h \ + core/box/../hakmem_tiny_integrity.h \ + core/box/../box/slab_freelist_atomic.h core/box/tiny_header_box.h \ core/box/front_gate_box.h core/hakmem_tiny.h core/hakmem_tiny_config.h: core/box/tls_sll_box.h: @@ -76,7 +85,20 @@ core/box/tiny_layout_box.h: core/box/tiny_header_box.h: core/box/tiny_layout_box.h: core/box/../tiny_debug_ring.h: +core/box/ss_addr_map_box.h: core/box/../superslab/superslab_inline.h: +core/box/tiny_ptr_bridge_box.h: +core/box/../hakmem_tiny_superslab_internal.h: +core/box/../box/ss_hot_cold_box.h: +core/box/../box/../superslab/superslab_types.h: +core/box/../box/ss_allocation_box.h: +core/hakmem_tiny_superslab.h: +core/box/../hakmem_debug_master.h: +core/box/../hakmem_shared_pool.h: +core/box/../hakmem_internal.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_tiny_integrity.h: +core/box/../box/slab_freelist_atomic.h: core/box/tiny_header_box.h: core/box/front_gate_box.h: core/hakmem_tiny.h: diff --git a/core/tiny_free_fast.inc.h b/core/tiny_free_fast.inc.h deleted file mode 100644 index f5fa5c09..00000000 --- a/core/tiny_free_fast.inc.h +++ /dev/null @@ -1,329 +0,0 @@ -// tiny_free_fast.inc.h - Box 6: Free Fast Path (same-thread free only) -// Purpose: Ultra-fast TLS freelist push (ownership check + 2-3 instructions) -// Invariant: owner_tid == my_tid → push to TLS, else → delegate to remote path -// Design: Clear boundary between same-thread (fast) and cross-thread (remote) -#pragma once -#include "tiny_atomic.h" -#include "hakmem_tiny.h" -#include "hakmem_tiny_superslab.h" -#include "slab_handle.h" -#include "tiny_alloc_fast_sfc.inc.h" // For sfc_free_push - -// ========== Debug Counters (compile-time gated) ========== -#if HAKMEM_DEBUG_COUNTERS -// Free pipeline counters (defined in hakmem_tiny.c) -extern unsigned long long g_free_via_ss_local[]; -extern unsigned long long g_free_via_ss_remote[]; -#endif - -// ========== Box 6: Free Fast Path ========== -// 箱理論の Fast Free 層。Same-thread free のみ処理(2-3命令 + ownership check)。 -// 不変条件: -// - owner_tid == my_tid → TLS freelist に push (no lock, no sync) -// - owner_tid != my_tid → Box 2 (Remote Queue) に委譲 -// - Cross-thread free は絶対に TLS freelist に入れない(A213 エラー防止) - -// External functions (Backend) -extern void hak_tiny_free(void* ptr); -extern void hak_tiny_free_with_slab(void* ptr, TinySlab* slab); -// hak_free_at signature: (void* ptr, size_t hint_sz, hak_callsite_t site) -// where hak_callsite_t is const void* -extern void hak_free_at(void* ptr, size_t hint_sz, const void* site); -// Note: hak_super_lookup() is defined in hakmem_super_registry.h (included transitively) -extern TinySlab* hak_tiny_owner_slab(void* ptr); -extern int g_use_superslab; - -// External helpers -extern uint32_t tiny_self_u32(void); -extern pthread_t tiny_self_pt(void); - -// External TLS variables (from Box 5) -// Phase 3d-B: TLS Cache Merge - Unified TLS SLL structure -extern __thread TinyTLSSLL g_tls_sll[TINY_NUM_CLASSES]; -extern __thread TinyTLSList g_tls_lists[TINY_NUM_CLASSES]; - -// Box 5 helper (TLS push) -extern void tiny_alloc_fast_push(int class_idx, hak_base_ptr_t ptr); - -// ========== Ownership Check ========== - -// Check if ptr belongs to current thread (SuperSlab path) -// Returns: 1 if same-thread, 0 if cross-thread -// -// Box Boundary: This is the critical check that prevents TOCTOU races -// - owner_tid == my_tid → Safe to push to TLS freelist -// - owner_tid != my_tid → MUST delegate to remote path -// -// Invariant: This check MUST be atomic (no TOCTOU between check and push) -static inline int tiny_free_is_same_thread_ss(SuperSlab* ss, int slab_idx, uint32_t my_tid) { - TinySlabMeta* meta = &ss->slabs[slab_idx]; - uint8_t my_tid_low = (uint8_t)((my_tid >> 8) & 0xFFu); - uint8_t owner = tiny_atomic_load_u8_relaxed(&meta->owner_tid_low); - return (owner == my_tid_low && owner != 0); -} - -// Check if ptr belongs to current thread (Legacy TinySlab path) -// Returns: 1 if same-thread, 0 if cross-thread -static inline int tiny_free_is_same_thread_legacy(TinySlab* slab) { - pthread_t my_tid = tiny_self_pt(); - return pthread_equal(slab->owner_tid, my_tid); -} - -// ========== Fast Path: Same-Thread Free (2-3 instructions) ========== - -// Free fast path for SuperSlab-backed allocation -// Returns: 1 on success (same-thread, pushed to TLS), 0 on failure (cross-thread) -// -// Assembly (x86-64, optimized): -// mov eax, DWORD PTR [meta->owner_tid] ; Load owner_tid -// cmp eax, my_tid ; Compare with my_tid -// jne .cross_thread ; If not equal, cross-thread -// mov rax, QWORD PTR g_tls_sll_head[cls] ; Load head -// mov QWORD PTR [ptr], rax ; ptr->next = head -// mov QWORD PTR g_tls_sll_head[cls], ptr ; head = ptr -// ret ; Done -// .cross_thread: -// ; Delegate to remote path -// -// Expected: 2-3 instructions on same-thread path (1 cmp, 1 load, 1 store) -// -// ⚠️ CRITICAL: ptr parameter must be BASE pointer (already converted from USER via ptr-1) -static inline int tiny_free_fast_ss(SuperSlab* ss, int slab_idx, void* base, uint32_t my_tid) { - // BUGFIX: Validate slab_idx before array access (prevents buffer overflow at ss->slabs[-1]) - int cap = ss_slabs_capacity(ss); - if (__builtin_expect(slab_idx < 0 || slab_idx >= cap, 0)) { - return 0; // Invalid index, reject - } - TinySlabMeta* meta = &ss->slabs[slab_idx]; - - // Debug: Track tiny_free_fast_ss calls -#if !HAKMEM_BUILD_RELEASE - static __thread int free_ss_debug_count = 0; - extern int g_sfc_debug; - if (__builtin_expect(g_sfc_debug, 0) && free_ss_debug_count < 20) { - free_ss_debug_count++; - int is_same = tiny_free_is_same_thread_ss(ss, slab_idx, my_tid); - extern int g_sfc_enabled; - fprintf(stderr, "[FREE_SS] base=%p, cls=%u, same_thread=%d, sfc_enabled=%d\n", - base, - meta->class_idx, - is_same, - g_sfc_enabled); - } -#endif - - // Box 6 Boundary: Ownership check (TOCTOU-safe) - if (__builtin_expect(!tiny_free_is_same_thread_ss(ss, slab_idx, my_tid), 0)) { -#if HAKMEM_DEBUG_COUNTERS - // Track cross-thread frees (compile-time gated) - g_free_via_ss_remote[meta->class_idx]++; -#endif - return 0; // Cross-thread → caller should delegate to remote path - } - - // Fast path: Same-thread free (2-3 instructions) - int class_idx = meta->class_idx; - // Phase E1-CORRECT: base pointer already converted by caller (no double conversion!) - -#if HAKMEM_DEBUG_COUNTERS - // Track same-thread frees (compile-time gated) - g_free_via_ss_local[class_idx]++; -#endif - - // Box 5 integration - extern int g_sfc_enabled; - if (g_sfc_enabled) { - // Box 5-NEW: Try SFC (128-256 slots) - if (!sfc_free_push(class_idx, base)) { - // SFC full → skip caching, use slow path (return 0) - // Do NOT fall back to SLL - it has no capacity check and would grow unbounded! - return 0; - } - } else { - // Box 5-OLD: Use SLL (16 slots) - tiny_alloc_fast_push(class_idx, base); - } - - return 1; // Success -} - -// Free fast path for Legacy TinySlab-backed allocation -// Returns: 1 on success (same-thread), 0 on failure (cross-thread) -// -// ⚠️ CRITICAL: ptr parameter must be BASE pointer (already converted from USER via ptr-1) -static inline int tiny_free_fast_legacy(TinySlab* slab, void* base) { - // Box 6 Boundary: Ownership check - if (__builtin_expect(!tiny_free_is_same_thread_legacy(slab), 0)) { - return 0; // Cross-thread → caller should delegate to precise path - } - - // Fast path: Same-thread free - int class_idx = slab->class_idx; - // Phase E1-CORRECT: base pointer already converted by caller (no double conversion!) - - // Box 5-NEW/5-OLD integration: Push to TLS freelist (SFC or SLL) - extern int g_sfc_enabled; - if (g_sfc_enabled) { - // Box 5-NEW: Try SFC (128 slots) - if (!sfc_free_push(class_idx, base)) { - // SFC full → skip caching, use slow path (return 0) - // Do NOT fall back to SLL - it has no capacity check and would grow unbounded! - return 0; - } - } else { - // Box 5-OLD: Use SLL (16 slots) - tiny_alloc_fast_push(class_idx, base); - } - - return 1; // Success -} - -// ========== Combined Fast Free (Lookup + Ownership + Push) ========== - -// Complete fast free path (inline for zero-cost) -// Returns: none (delegates to backend on cross-thread or non-tiny) -// -// Flow: -// 1. Lookup ptr → SuperSlab or TinySlab -// 2. Ownership check (owner_tid == my_tid) -// 3. Same-thread → TLS freelist push (2-3 instructions) -// 4. Cross-thread → Delegate to Box 2 (Remote Queue) -// 5. Not Tiny → Delegate to backend (Mid/Large) -// -// Example usage: -// tiny_free_fast(ptr); // Always succeeds (delegates on failure) -static inline void tiny_free_fast(void* ptr) { - // Optional runtime gate to disable fast free and route to slow path - // Env: HAKMEM_TINY_FREE_FAST (default: 1). Additionally, if - // HAKMEM_TINY_FREE_TO_SS=1 is set, prefer SS path by disabling fast free. - static int s_free_fast_en = -1; - if (__builtin_expect(s_free_fast_en == -1, 0)) { - const char* e = getenv("HAKMEM_TINY_FREE_FAST"); - int v = (e && *e && *e != '0') ? 1 : 1; // default ON - const char* to_ss = getenv("HAKMEM_TINY_FREE_TO_SS"); - if (to_ss && *to_ss && *to_ss != '0') v = 0; // FREE_TO_SS implies slow path - s_free_fast_en = v; - } - if (!s_free_fast_en) { - // Delegate to precise slow path (handles same/remote + publish) - hak_tiny_free(ptr); - return; - } - // 1. SuperSlab-backed tiny pointer? - if (__builtin_expect(g_use_superslab != 0, 1)) { - // NOTE: Use safe hak_super_lookup here (not ss_fast_lookup) because - // tiny_free_fast() can receive arbitrary pointers without prior validation. - // ss_fast_lookup masks to 1MB boundary and reads magic - would crash on unmapped memory. - SuperSlab* ss = hak_super_lookup(ptr); - if (__builtin_expect(ss != NULL && ss->magic == SUPERSLAB_MAGIC, 0)) { - // ROOT CAUSE FIX: slab_index_for() can handle USER pointers directly - // Avoid per-class offset conversion errors (C0/C7 have offset=0, C1-6 have offset=1) - int slab_idx = slab_index_for(ss, ptr); - - // Convert USER → BASE for tiny_free_fast_ss (needed for next pointer operations) - int class_idx = tiny_get_class_from_ss(ss, slab_idx); - void* base = (void*)((uint8_t*)ptr - tiny_user_offset(class_idx)); - uint32_t self_tid = tiny_self_u32(); - - // Box 6 Boundary: Try same-thread fast path - // CRITICAL: Pass BASE pointer (already converted above) - if (tiny_free_fast_ss(ss, slab_idx, base, self_tid)) { - return; // Success: same-thread, pushed to TLS - } - - // Cross-thread free → Box 2 (Remote Queue) - // Delegate to full tiny free (handles remote push) - hak_tiny_free(ptr); - return; - } - } - - // 2. Legacy TinySlab-backed pointer? - TinySlab* slab = hak_tiny_owner_slab(ptr); - if (__builtin_expect(slab != NULL, 0)) { - // Convert USER → BASE (for Legacy path) using centralized layout - int legacy_class = slab->class_idx; - size_t user_off = tiny_user_offset(legacy_class); - void* base_legacy = (void*)((uint8_t*)ptr - user_off); - - // Box 6 Boundary: Try same-thread fast path - // CRITICAL: Pass BASE pointer (already converted above) - if (tiny_free_fast_legacy(slab, base_legacy)) { - return; // Success: same-thread, pushed to TLS - } - - // Cross-thread free → precise path with known slab - hak_tiny_free_with_slab(ptr, slab); - return; - } - - // 3. Not a tiny allocation → Delegate to backend (Mid/Large/Mmap) - hak_free_at(ptr, 0, 0); -} - -// ========== Guard/Debug Variants ========== - -// Free with additional safety checks (for debugging/testing) -// This variant includes: -// - Sentinel checks (0xBADA55) -// - Double-free detection -// - Ownership validation -// -// Usage: Enable with HAKMEM_SAFE_FREE=1 environment variable -static inline void tiny_free_fast_guarded(void* ptr) { - // TODO: Implement guard checks if needed - // For now, delegate to standard fast path - tiny_free_fast(ptr); -} - -// ========== Statistics & Diagnostics ========== - -// Free fast path stats (for profiling) -typedef struct { - uint64_t same_thread_count; // Same-thread frees (TLS push) - uint64_t cross_thread_count; // Cross-thread frees (remote queue) - uint64_t non_tiny_count; // Non-tiny frees (backend) -} TinyFreeFastStats; - -// Get free fast path stats (TLS-local) -static __thread TinyFreeFastStats g_tiny_free_fast_stats = {0}; - -static inline TinyFreeFastStats tiny_free_fast_stats_get(void) { - return g_tiny_free_fast_stats; -} - -// Reset free fast path stats (for testing/benchmarking) -static inline void tiny_free_fast_stats_reset(void) { - g_tiny_free_fast_stats.same_thread_count = 0; - g_tiny_free_fast_stats.cross_thread_count = 0; - g_tiny_free_fast_stats.non_tiny_count = 0; -} - -// ========== Performance Notes ========== -// -// Expected metrics: -// - Same-thread hit rate: 80-90% (workload dependent) -// - Same-thread latency: 2-3 instructions (ownership check + push) -// - Cross-thread penalty: ~50-100 instructions (remote queue push) -// - Throughput improvement: +10-20% vs current multi-layer design -// -// Key optimizations: -// 1. Ownership check first (fail-fast on cross-thread) -// 2. `__builtin_expect` for branch prediction (same-thread is common) -// 3. `static inline` for zero-cost abstraction -// 4. TLS variables (no atomic ops in same-thread path) -// -// TOCTOU Race Prevention (Box 4 Boundary): -// - Ownership check is atomic (tiny_atomic_load_u32_relaxed) -// - No time window between check and push (single function) -// - Cross-thread frees are immediately delegated (no TLS touch) -// -// Comparison with current design: -// - Current: 38.43% free overhead (free.part.0 + mid_lookup + locks) -// - New: 2-3 instructions (ownership check + TLS push) -// - Reduction: -90% instructions in same-thread path -// -// Inspired by: -// - System tcache (glibc malloc) - fast same-thread free -// - Box Theory - Clear ownership boundaries -// - TOCTOU fix (Box 4) - Atomic ownership check diff --git a/hakmem.d b/hakmem.d index 414aa4dd..0ae5ff33 100644 --- a/hakmem.d +++ b/hakmem.d @@ -27,44 +27,55 @@ hakmem.o: core/hakmem.c core/hakmem.h core/hakmem_build_flags.h \ core/box/hak_core_init.inc.h core/hakmem_phase7_config.h \ core/box/ss_hot_prewarm_box.h core/box/hak_alloc_api.inc.h \ core/box/../hakmem_tiny.h core/box/../hakmem_pool.h \ - core/box/../hakmem_smallmid.h core/box/mid_large_config_box.h \ - core/box/../hakmem_config.h core/box/../hakmem_features.h \ - core/box/hak_free_api.inc.h core/hakmem_tiny_superslab.h \ - core/box/../tiny_free_fast_v2.inc.h core/box/../tiny_region_id.h \ - core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_config.h \ - core/box/../box/tls_sll_box.h core/box/../box/../hakmem_internal.h \ + core/box/../hakmem_smallmid.h core/box/../pool_tls.h \ + core/box/mid_large_config_box.h core/box/../hakmem_config.h \ + core/box/../hakmem_features.h core/box/hak_free_api.inc.h \ + core/hakmem_tiny_superslab.h core/box/front_gate_v2.h \ + core/box/external_guard_box.h core/box/ss_slab_meta_box.h \ + core/box/../superslab/superslab_types.h core/box/slab_freelist_atomic.h \ + core/box/fg_tiny_gate_box.h core/box/tiny_free_gate_box.h \ + core/box/ptr_type_box.h core/box/ptr_conversion_box.h \ + core/box/tiny_ptr_bridge_box.h \ + core/box/../hakmem_tiny_superslab_internal.h \ + core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_superslab.h \ + core/box/../box/ss_hot_cold_box.h \ + core/box/../box/../superslab/superslab_types.h \ + core/box/../box/ss_allocation_box.h core/box/../hakmem_debug_master.h \ + core/box/../hakmem_tiny.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_shared_pool.h core/box/../superslab/superslab_types.h \ + core/box/../hakmem_internal.h core/box/../tiny_region_id.h \ + core/box/../hakmem_tiny_integrity.h \ + core/box/../box/slab_freelist_atomic.h \ + core/box/../tiny_free_fast_v2.inc.h core/box/../box/tls_sll_box.h \ + core/box/../box/../hakmem_internal.h \ core/box/../box/../hakmem_tiny_config.h \ core/box/../box/../hakmem_build_flags.h \ core/box/../box/../hakmem_debug_master.h \ core/box/../box/../tiny_remote.h core/box/../box/../tiny_region_id.h \ core/box/../box/../hakmem_tiny_integrity.h \ - core/box/../box/../hakmem_tiny.h core/box/../box/../ptr_track.h \ - core/box/../box/../tiny_debug_ring.h \ + core/box/../box/../ptr_track.h core/box/../box/../tiny_debug_ring.h \ + core/box/../box/ss_addr_map_box.h \ core/box/../box/../superslab/superslab_inline.h \ - core/box/../box/tiny_header_box.h core/box/../box/tls_sll_drain_box.h \ - core/box/../box/tls_sll_box.h core/box/../box/slab_recycling_box.h \ + core/box/../box/tiny_ptr_bridge_box.h core/box/../box/tiny_header_box.h \ + core/box/../box/tls_sll_drain_box.h core/box/../box/tls_sll_box.h \ + core/box/../box/slab_recycling_box.h \ core/box/../box/../hakmem_tiny_superslab.h \ - core/box/../box/../hakmem_shared_pool.h \ - core/box/../box/../superslab/superslab_types.h \ - core/box/../box/ss_hot_cold_box.h \ - core/box/../box/../superslab/superslab_types.h \ + core/box/../box/ss_hot_cold_box.h core/box/../box/ss_release_guard_box.h \ + core/box/../box/../hakmem_tiny_superslab_internal.h \ core/box/../box/free_local_box.h core/box/../box/ptr_type_box.h \ core/box/../box/free_publish_box.h core/hakmem_tiny.h \ - core/tiny_region_id.h core/box/../hakmem_tiny_integrity.h \ - core/box/../hakmem_env_cache.h core/box/../superslab/superslab_inline.h \ - core/box/../box/ss_slab_meta_box.h \ - core/box/../box/slab_freelist_atomic.h core/box/../box/free_remote_box.h \ + core/tiny_region_id.h core/box/../hakmem_env_cache.h \ + core/box/../superslab/superslab_inline.h \ + core/box/../box/ss_slab_meta_box.h core/box/../box/free_remote_box.h \ core/hakmem_tiny_integrity.h core/box/../box/ptr_conversion_box.h \ - core/box/../box/tiny_layout_box.h core/box/front_gate_v2.h \ - core/box/external_guard_box.h core/box/ss_slab_meta_box.h \ - core/box/fg_tiny_gate_box.h core/box/hak_exit_debug.inc.h \ - core/box/hak_wrappers.inc.h core/box/front_gate_classifier.h \ - core/box/../front/malloc_tiny_fast.h \ + core/box/hak_exit_debug.inc.h core/box/hak_wrappers.inc.h \ + core/box/front_gate_classifier.h core/box/../front/malloc_tiny_fast.h \ core/box/../front/../hakmem_build_flags.h \ core/box/../front/../hakmem_tiny_config.h \ core/box/../front/../superslab/superslab_inline.h \ core/box/../front/../box/ss_slab_meta_box.h \ core/box/../front/tiny_unified_cache.h \ + core/box/../front/../box/ptr_type_box.h \ core/box/../front/../box/tiny_front_config_box.h \ core/box/../front/../box/../hakmem_build_flags.h \ core/box/../front/../tiny_region_id.h core/box/../front/../hakmem_tiny.h \ @@ -74,8 +85,9 @@ hakmem.o: core/hakmem.c core/hakmem.h core/hakmem_build_flags.h \ core/box/../front/../box/../front/tiny_unified_cache.h \ core/box/../front/../box/tiny_layout_box.h \ core/box/../front/../box/tiny_front_cold_box.h \ - core/box/tiny_front_config_box.h core/box/wrapper_env_box.h \ - core/box/../hakmem_internal.h core/box/../superslab/superslab_inline.h + core/box/tiny_alloc_gate_box.h core/box/tiny_front_config_box.h \ + core/box/wrapper_env_box.h core/box/../hakmem_internal.h \ + core/box/../superslab/superslab_inline.h core/hakmem.h: core/hakmem_build_flags.h: core/hakmem_config.h: @@ -144,15 +156,38 @@ core/box/hak_alloc_api.inc.h: core/box/../hakmem_tiny.h: core/box/../hakmem_pool.h: core/box/../hakmem_smallmid.h: +core/box/../pool_tls.h: core/box/mid_large_config_box.h: core/box/../hakmem_config.h: core/box/../hakmem_features.h: core/box/hak_free_api.inc.h: core/hakmem_tiny_superslab.h: -core/box/../tiny_free_fast_v2.inc.h: -core/box/../tiny_region_id.h: +core/box/front_gate_v2.h: +core/box/external_guard_box.h: +core/box/ss_slab_meta_box.h: +core/box/../superslab/superslab_types.h: +core/box/slab_freelist_atomic.h: +core/box/fg_tiny_gate_box.h: +core/box/tiny_free_gate_box.h: +core/box/ptr_type_box.h: +core/box/ptr_conversion_box.h: +core/box/tiny_ptr_bridge_box.h: +core/box/../hakmem_tiny_superslab_internal.h: core/box/../hakmem_build_flags.h: +core/box/../hakmem_tiny_superslab.h: +core/box/../box/ss_hot_cold_box.h: +core/box/../box/../superslab/superslab_types.h: +core/box/../box/ss_allocation_box.h: +core/box/../hakmem_debug_master.h: +core/box/../hakmem_tiny.h: core/box/../hakmem_tiny_config.h: +core/box/../hakmem_shared_pool.h: +core/box/../superslab/superslab_types.h: +core/box/../hakmem_internal.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_tiny_integrity.h: +core/box/../box/slab_freelist_atomic.h: +core/box/../tiny_free_fast_v2.inc.h: core/box/../box/tls_sll_box.h: core/box/../box/../hakmem_internal.h: core/box/../box/../hakmem_tiny_config.h: @@ -161,37 +196,30 @@ core/box/../box/../hakmem_debug_master.h: core/box/../box/../tiny_remote.h: core/box/../box/../tiny_region_id.h: core/box/../box/../hakmem_tiny_integrity.h: -core/box/../box/../hakmem_tiny.h: core/box/../box/../ptr_track.h: core/box/../box/../tiny_debug_ring.h: +core/box/../box/ss_addr_map_box.h: core/box/../box/../superslab/superslab_inline.h: +core/box/../box/tiny_ptr_bridge_box.h: core/box/../box/tiny_header_box.h: core/box/../box/tls_sll_drain_box.h: core/box/../box/tls_sll_box.h: core/box/../box/slab_recycling_box.h: core/box/../box/../hakmem_tiny_superslab.h: -core/box/../box/../hakmem_shared_pool.h: -core/box/../box/../superslab/superslab_types.h: core/box/../box/ss_hot_cold_box.h: -core/box/../box/../superslab/superslab_types.h: +core/box/../box/ss_release_guard_box.h: +core/box/../box/../hakmem_tiny_superslab_internal.h: core/box/../box/free_local_box.h: core/box/../box/ptr_type_box.h: core/box/../box/free_publish_box.h: core/hakmem_tiny.h: core/tiny_region_id.h: -core/box/../hakmem_tiny_integrity.h: core/box/../hakmem_env_cache.h: core/box/../superslab/superslab_inline.h: core/box/../box/ss_slab_meta_box.h: -core/box/../box/slab_freelist_atomic.h: core/box/../box/free_remote_box.h: core/hakmem_tiny_integrity.h: core/box/../box/ptr_conversion_box.h: -core/box/../box/tiny_layout_box.h: -core/box/front_gate_v2.h: -core/box/external_guard_box.h: -core/box/ss_slab_meta_box.h: -core/box/fg_tiny_gate_box.h: core/box/hak_exit_debug.inc.h: core/box/hak_wrappers.inc.h: core/box/front_gate_classifier.h: @@ -201,6 +229,7 @@ core/box/../front/../hakmem_tiny_config.h: core/box/../front/../superslab/superslab_inline.h: core/box/../front/../box/ss_slab_meta_box.h: core/box/../front/tiny_unified_cache.h: +core/box/../front/../box/ptr_type_box.h: core/box/../front/../box/tiny_front_config_box.h: core/box/../front/../box/../hakmem_build_flags.h: core/box/../front/../tiny_region_id.h: @@ -211,6 +240,7 @@ core/box/../front/../box/../tiny_region_id.h: core/box/../front/../box/../front/tiny_unified_cache.h: core/box/../front/../box/tiny_layout_box.h: core/box/../front/../box/tiny_front_cold_box.h: +core/box/tiny_alloc_gate_box.h: core/box/tiny_front_config_box.h: core/box/wrapper_env_box.h: core/box/../hakmem_internal.h: diff --git a/hakmem_shared_pool.d b/hakmem_shared_pool.d index ed79c3b0..28933e0d 100644 --- a/hakmem_shared_pool.d +++ b/hakmem_shared_pool.d @@ -26,13 +26,20 @@ hakmem_shared_pool.o: core/hakmem_shared_pool.c \ core/box/../hakmem_debug_master.h core/box/../tiny_remote.h \ core/box/../hakmem_tiny_integrity.h core/box/../hakmem_tiny.h \ core/box/../ptr_track.h core/box/../ptr_trace.h \ - core/box/../tiny_debug_ring.h core/box/../superslab/superslab_inline.h \ - core/box/tiny_header_box.h core/box/slab_recycling_box.h \ - core/box/../hakmem_tiny_superslab.h core/box/ss_hot_cold_box.h \ - core/box/free_local_box.h core/hakmem_tiny_superslab.h \ - core/box/ptr_type_box.h core/box/free_publish_box.h core/hakmem_tiny.h \ - core/tiny_region_id.h core/box/tls_slab_reuse_guard_box.h \ - core/hakmem_policy.h + core/box/../tiny_debug_ring.h core/box/ss_addr_map_box.h \ + core/box/../superslab/superslab_inline.h core/box/tiny_ptr_bridge_box.h \ + core/box/../hakmem_tiny_superslab_internal.h \ + core/box/../hakmem_tiny_superslab.h core/box/../box/ss_hot_cold_box.h \ + core/box/../box/ss_allocation_box.h core/hakmem_tiny_superslab.h \ + core/box/../hakmem_debug_master.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_internal.h core/box/../tiny_region_id.h \ + core/box/../hakmem_tiny_integrity.h \ + core/box/../box/slab_freelist_atomic.h core/box/tiny_header_box.h \ + core/box/slab_recycling_box.h core/box/../hakmem_tiny_superslab.h \ + core/box/ss_hot_cold_box.h core/box/ss_release_guard_box.h \ + core/box/free_local_box.h core/box/ptr_type_box.h \ + core/box/free_publish_box.h core/hakmem_tiny.h core/tiny_region_id.h \ + core/box/tls_slab_reuse_guard_box.h core/hakmem_policy.h core/hakmem_shared_pool_internal.h: core/hakmem_shared_pool.h: core/superslab/superslab_types.h: @@ -91,13 +98,26 @@ core/box/../hakmem_tiny.h: core/box/../ptr_track.h: core/box/../ptr_trace.h: core/box/../tiny_debug_ring.h: +core/box/ss_addr_map_box.h: core/box/../superslab/superslab_inline.h: +core/box/tiny_ptr_bridge_box.h: +core/box/../hakmem_tiny_superslab_internal.h: +core/box/../hakmem_tiny_superslab.h: +core/box/../box/ss_hot_cold_box.h: +core/box/../box/ss_allocation_box.h: +core/hakmem_tiny_superslab.h: +core/box/../hakmem_debug_master.h: +core/box/../hakmem_tiny_config.h: +core/box/../hakmem_internal.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_tiny_integrity.h: +core/box/../box/slab_freelist_atomic.h: core/box/tiny_header_box.h: core/box/slab_recycling_box.h: core/box/../hakmem_tiny_superslab.h: core/box/ss_hot_cold_box.h: +core/box/ss_release_guard_box.h: core/box/free_local_box.h: -core/hakmem_tiny_superslab.h: core/box/ptr_type_box.h: core/box/free_publish_box.h: core/hakmem_tiny.h: diff --git a/hakmem_tiny_sfc.d b/hakmem_tiny_sfc.d index 8b53d2fc..7a8e66d0 100644 --- a/hakmem_tiny_sfc.d +++ b/hakmem_tiny_sfc.d @@ -21,7 +21,17 @@ hakmem_tiny_sfc.o: core/hakmem_tiny_sfc.c core/tiny_alloc_fast_sfc.inc.h \ core/box/../tiny_remote.h core/box/../hakmem_tiny_integrity.h \ core/box/../hakmem_tiny.h core/box/../ptr_track.h \ core/box/../ptr_trace.h core/box/../tiny_debug_ring.h \ - core/box/../superslab/superslab_inline.h core/box/tiny_header_box.h \ + core/box/ss_addr_map_box.h core/box/../superslab/superslab_inline.h \ + core/box/tiny_ptr_bridge_box.h \ + core/box/../hakmem_tiny_superslab_internal.h \ + core/box/../hakmem_tiny_superslab.h core/box/../box/ss_hot_cold_box.h \ + core/box/../box/../superslab/superslab_types.h \ + core/box/../box/ss_allocation_box.h core/hakmem_tiny_superslab.h \ + core/box/../hakmem_debug_master.h core/box/../hakmem_tiny_config.h \ + core/box/../hakmem_shared_pool.h core/box/../superslab/superslab_types.h \ + core/box/../hakmem_internal.h core/box/../tiny_region_id.h \ + core/box/../hakmem_tiny_integrity.h \ + core/box/../box/slab_freelist_atomic.h core/box/tiny_header_box.h \ core/hakmem_env_cache.h core/tiny_alloc_fast_sfc.inc.h: core/hakmem_tiny.h: @@ -73,6 +83,22 @@ core/box/../hakmem_tiny.h: core/box/../ptr_track.h: core/box/../ptr_trace.h: core/box/../tiny_debug_ring.h: +core/box/ss_addr_map_box.h: core/box/../superslab/superslab_inline.h: +core/box/tiny_ptr_bridge_box.h: +core/box/../hakmem_tiny_superslab_internal.h: +core/box/../hakmem_tiny_superslab.h: +core/box/../box/ss_hot_cold_box.h: +core/box/../box/../superslab/superslab_types.h: +core/box/../box/ss_allocation_box.h: +core/hakmem_tiny_superslab.h: +core/box/../hakmem_debug_master.h: +core/box/../hakmem_tiny_config.h: +core/box/../hakmem_shared_pool.h: +core/box/../superslab/superslab_types.h: +core/box/../hakmem_internal.h: +core/box/../tiny_region_id.h: +core/box/../hakmem_tiny_integrity.h: +core/box/../box/slab_freelist_atomic.h: core/box/tiny_header_box.h: core/hakmem_env_cache.h: