diff --git a/Makefile b/Makefile index 14243c0b..e7791947 100644 --- a/Makefile +++ b/Makefile @@ -218,12 +218,12 @@ LDFLAGS += $(EXTRA_LDFLAGS) # Targets TARGET = test_hakmem -OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/capacity_box.o core/box/carve_push_box.o core/box/unified_batch_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/ss_tls_hint_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/page_arena.o core/front/tiny_unified_cache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o test_hakmem.o +OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o superslab_allocate.o superslab_stats.o superslab_cache.o superslab_ace.o superslab_slab.o superslab_backend.o superslab_head.o hakmem_smallmid.o hakmem_smallmid_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_super_registry.o hakmem_shared_pool.o hakmem_shared_pool_acquire.o hakmem_shared_pool_release.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/superslab_expansion_box.o core/box/integrity_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/capacity_box.o core/box/carve_push_box.o core/box/unified_batch_box.o core/box/prewarm_box.o core/box/ss_hot_prewarm_box.o core/box/front_metrics_box.o core/box/bench_fast_box.o core/box/ss_addr_map_box.o core/box/ss_tls_hint_box.o core/box/slab_recycling_box.o core/box/pagefault_telemetry_box.o core/box/tiny_sizeclass_hist_box.o core/box/tiny_env_box.o core/box/wrapper_env_box.o core/page_arena.o core/front/tiny_unified_cache.o core/tiny_alloc_fast_push.o core/link_stubs.o core/tiny_failfast.o test_hakmem.o OBJS = $(OBJS_BASE) # Shared library SHARED_LIB = libhakmem.so -SHARED_OBJS = hakmem_shared.o hakmem_config_shared.o hakmem_tiny_config_shared.o hakmem_ucb1_shared.o hakmem_bigcache_shared.o hakmem_pool_shared.o hakmem_l25_pool_shared.o hakmem_site_rules_shared.o hakmem_tiny_shared.o superslab_allocate_shared.o superslab_stats_shared.o superslab_cache_shared.o superslab_ace_shared.o superslab_slab_shared.o superslab_backend_shared.o superslab_head_shared.o hakmem_smallmid_shared.o hakmem_smallmid_superslab_shared.o core/box/superslab_expansion_box_shared.o core/box/integrity_box_shared.o core/box/mailbox_box_shared.o core/box/front_gate_box_shared.o core/box/front_gate_classifier_shared.o core/box/free_publish_box_shared.o core/box/capacity_box_shared.o core/box/carve_push_box_shared.o core/box/unified_batch_box_shared.o core/box/prewarm_box_shared.o core/box/ss_hot_prewarm_box_shared.o core/box/front_metrics_box_shared.o core/box/bench_fast_box_shared.o core/box/ss_addr_map_box_shared.o core/box/ss_tls_hint_box_shared.o core/box/slab_recycling_box_shared.o core/box/pagefault_telemetry_box_shared.o core/box/tiny_sizeclass_hist_box_shared.o core/page_arena_shared.o core/front/tiny_unified_cache_shared.o core/tiny_alloc_fast_push_shared.o core/link_stubs_shared.o core/tiny_failfast_shared.o tiny_sticky_shared.o tiny_remote_shared.o tiny_publish_shared.o tiny_debug_ring_shared.o hakmem_tiny_magazine_shared.o hakmem_tiny_stats_shared.o hakmem_tiny_sfc_shared.o hakmem_tiny_query_shared.o hakmem_tiny_rss_shared.o hakmem_tiny_registry_shared.o hakmem_tiny_remote_target_shared.o hakmem_tiny_bg_spill_shared.o tiny_adaptive_sizing_shared.o hakmem_super_registry_shared.o hakmem_shared_pool_shared.o hakmem_shared_pool_acquire_shared.o hakmem_shared_pool_release_shared.o hakmem_elo_shared.o hakmem_batch_shared.o hakmem_p2_shared.o hakmem_sizeclass_dist_shared.o hakmem_evo_shared.o hakmem_debug_shared.o hakmem_sys_shared.o hakmem_whale_shared.o hakmem_policy_shared.o hakmem_ace_shared.o hakmem_ace_stats_shared.o hakmem_ace_controller_shared.o hakmem_ace_metrics_shared.o hakmem_ace_ucb1_shared.o hakmem_prof_shared.o hakmem_learner_shared.o hakmem_size_hist_shared.o hakmem_learn_log_shared.o hakmem_syscall_shared.o tiny_fastcache_shared.o +SHARED_OBJS = hakmem_shared.o hakmem_config_shared.o hakmem_tiny_config_shared.o hakmem_ucb1_shared.o hakmem_bigcache_shared.o hakmem_pool_shared.o hakmem_l25_pool_shared.o hakmem_site_rules_shared.o hakmem_tiny_shared.o superslab_allocate_shared.o superslab_stats_shared.o superslab_cache_shared.o superslab_ace_shared.o superslab_slab_shared.o superslab_backend_shared.o superslab_head_shared.o hakmem_smallmid_shared.o hakmem_smallmid_superslab_shared.o core/box/superslab_expansion_box_shared.o core/box/integrity_box_shared.o core/box/mailbox_box_shared.o core/box/front_gate_box_shared.o core/box/front_gate_classifier_shared.o core/box/free_publish_box_shared.o core/box/capacity_box_shared.o core/box/carve_push_box_shared.o core/box/unified_batch_box_shared.o core/box/prewarm_box_shared.o core/box/ss_hot_prewarm_box_shared.o core/box/front_metrics_box_shared.o core/box/bench_fast_box_shared.o core/box/ss_addr_map_box_shared.o core/box/ss_tls_hint_box_shared.o core/box/slab_recycling_box_shared.o core/box/pagefault_telemetry_box_shared.o core/box/tiny_sizeclass_hist_box_shared.o core/box/tiny_env_box_shared.o core/box/wrapper_env_box_shared.o core/page_arena_shared.o core/front/tiny_unified_cache_shared.o core/tiny_alloc_fast_push_shared.o core/link_stubs_shared.o core/tiny_failfast_shared.o tiny_sticky_shared.o tiny_remote_shared.o tiny_publish_shared.o tiny_debug_ring_shared.o hakmem_tiny_magazine_shared.o hakmem_tiny_stats_shared.o hakmem_tiny_sfc_shared.o hakmem_tiny_query_shared.o hakmem_tiny_rss_shared.o hakmem_tiny_registry_shared.o hakmem_tiny_remote_target_shared.o hakmem_tiny_bg_spill_shared.o tiny_adaptive_sizing_shared.o hakmem_super_registry_shared.o hakmem_shared_pool_shared.o hakmem_shared_pool_acquire_shared.o hakmem_shared_pool_release_shared.o hakmem_elo_shared.o hakmem_batch_shared.o hakmem_p2_shared.o hakmem_sizeclass_dist_shared.o hakmem_evo_shared.o hakmem_debug_shared.o hakmem_sys_shared.o hakmem_whale_shared.o hakmem_policy_shared.o hakmem_ace_shared.o hakmem_ace_stats_shared.o hakmem_ace_controller_shared.o hakmem_ace_metrics_shared.o hakmem_ace_ucb1_shared.o hakmem_prof_shared.o hakmem_learner_shared.o hakmem_size_hist_shared.o hakmem_learn_log_shared.o hakmem_syscall_shared.o tiny_fastcache_shared.o # Pool TLS Phase 1 (enable with POOL_TLS_PHASE1=1) ifeq ($(POOL_TLS_PHASE1),1) diff --git a/core/box/hak_wrappers.inc.h b/core/box/hak_wrappers.inc.h index 752753da..1e97740e 100644 --- a/core/box/hak_wrappers.inc.h +++ b/core/box/hak_wrappers.inc.h @@ -33,6 +33,7 @@ void* realloc(void* ptr, size_t size) { #include "../hakmem_pool.h" // Mid registry lookup (failsafe for headerless Mid) #include "../front/malloc_tiny_fast.h" // Phase 26: Front Gate Unification #include "tiny_front_config_box.h" // Phase 4-Step3: Compile-time config for dead code elimination +#include "wrapper_env_box.h" // Wrapper env cache (step trace / LD safe / free trace) // malloc wrapper - intercepts system malloc() calls __thread uint64_t g_malloc_total_calls = 0; @@ -77,12 +78,8 @@ void* malloc(size_t size) { // This prevents infinite recursion when getenv/fprintf/dlopen call malloc g_hakmem_lock_depth++; // Debug step trace for 33KB: gated by env HAKMEM_STEP_TRACE (default: OFF) - static int g_step_trace = -1; - if (__builtin_expect(g_step_trace == -1, 0)) { - const char* e = getenv("HAKMEM_STEP_TRACE"); - g_step_trace = (e && *e && *e != '0') ? 1 : 0; - } - if (g_step_trace && size == 33000) write(2, "STEP:1 Lock++\n", 14); + const wrapper_env_cfg_t* wcfg = wrapper_env_cfg(); + if (wcfg->step_trace && size == 33000) write(2, "STEP:1 Lock++\n", 14); // Guard against recursion during initialization if (__builtin_expect(g_initializing != 0, 0)) { @@ -103,41 +100,36 @@ void* malloc(size_t size) { if (__builtin_expect(hak_force_libc_alloc(), 0)) { g_hakmem_lock_depth--; extern void* __libc_malloc(size_t); - if (size == 33000) write(2, "RET:ForceLibc\n", 14); + if (wcfg->step_trace && size == 33000) write(2, "RET:ForceLibc\n", 14); return __libc_malloc(size); } - if (g_step_trace && size == 33000) write(2, "STEP:2 ForceLibc passed\n", 24); + if (wcfg->step_trace && size == 33000) write(2, "STEP:2 ForceLibc passed\n", 24); int ld_mode = hak_ld_env_mode(); if (ld_mode) { - if (g_step_trace && size == 33000) write(2, "STEP:3 LD Mode\n", 15); + if (wcfg->step_trace && size == 33000) write(2, "STEP:3 LD Mode\n", 15); if (hak_ld_block_jemalloc() && g_jemalloc_loaded) { g_hakmem_lock_depth--; extern void* __libc_malloc(size_t); - if (size == 33000) write(2, "RET:Jemalloc\n", 13); + if (wcfg->step_trace && size == 33000) write(2, "RET:Jemalloc\n", 13); return __libc_malloc(size); } if (!g_initialized) { hak_init(); } if (g_initializing) { g_hakmem_lock_depth--; extern void* __libc_malloc(size_t); - if (size == 33000) write(2, "RET:Init2\n", 10); + if (wcfg->step_trace && size == 33000) write(2, "RET:Init2\n", 10); return __libc_malloc(size); } // Cache HAKMEM_LD_SAFE to avoid repeated getenv on hot path - static _Atomic int ld_safe_mode = -1; // -1 = uninitialized - if (__builtin_expect(ld_safe_mode < 0, 0)) { - const char* lds = getenv("HAKMEM_LD_SAFE"); - ld_safe_mode = (lds ? atoi(lds) : 1); - } - if (ld_safe_mode >= 2) { + if (wcfg->ld_safe_mode >= 2) { g_hakmem_lock_depth--; extern void* __libc_malloc(size_t); - if (size == 33000) write(2, "RET:LDSafe\n", 11); + if (wcfg->step_trace && size == 33000) write(2, "RET:LDSafe\n", 11); return __libc_malloc(size); } } - if (g_step_trace && size == 33000) write(2, "STEP:4 LD Check passed\n", 23); + if (wcfg->step_trace && size == 33000) write(2, "STEP:4 LD Check passed\n", 23); // Phase 26: CRITICAL - Ensure initialization before fast path // (fast path bypasses hak_alloc_at, so we need to init here) @@ -151,19 +143,19 @@ void* malloc(size_t size) { // Phase 4-Step3: Use config macro for compile-time optimization // Phase 7-Step1: Changed expect hint from 0→1 (unified path is now LIKELY) if (__builtin_expect(TINY_FRONT_UNIFIED_GATE_ENABLED, 1)) { - if (g_step_trace && size == 33000) write(2, "STEP:5 Unified Gate check\n", 26); + if (wcfg->step_trace && size == 33000) write(2, "STEP:5 Unified Gate check\n", 26); if (size <= tiny_get_max_size()) { - if (g_step_trace && size == 33000) write(2, "STEP:5.1 Inside Unified\n", 24); + if (wcfg->step_trace && size == 33000) write(2, "STEP:5.1 Inside Unified\n", 24); void* ptr = malloc_tiny_fast(size); if (__builtin_expect(ptr != NULL, 1)) { g_hakmem_lock_depth--; - if (size == 33000) write(2, "RET:TinyFast\n", 13); + if (wcfg->step_trace && size == 33000) write(2, "RET:TinyFast\n", 13); return ptr; } // Unified Cache miss → fallback to normal path (hak_alloc_at) } } - if (g_step_trace && size == 33000) write(2, "STEP:6 All checks passed\n", 25); + if (wcfg->step_trace && size == 33000) write(2, "STEP:6 All checks passed\n", 25); #if !HAKMEM_BUILD_RELEASE if (count > 14250 && count < 14280 && size <= 1024) { diff --git a/core/box/tiny_env_box.c b/core/box/tiny_env_box.c new file mode 100644 index 00000000..6bc976e5 --- /dev/null +++ b/core/box/tiny_env_box.c @@ -0,0 +1,71 @@ +#include "tiny_env_box.h" +#include +#include + +tiny_env_cfg_t g_tiny_env = {0}; + +static int env_flag(const char* name, int def_val) { + const char* e = getenv(name); + if (!e || *e == '\0') return def_val; + return (*e != '0'); +} + +static int env_int(const char* name, int def_val) { + const char* e = getenv(name); + if (!e || *e == '\0') return def_val; + return atoi(e); +} + +void tiny_env_init_once(void) { + static _Atomic int init_state = 0; // 0: uninit, 1: inited + int expected = 0; + if (!atomic_compare_exchange_strong_explicit(&init_state, &expected, 1, + memory_order_acq_rel, memory_order_relaxed)) { + return; // already initialized + } + + g_tiny_env.tiny_tls_sll = env_flag("HAKMEM_TINY_TLS_SLL", 1); + g_tiny_env.sll_multiplier = env_int("HAKMEM_SLL_MULTIPLIER", 2); + g_tiny_env.tiny_no_front_cache = env_flag("HAKMEM_TINY_NO_FRONT_CACHE", 0); + g_tiny_env.tiny_no_quick = env_flag("HAKMEM_TINY_NO_QUICK", 0); + g_tiny_env.tiny_prefetch = env_flag("HAKMEM_TINY_PREFETCH", 0); + g_tiny_env.front_direct = env_flag("HAKMEM_TINY_FRONT_DIRECT", 0); + g_tiny_env.use_class_map = env_flag("HAKMEM_TINY_NO_CLASS_MAP", 0) ? 0 : 1; + g_tiny_env.route_enable = env_flag("HAKMEM_ROUTE", 0); + g_tiny_env.route_sample_lg = env_int("HAKMEM_ROUTE_SAMPLE_LG", 10); + g_tiny_env.larson_fix = env_flag("HAKMEM_TINY_LARSON_FIX", 0); + g_tiny_env.active_track = env_flag("HAKMEM_TINY_ACTIVE_TRACK", 0); + g_tiny_env.drain_to_sll = env_int("HAKMEM_TINY_DRAIN_TO_SLL", 0); + g_tiny_env.free_to_ss = env_flag("HAKMEM_TINY_FREE_TO_SS", 0); + g_tiny_env.route_free = env_flag("HAKMEM_TINY_ROUTE_FREE", 0); + g_tiny_env.sll_diag = env_flag("HAKMEM_TINY_SLL_DIAG", 0); + g_tiny_env.sll_safeheader = env_flag("HAKMEM_TINY_SLL_SAFEHEADER", 0); + g_tiny_env.sll_ring = env_flag("HAKMEM_TINY_SLL_RING", 0); + g_tiny_env.free_fast = env_flag("HAKMEM_TINY_FREE_FAST", 1); + g_tiny_env.sll_canary_fast = env_flag("HAKMEM_TINY_SLL_CANARY_FAST", 0); + g_tiny_env.ss_free_debug = env_flag("HAKMEM_SS_FREE_DEBUG", 0); + g_tiny_env.ss_adopt = env_flag("HAKMEM_TINY_SS_ADOPT", 1); + g_tiny_env.disable_remote = env_flag("HAKMEM_TINY_DISABLE_REMOTE", 0); + g_tiny_env.freelist_mask = env_flag("HAKMEM_TINY_FREELIST_MASK", 0); + g_tiny_env.alloc_1024_metric = env_flag("HAKMEM_TINY_ALLOC_1024_METRIC", 0); + g_tiny_env.tiny_profile = env_flag("HAKMEM_TINY_PROFILE", 0); + g_tiny_env.tiny_fast_stats = env_flag("HAKMEM_TINY_FAST_STATS", 0); + g_tiny_env.heap_v2_stats = env_flag("HAKMEM_TINY_HEAP_V2_STATS", 0); + g_tiny_env.front_slim = env_flag("HAKMEM_TINY_FRONT_SLIM", 0); + { + int pct = env_int("HAKMEM_SFC_CASCADE_PCT", 50); + if (pct < 0) pct = 0; + if (pct > 100) pct = 100; + g_tiny_env.sfc_cascade_pct = pct; + } + g_tiny_env.sfc_cascade = env_flag("HAKMEM_TINY_SFC_CASCADE", 0); + g_tiny_env.alloc_remote_relax = env_flag("HAKMEM_TINY_ALLOC_REMOTE_RELAX", 0); + g_tiny_env.ss_empty_reuse = env_flag("HAKMEM_SS_EMPTY_REUSE", 1); + g_tiny_env.ss_empty_scan_limit = env_int("HAKMEM_SS_EMPTY_SCAN_LIMIT", 32); + g_tiny_env.ss_acquire_debug = env_flag("HAKMEM_SS_ACQUIRE_DEBUG", 0); + g_tiny_env.tension_drain_enable = env_flag("HAKMEM_TINY_TENSION_DRAIN_ENABLE", 1); + g_tiny_env.tension_drain_threshold = env_int("HAKMEM_TINY_TENSION_DRAIN_THRESHOLD", 1024); + + g_tiny_env.inited = 1; +} + diff --git a/core/box/tiny_env_box.h b/core/box/tiny_env_box.h new file mode 100644 index 00000000..37cf4af2 --- /dev/null +++ b/core/box/tiny_env_box.h @@ -0,0 +1,56 @@ +// tiny_env_box.h - Centralized Tiny env cache (hot-path safe) +#pragma once + +#include + +typedef struct { + int inited; + int tiny_tls_sll; // HAKMEM_TINY_TLS_SLL (default: 1) + int sll_multiplier; // HAKMEM_SLL_MULTIPLIER (default: 2) + int tiny_no_front_cache; // HAKMEM_TINY_NO_FRONT_CACHE (default: 0) + int tiny_no_quick; // HAKMEM_TINY_NO_QUICK (default: 0) + int tiny_prefetch; // HAKMEM_TINY_PREFETCH (default: 0) + int front_direct; // HAKMEM_TINY_FRONT_DIRECT (default: 0) + int use_class_map; // !HAKMEM_TINY_NO_CLASS_MAP (default: 1) + int route_enable; // HAKMEM_ROUTE (default: 0) + int route_sample_lg; // HAKMEM_ROUTE_SAMPLE_LG (default: 10 -> 1/1024) + int larson_fix; // HAKMEM_TINY_LARSON_FIX (default: 0) + int active_track; // HAKMEM_TINY_ACTIVE_TRACK (default: 0) + int drain_to_sll; // HAKMEM_TINY_DRAIN_TO_SLL (default: 0) + int free_to_ss; // HAKMEM_TINY_FREE_TO_SS (default: 0) + int route_free; // HAKMEM_TINY_ROUTE_FREE (default: 0) + int sll_diag; // HAKMEM_TINY_SLL_DIAG (default: 0) + int sll_safeheader; // HAKMEM_TINY_SLL_SAFEHEADER (default: 0) + int sll_ring; // HAKMEM_TINY_SLL_RING (default: 0) + int free_fast; // HAKMEM_TINY_FREE_FAST (default: 1) + int sll_canary_fast; // HAKMEM_TINY_SLL_CANARY_FAST (default: 0) + int ss_free_debug; // HAKMEM_SS_FREE_DEBUG (default: 0) + int ss_adopt; // HAKMEM_TINY_SS_ADOPT (default: 1) + int disable_remote; // HAKMEM_TINY_DISABLE_REMOTE (default: 0) + int freelist_mask; // HAKMEM_TINY_FREELIST_MASK (default: 0) + int alloc_1024_metric; // HAKMEM_TINY_ALLOC_1024_METRIC (default: 0) + int tiny_profile; // HAKMEM_TINY_PROFILE (default: 0) + int tiny_fast_stats; // HAKMEM_TINY_FAST_STATS (default: 0) + int heap_v2_stats; // HAKMEM_TINY_HEAP_V2_STATS (default: 0) + int front_slim; // HAKMEM_TINY_FRONT_SLIM (default: 0) + int sfc_cascade_pct; // HAKMEM_SFC_CASCADE_PCT (default: 50) + int sfc_cascade; // HAKMEM_TINY_SFC_CASCADE (default: 0) + int alloc_remote_relax; // HAKMEM_TINY_ALLOC_REMOTE_RELAX (default: 0) + int ss_empty_reuse; // HAKMEM_SS_EMPTY_REUSE (default: 1) + int ss_empty_scan_limit; // HAKMEM_SS_EMPTY_SCAN_LIMIT (default: 32) + int ss_acquire_debug; // HAKMEM_SS_ACQUIRE_DEBUG (default: 0) + int tension_drain_enable; // HAKMEM_TINY_TENSION_DRAIN_ENABLE (default: 1) + int tension_drain_threshold; // HAKMEM_TINY_TENSION_DRAIN_THRESHOLD (default: 1024) +} tiny_env_cfg_t; + +extern tiny_env_cfg_t g_tiny_env; + +void tiny_env_init_once(void); + +static inline const tiny_env_cfg_t* tiny_env_cfg(void) { + if (__builtin_expect(!g_tiny_env.inited, 0)) { + tiny_env_init_once(); + } + return &g_tiny_env; +} + diff --git a/core/box/wrapper_env_box.c b/core/box/wrapper_env_box.c new file mode 100644 index 00000000..ecf87115 --- /dev/null +++ b/core/box/wrapper_env_box.c @@ -0,0 +1,54 @@ +#define _GNU_SOURCE +#include "wrapper_env_box.h" +#include +#include + +wrapper_env_cfg_t g_wrapper_env = {.inited = 0, .step_trace = 0, .ld_safe_mode = 1, .free_wrap_trace = 0}; + +static inline int env_flag(const char* name, int def) { + const char* e = getenv(name); + if (!e || *e == '\0') return def; + return (*e != '0'); +} + +static inline int env_int(const char* name, int def) { + const char* e = getenv(name); + if (!e || *e == '\0') return def; + char* end; + long val = strtol(e, &end, 10); + return (end != e) ? (int)val : def; +} + +void wrapper_env_init_once(void) { + // Atomic CAS to ensure exactly-once initialization + static _Atomic int init_started = 0; + int expected = 0; + + if (!atomic_compare_exchange_strong_explicit(&init_started, &expected, 1, + memory_order_acq_rel, + memory_order_relaxed)) { + // Someone else is initializing or already initialized + // Spin until they're done + while (!__builtin_expect(g_wrapper_env.inited, 1)) { + __builtin_ia32_pause(); + } + return; + } + + // We own the initialization + g_wrapper_env.step_trace = env_flag("HAKMEM_STEP_TRACE", 0); + g_wrapper_env.ld_safe_mode = env_int("HAKMEM_LD_SAFE", 1); + g_wrapper_env.free_wrap_trace = env_flag("HAKMEM_FREE_WRAP_TRACE", 0); + + // Mark as initialized last with memory barrier + atomic_store_explicit(&g_wrapper_env.inited, 1, memory_order_release); +} + +__attribute__((constructor(101))) +static void wrapper_env_ctor(void) { + // Constructor priority 101 runs early (libc uses 100+) + // This ensures initialization before any malloc calls + if (!g_wrapper_env.inited) { + wrapper_env_init_once(); + } +} diff --git a/core/box/wrapper_env_box.h b/core/box/wrapper_env_box.h new file mode 100644 index 00000000..a1048512 --- /dev/null +++ b/core/box/wrapper_env_box.h @@ -0,0 +1,25 @@ +// wrapper_env_box.h - Environment variable cache for malloc/free wrappers +// Eliminates getenv() calls from malloc/free hot paths +#pragma once + +#include + +typedef struct { + int inited; + int step_trace; // HAKMEM_STEP_TRACE (default: 0) + int ld_safe_mode; // HAKMEM_LD_SAFE (default: 1) + int free_wrap_trace; // HAKMEM_FREE_WRAP_TRACE (default: 0) +} wrapper_env_cfg_t; + +extern wrapper_env_cfg_t g_wrapper_env; + +void wrapper_env_init_once(void); + +static inline const wrapper_env_cfg_t* wrapper_env_cfg(void) { + // Constructor ensures init at library load time + // This check prevents repeated initialization in multi-threaded context + if (__builtin_expect(!g_wrapper_env.inited, 0)) { + wrapper_env_init_once(); + } + return &g_wrapper_env; +} diff --git a/core/hakmem.c b/core/hakmem.c index 57f512e6..7265d833 100644 --- a/core/hakmem.c +++ b/core/hakmem.c @@ -128,11 +128,19 @@ static int g_ultra_debug_on_exit = 0; // HAKMEM_TINY_ULTRA_DEBUG=1 _Atomic uint64_t g_free_wrapper_calls = 0; // Cached LD_PRELOAD detection for wrapper hot paths (avoid getenv per call) static int g_ldpre_env_cached = -1; // -1 = unknown, 0/1 cached -static inline int hak_ld_env_mode(void) { +// Cached libc force flags +static int g_force_libc_alloc_init = -1; // HAKMEM_FORCE_LIBC_ALLOC_INIT +static inline void hak_ld_env_init(void) { if (g_ldpre_env_cached < 0) { const char* ldpre = getenv("LD_PRELOAD"); g_ldpre_env_cached = (ldpre && strstr(ldpre, "libhakmem.so")) ? 1 : 0; } +} +__attribute__((constructor)) +static void hak_ld_env_ctor(void) { + hak_ld_env_init(); +} +static inline int hak_ld_env_mode(void) { return g_ldpre_env_cached; } @@ -142,12 +150,40 @@ static int g_force_libc_alloc = 1; #else static int g_force_libc_alloc = -1; // 1=force libc, 0=use hakmem, -1=uninitialized #endif +__attribute__((constructor)) +static void hak_force_libc_ctor(void) { + // Cache FORCE_LIBC and WRAP_TINY at load time to avoid hot-path getenv +#ifndef HAKMEM_FORCE_LIBC_ALLOC_BUILD + if (g_force_libc_alloc < 0) { + const char* force = getenv("HAKMEM_FORCE_LIBC_ALLOC"); + if (force && *force) { + g_force_libc_alloc = (atoi(force) != 0); + } else { + const char* wrap = getenv("HAKMEM_WRAP_TINY"); + if (wrap && *wrap && atoi(wrap) == 0) { + g_force_libc_alloc = 1; + } else { + g_force_libc_alloc = 0; + } + } + } + if (g_force_libc_alloc_init < 0) { + const char* init_only = getenv("HAKMEM_FORCE_LIBC_ALLOC_INIT"); + g_force_libc_alloc_init = (init_only && atoi(init_only) != 0) ? 1 : 0; + } +#else + g_force_libc_alloc_init = 0; +#endif +} static inline int hak_force_libc_alloc(void) { // During early process start or allocator init, optionally force libc until init completes. // This avoids sanitizer -> dlsym -> malloc recursion before TLS is ready. if (!g_initialized) { - const char* init_only = getenv("HAKMEM_FORCE_LIBC_ALLOC_INIT"); - if (init_only && atoi(init_only) != 0) { + if (g_force_libc_alloc_init < 0) { + const char* init_only = getenv("HAKMEM_FORCE_LIBC_ALLOC_INIT"); + g_force_libc_alloc_init = (init_only && atoi(init_only) != 0) ? 1 : 0; + } + if (g_force_libc_alloc_init) { return 1; } }