tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 19-3b: pass down env snapshot in hot paths

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-15 12:50:16 +09:00
parent 8f4ada5bbd
commit e1a4561992
7 changed files with 207 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
core/front/malloc_tiny_fast.h
View File

@ -233,9 +233,10 @@ static inline void* malloc_tiny_fast_for_class(size_t size, int class_idx) {
// This is the most common hot path - avoids TLS policy overhead
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
const HakmemEnvSnapshot* env = NULL;
bool c7_ultra_on;
if (hakmem_env_snapshot_enabled()) {
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
env = hakmem_env_snapshot();
c7_ultra_on = env->tiny_c7_ultra_enabled;
} else {
c7_ultra_on = tiny_c7_ultra_enabled_env();
@ -269,7 +270,7 @@ static inline void* malloc_tiny_fast_for_class(size_t size, int class_idx) {
route_kind = tiny_static_route_get_kind_fast(class_idx);
} else {
// Phase 3 C2: Use policy hot cache if enabled (eliminates policy_snapshot() call)
route_kind = tiny_policy_hot_get_route(class_idx);
route_kind = tiny_policy_hot_get_route_with_env((uint32_t)class_idx, env);
}
// Phase 2 B3: Routing dispatch (ENV gate HAKMEM_TINY_ALLOC_ROUTE_SHAPE)
@ -381,7 +382,7 @@ static inline void* malloc_tiny_fast(size_t size) {
// Cold path: Cross-thread free, TinyHeap routes, and legacy fallback
// (noinline,cold to keep hot path small and I-cache clean)
__attribute__((noinline,cold))
static int free_tiny_fast_cold(void* ptr, void* base, int class_idx)
static int free_tiny_fast_cold(void* ptr, void* base, int class_idx, const HakmemEnvSnapshot* env)
{
FREE_TINY_FAST_HOTCOLD_STAT_INC(cold_hit);
@ -399,15 +400,6 @@ static int free_tiny_fast_cold(void* ptr, void* base, int class_idx)
route = tiny_route_for_class((uint8_t)class_idx);
}
const int use_tiny_heap = tiny_route_is_heap_kind(route);
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
const TinyFrontV3Snapshot* front_snap;
if (hakmem_env_snapshot_enabled()) {
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
front_snap = env->tiny_front_v3_enabled ? tiny_front_v3_snapshot_get() : NULL;
} else {
front_snap = __builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL;
}
// TWO-SPEED: SuperSlab registration check is DEBUG-ONLY to keep HOT PATH fast.
// In Release builds, we trust header magic (0xA0) as sufficient validation.
@ -571,7 +563,7 @@ static int free_tiny_fast_cold(void* ptr, void* base, int class_idx)
// Phase REFACTOR-2: Legacy fallback (use unified helper)
legacy_fallback:
FREE_TINY_FAST_HOTCOLD_STAT_INC(cold_legacy_fallback);
tiny_legacy_fallback_free_base(base, class_idx);
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
return 1;
}
@ -623,13 +615,8 @@ static inline int free_tiny_fast_hot(void* ptr) {
// Phase v11b-1: C7 ULTRA early-exit (skip policy snapshot for most common case)
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
bool c7_ultra_free;
if (hakmem_env_snapshot_enabled()) {
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
c7_ultra_free = env->tiny_c7_ultra_enabled;
} else {
c7_ultra_free = tiny_c7_ultra_enabled_env();
}
const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;
const bool c7_ultra_free = env ? env->tiny_c7_ultra_enabled : tiny_c7_ultra_enabled_env();
if (class_idx == 7 && c7_ultra_free) {
FREE_TINY_FAST_HOTCOLD_STAT_INC(hot_c7_ultra);
@ -650,7 +637,7 @@ static inline int free_tiny_fast_hot(void* ptr) {
if (__builtin_expect(class_idx <= 3 && !g_larson_fix, 1)) {
// C0-C3 + Larson mode OFF → Direct to legacy (no policy snapshot overhead)
tiny_legacy_fallback_free_base(base, class_idx);
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
FREE_TINY_FAST_HOTCOLD_STAT_INC(hot_hit);
return 1;
}
@ -717,7 +704,7 @@ static inline int free_tiny_fast_hot(void* ptr) {
cold_path:
// Delegate to cold path for cross-thread, TinyHeap, and legacy handling
return free_tiny_fast_cold(ptr, base, class_idx);
return free_tiny_fast_cold(ptr, base, class_idx, env);
#else
// No header mode - fall back to normal free
@ -773,6 +760,9 @@ static inline int free_tiny_fast(void* ptr) {
// Phase FREE-LEGACY-BREAKDOWN-1: カウンタ散布 (1. 関数入口)
FREE_PATH_STAT_INC(total_calls);
// Phase 19-3b: Consolidate ENV snapshot reads (capture once per free_tiny_fast call).
const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;
// Phase 9: MONO DUALHOT early-exit for C0-C3 (skip policy snapshot, direct to legacy)
// Conditions:
// - ENV: HAKMEM_FREE_TINY_FAST_MONO_DUALHOT=1
@ -792,7 +782,7 @@ static inline int free_tiny_fast(void* ptr) {
g_tiny_route_class[class_idx] == TINY_ROUTE_LEGACY) {
// Direct path: Skip policy snapshot, go straight to legacy fallback
FREE_PATH_STAT_INC(mono_dualhot_hit);
tiny_legacy_fallback_free_base(base, class_idx);
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
return 1;
}
}
@ -820,7 +810,7 @@ static inline int free_tiny_fast(void* ptr) {
if (!g_larson_fix) {
// Direct path: Skip policy snapshot, go straight to legacy fallback
FREE_PATH_STAT_INC(mono_legacy_direct_hit);
tiny_legacy_fallback_free_base(base, class_idx);
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
return 1;
}
}
@ -830,13 +820,7 @@ static inline int free_tiny_fast(void* ptr) {
// Phase v11b-1: C7 ULTRA early-exit (skip policy snapshot for most common case)
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
bool c7_ultra_free;
if (hakmem_env_snapshot_enabled()) {
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
c7_ultra_free = env->tiny_c7_ultra_enabled;
} else {
c7_ultra_free = tiny_c7_ultra_enabled_env();
}
const bool c7_ultra_free = env ? env->tiny_c7_ultra_enabled : tiny_c7_ultra_enabled_env();
if (class_idx == 7 && c7_ultra_free) {
tiny_c7_ultra_free(ptr);
@ -909,15 +893,6 @@ legacy_fallback:
route = tiny_route_for_class((uint8_t)class_idx);
}
const int use_tiny_heap = tiny_route_is_heap_kind(route);
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
const TinyFrontV3Snapshot* front_snap;
if (hakmem_env_snapshot_enabled()) {
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
front_snap = env->tiny_front_v3_enabled ? tiny_front_v3_snapshot_get() : NULL;
} else {
front_snap = __builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL;
}
// TWO-SPEED: SuperSlab registration check is DEBUG-ONLY to keep HOT PATH fast.
// In Release builds, we trust header magic (0xA0) as sufficient validation.
@ -1055,7 +1030,7 @@ legacy_fallback:
#endif
// Phase REFACTOR-2: Legacy fallback (use unified helper)
tiny_legacy_fallback_free_base(base, class_idx);
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
return 1;
#else
// No header mode - fall back to normal free