Phase 19-3b: pass down env snapshot in hot paths
This commit is contained in:
Moe Charm (CI)
@ -1,5 +1,29 @@
|
||||
# 本線タスク（現在）
|
||||
|
||||
## 更新メモ（2025-12-15 Phase 19-3b ENV-SNAPSHOT-PASSDOWN）
|
||||
|
||||
### Phase 19-3b ENV-SNAPSHOT-PASSDOWN: Consolidate ENV snapshot reads across hot helpers — ✅ GO (+2.76%)
|
||||
|
||||
**A/B Test Results** (`scripts/run_mixed_10_cleanenv.sh`, iter=20M ws=400):
|
||||
- Baseline (Phase 19-3a): mean **55.56M** ops/s, median **55.65M**
|
||||
- Optimized (Phase 19-3b): mean **57.10M** ops/s, median **57.09M**
|
||||
- Delta: **+2.76% mean** / **+2.57% median** → ✅ GO
|
||||
|
||||
**Change**:
|
||||
- `core/front/malloc_tiny_fast.h`: capture `env` once in `free_tiny_fast()` / `free_tiny_fast_hot()` and pass into cold/legacy helpers; use `tiny_policy_hot_get_route_with_env()` to avoid a second snapshot gate.
|
||||
- `core/box/tiny_legacy_fallback_box.h`: add `tiny_legacy_fallback_free_base_with_env(...)` and use it from hot paths to avoid redundant `hakmem_env_snapshot_enabled()` checks.
|
||||
- `core/box/tiny_metadata_cache_hot_box.h`: add `tiny_policy_hot_get_route_with_env(...)` so `malloc_tiny_fast_for_class()` can reuse the already-fetched snapshot.
|
||||
- Remove dead `front_snap` computations (set-but-unused) from the free hot paths.
|
||||
|
||||
**Why it works**:
|
||||
- Hot call chains had multiple redundant `hakmem_env_snapshot_enabled()` gates (branch + loads) across nested helpers.
|
||||
- Capture once → pass-down keeps the “ENV decision” at a single boundary per operation and removes duplicated work.
|
||||
|
||||
**Next**:
|
||||
- Phase 19-3c (optional): if needed, also pass `env` into alloc-side call chains to remove the remaining `malloc_tiny_fast_for_class()` gate.
|
||||
|
||||
---
|
||||
|
||||
## 更新メモ（2025-12-15 Phase 19-3a UNLIKELY-HINT-REMOVAL）
|
||||
|
||||
### Phase 19-3a UNLIKELY-HINT-REMOVAL: ENV Snapshot UNLIKELY Hint Removal — ✅ GO (+4.42%)
|
||||
|
||||
@ -21,18 +21,11 @@
|
||||
// - Returns: void (always succeeds or falls back to tiny_hot_free_fast)
|
||||
//
|
||||
__attribute__((always_inline))
|
||||
static inline void tiny_legacy_fallback_free_base(void* base, uint32_t class_idx) {
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
const TinyFrontV3Snapshot* front_snap;
|
||||
bool metadata_cache_on;
|
||||
if (__builtin_expect(hakmem_env_snapshot_enabled(), 0)) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
front_snap = env->tiny_front_v3_enabled ? tiny_front_v3_snapshot_get() : NULL;
|
||||
metadata_cache_on = env->tiny_metadata_cache_eff; // Uses effective (cache && !learner)
|
||||
} else {
|
||||
front_snap = __builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL;
|
||||
metadata_cache_on = tiny_metadata_cache_enabled();
|
||||
}
|
||||
static inline void tiny_legacy_fallback_free_base_with_env(void* base, uint32_t class_idx, const HakmemEnvSnapshot* env) {
|
||||
const TinyFrontV3Snapshot* front_snap =
|
||||
env ? (env->tiny_front_v3_enabled ? tiny_front_v3_snapshot_get() : NULL)
|
||||
: (__builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL);
|
||||
const bool metadata_cache_on = env ? env->tiny_metadata_cache_eff : tiny_metadata_cache_enabled();
|
||||
|
||||
// Phase 3 C2 Patch 2: First page cache hint (optional fast-path)
|
||||
// Check if pointer is in cached page (avoids metadata lookup in future optimizations)
|
||||
@ -64,4 +57,10 @@ static inline void tiny_legacy_fallback_free_base(void* base, uint32_t class_idx
|
||||
tiny_hot_free_fast(class_idx, base);
|
||||
}
|
||||
|
||||
__attribute__((always_inline))
|
||||
static inline void tiny_legacy_fallback_free_base(void* base, uint32_t class_idx) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;
|
||||
tiny_legacy_fallback_free_base_with_env(base, class_idx, env);
|
||||
}
|
||||
|
||||
#endif // HAKMEM_TINY_LEGACY_FALLBACK_BOX_H
|
||||
|
||||
@ -58,15 +58,9 @@ static inline void tiny_policy_hot_refresh(void) {
|
||||
/// @param class_idx: Size class (0-7)
|
||||
/// @return: Route kind for this class
|
||||
__attribute__((always_inline))
|
||||
static inline SmallRouteKind tiny_policy_hot_get_route(uint32_t class_idx) {
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
bool metadata_cache_eff;
|
||||
if (__builtin_expect(hakmem_env_snapshot_enabled(), 0)) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
metadata_cache_eff = env->tiny_metadata_cache_eff; // Already includes learner check
|
||||
} else {
|
||||
metadata_cache_eff = tiny_metadata_cache_enabled() && !g_policy_hot.learner_v7_enabled;
|
||||
}
|
||||
static inline SmallRouteKind tiny_policy_hot_get_route_with_env(uint32_t class_idx, const HakmemEnvSnapshot* env) {
|
||||
const bool metadata_cache_eff =
|
||||
env ? env->tiny_metadata_cache_eff : (tiny_metadata_cache_enabled() && !g_policy_hot.learner_v7_enabled);
|
||||
|
||||
if (__builtin_expect(metadata_cache_eff, 0)) {
|
||||
// Fast path: use cached route_kind
|
||||
@ -80,4 +74,10 @@ static inline SmallRouteKind tiny_policy_hot_get_route(uint32_t class_idx) {
|
||||
return policy->route_kind[class_idx];
|
||||
}
|
||||
|
||||
__attribute__((always_inline))
|
||||
static inline SmallRouteKind tiny_policy_hot_get_route(uint32_t class_idx) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;
|
||||
return tiny_policy_hot_get_route_with_env(class_idx, env);
|
||||
}
|
||||
|
||||
#endif // HAK_TINY_METADATA_CACHE_HOT_BOX_H
|
||||
|
||||
@ -233,9 +233,10 @@ static inline void* malloc_tiny_fast_for_class(size_t size, int class_idx) {
|
||||
// This is the most common hot path - avoids TLS policy overhead
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
|
||||
const HakmemEnvSnapshot* env = NULL;
|
||||
bool c7_ultra_on;
|
||||
if (hakmem_env_snapshot_enabled()) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
env = hakmem_env_snapshot();
|
||||
c7_ultra_on = env->tiny_c7_ultra_enabled;
|
||||
} else {
|
||||
c7_ultra_on = tiny_c7_ultra_enabled_env();
|
||||
@ -269,7 +270,7 @@ static inline void* malloc_tiny_fast_for_class(size_t size, int class_idx) {
|
||||
route_kind = tiny_static_route_get_kind_fast(class_idx);
|
||||
} else {
|
||||
// Phase 3 C2: Use policy hot cache if enabled (eliminates policy_snapshot() call)
|
||||
route_kind = tiny_policy_hot_get_route(class_idx);
|
||||
route_kind = tiny_policy_hot_get_route_with_env((uint32_t)class_idx, env);
|
||||
}
|
||||
|
||||
// Phase 2 B3: Routing dispatch (ENV gate HAKMEM_TINY_ALLOC_ROUTE_SHAPE)
|
||||
@ -381,7 +382,7 @@ static inline void* malloc_tiny_fast(size_t size) {
|
||||
// Cold path: Cross-thread free, TinyHeap routes, and legacy fallback
|
||||
// (noinline,cold to keep hot path small and I-cache clean)
|
||||
__attribute__((noinline,cold))
|
||||
static int free_tiny_fast_cold(void* ptr, void* base, int class_idx)
|
||||
static int free_tiny_fast_cold(void* ptr, void* base, int class_idx, const HakmemEnvSnapshot* env)
|
||||
{
|
||||
FREE_TINY_FAST_HOTCOLD_STAT_INC(cold_hit);
|
||||
|
||||
@ -399,15 +400,6 @@ static int free_tiny_fast_cold(void* ptr, void* base, int class_idx)
|
||||
route = tiny_route_for_class((uint8_t)class_idx);
|
||||
}
|
||||
const int use_tiny_heap = tiny_route_is_heap_kind(route);
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
|
||||
const TinyFrontV3Snapshot* front_snap;
|
||||
if (hakmem_env_snapshot_enabled()) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
front_snap = env->tiny_front_v3_enabled ? tiny_front_v3_snapshot_get() : NULL;
|
||||
} else {
|
||||
front_snap = __builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL;
|
||||
}
|
||||
|
||||
// TWO-SPEED: SuperSlab registration check is DEBUG-ONLY to keep HOT PATH fast.
|
||||
// In Release builds, we trust header magic (0xA0) as sufficient validation.
|
||||
@ -571,7 +563,7 @@ static int free_tiny_fast_cold(void* ptr, void* base, int class_idx)
|
||||
// Phase REFACTOR-2: Legacy fallback (use unified helper)
|
||||
legacy_fallback:
|
||||
FREE_TINY_FAST_HOTCOLD_STAT_INC(cold_legacy_fallback);
|
||||
tiny_legacy_fallback_free_base(base, class_idx);
|
||||
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -623,13 +615,8 @@ static inline int free_tiny_fast_hot(void* ptr) {
|
||||
// Phase v11b-1: C7 ULTRA early-exit (skip policy snapshot for most common case)
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
|
||||
bool c7_ultra_free;
|
||||
if (hakmem_env_snapshot_enabled()) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
c7_ultra_free = env->tiny_c7_ultra_enabled;
|
||||
} else {
|
||||
c7_ultra_free = tiny_c7_ultra_enabled_env();
|
||||
}
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;
|
||||
const bool c7_ultra_free = env ? env->tiny_c7_ultra_enabled : tiny_c7_ultra_enabled_env();
|
||||
|
||||
if (class_idx == 7 && c7_ultra_free) {
|
||||
FREE_TINY_FAST_HOTCOLD_STAT_INC(hot_c7_ultra);
|
||||
@ -650,7 +637,7 @@ static inline int free_tiny_fast_hot(void* ptr) {
|
||||
|
||||
if (__builtin_expect(class_idx <= 3 && !g_larson_fix, 1)) {
|
||||
// C0-C3 + Larson mode OFF → Direct to legacy (no policy snapshot overhead)
|
||||
tiny_legacy_fallback_free_base(base, class_idx);
|
||||
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
|
||||
FREE_TINY_FAST_HOTCOLD_STAT_INC(hot_hit);
|
||||
return 1;
|
||||
}
|
||||
@ -717,7 +704,7 @@ static inline int free_tiny_fast_hot(void* ptr) {
|
||||
|
||||
cold_path:
|
||||
// Delegate to cold path for cross-thread, TinyHeap, and legacy handling
|
||||
return free_tiny_fast_cold(ptr, base, class_idx);
|
||||
return free_tiny_fast_cold(ptr, base, class_idx, env);
|
||||
|
||||
#else
|
||||
// No header mode - fall back to normal free
|
||||
@ -773,6 +760,9 @@ static inline int free_tiny_fast(void* ptr) {
|
||||
// Phase FREE-LEGACY-BREAKDOWN-1: カウンタ散布 (1. 関数入口)
|
||||
FREE_PATH_STAT_INC(total_calls);
|
||||
|
||||
// Phase 19-3b: Consolidate ENV snapshot reads (capture once per free_tiny_fast call).
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;
|
||||
|
||||
// Phase 9: MONO DUALHOT early-exit for C0-C3 (skip policy snapshot, direct to legacy)
|
||||
// Conditions:
|
||||
// - ENV: HAKMEM_FREE_TINY_FAST_MONO_DUALHOT=1
|
||||
@ -792,7 +782,7 @@ static inline int free_tiny_fast(void* ptr) {
|
||||
g_tiny_route_class[class_idx] == TINY_ROUTE_LEGACY) {
|
||||
// Direct path: Skip policy snapshot, go straight to legacy fallback
|
||||
FREE_PATH_STAT_INC(mono_dualhot_hit);
|
||||
tiny_legacy_fallback_free_base(base, class_idx);
|
||||
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -820,7 +810,7 @@ static inline int free_tiny_fast(void* ptr) {
|
||||
if (!g_larson_fix) {
|
||||
// Direct path: Skip policy snapshot, go straight to legacy fallback
|
||||
FREE_PATH_STAT_INC(mono_legacy_direct_hit);
|
||||
tiny_legacy_fallback_free_base(base, class_idx);
|
||||
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -830,13 +820,7 @@ static inline int free_tiny_fast(void* ptr) {
|
||||
// Phase v11b-1: C7 ULTRA early-exit (skip policy snapshot for most common case)
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
|
||||
bool c7_ultra_free;
|
||||
if (hakmem_env_snapshot_enabled()) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
c7_ultra_free = env->tiny_c7_ultra_enabled;
|
||||
} else {
|
||||
c7_ultra_free = tiny_c7_ultra_enabled_env();
|
||||
}
|
||||
const bool c7_ultra_free = env ? env->tiny_c7_ultra_enabled : tiny_c7_ultra_enabled_env();
|
||||
|
||||
if (class_idx == 7 && c7_ultra_free) {
|
||||
tiny_c7_ultra_free(ptr);
|
||||
@ -909,15 +893,6 @@ legacy_fallback:
|
||||
route = tiny_route_for_class((uint8_t)class_idx);
|
||||
}
|
||||
const int use_tiny_heap = tiny_route_is_heap_kind(route);
|
||||
// Phase 4 E1: Use ENV snapshot when enabled (consolidates 3 TLS reads → 1)
|
||||
// Phase 19-3a: Remove UNLIKELY hint (snapshot is ON by default in presets, hint is backwards)
|
||||
const TinyFrontV3Snapshot* front_snap;
|
||||
if (hakmem_env_snapshot_enabled()) {
|
||||
const HakmemEnvSnapshot* env = hakmem_env_snapshot();
|
||||
front_snap = env->tiny_front_v3_enabled ? tiny_front_v3_snapshot_get() : NULL;
|
||||
} else {
|
||||
front_snap = __builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL;
|
||||
}
|
||||
|
||||
// TWO-SPEED: SuperSlab registration check is DEBUG-ONLY to keep HOT PATH fast.
|
||||
// In Release builds, we trust header magic (0xA0) as sufficient validation.
|
||||
@ -1055,7 +1030,7 @@ legacy_fallback:
|
||||
#endif
|
||||
|
||||
// Phase REFACTOR-2: Legacy fallback (use unified helper)
|
||||
tiny_legacy_fallback_free_base(base, class_idx);
|
||||
tiny_legacy_fallback_free_base_with_env(base, (uint32_t)class_idx, env);
|
||||
return 1;
|
||||
#else
|
||||
// No header mode - fall back to normal free
|
||||
|
||||
@ -0,0 +1,76 @@
|
||||
# Phase 19-3b: ENV Snapshot Pass-Down — A/B Test Results
|
||||
|
||||
## Summary
|
||||
|
||||
Verdict: ✅ **GO**
|
||||
|
||||
- Baseline mean: **55.56M ops/s**
|
||||
- Optimized mean: **57.10M ops/s**
|
||||
- Delta (mean): **+2.76%**
|
||||
|
||||
- Baseline median: **55.65M ops/s**
|
||||
- Optimized median: **57.09M ops/s**
|
||||
- Delta (median): **+2.57%**
|
||||
|
||||
## What Changed
|
||||
|
||||
- `core/front/malloc_tiny_fast.h`
|
||||
- Capture `const HakmemEnvSnapshot* env` once per hot call and pass it down:
|
||||
- `free_tiny_fast()` / `free_tiny_fast_hot()` capture once
|
||||
- `free_tiny_fast_cold(..., env)` consumes it
|
||||
- `tiny_legacy_fallback_free_base_with_env(..., env)` consumes it
|
||||
- Reuse the same snapshot in the alloc route selection:
|
||||
- `tiny_policy_hot_get_route_with_env(class_idx, env)`
|
||||
- Remove dead `front_snap` computations (were set-but-unused).
|
||||
|
||||
- `core/box/tiny_legacy_fallback_box.h`
|
||||
- Add `tiny_legacy_fallback_free_base_with_env(...)` and keep the old wrapper for compatibility.
|
||||
|
||||
- `core/box/tiny_metadata_cache_hot_box.h`
|
||||
- Add `tiny_policy_hot_get_route_with_env(...)` and keep the old wrapper for compatibility.
|
||||
|
||||
## Bench Setup
|
||||
|
||||
- Command: `scripts/run_mixed_10_cleanenv.sh`
|
||||
- Profile: `HAKMEM_PROFILE=MIXED_TINYV3_C7_SAFE` (script default)
|
||||
- Params: `iter=20000000 ws=400`
|
||||
- Host: same machine / same build flags, back-to-back runs.
|
||||
|
||||
## Raw Results
|
||||
|
||||
### Baseline (Phase 19-3a)
|
||||
|
||||
```
|
||||
56215204
|
||||
55685609
|
||||
55968309
|
||||
55866150
|
||||
54795835
|
||||
55113419
|
||||
55659129
|
||||
55645869
|
||||
55286223
|
||||
55409488
|
||||
```
|
||||
|
||||
Mean: 55,564,523.5
|
||||
Median: 55,652,499.0
|
||||
|
||||
### Optimized (Phase 19-3b)
|
||||
|
||||
```
|
||||
57413912
|
||||
57291780
|
||||
56913158
|
||||
57044292
|
||||
57219468
|
||||
56609810
|
||||
56995683
|
||||
57027125
|
||||
57350810
|
||||
57126094
|
||||
```
|
||||
|
||||
Mean: 57,099,213.2
|
||||
Median: 57,085,193.0
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
## 0. Goal
|
||||
|
||||
**Objective**: Reduce ENV check overhead from per-operation 3+ TLS reads to 1 TLS read
|
||||
**Expected Impact**: -10.0 instructions/op, -4.0 branches/op, +5-8% throughput
|
||||
**Expected Impact (target)**: -10.0 instructions/op, -4.0 branches/op, +3-8% throughput
|
||||
**Risk Level**: MEDIUM (ENV invalidation handling required)
|
||||
**Box Name**: EnvSnapshotConsolidationBox (Phase 19-3)
|
||||
|
||||
@ -32,6 +32,12 @@ Phase 19-3a removed the call-site UNLIKELY hint:
|
||||
Observed impact: **GO (+4.42% throughput)** on Mixed.
|
||||
This validates that the remaining ENV work is dominated by branch/layout effects, not just raw "read cost".
|
||||
|
||||
### Phase 19-3b Result (validated)
|
||||
|
||||
Phase 19-3b consolidated snapshot reads by capturing `env` once per hot call and passing it down into nested helpers.
|
||||
|
||||
Observed impact: **GO (+2.76% mean / +2.57% median)** on Mixed 10-run (`scripts/run_mixed_10_cleanenv.sh`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Current State Analysis
|
||||
@ -235,19 +241,20 @@ if (class_idx == 7 && g_c7_ultra_enabled_fixed) {
|
||||
**Files**:
|
||||
- Modified: `core/front/malloc_tiny_fast.h`
|
||||
- Phase 19-3a: remove backwards `__builtin_expect(..., 0)` hints (DONE, +4.42% GO).
|
||||
- Phase 19-3b/c: thread `const HakmemEnvSnapshot* env` down to eliminate repeated `hakmem_env_snapshot_enabled()` checks.
|
||||
- Modified: `core/box/hak_wrappers.inc.h`
|
||||
- Compute `env` once per wrapper entry and pass it to hot helpers (especially when `HAKMEM_FASTLANE_DIRECT=1`).
|
||||
- Optional (only if separate rollback gate is desired):
|
||||
- New: `core/box/env_snapshot_consolidation_env_box.{h,c}` (cached gate + refresh hook)
|
||||
- Phase 19-3b: thread `const HakmemEnvSnapshot* env` down to eliminate repeated `hakmem_env_snapshot_enabled()` checks (DONE, +2.76% GO).
|
||||
- Modified: `core/box/tiny_legacy_fallback_box.h`
|
||||
- Add `_with_env` helper (Phase 19-3b).
|
||||
- Modified: `core/box/tiny_metadata_cache_hot_box.h`
|
||||
- Add `_with_env` helper (Phase 19-3b).
|
||||
- Optional: `core/box/hak_wrappers.inc.h`
|
||||
- If needed, compute `env` once per wrapper entry and pass it down (removes the remaining alloc-side gate in `malloc_tiny_fast_for_class()`).
|
||||
|
||||
**ENV Gate**:
|
||||
- Base: `HAKMEM_ENV_SNAPSHOT=0/1` (Phase 4 E1 gate; promoted ON in presets)
|
||||
- Optional: `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0/1` (default 0, opt-in) — gates only the “pass-down” refactor for A/B safety.
|
||||
|
||||
**Rollback**:
|
||||
- If using `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION`: set it to `0`.
|
||||
- Otherwise: `HAKMEM_FASTLANE_DIRECT=0` falls back to the wrapper/FastLane path (still safe).
|
||||
- Snapshot behavior: set `HAKMEM_ENV_SNAPSHOT=0` to fall back to per-feature env gates.
|
||||
- Pass-down refactor: revert the Phase 19-3b commit (or add a dedicated pass-down gate if future A/B is needed).
|
||||
|
||||
### 3.2 API Design
|
||||
|
||||
@ -268,29 +275,28 @@ int free_tiny_fast_with_env(void* ptr, const HakmemEnvSnapshot* env);
|
||||
- `__builtin_expect(hakmem_env_snapshot_enabled(), 0)` → `hakmem_env_snapshot_enabled()`
|
||||
- Measured: **GO (+4.42%)**
|
||||
|
||||
**Phase 19-3b (NEXT)**: wrapper entry computes `const HakmemEnvSnapshot* env` once and passes it down.
|
||||
- Wrapper entry:
|
||||
- `const HakmemEnvSnapshot* env = hakmem_env_snapshot_enabled() ? hakmem_env_snapshot() : NULL;`
|
||||
- Hot helpers:
|
||||
- Replace repeated `hakmem_env_snapshot_enabled()` checks with `if (env) { ... } else { ... }`
|
||||
- Keep `env==NULL` fallback path unchanged.
|
||||
- Target: reduce repeated gate checks across hot helpers (especially inside `free_tiny_fast*`).
|
||||
**Phase 19-3b (DONE)**: capture `env` once per hot call and pass it down into nested helpers.
|
||||
- In `core/front/malloc_tiny_fast.h`:
|
||||
- `free_tiny_fast()` / `free_tiny_fast_hot()` capture `env` once and pass it to cold + legacy helpers.
|
||||
- `malloc_tiny_fast_for_class()` reuses the same snapshot for `tiny_policy_hot_get_route_with_env(...)`.
|
||||
- In `core/box/tiny_legacy_fallback_box.h` and `core/box/tiny_metadata_cache_hot_box.h`:
|
||||
- add `_with_env` helpers to consume the pass-down pointer.
|
||||
- Measured: **GO (+2.76% mean / +2.57% median)** on Mixed 10-run.
|
||||
|
||||
**Phase 19-3c (OPTIONAL)**: propagate `env` into legacy fallback + metadata cache helpers to eliminate the remaining call sites:
|
||||
- `core/box/tiny_legacy_fallback_box.h`
|
||||
- `core/box/tiny_metadata_cache_hot_box.h`
|
||||
- (Already done in Phase 19-3b.) Optional next: pass `env` down from wrapper entry to remove the remaining alloc-side gate.
|
||||
|
||||
### 3.4 Files to Modify
|
||||
|
||||
1. `core/front/malloc_tiny_fast.h`
|
||||
- Phase 19-3b: add `_with_env` helper variants or thread `env` through internal helpers.
|
||||
- Replace the remaining repeated `hakmem_env_snapshot_enabled()` call sites with `env`-based checks.
|
||||
2. `core/box/hak_wrappers.inc.h`
|
||||
- Compute `env` once per entry and pass it down (especially for `HAKMEM_FASTLANE_DIRECT` path).
|
||||
3. (Optional) `core/box/tiny_legacy_fallback_box.h`
|
||||
- Thread `env` into the legacy fallback helper to eliminate an extra gate check.
|
||||
4. (Optional) `core/box/tiny_metadata_cache_hot_box.h`
|
||||
- Same for metadata-cache effective checks.
|
||||
- Phase 19-3a: UNLIKELY hint removal.
|
||||
- Phase 19-3b: pass-down `env` to cold + legacy helpers.
|
||||
2. `core/box/tiny_legacy_fallback_box.h`
|
||||
- Phase 19-3b: add `_with_env` helper + keep wrapper.
|
||||
3. `core/box/tiny_metadata_cache_hot_box.h`
|
||||
- Phase 19-3b: add `_with_env` helper + keep wrapper.
|
||||
4. (Optional) `core/box/hak_wrappers.inc.h`
|
||||
- Pass `env` down from wrapper entry (alloc-side; removes one remaining gate).
|
||||
|
||||
---
|
||||
|
||||
@ -299,9 +305,9 @@ int free_tiny_fast_with_env(void* ptr, const HakmemEnvSnapshot* env);
|
||||
### 4.1 Boundary Preservation
|
||||
|
||||
**L0 (ENV gate)**:
|
||||
- `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0` → `env==NULL` (or not passed down), fallback to existing path
|
||||
- `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=1` → `env` is passed down, new path
|
||||
- Compile-time flag: `#if HAKMEM_ENV_SNAPSHOT_CONSOLIDATION` for complete removal
|
||||
- `HAKMEM_ENV_SNAPSHOT=0` → `env==NULL` → fallback to per-feature env gates
|
||||
- `HAKMEM_ENV_SNAPSHOT=1` → `env!=NULL` → snapshot-based checks
|
||||
- (Optional) A dedicated “pass-down gate” can be introduced for A/B safety, but avoid adding a new hot-branch unless needed.
|
||||
|
||||
**L1 (Hot inline)**:
|
||||
- No algorithmic changes, only ENV check consolidation
|
||||
@ -333,23 +339,13 @@ int free_tiny_fast_with_env(void* ptr, const HakmemEnvSnapshot* env);
|
||||
|
||||
**Runtime rollback**:
|
||||
```sh
|
||||
HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0 # Disable new path
|
||||
```
|
||||
|
||||
**Compile-time rollback**:
|
||||
```c
|
||||
#if HAKMEM_ENV_SNAPSHOT_CONSOLIDATION
|
||||
// New context path
|
||||
#else
|
||||
// Old scattered ENV checks (preserved)
|
||||
#endif
|
||||
HAKMEM_ENV_SNAPSHOT=0 # Disable snapshot path (falls back to per-feature env gates)
|
||||
```
|
||||
|
||||
**Gradual rollout**:
|
||||
1. Phase 19-3a: UNLIKELY hint removal (DONE, GO)
|
||||
2. Phase 19-3b: wrapper pass-down to hot helpers (measure)
|
||||
3. Phase 19-3c: legacy + metadata pass-down (measure)
|
||||
4. Graduate: add to `MIXED_TINYV3_C7_SAFE` preset if GO
|
||||
2. Phase 19-3b: hot helper pass-down (DONE, GO)
|
||||
3. Phase 19-3c: optional wrapper-entry pass-down (alloc-side; measure)
|
||||
|
||||
### 4.4 Observability
|
||||
|
||||
@ -366,15 +362,12 @@ typedef struct {
|
||||
**Perf validation**:
|
||||
- Before: `perf record` shows `hakmem_env_snapshot_enabled` at ~7%
|
||||
- After: `hakmem_env_snapshot_enabled` should drop to <1%
|
||||
- Expected: deep helpers stop calling `hakmem_env_snapshot_enabled()` repeatedly (only wrapper entry remains)
|
||||
- Expected: deep helpers stop calling `hakmem_env_snapshot_enabled()` repeatedly (single capture per hot call)
|
||||
|
||||
**A/B testing**:
|
||||
```sh
|
||||
# Baseline (Phase 19-2 state)
|
||||
HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0 ./bench_random_mixed_hakmem 200000000 400 1
|
||||
|
||||
# Optimized (Phase 19-3)
|
||||
HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=1 ./bench_random_mixed_hakmem 200000000 400 1
|
||||
# Recommended: compare baseline vs optimized commits with the same bench script
|
||||
scripts/run_mixed_10_cleanenv.sh
|
||||
```
|
||||
|
||||
---
|
||||
@ -501,7 +494,7 @@ HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=1 ./bench_random_mixed_hakmem 200000000 400 1
|
||||
- [ ] Add `malloc_tiny_fast_ctx()` variant (Phase 19-3a)
|
||||
- [ ] Add `free_tiny_fast_ctx()` variant (Phase 19-3b)
|
||||
- [ ] Propagate context to `tiny_legacy_fallback_box.h` (Phase 19-3c)
|
||||
- [ ] Add ENV gate `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0/1`
|
||||
- [ ] (Optional) Add a dedicated pass-down gate if A/B within a single binary is needed
|
||||
- [ ] Add stats counters (debug builds)
|
||||
|
||||
### 7.3 Testing (Per Phase)
|
||||
@ -528,13 +521,8 @@ HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=1 ./bench_random_mixed_hakmem 200000000 400 1
|
||||
|
||||
**Benchmark suite**:
|
||||
```sh
|
||||
# Baseline (Phase 19-2)
|
||||
HAKMEM_PROFILE=MIXED_TINYV3_C7_SAFE HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0 \
|
||||
scripts/run_mixed_10_cleanenv.sh
|
||||
|
||||
# Optimized (Phase 19-3)
|
||||
HAKMEM_PROFILE=MIXED_TINYV3_C7_SAFE HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=1 \
|
||||
scripts/run_mixed_10_cleanenv.sh
|
||||
# Run the same cleanenv script on baseline vs optimized commits
|
||||
scripts/run_mixed_10_cleanenv.sh
|
||||
```
|
||||
|
||||
**GO/NO-GO criteria**:
|
||||
@ -568,7 +556,7 @@ perf stat -e cycles,instructions,branches,branch-misses,L1-icache-load-misses \
|
||||
|
||||
**Timeline**: 4-6 hours implementation + 2 hours testing
|
||||
**Risk**: LOW (isolated to alloc path)
|
||||
**Rollback**: `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0`
|
||||
**Rollback**: revert Phase 19-3b commit (or set `HAKMEM_ENV_SNAPSHOT=0` to disable snapshot path)
|
||||
|
||||
### 8.2 Phase 19-3b: free Path (Week 1)
|
||||
|
||||
@ -580,7 +568,7 @@ perf stat -e cycles,instructions,branches,branch-misses,L1-icache-load-misses \
|
||||
|
||||
**Timeline**: 4-6 hours implementation + 2 hours testing
|
||||
**Risk**: LOW-MEDIUM (more call sites than malloc)
|
||||
**Rollback**: `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0`
|
||||
**Rollback**: revert Phase 19-3b commit (or set `HAKMEM_ENV_SNAPSHOT=0` to disable snapshot path)
|
||||
|
||||
### 8.3 Phase 19-3c: Legacy + Metadata (Week 2)
|
||||
|
||||
@ -591,7 +579,7 @@ perf stat -e cycles,instructions,branches,branch-misses,L1-icache-load-misses \
|
||||
|
||||
**Timeline**: 3-4 hours implementation + 2 hours testing
|
||||
**Risk**: MEDIUM (touches multiple boxes)
|
||||
**Rollback**: `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0`
|
||||
**Rollback**: revert Phase 19-3b commit (or set `HAKMEM_ENV_SNAPSHOT=0` to disable snapshot path)
|
||||
|
||||
### 8.4 Graduate (Week 2-3)
|
||||
|
||||
@ -602,15 +590,15 @@ perf stat -e cycles,instructions,branches,branch-misses,L1-icache-load-misses \
|
||||
- Perf validation confirms instruction reduction
|
||||
|
||||
**Promotion actions**:
|
||||
1. Add to `MIXED_TINYV3_C7_SAFE` preset: `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=1`
|
||||
1. Ensure `MIXED_TINYV3_C7_SAFE` preset keeps `HAKMEM_ENV_SNAPSHOT=1` (already)
|
||||
2. Document in optimization roadmap
|
||||
3. Update Box Theory index
|
||||
4. Keep ENV default=0 (opt-in) until production validation
|
||||
|
||||
**Rollback strategy**:
|
||||
- Preset level: Remove from preset, keep code
|
||||
- Code level: `#if HAKMEM_ENV_SNAPSHOT_CONSOLIDATION` → disable at compile time
|
||||
- Emergency: `HAKMEM_ENV_SNAPSHOT_CONSOLIDATION=0` global default
|
||||
- Code level: revert the Phase 19-3b commit
|
||||
- Emergency: set `HAKMEM_ENV_SNAPSHOT=0` (falls back to per-feature env gates)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -46,14 +46,33 @@ scripts/run_mixed_10_cleanenv.sh
|
||||
|
||||
**GO Criteria**: No regression (±1%)
|
||||
|
||||
**Status**: DONE ✅ (GO +4.42%)
|
||||
|
||||
---
|
||||
|
||||
### Phase 19-3b: Pass snapshot down from wrapper entry
|
||||
### Phase 19-3b: Pass snapshot down (hot helper pass-down)
|
||||
|
||||
**Status**: DONE ✅ (GO +2.76% mean / +2.57% median)
|
||||
Results: `docs/analysis/PHASE19_FASTLANE_INSTRUCTION_REDUCTION_3B_ENV_SNAPSHOT_PASSDOWN_AB_TEST_RESULTS.md`
|
||||
|
||||
**Current state**: Each callee calls `hakmem_env_snapshot_enabled()` independently
|
||||
- 5 calls × 2 TLS reads each = **10 TLS reads/op**
|
||||
|
||||
**Proposed**:
|
||||
**Implementation (landed)**:
|
||||
- `core/front/malloc_tiny_fast.h`
|
||||
- Capture `env` once per hot call and pass it down:
|
||||
- `free_tiny_fast()` / `free_tiny_fast_hot()` capture `env` once
|
||||
- `free_tiny_fast_cold(..., env)` consumes it
|
||||
- `tiny_legacy_fallback_free_base_with_env(..., env)` consumes it
|
||||
- Reuse the same snapshot for alloc route selection:
|
||||
- `tiny_policy_hot_get_route_with_env(class_idx, env)`
|
||||
- `core/box/tiny_legacy_fallback_box.h`: add `tiny_legacy_fallback_free_base_with_env(...)`
|
||||
- `core/box/tiny_metadata_cache_hot_box.h`: add `tiny_policy_hot_get_route_with_env(...)`
|
||||
|
||||
**Optional extension (if chasing the last alloc-side gate)**:
|
||||
Pass `env` down from `core/box/hak_wrappers.inc.h` entry. Keep the invariant: malloc miss must fall through (do not call `malloc_cold()` directly).
|
||||
|
||||
**Example (optional)**:
|
||||
```c
|
||||
// core/box/hak_wrappers.inc.h (malloc wrapper)
|
||||
void* malloc(size_t size) {
|
||||
|
||||
Reference in New Issue
Block a user