#!/usr/bin/env python3 import statistics # Phase 3 Baseline (10-run) baseline_10run = [ 45753693, 46285007, 45977011, 46142131, 46068493, 45920245, 46143884, 46011560, 45995670, 46084818 ] # D1 Baseline (20-run, ROUTE=0) d1_baseline = [ 46264909, 46143884, 46296296, 46439628, 46296296, 46189376, 46296296, 46499548, 46296296, 46387832, 46143884, 46296296, 46143884, 46296296, 46439628, 46296296, 46296296, 46439628, 46296296, 46296296 ] # D1 Optimized (20-run, ROUTE=1) d1_optimized = [ 47259147, 47259147, 47501710, 47393365, 47165991, 47165991, 47393365, 47165991, 47393365, 47393365, 47165991, 47393365, 47165991, 47393365, 47393365, 47393365, 47393365, 47393365, 47165991, 47393365 ] def analyze(data, label): mean = statistics.mean(data) median = statistics.median(data) stddev = statistics.stdev(data) if len(data) > 1 else 0 min_val = min(data) max_val = max(data) print(f"\n{label}:") print(f" Mean: {mean:>12.2f} ops/s ({mean/1e6:.2f}M ops/s)") print(f" Median: {median:>12.2f} ops/s ({median/1e6:.2f}M ops/s)") print(f" StdDev: {stddev:>12.2f} ops/s ({stddev/1e6:.2f}M ops/s)") print(f" Min: {min_val:>12.2f} ops/s ({min_val/1e6:.2f}M ops/s)") print(f" Max: {max_val:>12.2f} ops/s ({max_val/1e6:.2f}M ops/s)") return mean, median, stddev print("=" * 70) print("PHASE 3 FINALIZATION - D1 STATISTICAL ANALYSIS") print("=" * 70) baseline_mean, baseline_median, baseline_stddev = analyze(baseline_10run, "BASELINE_PHASE3 (10-run)") d1_base_mean, d1_base_median, d1_base_stddev = analyze(d1_baseline, "D1_BASELINE (20-run, ROUTE=0)") d1_opt_mean, d1_opt_median, d1_opt_stddev = analyze(d1_optimized, "D1_OPTIMIZED (20-run, ROUTE=1)") print("\n" + "=" * 70) print("D1 GAIN ANALYSIS") print("=" * 70) mean_gain = ((d1_opt_mean - d1_base_mean) / d1_base_mean) * 100 median_gain = ((d1_opt_median - d1_base_median) / d1_base_median) * 100 variance_ratio = d1_opt_stddev / d1_base_stddev if d1_base_stddev > 0 else 0 print(f"\nMean gain: {mean_gain:>+6.2f}%") print(f"Median gain: {median_gain:>+6.2f}%") print(f"Variance ratio: {variance_ratio:>6.2f}x (optimized/baseline)") print("\n" + "=" * 70) print("D1 DECISION CRITERIA") print("=" * 70) print("\nCriteria from PHASE3_CACHE_LOCALITY_NEXT_INSTRUCTIONS.md:65:") print(" IF mean_gain >= +1.0% AND median_gain >= +0.0%:") print(" → GO: Promote HAKMEM_FREE_STATIC_ROUTE=1 to default") print(" ELSE IF both within ±1.0%:") print(" → NEUTRAL: Keep as opt-in research box") print(" ELSE:") print(" → FLAG_FOR_INVESTIGATION") print(f"\nActual results:") print(f" Mean gain: {mean_gain:>+6.2f}% {'✓ >= +1.0%' if mean_gain >= 1.0 else '✗ < +1.0%'}") print(f" Median gain: {median_gain:>+6.2f}% {'✓ >= +0.0%' if median_gain >= 0.0 else '✗ < +0.0%'}") if mean_gain >= 1.0 and median_gain >= 0.0: decision = "GO" explanation = "Both criteria met: mean >= +1.0% AND median >= +0.0%" elif abs(mean_gain) <= 1.0 and abs(median_gain) <= 1.0: decision = "NEUTRAL" explanation = "Both gains within ±1.0% threshold" else: decision = "FLAG_FOR_INVESTIGATION" explanation = "Criteria not met, requires investigation" print(f"\n{'=' * 70}") print(f"DECISION: {decision}") print(f"{'=' * 70}") print(f"Reason: {explanation}") print() if decision == "GO": print("ACTION REQUIRED:") print(" 1. Update core/bench_profile.h MIXED_TINYV3_C7_SAFE preset") print(" Add: bench_setenv_default(\"HAKMEM_FREE_STATIC_ROUTE\", \"1\");") print(" 2. Rebuild: make clean && make bench_random_mixed_hakmem") print(" 3. Sanity check: 1-run verification") elif decision == "NEUTRAL": print("ACTION REQUIRED:") print(" 1. Keep HAKMEM_FREE_STATIC_ROUTE as opt-in research box") print(" 2. Document as available but not default in MIXED preset") else: print("ACTION REQUIRED:") print(" 1. Investigate why criteria not met") print(" 2. Review benchmark variance and methodology") print("\n" + "=" * 70) print("SUMMARY FOR DOCUMENTATION") print("=" * 70) print(f""" BASELINE_PHASE3 (10-run): Mean: {baseline_mean/1e6:.2f}M ops/s, Median: {baseline_median/1e6:.2f}M ops/s, StdDev: {baseline_stddev/1e6:.2f}M D1_BASELINE (20-run, ROUTE=0): Mean: {d1_base_mean/1e6:.2f}M ops/s, Median: {d1_base_median/1e6:.2f}M ops/s, StdDev: {d1_base_stddev/1e6:.2f}M D1_OPTIMIZED (20-run, ROUTE=1): Mean: {d1_opt_mean/1e6:.2f}M ops/s, Median: {d1_opt_median/1e6:.2f}M ops/s, StdDev: {d1_opt_stddev/1e6:.2f}M D1 GAIN: Mean: {mean_gain:+.2f}%, Median: {median_gain:+.2f}% D1 DECISION: {decision} """)