hakmem/analyze_d1_results.py

#!/usr/bin/env python3
import statistics

# Phase 3 Baseline (10-run)
baseline_10run = [
    45753693, 46285007, 45977011, 46142131, 46068493,
    45920245, 46143884, 46011560, 45995670, 46084818
]

# D1 Baseline (20-run, ROUTE=0)
d1_baseline = [
    46264909, 46143884, 46296296, 46439628, 46296296,
    46189376, 46296296, 46499548, 46296296, 46387832,
    46143884, 46296296, 46143884, 46296296, 46439628,
    46296296, 46296296, 46439628, 46296296, 46296296
]

# D1 Optimized (20-run, ROUTE=1)
d1_optimized = [
    47259147, 47259147, 47501710, 47393365, 47165991,
    47165991, 47393365, 47165991, 47393365, 47393365,
    47165991, 47393365, 47165991, 47393365, 47393365,
    47393365, 47393365, 47393365, 47165991, 47393365
]

def analyze(data, label):
    mean = statistics.mean(data)
    median = statistics.median(data)
    stddev = statistics.stdev(data) if len(data) > 1 else 0
    min_val = min(data)
    max_val = max(data)

    print(f"\n{label}:")
    print(f"  Mean:   {mean:>12.2f} ops/s ({mean/1e6:.2f}M ops/s)")
    print(f"  Median: {median:>12.2f} ops/s ({median/1e6:.2f}M ops/s)")
    print(f"  StdDev: {stddev:>12.2f} ops/s ({stddev/1e6:.2f}M ops/s)")
    print(f"  Min:    {min_val:>12.2f} ops/s ({min_val/1e6:.2f}M ops/s)")
    print(f"  Max:    {max_val:>12.2f} ops/s ({max_val/1e6:.2f}M ops/s)")

    return mean, median, stddev

print("=" * 70)
print("PHASE 3 FINALIZATION - D1 STATISTICAL ANALYSIS")
print("=" * 70)

baseline_mean, baseline_median, baseline_stddev = analyze(baseline_10run, "BASELINE_PHASE3 (10-run)")
d1_base_mean, d1_base_median, d1_base_stddev = analyze(d1_baseline, "D1_BASELINE (20-run, ROUTE=0)")
d1_opt_mean, d1_opt_median, d1_opt_stddev = analyze(d1_optimized, "D1_OPTIMIZED (20-run, ROUTE=1)")

print("\n" + "=" * 70)
print("D1 GAIN ANALYSIS")
print("=" * 70)

mean_gain = ((d1_opt_mean - d1_base_mean) / d1_base_mean) * 100
median_gain = ((d1_opt_median - d1_base_median) / d1_base_median) * 100
variance_ratio = d1_opt_stddev / d1_base_stddev if d1_base_stddev > 0 else 0

print(f"\nMean gain:     {mean_gain:>+6.2f}%")
print(f"Median gain:   {median_gain:>+6.2f}%")
print(f"Variance ratio: {variance_ratio:>6.2f}x (optimized/baseline)")

print("\n" + "=" * 70)
print("D1 DECISION CRITERIA")
print("=" * 70)

print("\nCriteria from PHASE3_CACHE_LOCALITY_NEXT_INSTRUCTIONS.md:65:")
print("  IF mean_gain >= +1.0% AND median_gain >= +0.0%:")
print("    → GO: Promote HAKMEM_FREE_STATIC_ROUTE=1 to default")
print("  ELSE IF both within ±1.0%:")
print("    → NEUTRAL: Keep as opt-in research box")
print("  ELSE:")
print("    → FLAG_FOR_INVESTIGATION")

print(f"\nActual results:")
print(f"  Mean gain:   {mean_gain:>+6.2f}% {'✓ >= +1.0%' if mean_gain >= 1.0 else '✗ < +1.0%'}")
print(f"  Median gain: {median_gain:>+6.2f}% {'✓ >= +0.0%' if median_gain >= 0.0 else '✗ < +0.0%'}")

if mean_gain >= 1.0 and median_gain >= 0.0:
    decision = "GO"
    explanation = "Both criteria met: mean >= +1.0% AND median >= +0.0%"
elif abs(mean_gain) <= 1.0 and abs(median_gain) <= 1.0:
    decision = "NEUTRAL"
    explanation = "Both gains within ±1.0% threshold"
else:
    decision = "FLAG_FOR_INVESTIGATION"
    explanation = "Criteria not met, requires investigation"

print(f"\n{'=' * 70}")
print(f"DECISION: {decision}")
print(f"{'=' * 70}")
print(f"Reason: {explanation}")
print()

if decision == "GO":
    print("ACTION REQUIRED:")
    print("  1. Update core/bench_profile.h MIXED_TINYV3_C7_SAFE preset")
    print("     Add: bench_setenv_default(\"HAKMEM_FREE_STATIC_ROUTE\", \"1\");")
    print("  2. Rebuild: make clean && make bench_random_mixed_hakmem")
    print("  3. Sanity check: 1-run verification")
elif decision == "NEUTRAL":
    print("ACTION REQUIRED:")
    print("  1. Keep HAKMEM_FREE_STATIC_ROUTE as opt-in research box")
    print("  2. Document as available but not default in MIXED preset")
else:
    print("ACTION REQUIRED:")
    print("  1. Investigate why criteria not met")
    print("  2. Review benchmark variance and methodology")

print("\n" + "=" * 70)
print("SUMMARY FOR DOCUMENTATION")
print("=" * 70)

print(f"""
BASELINE_PHASE3 (10-run):
  Mean: {baseline_mean/1e6:.2f}M ops/s, Median: {baseline_median/1e6:.2f}M ops/s, StdDev: {baseline_stddev/1e6:.2f}M

D1_BASELINE (20-run, ROUTE=0):
  Mean: {d1_base_mean/1e6:.2f}M ops/s, Median: {d1_base_median/1e6:.2f}M ops/s, StdDev: {d1_base_stddev/1e6:.2f}M

D1_OPTIMIZED (20-run, ROUTE=1):
  Mean: {d1_opt_mean/1e6:.2f}M ops/s, Median: {d1_opt_median/1e6:.2f}M ops/s, StdDev: {d1_opt_stddev/1e6:.2f}M

D1 GAIN:
  Mean: {mean_gain:+.2f}%, Median: {median_gain:+.2f}%

D1 DECISION: {decision}
""")