#!/usr/bin/env bash set -euo pipefail # Phase 7 Full Benchmark Suite Runner # Executes all benchmarks and generates summary report echo "=========================================" echo "Phase 7 Full Benchmark Suite" echo "=========================================" echo "" # Color codes for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Step 1: Verify build status echo -e "${YELLOW}Step 1: Verifying build status...${NC}" echo "" if ! grep -q "HAKMEM_TINY_HEADER_CLASSIDX=1" Makefile; then echo -e "${RED}ERROR: HEADER_CLASSIDX=1 not enabled in Makefile!${NC}" exit 1 fi echo -e "${GREEN}✓ HEADER_CLASSIDX=1 is enabled${NC}" echo "" # Step 2: Quick sanity test echo -e "${YELLOW}Step 2: Running sanity tests...${NC}" echo "" tests_passed=0 tests_total=5 echo "Testing larson_hakmem..." if ./larson_hakmem 1 8 128 1024 1 12345 1 >/dev/null 2>&1; then echo -e "${GREEN}✓ larson_hakmem OK${NC}" ((tests_passed++)) else echo -e "${RED}✗ larson_hakmem FAILED${NC}" fi echo "Testing bench_random_mixed_hakmem..." if ./bench_random_mixed_hakmem 1000 128 1234567 >/dev/null 2>&1; then echo -e "${GREEN}✓ bench_random_mixed_hakmem OK${NC}" ((tests_passed++)) else echo -e "${RED}✗ bench_random_mixed_hakmem FAILED${NC}" fi echo "Testing bench_mid_large_mt_hakmem..." if ./bench_mid_large_mt_hakmem 2 1000 2048 42 >/dev/null 2>&1; then echo -e "${GREEN}✓ bench_mid_large_mt_hakmem OK${NC}" ((tests_passed++)) else echo -e "${RED}✗ bench_mid_large_mt_hakmem FAILED${NC}" fi echo "Testing bench_vm_mixed_hakmem..." if ./bench_vm_mixed_hakmem 100 256 424242 >/dev/null 2>&1; then echo -e "${GREEN}✓ bench_vm_mixed_hakmem OK${NC}" ((tests_passed++)) else echo -e "${RED}✗ bench_vm_mixed_hakmem FAILED${NC}" fi echo "Testing bench_tiny_hot_hakmem..." if ./bench_tiny_hot_hakmem 32 10 1000 >/dev/null 2>&1; then echo -e "${GREEN}✓ bench_tiny_hot_hakmem OK${NC}" ((tests_passed++)) else echo -e "${RED}✗ bench_tiny_hot_hakmem FAILED${NC}" fi echo "" echo "Sanity tests: ${tests_passed}/${tests_total} passed" if [ $tests_passed -ne $tests_total ]; then echo -e "${RED}ERROR: Some sanity tests failed. Aborting.${NC}" exit 1 fi echo "" # Step 3: Run full benchmark suite echo -e "${YELLOW}Step 3: Running full benchmark suite (this will take ~15-20 minutes)...${NC}" echo "" if [ ! -x "./scripts/bench_suite_matrix.sh" ]; then echo -e "${RED}ERROR: bench_suite_matrix.sh not found or not executable${NC}" exit 1 fi ./scripts/bench_suite_matrix.sh # Step 4: Analyze results echo "" echo -e "${YELLOW}Step 4: Analyzing results...${NC}" echo "" latest=$(ls -td bench_results/suite/* 2>/dev/null | head -1) if [ -z "$latest" ] || [ ! -f "$latest/results.csv" ]; then echo -e "${RED}ERROR: No results found!${NC}" exit 1 fi echo "Results location: $latest" echo "" # Quick summary echo "=========================================" echo "Quick Summary (Average Performance)" echo "=========================================" echo "" awk -F, 'NR>1 { if ($2=="hakmem") { hakmem[$1]+=$4; count_h[$1]++ } if ($2=="system") { system[$1]+=$4; count_s[$1]++ } if ($2=="mi") { mi[$1]+=$4; count_m[$1]++ } } END { for (b in hakmem) { h = hakmem[b]/count_h[b] s = system[b]/count_s[b] m = mi[b]/count_m[b] pct_sys = (h/s - 1) * 100 pct_mi = (h/m - 1) * 100 printf "%-20s HAKMEM: %8.2f M/s System: %8.2f M/s mimalloc: %8.2f M/s\n", b ":", h/1e6, s/1e6, m/1e6 printf "%-20s vs System: %+6.1f%% vs mimalloc: %+6.1f%%\n", "", pct_sys, pct_mi printf "\n" } }' "$latest/results.csv" echo "=========================================" echo "Detailed Comparison (HAKMEM vs System)" echo "=========================================" echo "" awk -F, 'NR>1 && ($2=="hakmem" || $2=="system") { key=$1 "," $3 if ($2=="hakmem") h[key]=$4 if ($2=="system") s[key]=$4 } END { for (k in h) { if (s[k]) { pct = (h[k]/s[k] - 1) * 100 status = pct > 0 ? "WIN" : "LOSS" printf "%-50s HAKMEM: %8.2f M/s System: %8.2f M/s %+6.1f%% [%s]\n", k ":", h[k]/1e6, s[k]/1e6, pct, status } } }' "$latest/results.csv" | sort echo "" echo "=========================================" echo "Full results saved to:" echo " CSV: $latest/results.csv" echo " Logs: $latest/raw/" echo "=========================================" echo "" # Generate summary markdown summary_file="PHASE7_RESULTS_SUMMARY_$(date +%Y%m%d_%H%M%S).md" cat > "$summary_file" << REPORT # Phase 7 Benchmark Results Summary **Date**: $(date +%Y-%m-%d) **Phase**: 7-1.3 (HEADER_CLASSIDX=1) **Suite**: $(basename $latest) ## Quick Summary \`\`\` $(awk -F, 'NR>1 { if ($2=="hakmem") { hakmem[$1]+=$4; count_h[$1]++ } if ($2=="system") { system[$1]+=$4; count_s[$1]++ } if ($2=="mi") { mi[$1]+=$4; count_m[$1]++ } } END { for (b in hakmem) { h = hakmem[b]/count_h[b] s = system[b]/count_s[b] m = mi[b]/count_m[b] pct_sys = (h/s - 1) * 100 pct_mi = (h/m - 1) * 100 printf "%-20s HAKMEM: %8.2f M/s System: %8.2f M/s mimalloc: %8.2f M/s\n", b ":", h/1e6, s/1e6, m/1e6 printf "%-20s vs System: %+6.1f%% vs mimalloc: %+6.1f%%\n\n", "", pct_sys, pct_mi } }' "$latest/results.csv") \`\`\` ## Detailed Results \`\`\` $(cat "$latest/results.csv") \`\`\` ## Analysis ### Strengths [To be filled in based on results] ### Weaknesses [To be filled in based on results] ### Next Steps [To be determined] --- **Full results**: $latest REPORT echo -e "${GREEN}Summary report saved to: $summary_file${NC}" echo "" echo -e "${GREEN}Benchmark suite completed successfully!${NC}"