hakmem/run_comprehensive_benchmark.sh

#!/bin/bash
# Comprehensive Benchmark Script for HAKMEM
# Runs multiple iterations and calculates statistics (mean, median, stddev)

set -e

# Configuration
ITERATIONS=10
RESULTS_DIR="benchmark_results_$(date +%Y%m%d_%H%M%S)"

# Create results directory
mkdir -p "$RESULTS_DIR"

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}HAKMEM Comprehensive Benchmark Suite${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo "Iterations per benchmark: $ITERATIONS"
echo "Results directory: $RESULTS_DIR"
echo ""

# Function to extract throughput value from output
extract_throughput() {
    grep "Throughput" | grep -oP '\d+' | head -1
}

# Function to calculate statistics
calculate_stats() {
    local values_file=$1
    local benchmark_name=$2

    if [ ! -f "$values_file" ]; then
        echo -e "${RED}Error: $values_file not found${NC}"
        return
    fi

    # Read values into array
    mapfile -t values < "$values_file"

    # Calculate mean
    local sum=0
    for val in "${values[@]}"; do
        sum=$((sum + val))
    done
    local mean=$((sum / ${#values[@]}))

    # Calculate median (sort and take middle value)
    IFS=$'\n' sorted=($(sort -n <<<"${values[*]}"))
    unset IFS
    local len=${#sorted[@]}
    local median
    if [ $((len % 2)) -eq 0 ]; then
        median=$(( (sorted[len/2-1] + sorted[len/2]) / 2 ))
    else
        median=${sorted[len/2]}
    fi

    # Calculate min/max
    local min=${sorted[0]}
    local max=${sorted[len-1]}

    # Calculate standard deviation
    local sum_sq_diff=0
    for val in "${values[@]}"; do
        local diff=$((val - mean))
        sum_sq_diff=$((sum_sq_diff + diff * diff))
    done
    local variance=$((sum_sq_diff / ${#values[@]}))
    local stddev=$(echo "scale=2; sqrt($variance)" | bc)

    # Calculate coefficient of variation (CV)
    local cv=$(echo "scale=2; ($stddev / $mean) * 100" | bc)

    # Print results
    echo -e "${GREEN}Statistics for $benchmark_name:${NC}"
    echo "  Mean:     $(printf "%'d" $mean) ops/s"
    echo "  Median:   $(printf "%'d" $median) ops/s"
    echo "  Stddev:   $(printf "%'d" $stddev) ops/s (CV: ${cv}%)"
    echo "  Min:      $(printf "%'d" $min) ops/s"
    echo "  Max:      $(printf "%'d" $max) ops/s"
    echo "  Range:    $(printf "%'d" $((max - min))) ops/s ($(echo "scale=2; (($max - $min) * 100.0 / $mean)" | bc)%)"
    echo ""

    # Save summary
    cat >> "$RESULTS_DIR/summary.txt" << EOF
$benchmark_name:
  Mean:     $mean ops/s
  Median:   $median ops/s
  Stddev:   $stddev ops/s (CV: ${cv}%)
  Min:      $min ops/s
  Max:      $max ops/s
  Range:    $((max - min)) ops/s

EOF
}

# Function to run benchmark multiple times
run_benchmark() {
    local name=$1
    local cmd=$2
    local output_file="$RESULTS_DIR/${name}_values.txt"
    local log_file="$RESULTS_DIR/${name}_full.log"

    echo -e "${YELLOW}Running: $name${NC}"
    echo "Command: $cmd"
    echo ""

    > "$output_file"  # Clear output file
    > "$log_file"     # Clear log file

    for i in $(seq 1 $ITERATIONS); do
        echo -n "  Run $i/$ITERATIONS... "

        # Run benchmark and extract throughput
        local output=$(eval "$cmd" 2>&1)
        echo "$output" >> "$log_file"

        local throughput=$(echo "$output" | extract_throughput)

        if [ -n "$throughput" ]; then
            echo "$throughput" >> "$output_file"
            echo -e "${GREEN}${throughput} ops/s${NC}"
        else
            echo -e "${RED}FAILED (no throughput found)${NC}"
        fi
    done

    echo ""
    calculate_stats "$output_file" "$name"
}

# Benchmark 1: Random Mixed 256B (HAKMEM)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 1: Random Mixed 256B (HAKMEM)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
run_benchmark "random_mixed_256b_hakmem" \
    "./out/release/bench_random_mixed_hakmem 100000 256 42"

# Benchmark 2: Random Mixed 256B (System)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 2: Random Mixed 256B (System)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
if [ -f "./out/release/bench_random_mixed_system" ]; then
    run_benchmark "random_mixed_256b_system" \
        "./out/release/bench_random_mixed_system 100000 256 42"
else
    echo -e "${YELLOW}Skipping: bench_random_mixed_system not found${NC}"
    echo ""
fi

# Benchmark 3: Larson 1T (HAKMEM)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 3: Larson 1T (HAKMEM)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
run_benchmark "larson_1t_hakmem" \
    "./out/release/larson_hakmem 10 1 1 10000 10000 1 42"

# Benchmark 4: Larson 8T (HAKMEM)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 4: Larson 8T (HAKMEM)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
run_benchmark "larson_8t_hakmem" \
    "./out/release/larson_hakmem 10 8 8 10000 10000 1 42"

# Benchmark 5: Random Mixed 128B (HAKMEM)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 5: Random Mixed 128B (HAKMEM)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
run_benchmark "random_mixed_128b_hakmem" \
    "./out/release/bench_random_mixed_hakmem 100000 128 42"

# Benchmark 6: Random Mixed 512B (HAKMEM)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 6: Random Mixed 512B (HAKMEM)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
run_benchmark "random_mixed_512b_hakmem" \
    "./out/release/bench_random_mixed_hakmem 100000 512 42"

# Benchmark 7: Random Mixed 1024B (HAKMEM)
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark 7: Random Mixed 1024B (HAKMEM)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
run_benchmark "random_mixed_1024b_hakmem" \
    "./out/release/bench_random_mixed_hakmem 100000 1024 42"

# Final Summary
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Benchmark Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
cat "$RESULTS_DIR/summary.txt"

echo -e "${GREEN}All benchmarks completed!${NC}"
echo "Results saved to: $RESULTS_DIR"
echo ""