hakorune/tools/perf/bench_compare_c_vs_hako.sh

#!/usr/bin/env bash
set -euo pipefail

# Compare a C baseline vs Hakorune VM for a given bench key.
# Usage: bench_compare_c_vs_hako.sh <bench_key> [warmup] [repeat]
#   bench_key: box_create_destroy_small | method_call_only_small
# Output: [bench] name=<key> c_ms=<med> ny_ms=<med> ratio=<c/ny>

KEY=${1:-}
WARMUP=${2:-2}
REPEAT=${3:-5}

if [[ -z "${KEY}" ]]; then
  echo "Usage: $0 <bench_key> [warmup] [repeat]" >&2
  exit 2
fi

ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd)
TARGET_DIR="${ROOT_DIR}/target"
C_SRC="${ROOT_DIR}/benchmarks/c/bench_${KEY}.c"
C_BIN="${TARGET_DIR}/perf_c_${KEY}"
HAKO_PROG="${ROOT_DIR}/benchmarks/bench_${KEY}.hako"
HAKORUNE_BIN="${TARGET_DIR}/release/hakorune"

if [[ ! -x "${HAKORUNE_BIN}" ]]; then
  echo "[hint] hakorune not built. Run: cargo build --release" >&2
  exit 2
fi

if [[ ! -f "${C_SRC}" ]]; then
  echo "[error] C source not found: ${C_SRC}" >&2
  exit 2
fi
if [[ ! -f "${HAKO_PROG}" ]]; then
  echo "[error] Hako program not found: ${HAKO_PROG}" >&2
  exit 2
fi

mkdir -p "${TARGET_DIR}"

# Build C baseline
cc -O3 -march=native -mtune=native -o "${C_BIN}" "${C_SRC}" 2>/dev/null || cc -O3 -o "${C_BIN}" "${C_SRC}"

time_ms() {
  date +%s%3N
}

measure_cmd_ms() {
  local cmd=("$@")
  local t1 t2 dt
  t1=$(time_ms)
  "${cmd[@]}" >/dev/null 2>&1 || true
  t2=$(time_ms)
  dt=$((t2 - t1))
  echo "$dt"
}

median_ms() {
  # read numbers from stdin
  awk 'NF{print $1}' | sort -n | awk ' { a[NR]=$1 } END { if (NR==0) {print 0; exit} n=int((NR+1)/2); print a[n] }'
}

collect_series() {
  local label=$1; shift
  local warmup=$1; shift
  local repeat=$1; shift
  local -a cmd=("$@")
  # warmup
  for _ in $(seq 1 "${warmup}"); do
    measure_cmd_ms "${cmd[@]}" >/dev/null || true
  done
  # samples
  for _ in $(seq 1 "${repeat}"); do
    measure_cmd_ms "${cmd[@]}"
  done
}

# C series
C_SERIES=$(collect_series C "${WARMUP}" "${REPEAT}" "${C_BIN}")
C_MED=$(printf "%s\n" "${C_SERIES}" | median_ms)

# Hako series (VM)
# Minimal env for fast VM bring-up (bench profile)
# - Disable dynamic plugins and nyash.toml env injection to avoid startup scans
HAKO_ENV=(
  NYASH_PARSER_STAGE3=1
  HAKO_PARSER_STAGE3=1
  NYASH_PARSER_ALLOW_SEMICOLON=1
  NYASH_DISABLE_PLUGINS=1
  NYASH_SKIP_TOML_ENV=1
  NYASH_USE_NY_COMPILER=0
  NYASH_ENABLE_USING=0
  NYASH_STR_CP=0
)
HAKO_SERIES=$(collect_series HAKO "${WARMUP}" "${REPEAT}" env "${HAKO_ENV[@]}" timeout 20s "${HAKORUNE_BIN}" --backend vm "${HAKO_PROG}")
HAKO_MED=$(printf "%s\n" "${HAKO_SERIES}" | median_ms)
if [[ "${PERF_SUBTRACT_STARTUP:-0}" == "1" ]]; then
  tmp_ret0=$(mktemp --suffix .hako)
  cat >"${tmp_ret0}" <<'HAKO'
static box Main { main() { return 0 } }
HAKO
  base_series=$(collect_series 1 3 env "${HAKO_ENV[@]}" timeout 20s "${HAKORUNE_BIN}" --backend vm "${tmp_ret0}")
  base_med=$(printf "%s\n" "${base_series}" | median_ms)
  rm -f "${tmp_ret0}" || true
  if [[ "${base_med}" =~ ^[0-9]+$ ]]; then
    HAKO_MED=$(( HAKO_MED>base_med ? HAKO_MED-base_med : 0 ))
  fi
fi

# ratio = c / ny (1.0 means parity)
ratio() {
  python3 - "$@" <<'PY'
import sys
c, n = map(float, sys.argv[1:3])
print(f"{(c/n) if n>0 else 0.0:.2f}")
PY
}

RATIO=$(ratio "${C_MED}" "${HAKO_MED}")

printf "[bench] name=%-24s c_ms=%s ny_ms=%s ratio=%s\n" "${KEY}" "${C_MED}" "${HAKO_MED}" "${RATIO}"

# Optional AOT
if [[ "${PERF_AOT:-0}" == "1" ]]; then
  TMP_JSON=$(mktemp --suffix .json)
  EXE_OUT="${TARGET_DIR}/perf_ny_${KEY}.exe"
  # Use Stage‑B wrapper (robust, stable) to emit MIR JSON
  if HAKO_SELFHOST_BUILDER_FIRST=1 HAKO_SELFHOST_NO_DELEGATE=1 bash "${ROOT_DIR}/tools/hakorune_emit_mir.sh" "${HAKO_PROG}" "${TMP_JSON}" >/dev/null 2>&1; then
    if ! bash "${ROOT_DIR}/tools/ny_mir_builder.sh" --in "${TMP_JSON}" --emit exe -o "${EXE_OUT}" --quiet >/dev/null 2>&1; then
      rm -f "${TMP_JSON}" || true
      exit 0
    fi
    AOT_SERIES=$(collect_series HAKO "${WARMUP}" "${REPEAT}" timeout 20s "${EXE_OUT}")
    AOT_MED=$(printf "%s\n" "${AOT_SERIES}" | median_ms)
    if [[ "${PERF_SUBTRACT_STARTUP:-0}" == "1" ]]; then
      # Build ret0
      TMP_R0=$(mktemp --suffix .hako); cat >"${TMP_R0}" <<'HAKO'
static box Main { main() { return 0 } }
HAKO
      TMP_R0_JSON=$(mktemp --suffix .json)
      EXE_R0="${TARGET_DIR}/perf_ny_ret0.exe"
      if bash "${ROOT_DIR}/tools/hakorune_emit_mir.sh" "${TMP_R0}" "${TMP_R0_JSON}" >/dev/null 2>&1 \
         && bash "${ROOT_DIR}/tools/ny_mir_builder.sh" --in "${TMP_R0_JSON}" --emit exe -o "${EXE_R0}" --quiet >/dev/null 2>&1; then
        BASE_SERIES=$(collect_series 1 3 timeout 20s "${EXE_R0}")
        BASE_MED=$(printf "%s\n" "${BASE_SERIES}" | median_ms)
        if [[ "${BASE_MED}" =~ ^[0-9]+$ ]]; then
          AOT_MED=$(( AOT_MED>BASE_MED ? AOT_MED-BASE_MED : 0 ))
        fi
      fi
      rm -f "${TMP_R0}" "${TMP_R0_JSON}" || true
    fi
    RATIO_AOT=$(ratio "${C_MED}" "${AOT_MED}")
    printf "[bench] name=%-24s c_ms=%s ny_aot_ms=%s ratio=%s\n" "${KEY} (aot)" "${C_MED}" "${AOT_MED}" "${RATIO_AOT}"
  fi
  rm -f "${TMP_JSON}" || true
fi