#!/usr/bin/env bash set -euo pipefail # Compare a C baseline vs Hakorune VM for a given bench key. # Usage: bench_compare_c_vs_hako.sh [warmup] [repeat] # bench_key: box_create_destroy_small | method_call_only_small # Output: [bench] name= c_ms= ny_ms= ratio= KEY=${1:-} WARMUP=${2:-2} REPEAT=${3:-5} if [[ -z "${KEY}" ]]; then echo "Usage: $0 [warmup] [repeat]" >&2 exit 2 fi ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) TARGET_DIR="${ROOT_DIR}/target" C_SRC="${ROOT_DIR}/benchmarks/c/bench_${KEY}.c" C_BIN="${TARGET_DIR}/perf_c_${KEY}" HAKO_PROG="${ROOT_DIR}/benchmarks/bench_${KEY}.hako" HAKORUNE_BIN="${TARGET_DIR}/release/hakorune" if [[ ! -x "${HAKORUNE_BIN}" ]]; then echo "[hint] hakorune not built. Run: cargo build --release" >&2 exit 2 fi if [[ ! -f "${C_SRC}" ]]; then echo "[error] C source not found: ${C_SRC}" >&2 exit 2 fi if [[ ! -f "${HAKO_PROG}" ]]; then echo "[error] Hako program not found: ${HAKO_PROG}" >&2 exit 2 fi mkdir -p "${TARGET_DIR}" # Build C baseline cc -O3 -march=native -mtune=native -o "${C_BIN}" "${C_SRC}" 2>/dev/null || cc -O3 -o "${C_BIN}" "${C_SRC}" time_ms() { date +%s%3N } measure_cmd_ms() { local cmd=("$@") local t1 t2 dt t1=$(time_ms) "${cmd[@]}" >/dev/null 2>&1 || true t2=$(time_ms) dt=$((t2 - t1)) echo "$dt" } median_ms() { # read numbers from stdin awk 'NF{print $1}' | sort -n | awk ' { a[NR]=$1 } END { if (NR==0) {print 0; exit} n=int((NR+1)/2); print a[n] }' } collect_series() { local label=$1; shift local warmup=$1; shift local repeat=$1; shift local -a cmd=("$@") # warmup for _ in $(seq 1 "${warmup}"); do measure_cmd_ms "${cmd[@]}" >/dev/null || true done # samples for _ in $(seq 1 "${repeat}"); do measure_cmd_ms "${cmd[@]}" done } # C series C_SERIES=$(collect_series C "${WARMUP}" "${REPEAT}" "${C_BIN}") C_MED=$(printf "%s\n" "${C_SERIES}" | median_ms) # Hako series (VM) # Minimal env for fast VM bring-up (bench profile) # - Disable dynamic plugins and nyash.toml env injection to avoid startup scans HAKO_ENV=( NYASH_FEATURES=stage3 NYASH_FEATURES=stage3 NYASH_PARSER_ALLOW_SEMICOLON=1 NYASH_DISABLE_PLUGINS=1 NYASH_SKIP_TOML_ENV=1 NYASH_USE_NY_COMPILER=0 NYASH_ENABLE_USING=0 NYASH_STR_CP=0 ) HAKO_SERIES=$(collect_series HAKO "${WARMUP}" "${REPEAT}" env "${HAKO_ENV[@]}" timeout 20s "${HAKORUNE_BIN}" --backend vm "${HAKO_PROG}") HAKO_MED=$(printf "%s\n" "${HAKO_SERIES}" | median_ms) if [[ "${PERF_SUBTRACT_STARTUP:-0}" == "1" ]]; then tmp_ret0=$(mktemp --suffix .hako) cat >"${tmp_ret0}" <<'HAKO' static box Main { main() { return 0 } } HAKO base_series=$(collect_series 1 3 env "${HAKO_ENV[@]}" timeout 20s "${HAKORUNE_BIN}" --backend vm "${tmp_ret0}") base_med=$(printf "%s\n" "${base_series}" | median_ms) rm -f "${tmp_ret0}" || true if [[ "${base_med}" =~ ^[0-9]+$ ]]; then HAKO_MED=$(( HAKO_MED>base_med ? HAKO_MED-base_med : 0 )) fi fi # ratio = c / ny (1.0 means parity) ratio() { python3 - "$@" <<'PY' import sys c, n = map(float, sys.argv[1:3]) print(f"{(c/n) if n>0 else 0.0:.2f}") PY } RATIO=$(ratio "${C_MED}" "${HAKO_MED}") printf "[bench] name=%-24s c_ms=%s ny_ms=%s ratio=%s\n" "${KEY}" "${C_MED}" "${HAKO_MED}" "${RATIO}" # Optional AOT if [[ "${PERF_AOT:-0}" == "1" ]]; then TMP_JSON=$(mktemp --suffix .json) EXE_OUT="${TARGET_DIR}/perf_ny_${KEY}.exe" # Use Stage‑B wrapper (robust, stable) to emit MIR JSON if HAKO_SELFHOST_BUILDER_FIRST=1 HAKO_SELFHOST_NO_DELEGATE=1 bash "${ROOT_DIR}/tools/hakorune_emit_mir.sh" "${HAKO_PROG}" "${TMP_JSON}" >/dev/null 2>&1; then if ! bash "${ROOT_DIR}/tools/ny_mir_builder.sh" --in "${TMP_JSON}" --emit exe -o "${EXE_OUT}" --quiet >/dev/null 2>&1; then rm -f "${TMP_JSON}" || true exit 0 fi AOT_SERIES=$(collect_series HAKO "${WARMUP}" "${REPEAT}" timeout 20s "${EXE_OUT}") AOT_MED=$(printf "%s\n" "${AOT_SERIES}" | median_ms) if [[ "${PERF_SUBTRACT_STARTUP:-0}" == "1" ]]; then # Build ret0 TMP_R0=$(mktemp --suffix .hako); cat >"${TMP_R0}" <<'HAKO' static box Main { main() { return 0 } } HAKO TMP_R0_JSON=$(mktemp --suffix .json) EXE_R0="${TARGET_DIR}/perf_ny_ret0.exe" if bash "${ROOT_DIR}/tools/hakorune_emit_mir.sh" "${TMP_R0}" "${TMP_R0_JSON}" >/dev/null 2>&1 \ && bash "${ROOT_DIR}/tools/ny_mir_builder.sh" --in "${TMP_R0_JSON}" --emit exe -o "${EXE_R0}" --quiet >/dev/null 2>&1; then BASE_SERIES=$(collect_series 1 3 timeout 20s "${EXE_R0}") BASE_MED=$(printf "%s\n" "${BASE_SERIES}" | median_ms) if [[ "${BASE_MED}" =~ ^[0-9]+$ ]]; then AOT_MED=$(( AOT_MED>BASE_MED ? AOT_MED-BASE_MED : 0 )) fi fi rm -f "${TMP_R0}" "${TMP_R0_JSON}" || true fi RATIO_AOT=$(ratio "${C_MED}" "${AOT_MED}") printf "[bench] name=%-24s c_ms=%s ny_aot_ms=%s ratio=%s\n" "${KEY} (aot)" "${C_MED}" "${AOT_MED}" "${RATIO_AOT}" fi rm -f "${TMP_JSON}" || true fi