docs/ci: selfhost bootstrap/exe-first workflows; add ny-llvmc scaffolding + JSON v0 schema validation; plan: unify to Nyash ABI v2 (no backwards compat)

This commit is contained in:
Selfhosting Dev
2025-09-17 20:33:19 +09:00
parent a5054a271b
commit 4ea3ca2685
56 changed files with 2275 additions and 1623 deletions

View File

@ -11,8 +11,6 @@ use std::thread::sleep;
use crate::runner::pipeline::{suggest_in_base, resolve_using_target};
use crate::runner::trace::cli_verbose;
use crate::cli_v;
use crate::runner::trace::cli_verbose;
use crate::cli_v;
// (moved) suggest_in_base is now in runner/pipeline.rs
@ -131,97 +129,13 @@ impl NyashRunner {
/// Helper: run PyVM harness over a MIR module, returning the exit code
fn run_pyvm_harness(&self, module: &nyash_rust::mir::MirModule, tag: &str) -> Result<i32, String> {
let py3 = which::which("python3").map_err(|e| format!("python3 not found: {}", e))?;
let runner = std::path::Path::new("tools/pyvm_runner.py");
if !runner.exists() { return Err(format!("PyVM runner not found: {}", runner.display())); }
let tmp_dir = std::path::Path::new("tmp");
let _ = std::fs::create_dir_all(tmp_dir);
let mir_json_path = tmp_dir.join("nyash_pyvm_mir.json");
crate::runner::mir_json_emit::emit_mir_json_for_harness_bin(module, &mir_json_path)
.map_err(|e| format!("PyVM MIR JSON emit error: {}", e))?;
cli_v!("[ny-compiler] using PyVM ({} ) → {}", tag, mir_json_path.display());
// Determine entry function hint (prefer Main.main if present)
let entry = if module.functions.contains_key("Main.main") { "Main.main" }
else if module.functions.contains_key("main") { "main" } else { "Main.main" };
let status = std::process::Command::new(py3)
.args([
runner.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--entry",
entry,
])
.status()
.map_err(|e| format!("spawn pyvm: {}", e))?;
let code = status.code().unwrap_or(1);
if !status.success() { cli_v!("❌ PyVM ({}) failed (status={})", tag, code); }
Ok(code)
super::common_util::pyvm::run_pyvm_harness(module, tag)
}
/// Helper: try external selfhost compiler EXE to parse Ny -> JSON v0 and return MIR module
/// Returns Some(module) on success, None on failure (timeout/invalid output/missing exe)
fn exe_try_parse_json_v0(&self, filename: &str, timeout_ms: u64) -> Option<nyash_rust::mir::MirModule> {
// Resolve parser EXE path
let exe_path = if let Ok(p) = std::env::var("NYASH_NY_COMPILER_EXE_PATH") {
std::path::PathBuf::from(p)
} else {
let mut p = std::path::PathBuf::from("dist/nyash_compiler");
#[cfg(windows)]
{ p.push("nyash_compiler.exe"); }
#[cfg(not(windows))]
{ p.push("nyash_compiler"); }
if !p.exists() {
if let Ok(w) = which::which("nyash_compiler") { w } else { p }
} else { p }
};
if !exe_path.exists() { cli_v!("[ny-compiler] exe not found at {}", exe_path.display()); return None; }
// Build command
let mut cmd = std::process::Command::new(&exe_path);
cmd.arg(filename);
if crate::config::env::ny_compiler_min_json() { cmd.arg("--min-json"); }
if crate::config::env::selfhost_read_tmp() { cmd.arg("--read-tmp"); }
if let Some(raw) = crate::config::env::ny_compiler_child_args() { for tok in raw.split_whitespace() { cmd.arg(tok); } }
let mut cmd = cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = match cmd.spawn() { Ok(c) => c, Err(e) => { eprintln!("[ny-compiler] exe spawn failed: {}", e); return None; } };
let mut ch_stdout = child.stdout.take();
let mut ch_stderr = child.stderr.take();
let start = Instant::now();
let mut timed_out = false;
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => {
if start.elapsed() >= Duration::from_millis(timeout_ms) { let _ = child.kill(); let _ = child.wait(); timed_out = true; break; }
sleep(Duration::from_millis(10));
}
Err(e) => { eprintln!("[ny-compiler] exe wait error: {}", e); return None; }
}
}
let mut out_buf = Vec::new();
let mut err_buf = Vec::new();
if let Some(mut s) = ch_stdout { let _ = s.read_to_end(&mut out_buf); }
if let Some(mut s) = ch_stderr { let _ = s.read_to_end(&mut err_buf); }
if timed_out {
let head = String::from_utf8_lossy(&out_buf).chars().take(200).collect::<String>();
eprintln!("[ny-compiler] exe timeout after {} ms; stdout(head)='{}'", timeout_ms, head.replace('\n', "\\n"));
return None;
}
let stdout = match String::from_utf8(out_buf) { Ok(s) => s, Err(_) => String::new() };
let mut json_line = String::new();
for line in stdout.lines() { let t = line.trim(); if t.starts_with('{') && t.contains("\"version\"") && t.contains("\"kind\"") { json_line = t.to_string(); break; } }
if json_line.is_empty() {
if cli_verbose() {
let head: String = stdout.chars().take(200).collect();
let errh: String = String::from_utf8_lossy(&err_buf).chars().take(200).collect();
cli_v!("[ny-compiler] exe produced no JSON; stdout(head)='{}' stderr(head)='{}'", head.replace('\n', "\\n"), errh.replace('\n', "\\n"));
}
return None;
}
match json_v0_bridge::parse_json_v0_to_module(&json_line) {
Ok(module) => Some(module),
Err(e) => { eprintln!("[ny-compiler] JSON parse failed (exe): {}", e); None }
}
super::common_util::selfhost_exe::exe_try_parse_json_v0(filename, timeout_ms)
}
/// Phase-15.3: Attempt Ny compiler pipeline (Ny -> JSON v0 via Ny program), then execute MIR
@ -324,38 +238,22 @@ impl NyashRunner {
if crate::config::env::selfhost_read_tmp() { cmd.arg("--read-tmp"); }
if let Some(raw) = crate::config::env::ny_compiler_child_args() { for tok in raw.split_whitespace() { cmd.arg(tok); } }
let timeout_ms: u64 = crate::config::env::ny_compiler_timeout_ms();
let mut cmd = cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = match cmd.spawn() { Ok(c) => c, Err(e) => { eprintln!("[ny-compiler] exe spawn failed: {}", e); return false; } };
let mut ch_stdout = child.stdout.take();
let mut ch_stderr = child.stderr.take();
let start = Instant::now();
let mut timed_out = false;
loop {
match child.try_wait() {
Ok(Some(_status)) => { break; }
Ok(None) => {
if start.elapsed() >= Duration::from_millis(timeout_ms) { let _ = child.kill(); let _ = child.wait(); timed_out = true; break; }
sleep(Duration::from_millis(10));
}
Err(e) => { eprintln!("[ny-compiler] exe wait error: {}", e); return false; }
}
}
let mut out_buf = Vec::new();
let mut err_buf = Vec::new();
if let Some(mut s) = ch_stdout { let _ = s.read_to_end(&mut out_buf); }
if let Some(mut s) = ch_stderr { let _ = s.read_to_end(&mut err_buf); }
if timed_out {
let head = String::from_utf8_lossy(&out_buf).chars().take(200).collect::<String>();
let out = match super::common_util::io::spawn_with_timeout(cmd, timeout_ms) {
Ok(o) => o,
Err(e) => { eprintln!("[ny-compiler] exe spawn failed: {}", e); return false; }
};
if out.timed_out {
let head = String::from_utf8_lossy(&out.stdout).chars().take(200).collect::<String>();
eprintln!("[ny-compiler] exe timeout after {} ms; stdout(head)='{}'", timeout_ms, head.replace('\n', "\\n"));
return false;
}
let stdout = match String::from_utf8(out_buf) { Ok(s) => s, Err(_) => String::new() };
let stdout = match String::from_utf8(out.stdout) { Ok(s) => s, Err(_) => String::new() };
let mut json_line = String::new();
for line in stdout.lines() { let t = line.trim(); if t.starts_with('{') && t.contains("\"version\"") && t.contains("\"kind\"") { json_line = t.to_string(); break; } }
if json_line.is_empty() {
if crate::config::env::cli_verbose() {
let head: String = stdout.chars().take(200).collect();
let errh: String = String::from_utf8_lossy(&err_buf).chars().take(200).collect();
let errh: String = String::from_utf8_lossy(&out.stderr).chars().take(200).collect();
eprintln!("[ny-compiler] exe produced no JSON; stdout(head)='{}' stderr(head)='{}'", head.replace('\n', "\\n"), errh.replace('\n', "\\n"));
}
return false;
@ -450,40 +348,15 @@ impl NyashRunner {
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(2000);
let mut cmd = cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = match cmd.spawn() {
Ok(c) => c,
let out = match super::common_util::io::spawn_with_timeout(cmd, timeout_ms) {
Ok(o) => o,
Err(e) => { eprintln!("[ny-compiler] spawn failed: {}", e); return false; }
};
let mut ch_stdout = child.stdout.take();
let mut ch_stderr = child.stderr.take();
let start = Instant::now();
let mut timed_out = false;
loop {
match child.try_wait() {
Ok(Some(_status)) => { break; }
Ok(None) => {
if start.elapsed() >= Duration::from_millis(timeout_ms) {
let _ = child.kill();
let _ = child.wait();
timed_out = true;
break;
}
sleep(Duration::from_millis(10));
}
Err(e) => { eprintln!("[ny-compiler] wait error: {}", e); return false; }
}
}
// Collect any available output
let mut out_buf = Vec::new();
let mut err_buf = Vec::new();
if let Some(mut s) = ch_stdout { let _ = s.read_to_end(&mut out_buf); }
if let Some(mut s) = ch_stderr { let _ = s.read_to_end(&mut err_buf); }
if timed_out {
let head = String::from_utf8_lossy(&out_buf).chars().take(200).collect::<String>();
if out.timed_out {
let head = String::from_utf8_lossy(&out.stdout).chars().take(200).collect::<String>();
eprintln!("[ny-compiler] child timeout after {} ms; stdout(head)='{}'", timeout_ms, head.replace('\n', "\\n"));
}
let stdout = match String::from_utf8(out_buf) { Ok(s) => s, Err(_) => String::new() };
let stdout = match String::from_utf8(out.stdout.clone()) { Ok(s) => s, Err(_) => String::new() };
if timed_out {
// Fall back path will be taken below when json_line remains empty
} else if let Ok(s) = String::from_utf8(err_buf.clone()) {

View File

@ -0,0 +1,55 @@
use std::io::Read;
use std::process::{Command, Stdio};
use std::thread::sleep;
use std::time::{Duration, Instant};
pub struct ChildOutput {
pub stdout: Vec<u8>,
pub stderr: Vec<u8>,
pub status_ok: bool,
pub exit_code: Option<i32>,
pub timed_out: bool,
}
/// Spawn command with timeout (ms), capture stdout/stderr, and return ChildOutput.
pub fn spawn_with_timeout(mut cmd: Command, timeout_ms: u64) -> std::io::Result<ChildOutput> {
let mut cmd = cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = cmd.spawn()?;
let mut ch_stdout = child.stdout.take();
let mut ch_stderr = child.stderr.take();
let start = Instant::now();
let mut timed_out = false;
let mut exit_status: Option<std::process::ExitStatus> = None;
loop {
match child.try_wait()? {
Some(status) => { exit_status = Some(status); break },
None => {
if start.elapsed() >= Duration::from_millis(timeout_ms) {
let _ = child.kill();
let _ = child.wait();
timed_out = true;
break;
}
sleep(Duration::from_millis(10));
}
}
}
let mut out_buf = Vec::new();
let mut err_buf = Vec::new();
if let Some(mut s) = ch_stdout {
let _ = s.read_to_end(&mut out_buf);
}
if let Some(mut s) = ch_stderr {
let _ = s.read_to_end(&mut err_buf);
}
let (status_ok, exit_code) = if let Some(st) = exit_status {
(st.success(), st.code())
} else { (false, None) };
Ok(ChildOutput {
stdout: out_buf,
stderr: err_buf,
status_ok,
exit_code,
timed_out,
})
}

View File

@ -0,0 +1,9 @@
/*!
* Shared helpers for runner/modes/common.rs
*
* Minimal extraction to reduce duplication and prepare for full split.
*/
pub mod pyvm;
pub mod selfhost_exe;
pub mod io;

View File

@ -0,0 +1,39 @@
use std::process::Stdio;
/// Run PyVM harness over a MIR module, returning the exit code
pub fn run_pyvm_harness(module: &crate::mir::MirModule, tag: &str) -> Result<i32, String> {
let py3 = which::which("python3").map_err(|e| format!("python3 not found: {}", e))?;
let runner = std::path::Path::new("tools/pyvm_runner.py");
if !runner.exists() {
return Err(format!("PyVM runner not found: {}", runner.display()));
}
let tmp_dir = std::path::Path::new("tmp");
let _ = std::fs::create_dir_all(tmp_dir);
let mir_json_path = tmp_dir.join("nyash_pyvm_mir.json");
crate::runner::mir_json_emit::emit_mir_json_for_harness_bin(module, &mir_json_path)
.map_err(|e| format!("PyVM MIR JSON emit error: {}", e))?;
crate::cli_v!("[ny-compiler] using PyVM ({} ) → {}", tag, mir_json_path.display());
// Determine entry function hint (prefer Main.main if present)
let entry = if module.functions.contains_key("Main.main") {
"Main.main"
} else if module.functions.contains_key("main") {
"main"
} else {
"Main.main"
};
let status = std::process::Command::new(py3)
.args([
runner.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--entry",
entry,
])
.status()
.map_err(|e| format!("spawn pyvm: {}", e))?;
let code = status.code().unwrap_or(1);
if !status.success() {
crate::cli_v!("❌ PyVM ({}) failed (status={})", tag, code);
}
Ok(code)
}

View File

@ -0,0 +1,131 @@
use std::io::Read;
use std::process::Stdio;
use std::thread::sleep;
use std::time::{Duration, Instant};
/// Try external selfhost compiler EXE to parse Ny -> JSON v0 and return MIR module.
/// Returns Some(module) on success, None on failure (timeout/invalid output/missing exe)
pub fn exe_try_parse_json_v0(filename: &str, timeout_ms: u64) -> Option<crate::mir::MirModule> {
// Resolve parser EXE path
let exe_path = if let Ok(p) = std::env::var("NYASH_NY_COMPILER_EXE_PATH") {
std::path::PathBuf::from(p)
} else {
let mut p = std::path::PathBuf::from("dist/nyash_compiler");
#[cfg(windows)]
{
p.push("nyash_compiler.exe");
}
#[cfg(not(windows))]
{
p.push("nyash_compiler");
}
if !p.exists() {
if let Ok(w) = which::which("nyash_compiler") {
w
} else {
p
}
} else {
p
}
};
if !exe_path.exists() {
crate::cli_v!("[ny-compiler] exe not found at {}", exe_path.display());
return None;
}
// Build command
let mut cmd = std::process::Command::new(&exe_path);
cmd.arg(filename);
if crate::config::env::ny_compiler_min_json() {
cmd.arg("--min-json");
}
if crate::config::env::selfhost_read_tmp() {
cmd.arg("--read-tmp");
}
if let Some(raw) = crate::config::env::ny_compiler_child_args() {
for tok in raw.split_whitespace() {
cmd.arg(tok);
}
}
let mut cmd = cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = match cmd.spawn() {
Ok(c) => c,
Err(e) => {
eprintln!("[ny-compiler] exe spawn failed: {}", e);
return None;
}
};
let mut ch_stdout = child.stdout.take();
let mut ch_stderr = child.stderr.take();
let start = Instant::now();
let mut timed_out = false;
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => {
if start.elapsed() >= Duration::from_millis(timeout_ms) {
let _ = child.kill();
let _ = child.wait();
timed_out = true;
break;
}
sleep(Duration::from_millis(10));
}
Err(e) => {
eprintln!("[ny-compiler] exe wait error: {}", e);
return None;
}
}
}
let mut out_buf = Vec::new();
let mut err_buf = Vec::new();
if let Some(mut s) = ch_stdout {
let _ = s.read_to_end(&mut out_buf);
}
if let Some(mut s) = ch_stderr {
let _ = s.read_to_end(&mut err_buf);
}
if timed_out {
let head = String::from_utf8_lossy(&out_buf)
.chars()
.take(200)
.collect::<String>();
eprintln!(
"[ny-compiler] exe timeout after {} ms; stdout(head)='{}'",
timeout_ms,
head.replace('\n', "\\n")
);
return None;
}
let stdout = match String::from_utf8(out_buf) {
Ok(s) => s,
Err(_) => String::new(),
};
let mut json_line = String::new();
for line in stdout.lines() {
let t = line.trim();
if t.starts_with('{') && t.contains("\"version\"") && t.contains("\"kind\"") {
json_line = t.to_string();
break;
}
}
if json_line.is_empty() {
if crate::config::env::cli_verbose() {
let head: String = stdout.chars().take(200).collect();
let errh: String = String::from_utf8_lossy(&err_buf).chars().take(200).collect();
crate::cli_v!(
"[ny-compiler] exe produced no JSON; stdout(head)='{}' stderr(head)='{}'",
head.replace('\n', "\\n"),
errh.replace('\n', "\\n")
);
}
return None;
}
match crate::runner::json_v0_bridge::parse_json_v0_to_module(&json_line) {
Ok(module) => Some(module),
Err(e) => {
eprintln!("[ny-compiler] JSON parse failed: {}", e);
None
}
}
}

View File

@ -84,21 +84,22 @@ impl NyashRunner {
);
}
// 2) Run harness with --in/--out失敗時は即エラー
let status = std::process::Command::new(py3)
.args([
harness.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--out",
&_out_path,
])
.status()
let mut cmd = std::process::Command::new(py3);
cmd.args([
harness.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--out",
&_out_path,
]);
let out = crate::runner::modes::common_util::io::spawn_with_timeout(cmd, 20_000)
.map_err(|e| format!("spawn harness: {}", e))
.unwrap();
if !status.success() {
if out.timed_out || !out.status_ok {
eprintln!(
"❌ llvmlite harness failed (status={})",
status.code().unwrap_or(-1)
"❌ llvmlite harness failed (timeout={} code={:?})",
out.timed_out,
out.exit_code
);
process::exit(1);
}

View File

@ -8,5 +8,8 @@ pub mod mir;
pub mod vm;
pub mod pyvm;
// Shared helpers extracted from common.rs (in progress)
pub mod common_util;
#[cfg(feature = "cranelift-jit")]
pub mod aot;

View File

@ -76,22 +76,20 @@ pub fn execute_pyvm_only(_runner: &NyashRunner, filename: &str) {
mir_json_path.display()
);
}
let status = std::process::Command::new(py3)
.args([
runner.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--entry",
entry,
])
.status()
let mut cmd = std::process::Command::new(py3);
cmd.args([
runner.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--entry",
entry,
]);
let out = crate::runner::modes::common_util::io::spawn_with_timeout(cmd, 10_000)
.map_err(|e| format!("spawn pyvm: {}", e))
.unwrap();
let code = status.code().unwrap_or(1);
if !status.success() {
if std::env::var("NYASH_CLI_VERBOSE").ok().as_deref() == Some("1") {
eprintln!("❌ PyVM failed (status={})", code);
}
let code = if out.timed_out { 1 } else { out.exit_code.unwrap_or(1) };
if out.timed_out && std::env::var("NYASH_CLI_VERBOSE").ok().as_deref() == Some("1") {
eprintln!("❌ PyVM timeout");
}
process::exit(code);
} else {
@ -99,4 +97,3 @@ pub fn execute_pyvm_only(_runner: &NyashRunner, filename: &str) {
process::exit(1);
}
}

View File

@ -207,23 +207,20 @@ impl NyashRunner {
"Main.main"
};
// Spawn runner
let status = std::process::Command::new(py3)
.args([
runner.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--entry",
entry,
])
.status()
let mut cmd = std::process::Command::new(py3);
cmd.args([
runner.to_string_lossy().as_ref(),
"--in",
&mir_json_path.display().to_string(),
"--entry",
entry,
]);
let out = super::common_util::io::spawn_with_timeout(cmd, 10_000)
.map_err(|e| format!("spawn pyvm: {}", e))
.unwrap();
// Always propagate PyVM exit code to match llvmlite semantics
let code = status.code().unwrap_or(1);
if !status.success() {
if std::env::var("NYASH_CLI_VERBOSE").ok().as_deref() == Some("1") {
eprintln!("❌ PyVM failed (status={})", code);
}
let code = if out.timed_out { 1 } else { out.exit_code.unwrap_or(1) };
if out.timed_out && std::env::var("NYASH_CLI_VERBOSE").ok().as_deref() == Some("1") {
eprintln!("❌ PyVM timeout");
}
process::exit(code);
} else {