feat(joinir): Phase 53 - SELFHOST-NORM-DEV-EXPAND implementation

Expanded selfhost dev Normalized target with 2 practical P2/P3 loop variations, strengthened structural signature axis, and implemented two-stage detection. Key Changes: 1. Documentation (phase49-selfhost-joinir-depth2-design.md +128 lines): - Added Phase 53 section with candidate selection rationale - Documented two-stage detector strategy (structural primary + dev-only name guard) - Defined structural axis strengthening (carrier count/type, branch patterns) 2. Fixtures (+210 lines): - selfhost_args_parse_p2.program.json (60 lines): P2 with String carrier + conditional branching - selfhost_stmt_count_p3.program.json (150 lines): P3 with 5 carriers + multi-branch if-else 3. Structured Builders (fixtures.rs +48 lines): - build_selfhost_args_parse_p2_structured_for_normalized_dev() - build_selfhost_stmt_count_p3_structured_for_normalized_dev() 4. ShapeGuard Two-Stage Detection (shape_guard.rs +80 lines): - Added SelfhostArgsParseP2/SelfhostStmtCountP3 to NormalizedDevShape enum - Implemented is_selfhost_args_parse_p2(): P2 core family + name guard - Implemented is_selfhost_stmt_count_p3(): 2-10 carrier check + name guard - Updated capability_for_shape() mappings 5. Bridge Integration (bridge.rs +8 lines, normalized.rs +10 lines): - Added shape handlers delegating to existing normalizers - Added roundtrip reconstruction handlers 6. Entry Point Registration (ast_lowerer/mod.rs +2 lines): - Registered selfhost_args_parse_p2/selfhost_stmt_count_p3 as LoopFrontend routes 7. Dev VM Comparison Tests (normalized_joinir_min.rs +40 lines): - normalized_selfhost_args_parse_p2_vm_bridge_direct_matches_structured() - normalized_selfhost_stmt_count_p3_vm_bridge_direct_matches_structured() 8. Test Context Fix (dev_env.rs): - Added thread-local test context depth counter - Fixed deadlock in nested test_ctx() calls via reentrant with_dev_env_if_unset() Structural Axis Growth: P2 family: - Carrier count: 1-3 (unchanged) - NEW: Type diversity (Integer/String mixed) - NEW: Conditional branching patterns (Eq-heavy comparisons) P3 family: - NEW: Carrier count upper bound: 2-10 (was 2-4) - NEW: Multi-branch if-else (5+ branches with nested structure) - NEW: Complex conditional patterns Test Results: - normalized_dev: 40/40 PASS (including 2 new tests) - lib regression: 939 PASS, 56 ignored - Existing behavior unchanged (normalized_dev feature-gated) Phase 53 Achievements: ✅ P2/P3 each gained 1 practical variation (2 total) ✅ Two-stage detection: structural primary + dev-only name guard ✅ Structural axis expanded: 4 axes (carrier count/type/Compare/branch patterns) ✅ All tests PASS, no regressions ✅ Test context deadlock fixed (0.04s for 29 tests) Files Modified: 14 files Lines Added: ~516 lines (net) Implementation: Pure additive (feature-gated) Next Phase (54+): - Accumulate 6+ loops per P2/P3 family - Achieve 5+ stable structural axes - Target < 5% false positive rate - Then shrink/remove name guard scope
2025-12-12 16:40:20 +09:00
parent 386cbc1915
commit 7b0db59100
14 changed files with 1580 additions and 64 deletions
--- a/tests/normalized_joinir_min.rs
+++ b/tests/normalized_joinir_min.rs
@ -1,6 +1,6 @@
 #![cfg(all(feature = "normalized_dev", debug_assertions))]

-use nyash_rust::backend::mir_interpreter::MirInterpreter;
+use nyash_rust::backend::{mir_interpreter::MirInterpreter, VMValue};
 use nyash_rust::mir::join_ir::{
    normalize_pattern1_minimal, normalize_pattern2_minimal, normalized_pattern1_to_structured,
    normalized_pattern2_to_structured, BinOpKind, ConstValue, JoinContId, JoinFuncId,
@ -12,7 +12,9 @@ use nyash_rust::mir::join_ir::normalized::dev_env::{
 use nyash_rust::mir::join_ir::normalized::fixtures::{
    build_jsonparser_atoi_structured_for_normalized_dev,
    build_jsonparser_atoi_real_structured_for_normalized_dev,
+    build_jsonparser_parse_array_continue_skip_ws_structured_for_normalized_dev,
    build_jsonparser_parse_number_real_structured_for_normalized_dev,
+    build_jsonparser_parse_object_continue_skip_ws_structured_for_normalized_dev,
    build_jsonparser_skip_ws_real_structured_for_normalized_dev,
    build_jsonparser_skip_ws_structured_for_normalized_dev,
    build_pattern2_break_fixture_structured, build_pattern2_minimal_structured,
@ -20,11 +22,20 @@ use nyash_rust::mir::join_ir::normalized::fixtures::{
    build_pattern3_if_sum_multi_min_structured_for_normalized_dev,
    build_pattern3_json_if_sum_min_structured_for_normalized_dev,
    build_pattern4_continue_min_structured_for_normalized_dev,
+    build_selfhost_args_parse_p2_structured_for_normalized_dev,
+    build_selfhost_if_sum_p3_ext_structured_for_normalized_dev,
+    build_selfhost_if_sum_p3_structured_for_normalized_dev,
+    build_selfhost_stmt_count_p3_structured_for_normalized_dev,
+    build_selfhost_token_scan_p2_accum_structured_for_normalized_dev,
+    build_selfhost_token_scan_p2_structured_for_normalized_dev,
 };
 use nyash_rust::mir::join_ir_runner::run_joinir_function;
 use nyash_rust::mir::join_ir_ops::JoinValue;
-use nyash_rust::mir::join_ir_vm_bridge::run_joinir_via_vm;
+use nyash_rust::mir::join_ir_vm_bridge::{
+    convert_join_module_to_mir_with_meta, run_joinir_via_vm,
+};
 use nyash_rust::mir::ValueId;
+use std::collections::BTreeMap;
 fn normalized_dev_test_ctx() -> NormalizedTestContext<'static> {
    let ctx = test_ctx();
    assert!(
@ -68,6 +79,22 @@ fn run_joinir_vm_bridge(
    run_joinir_via_vm(module, entry, args).expect("JoinIR→MIR execution should succeed")
 }

+fn run_joinir_vm_bridge_structured_only(
+    module: &JoinModule,
+    entry: JoinFuncId,
+    args: &[JoinValue],
+) -> JoinValue {
+    let mir =
+        convert_join_module_to_mir_with_meta(module, &BTreeMap::new()).expect("structured bridge");
+    let mut vm = MirInterpreter::new();
+    let entry_name = format!("join_func_{}", entry.0);
+    let vm_args: Vec<VMValue> = args.iter().cloned().map(|v| v.into_vm_value()).collect();
+    let result = vm
+        .execute_function_with_args(&mir, &entry_name, &vm_args)
+        .expect("VM execution should succeed");
+    JoinValue::from_vm_value(&result).expect("result conversion")
+}
+
 fn build_structured_pattern1() -> JoinModule {
    let mut module = JoinModule::new();
    let mut loop_fn = JoinFunction::new(
@ -512,6 +539,44 @@ fn normalized_pattern2_jsonparser_atoi_real_vm_bridge_direct_matches_structured(
    }
 }

+#[test]
+fn normalized_selfhost_token_scan_p2_vm_bridge_direct_matches_structured() {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_selfhost_token_scan_p2_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+    let cases = [0, 1, 3, 5];
+
+    for n in cases {
+        let input = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &input, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &input, true);
+
+        assert_eq!(base, dev, "vm bridge mismatch for n={}", n);
+        assert_eq!(
+            dev,
+            JoinValue::Int(n),
+            "unexpected result for selfhost_token_scan_p2 n={}",
+            n
+        );
+    }
+}
+
+#[test]
+fn normalized_selfhost_token_scan_p2_accum_vm_bridge_direct_matches_structured() {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_selfhost_token_scan_p2_accum_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+    let cases = [0, 1, 3, 5];
+
+    for n in cases {
+        let input = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &input, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &input, true);
+
+        assert_eq!(base, dev, "vm bridge mismatch for selfhost_token_scan_p2_accum n={}", n);
+    }
+}
+
 #[test]
 fn normalized_pattern3_if_sum_minimal_runner_dev_switch_matches_structured() {
    let _ctx = normalized_dev_test_ctx();
@ -568,6 +633,74 @@ fn normalized_pattern3_json_if_sum_min_vm_bridge_direct_matches_structured() {
    );
 }

+#[test]
+fn normalized_selfhost_if_sum_p3_vm_bridge_direct_matches_structured() {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_selfhost_if_sum_p3_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+    let cases = [0, 1, 3, 4];
+
+    for n in cases {
+        let input = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &input, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &input, true);
+
+        assert_eq!(base, dev, "vm bridge mismatch for selfhost_if_sum_p3 n={}", n);
+    }
+}
+
+#[test]
+fn normalized_selfhost_if_sum_p3_ext_vm_bridge_direct_matches_structured() {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_selfhost_if_sum_p3_ext_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+    let cases = [0, 1, 3, 4];
+
+    for n in cases {
+        let input = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &input, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &input, true);
+
+        assert_eq!(base, dev, "vm bridge mismatch for selfhost_if_sum_p3_ext n={}", n);
+    }
+}
+
+/// Phase 53: selfhost args-parse P2 (practical variation with string carrier)
+#[test]
+fn normalized_selfhost_args_parse_p2_vm_bridge_direct_matches_structured() {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_selfhost_args_parse_p2_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+    // Test different argc values: 0, 1, 2, 3
+    let cases = [0, 1, 2, 3];
+
+    for argc in cases {
+        let input = [JoinValue::Int(argc)];
+        let base = run_joinir_vm_bridge(&structured, entry, &input, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &input, true);
+
+        assert_eq!(base, dev, "vm bridge mismatch for selfhost_args_parse_p2 argc={}", argc);
+    }
+}
+
+/// Phase 53: selfhost stmt-count P3 (practical variation with multi-branch if-else)
+#[test]
+fn normalized_selfhost_stmt_count_p3_vm_bridge_direct_matches_structured() {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_selfhost_stmt_count_p3_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+    // Test different statement counts: 0, 5, 10, 15
+    let cases = [0, 5, 10, 15];
+
+    for n in cases {
+        let input = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &input, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &input, true);
+
+        assert_eq!(base, dev, "vm bridge mismatch for selfhost_stmt_count_p3 n={}", n);
+    }
+}
+
 #[cfg(feature = "normalized_dev")]
 #[test]
 fn test_phase46_canonical_set_includes_p2_mid() {
@ -724,3 +857,106 @@ fn test_normalized_pattern4_continue_minimal_vm_bridge_direct_matches_structured
        "unexpected result for P4 minimal continue (expected acc=4)",
    );
 }
+
+/// Phase 48-C: P4 minimal should use canonical normalized route even without env
+#[test]
+fn test_normalized_pattern4_continue_minimal_canonical_matches_structured() {
+    let structured = build_pattern4_continue_min_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+
+    let input = [JoinValue::Int(5)];
+    let structured_res = run_joinir_vm_bridge_structured_only(&structured, entry, &input);
+    let canonical = run_joinir_vm_bridge(&structured, entry, &input, false);
+
+    assert_eq!(
+        structured_res, canonical,
+        "canonical P4 minimal result mismatch"
+    );
+    assert_eq!(canonical, JoinValue::Int(4));
+}
+
+/// Phase 48-B: JsonParser _parse_array continue skip_ws (dev-only) VM Bridge comparison
+#[test]
+fn test_normalized_pattern4_jsonparser_parse_array_continue_skip_ws_vm_bridge_direct_matches_structured(
+) {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_jsonparser_parse_array_continue_skip_ws_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+
+    // Fixture mirrors pattern4_continue_min: skip i == 2
+    let cases = [3, 5, 7];
+
+    for n in cases {
+        let args = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &args, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &args, true);
+        assert_eq!(
+            base, dev,
+            "vm bridge mismatch for array continue case n={}",
+            n
+        );
+    }
+}
+
+/// Phase 48-C: JsonParser _parse_array continue skip_ws canonical route should match Structured
+#[test]
+fn test_normalized_pattern4_jsonparser_parse_array_continue_skip_ws_canonical_matches_structured()
+{
+    let structured = build_jsonparser_parse_array_continue_skip_ws_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+
+    let cases = [3, 5, 7];
+    for n in cases {
+        let args = [JoinValue::Int(n)];
+        let structured_res = run_joinir_vm_bridge_structured_only(&structured, entry, &args);
+        let canonical = run_joinir_vm_bridge(&structured, entry, &args, false);
+        assert_eq!(
+            structured_res, canonical,
+            "canonical array continue mismatch n={}",
+            n
+        );
+    }
+}
+
+/// Phase 48-B: JsonParser _parse_object continue skip_ws (dev-only) VM Bridge comparison
+#[test]
+fn test_normalized_pattern4_jsonparser_parse_object_continue_skip_ws_vm_bridge_direct_matches_structured(
+) {
+    let _ctx = normalized_dev_test_ctx();
+    let structured = build_jsonparser_parse_object_continue_skip_ws_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+
+    // Fixture mirrors pattern4_continue_min: skip i == 2
+    let cases = [4, 6, 8];
+
+    for n in cases {
+        let args = [JoinValue::Int(n)];
+        let base = run_joinir_vm_bridge(&structured, entry, &args, false);
+        let dev = run_joinir_vm_bridge(&structured, entry, &args, true);
+        assert_eq!(
+            base, dev,
+            "vm bridge mismatch for object continue case n={}",
+            n
+        );
+    }
+}
+
+/// Phase 48-C: JsonParser _parse_object continue skip_ws canonical route should match Structured
+#[test]
+fn test_normalized_pattern4_jsonparser_parse_object_continue_skip_ws_canonical_matches_structured()
+{
+    let structured = build_jsonparser_parse_object_continue_skip_ws_structured_for_normalized_dev();
+    let entry = structured.entry.expect("structured entry required");
+
+    let cases = [4, 6, 8];
+    for n in cases {
+        let args = [JoinValue::Int(n)];
+        let structured_res = run_joinir_vm_bridge_structured_only(&structured, entry, &args);
+        let canonical = run_joinir_vm_bridge(&structured, entry, &args, false);
+        assert_eq!(
+            structured_res, canonical,
+            "canonical object continue mismatch n={}",
+            n
+        );
+    }
+}