Phase 21.3 WIP: Hako Source Checker improvements - HC011/HC016/HC017 実装完了
主な変更: - ✅ HC011 (dead methods) 実装・テスト緑 - ✅ HC016 (unused alias) 実装・テスト緑 - ✅ HC017 (non-ascii quotes) 実装完了 - 🔧 tokenizer/parser_core 強化（AST優先ルート） - 🛡️ plugin_guard.rs 追加（stderr専用出力） - 📋 テストインフラ整備（run_tests.sh改善） 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
nyash-codex
@ -1,14 +1,80 @@
|
||||
// tools/hako_parser/ast_emit.hako — HakoAstEmitBox (MVP skeleton)
|
||||
// tools/hako_parser/ast_emit.hako — HakoAstEmitBox (MVP)
|
||||
using selfhost.shared.common.string_helpers as Str
|
||||
|
||||
static box HakoAstEmitBox {
|
||||
// Emit minimal AST JSON v0 from MapBox
|
||||
// Emit minimal AST JSON v0 from MapBox (stable order)
|
||||
to_json(ast) {
|
||||
// NOTE: MVP naive stringify; replace with proper JsonEmitBox if needed
|
||||
local s = "{\"boxes\":[],\"uses\":[]}"
|
||||
return s
|
||||
if ast == null { return "{\"boxes\":[],\"uses\":[]}" }
|
||||
local uses = me._sort_strings(ast.get("uses"))
|
||||
local boxes = me._sort_boxes(ast.get("boxes"))
|
||||
local out = "{\"uses\":" + me._emit_array(uses) + ",\"boxes\":" + me._emit_boxes(boxes) + "}"
|
||||
return out
|
||||
}
|
||||
_emit_array(arr) {
|
||||
if arr == null { return "[]" }
|
||||
local s = "["
|
||||
local n = arr.size()
|
||||
local i = 0
|
||||
while i < n {
|
||||
local v = arr.get(i)
|
||||
s = s + Str.json_quote(v)
|
||||
if i != n-1 { s = s + "," }
|
||||
i = i + 1
|
||||
}
|
||||
return s + "]"
|
||||
}
|
||||
_emit_boxes(boxes) {
|
||||
if boxes == null { return "[]" }
|
||||
local s = "["
|
||||
local n = boxes.size()
|
||||
local i = 0
|
||||
while i < n {
|
||||
local b = boxes.get(i)
|
||||
local name = Str.json_quote(b.get("name"))
|
||||
local is_static = b.get("is_static")
|
||||
local methods = me._emit_methods(me._sort_methods(b.get("methods")))
|
||||
s = s + "{\"name\":" + name + ",\"is_static\":" + Str.int_to_str(is_static) + ",\"methods\":" + methods + "}"
|
||||
if i != n-1 { s = s + "," }
|
||||
i = i + 1
|
||||
}
|
||||
return s + "]"
|
||||
}
|
||||
_emit_methods(methods) {
|
||||
if methods == null { return "[]" }
|
||||
local s = "["
|
||||
local n = methods.size()
|
||||
local i = 0
|
||||
while i < n {
|
||||
local m = methods.get(i)
|
||||
local name = Str.json_quote(m.get("name"))
|
||||
local arity = Str.int_to_str(m.get("arity"))
|
||||
// span is integer line number
|
||||
local spanv = m.get("span"); if spanv == null { spanv = 0 }
|
||||
s = s + "{\"name\":" + name + ",\"arity\":" + arity + ",\"span\":" + Str.int_to_str(spanv) + "}"
|
||||
if i != n-1 { s = s + "," }
|
||||
i = i + 1
|
||||
}
|
||||
return s + "]"
|
||||
}
|
||||
// Helpers: sorting (naive O(n^2))
|
||||
_sort_strings(arr) { if arr == null { return new ArrayBox() }
|
||||
local out = new ArrayBox(); local i=0; while i<arr.size() { out.push(arr.get(i)); i=i+1 }
|
||||
// selection sort
|
||||
local n = out.size(); local a=0; while a<n { local b=a+1; while b<n { if out.get(b) < out.get(a) { local tmp=out.get(a); out.set(a,out.get(b)); out.set(b,tmp) } b=b+1 } a=a+1 }
|
||||
return out }
|
||||
_sort_boxes(boxes) { if boxes == null { return new ArrayBox() }
|
||||
local out = new ArrayBox(); local i=0; while i<boxes.size() { out.push(boxes.get(i)); i=i+1 }
|
||||
local n=out.size(); local a=0; while a<n { local b=a+1; while b<n { if out.get(b).get("name") < out.get(a).get("name") { local tmp=out.get(a); out.set(a,out.get(b)); out.set(b,tmp) } b=b+1 } a=a+1 }
|
||||
return out }
|
||||
_sort_methods(methods) { if methods == null { return new ArrayBox() }
|
||||
local out = new ArrayBox(); local i=0; while i<methods.size() { out.push(methods.get(i)); i=i+1 }
|
||||
local n=out.size(); local a=0; while a<n { local b=a+1; while b<n {
|
||||
local ma=out.get(a); local mb=out.get(b)
|
||||
local ka = ma.get("name") + "/" + Str.int_to_str(ma.get("arity"))
|
||||
local kb = mb.get("name") + "/" + Str.int_to_str(mb.get("arity"))
|
||||
if kb < ka { local tmp=out.get(a); out.set(a,out.get(b)); out.set(b,tmp) }
|
||||
b=b+1 } a=a+1 }
|
||||
return out }
|
||||
}
|
||||
|
||||
static box HakoAstEmitMain { method main(args) { return 0 } }
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
// tools/hako_parser/cli.hako — HakoParserBox CLI (MVP skeleton)
|
||||
using selfhost.tools.hako_parser.parser_core as HakoParserCoreBox
|
||||
using selfhost.tools.hako_parser.ast_emit as HakoAstEmitBox
|
||||
using tools.hako_parser.parser_core as HakoParserCoreBox
|
||||
using tools.hako_parser.ast_emit as HakoAstEmitBox
|
||||
|
||||
static box HakoParserBox {
|
||||
run(args) {
|
||||
@ -16,4 +16,3 @@ static box HakoParserBox {
|
||||
}
|
||||
|
||||
static box HakoParserCliMain { method main(args) { return HakoParserBox.run(args) } }
|
||||
|
||||
|
||||
@ -1,17 +1,97 @@
|
||||
// tools/hako_parser/parser_core.hako — HakoParserCoreBox (MVP skeleton)
|
||||
// tools/hako_parser/parser_core.hako — HakoParserCoreBox (token-based MVP)
|
||||
using selfhost.shared.common.string_helpers as Str
|
||||
using selfhost.tools.hako_parser.tokenizer as HakoTokenizerBox
|
||||
using tools.hako_parser.tokenizer as HakoTokenizerBox
|
||||
|
||||
static box HakoParserCoreBox {
|
||||
// Parse .hako source into minimal AST map:
|
||||
// {
|
||||
// uses: Array<String>,
|
||||
// boxes: Array<{name,is_static,methods:Array<{name,arity,span}>}>
|
||||
// }
|
||||
parse(text) {
|
||||
local toks = HakoTokenizerBox.tokenize(text)
|
||||
// TODO: implement real parser; MVP returns a minimal AST map
|
||||
local ast = new MapBox()
|
||||
ast.set("boxes", new ArrayBox())
|
||||
ast.set("uses", new ArrayBox())
|
||||
ast.set("boxes", new ArrayBox())
|
||||
ast.set("includes", new ArrayBox())
|
||||
if text == null { return ast }
|
||||
|
||||
local toks = HakoTokenizerBox.tokenize(text)
|
||||
local p = 0
|
||||
local N = toks.size()
|
||||
|
||||
// Parse stream (single pass, tolerant)
|
||||
while p < N {
|
||||
local t = me._peek(toks, p, N)
|
||||
if me._eq(t, "USING") == 1 {
|
||||
// using "mod" (as Alias)?
|
||||
p = me._advance(p, N)
|
||||
local t1 = me._peek(toks, p, N)
|
||||
if me._eq(t1, "STRING") == 1 {
|
||||
ast.get("uses").push(t1.get("lexeme")); p = me._advance(p, N)
|
||||
// optional: as Alias
|
||||
local t2 = me._peek(toks, p, N); if me._eq(t2, "AS") == 1 { p = me._advance(p, N); local t3=me._peek(toks, p, N); if me._eq(t3, "IDENT")==1 { p = me._advance(p, N) } }
|
||||
} else {
|
||||
// tolerate malformed using; skip token
|
||||
}
|
||||
continue
|
||||
}
|
||||
if me._eq(t, "INCLUDE") == 1 {
|
||||
// include "path"
|
||||
p = me._advance(p, N); local s=me._peek(toks, p, N); if me._eq(s, "STRING") == 1 { ast.get("includes").push(Str.int_to_str(s.get("line"))); p = me._advance(p, N) }
|
||||
continue
|
||||
}
|
||||
if me._eq(t, "STATIC") == 1 {
|
||||
// static box Name { methods }
|
||||
// STATIC BOX IDENT LBRACE ... RBRACE
|
||||
local save = p
|
||||
p = me._advance(p, N) // STATIC
|
||||
local tb = me._peek(toks, p, N); if me._eq(tb, "BOX") == 0 { p = save + 1; continue } p = me._advance(p, N)
|
||||
local tn = me._peek(toks, p, N); if me._eq(tn, "IDENT") == 0 { continue }
|
||||
local box_name = tn.get("lexeme"); p = me._advance(p, N)
|
||||
// expect '{'
|
||||
local tl = me._peek(toks, p, N); if me._eq(tl, "LBRACE") == 0 { continue } p = me._advance(p, N)
|
||||
// register box
|
||||
local b = new MapBox(); b.set("name", box_name); b.set("is_static", 1); b.set("methods", new ArrayBox()); ast.get("boxes").push(b)
|
||||
// scan until matching RBRACE (flat, tolerate nested braces count)
|
||||
local depth = 1
|
||||
while p < N && depth > 0 {
|
||||
local tk = me._peek(toks, p, N)
|
||||
if me._eq(tk, "LBRACE") == 1 { depth = depth + 1; p = me._advance(p, N); continue }
|
||||
if me._eq(tk, "RBRACE") == 1 { depth = depth - 1; p = me._advance(p, N); if depth == 0 { break } else { continue } }
|
||||
// method
|
||||
if me._eq(tk, "METHOD") == 1 {
|
||||
local mline = tk.get("line"); p = me._advance(p, N)
|
||||
local mid = me._peek(toks, p, N); if me._eq(mid, "IDENT") == 0 { continue }
|
||||
local mname = mid.get("lexeme"); p = me._advance(p, N)
|
||||
// params
|
||||
local lp = me._peek(toks, p, N); if me._eq(lp, "LPAREN") == 0 { continue } p = me._advance(p, N)
|
||||
// count commas until RPAREN (no nesting inside params for MVP)
|
||||
local arity = 0; local any = 0
|
||||
while p < N {
|
||||
local tt = me._peek(toks, p, N)
|
||||
if me._eq(tt, "RPAREN") == 1 { p = me._advance(p, N); break }
|
||||
if me._eq(tt, "COMMA") == 1 { arity = arity + 1; p = me._advance(p, N); any = 1; continue }
|
||||
// consume any token inside params
|
||||
p = me._advance(p, N); any = 1
|
||||
}
|
||||
if any == 1 && arity == 0 { arity = 1 }
|
||||
// record method
|
||||
local m = new MapBox(); m.set("name", mname); m.set("arity", arity); m.set("span", mline)
|
||||
b.get("methods").push(m)
|
||||
continue
|
||||
}
|
||||
p = me._advance(p, N)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// skip unhandled token
|
||||
p = me._advance(p, N)
|
||||
}
|
||||
return ast
|
||||
}
|
||||
_peek(toks, idx, N) { if idx >= N { return null } return toks.get(idx) }
|
||||
_eq(t, kind) { if t == null { return 0 } if t.get("type") == kind { return 1 } return 0 }
|
||||
_advance(p, N) { if p < N { return p + 1 } return p }
|
||||
}
|
||||
|
||||
static box HakoParserCoreMain { method main(args) { return 0 } }
|
||||
|
||||
|
||||
@ -1,13 +1,136 @@
|
||||
// tools/hako_parser/tokenizer.hako — HakoTokenizerBox (MVP skeleton)
|
||||
// tools/hako_parser/tokenizer.hako — HakoTokenizerBox (Stage-3 aware tokenizer, MVP)
|
||||
// Produces tokens with type, lexeme, line, col. Handles strings (escapes), numbers,
|
||||
// identifiers, and punctuation. Keywords are normalized to upper-case kinds.
|
||||
using selfhost.shared.common.string_helpers as Str
|
||||
|
||||
static box HakoTokenizerBox {
|
||||
// Returns ArrayBox of tokens (MVP: string list)
|
||||
// Token: Map { type, lexeme, line, col }
|
||||
tokenize(text) {
|
||||
// TODO: implement real tokenizer; MVP returns lines as stub
|
||||
return text.split("\n")
|
||||
local out = new ArrayBox()
|
||||
if text == null { return out }
|
||||
local n = text.length()
|
||||
local i = 0
|
||||
local line = 1
|
||||
local col = 1
|
||||
while i < n {
|
||||
local ch = text.substring(i,i+1)
|
||||
// whitespace and newlines
|
||||
if ch == " " || ch == "\t" { i = i + 1; col = col + 1; continue }
|
||||
if ch == "\r" { i = i + 1; continue }
|
||||
if ch == "\n" { i = i + 1; line = line + 1; col = 1; continue }
|
||||
// line comment // ... (consume until EOL)
|
||||
if ch == "/" && i+1 < n && text.substring(i+1,i+2) == "/" {
|
||||
// skip until newline
|
||||
i = i + 2; col = col + 2
|
||||
while i < n {
|
||||
local c2 = text.substring(i,i+1)
|
||||
if c2 == "\n" { break }
|
||||
i = i + 1; col = col + 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
// block comment /* ... */ (consume until closing, track newlines)
|
||||
if ch == "/" && i+1 < n && text.substring(i+1,i+2) == "*" {
|
||||
i = i + 2; col = col + 2
|
||||
local closed = 0
|
||||
while i < n {
|
||||
local c2 = text.substring(i,i+1)
|
||||
if c2 == "*" && i+1 < n && text.substring(i+1,i+2) == "/" { i = i + 2; col = col + 2; closed = 1; break }
|
||||
if c2 == "\n" { i = i + 1; line = line + 1; col = 1; continue }
|
||||
i = i + 1; col = col + 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
// string literal "..." with escapes \" \\ \n \t
|
||||
if ch == '"' {
|
||||
local start_col = col
|
||||
local buf = ""
|
||||
i = i + 1; col = col + 1
|
||||
local closed = 0
|
||||
while i < n {
|
||||
local c3 = text.substring(i,i+1)
|
||||
if c3 == '"' { closed = 1; i = i + 1; col = col + 1; break }
|
||||
if c3 == "\\" {
|
||||
if i+1 < n {
|
||||
local esc = text.substring(i+1,i+2)
|
||||
if esc == '"' { buf = buf.concat('"') }
|
||||
else if esc == "\\" { buf = buf.concat("\\") }
|
||||
else if esc == "n" { buf = buf.concat("\n") }
|
||||
else if esc == "t" { buf = buf.concat("\t") }
|
||||
else { buf = buf.concat(esc) }
|
||||
i = i + 2; col = col + 2
|
||||
continue
|
||||
} else { i = i + 1; col = col + 1; break }
|
||||
}
|
||||
buf = buf.concat(c3)
|
||||
i = i + 1; col = col + 1
|
||||
}
|
||||
local tok = new MapBox(); tok.set("type","STRING"); tok.set("lexeme", buf); tok.set("line", line); tok.set("col", start_col)
|
||||
out.push(tok); continue
|
||||
}
|
||||
// number (integer only for MVP)
|
||||
if ch >= "0" && ch <= "9" {
|
||||
local start = i; local start_col = col
|
||||
while i < n {
|
||||
local c4 = text.substring(i,i+1)
|
||||
if !(c4 >= "0" && c4 <= "9") { break }
|
||||
i = i + 1; col = col + 1
|
||||
}
|
||||
local lex = text.substring(start, i)
|
||||
local tok = new MapBox(); tok.set("type","NUMBER"); tok.set("lexeme", lex); tok.set("line", line); tok.set("col", start_col)
|
||||
out.push(tok); continue
|
||||
}
|
||||
// identifier or keyword
|
||||
if me._is_ident_start(ch) == 1 {
|
||||
local start = i; local start_col = col
|
||||
while i < n {
|
||||
local c5 = text.substring(i,i+1)
|
||||
if me._is_ident_char(c5) == 0 { break }
|
||||
i = i + 1; col = col + 1
|
||||
}
|
||||
local lex = text.substring(start, i)
|
||||
local kind = me._kw_kind(lex)
|
||||
local tok = new MapBox(); tok.set("type", kind); tok.set("lexeme", lex); tok.set("line", line); tok.set("col", start_col)
|
||||
out.push(tok); continue
|
||||
}
|
||||
// punctuation / symbols we care about
|
||||
local sym_kind = me._sym_kind(ch)
|
||||
if sym_kind != null {
|
||||
local tok = new MapBox(); tok.set("type", sym_kind); tok.set("lexeme", ch); tok.set("line", line); tok.set("col", col)
|
||||
out.push(tok); i = i + 1; col = col + 1; continue
|
||||
}
|
||||
// unknown char → emit as PUNC so parser can skip gracefully
|
||||
local tok = new MapBox(); tok.set("type","PUNC"); tok.set("lexeme", ch); tok.set("line", line); tok.set("col", col)
|
||||
out.push(tok); i = i + 1; col = col + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
_is_ident_start(c) { if c=="_" {return 1}; if c>="A"&&c<="Z" {return 1}; if c>="a"&&c<="z" {return 1}; return 0 }
|
||||
_is_ident_char(c) { if me._is_ident_start(c)==1 { return 1 }; if c>="0"&&c<="9" { return 1 }; return 0 }
|
||||
_kw_kind(lex) {
|
||||
if lex == "using" { return "USING" }
|
||||
if lex == "as" { return "AS" }
|
||||
if lex == "static" { return "STATIC" }
|
||||
if lex == "box" { return "BOX" }
|
||||
if lex == "method" { return "METHOD" }
|
||||
if lex == "include" { return "INCLUDE" }
|
||||
if lex == "while" { return "WHILE" } // Stage-3 tokens (MVP)
|
||||
if lex == "for" { return "FOR" }
|
||||
if lex == "in" { return "IN" }
|
||||
return "IDENT"
|
||||
}
|
||||
_sym_kind(c) {
|
||||
if c == "{" { return "LBRACE" }
|
||||
if c == "}" { return "RBRACE" }
|
||||
if c == "(" { return "LPAREN" }
|
||||
if c == ")" { return "RPAREN" }
|
||||
if c == "," { return "COMMA" }
|
||||
if c == "." { return "DOT" }
|
||||
if c == ":" { return "COLON" }
|
||||
if c == "=" { return "EQ" }
|
||||
if c == ";" { return "SEMI" }
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
static box HakoTokenizerMain { method main(args) { return 0 } }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user