14 lines
389 B
Plaintext
14 lines
389 B
Plaintext
|
// tools/hako_parser/tokenizer.hako — HakoTokenizerBox (MVP skeleton)
|
||
|
|
using selfhost.shared.common.string_helpers as Str
|
||
|
|
|
||
|
|
static box HakoTokenizerBox {
|
||
|
|
// Returns ArrayBox of tokens (MVP: string list)
|
||
|
|
tokenize(text) {
|
||
|
|
// TODO: implement real tokenizer; MVP returns lines as stub
|
||
|
|
return text.split("\n")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static box HakoTokenizerMain { method main(args) { return 0 } }
|
||
|
|
|