34 lines
1.0 KiB
Rust
34 lines
1.0 KiB
Rust
|
use crate::tokenizer::{NyashTokenizer, TokenType};
|
||
|
|
|
||
|
|
fn collect_string_token(src: &str) -> String {
|
||
|
|
let mut t = NyashTokenizer::new(src);
|
||
|
|
let tokens = t.tokenize().expect("tokenize");
|
||
|
|
// Expect first non-EOF token to be STRING
|
||
|
|
for tok in tokens {
|
||
|
|
if let TokenType::STRING(s) = tok.token_type { return s; }
|
||
|
|
}
|
||
|
|
panic!("no STRING token found");
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn unicode_decode_toggle_off_keeps_literal() {
|
||
|
|
// OFF by default
|
||
|
|
std::env::remove_var("NYASH_PARSER_DECODE_UNICODE");
|
||
|
|
std::env::remove_var("HAKO_PARSER_DECODE_UNICODE");
|
||
|
|
let s = collect_string_token("\"\\u0041\"");
|
||
|
|
assert_eq!(s, "\\u0041");
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn unicode_decode_toggle_on_decodes_basic_and_surrogate() {
|
||
|
|
// ON: enable decode
|
||
|
|
std::env::set_var("NYASH_PARSER_DECODE_UNICODE", "1");
|
||
|
|
let s = collect_string_token("\"\\u0041\"");
|
||
|
|
assert_eq!(s, "A");
|
||
|
|
|
||
|
|
let s2 = collect_string_token("\"\\uD83D\\uDE00\"");
|
||
|
|
// Expect surrogate pair to decode into one char (😀)
|
||
|
|
assert_eq!(s2.chars().count(), 1);
|
||
|
|
}
|
||
|
|
|