hakorune/tests/tokenizer_unit.rs

use nyash_rust::tokenizer::{NyashTokenizer, Token, TokenType, TokenizeError};

#[test]
fn test_basic_tokens() {
    let mut tokenizer = NyashTokenizer::new("box new = + - *");
    let tokens = tokenizer.tokenize().unwrap();
    assert!(matches!(tokens[0].token_type, TokenType::BOX));
    assert!(matches!(tokens[1].token_type, TokenType::NEW));
    assert!(matches!(tokens[2].token_type, TokenType::ASSIGN));
    assert!(matches!(tokens[3].token_type, TokenType::PLUS));
    assert!(matches!(tokens[4].token_type, TokenType::MINUS));
    assert!(matches!(tokens[5].token_type, TokenType::MULTIPLY));
}

#[test]
fn test_string_literal() {
    let mut tokenizer = NyashTokenizer::new(r#""Hello, World!""#);
    let tokens = tokenizer.tokenize().unwrap();
    assert!(matches!(tokens[0].token_type, TokenType::STRING(_)));
}

#[test]
fn test_numeric_literals() {
    let mut tokenizer = NyashTokenizer::new("42 123 0");
    let tokens = tokenizer.tokenize().unwrap();
    assert!(matches!(tokens[0].token_type, TokenType::NUMBER(42)));
    assert!(matches!(tokens[1].token_type, TokenType::NUMBER(123)));
    assert!(matches!(tokens[2].token_type, TokenType::NUMBER(0)));
}

#[test]
fn test_identifiers_and_keywords() {
    let mut tokenizer = NyashTokenizer::new("test_var myBox getValue");
    let tokens = tokenizer.tokenize().unwrap();
    assert!(matches!(tokens[0].token_type, TokenType::IDENTIFIER(_)));
    assert!(matches!(tokens[1].token_type, TokenType::IDENTIFIER(_)));
    assert!(matches!(tokens[2].token_type, TokenType::IDENTIFIER(_)));
}

#[test]
fn test_operators() {
    let mut tokenizer = NyashTokenizer::new(">> == != <= >= < >");
    let tokens = tokenizer.tokenize().unwrap();
    assert_eq!(tokens[0].token_type, TokenType::ShiftRight);
    assert_eq!(tokens[1].token_type, TokenType::EQUALS);
    assert_eq!(tokens[2].token_type, TokenType::NotEquals);
    assert_eq!(tokens[3].token_type, TokenType::LessEquals);
    assert_eq!(tokens[4].token_type, TokenType::GreaterEquals);
    assert_eq!(tokens[5].token_type, TokenType::LESS);
    assert_eq!(tokens[6].token_type, TokenType::GREATER);
}

#[test]
fn test_complex_code() {
    let code = r#"
    box TestBox {
        value
        
        getValue() {
            return this.value
        }
    }
    
    obj = new TestBox()
    obj.value = "test123"
    "#;

    let mut tokenizer = NyashTokenizer::new(code);
    let tokens = tokenizer.tokenize().unwrap();

    // 基本的なトークンがある事を確認
    let token_types: Vec<_> = tokens.iter().map(|t| &t.token_type).collect();
    assert!(token_types.contains(&&TokenType::BOX));
    assert!(token_types.contains(&&TokenType::NEW));
    assert!(token_types.contains(&&TokenType::THIS));
    assert!(token_types.contains(&&TokenType::RETURN));
    assert!(token_types.contains(&&TokenType::DOT));
}

#[test]
fn test_line_numbers() {
    let code = "box\ntest\nvalue";
    let mut tokenizer = NyashTokenizer::new(code);
    let tokens = tokenizer.tokenize().unwrap();

    // NEWLINEトークンを除外して確認
    let non_newline: Vec<&Token> = tokens
        .iter()
        .filter(|t| !matches!(t.token_type, TokenType::NEWLINE))
        .collect();
    assert_eq!(non_newline[0].line, 1); // box
    assert_eq!(non_newline[1].line, 2); // test
    assert_eq!(non_newline[2].line, 3); // value
}

#[test]
fn test_comments() {
    let code = r#"box Test // this is a comment
# this is also a comment
value"#;

    let mut tokenizer = NyashTokenizer::new(code);
    let tokens = tokenizer.tokenize().unwrap();

    // コメントは除外されている
    let token_types: Vec<_> = tokens
        .iter()
        .filter(|t| !matches!(t.token_type, TokenType::NEWLINE))
        .map(|t| &t.token_type)
        .collect();
    assert_eq!(token_types.len(), 4); // box, Test, value, EOF
}

#[test]
fn test_error_handling() {
    let mut tokenizer = NyashTokenizer::new("@#$%");
    let result = tokenizer.tokenize();

    assert!(result.is_err());
    match result {
        Err(TokenizeError::UnexpectedCharacter { char, line, column }) => {
            assert_eq!(char, '@');
            assert_eq!(line, 1);
            assert_eq!(column, 1);
        }
        _ => panic!("Expected UnexpectedCharacter error"),
    }
}

#[test]
fn test_basic_sugar_tokens() {
    let mut t2 = NyashTokenizer::new("|> ?.? ?? += -= *= /= ..");
    let toks = t2.tokenize().unwrap();
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::PipeForward)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::QmarkDot)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::QmarkQmark)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::PlusAssign)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::MinusAssign)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::MulAssign)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::DivAssign)));
    assert!(toks
        .iter()
        .any(|k| matches!(k.token_type, TokenType::RANGE)));
}

#[test]
fn test_longest_match_sequences() {
    // '??' は '?' より優先、'?.' は '.' より優先、'..' は '.' より優先
    let mut t = NyashTokenizer::new("?? ? ?. .. .");
    let toks = t.tokenize().unwrap();
    let kinds: Vec<&TokenType> = toks.iter().map(|k| &k.token_type).collect();
    assert!(matches!(kinds[0], TokenType::QmarkQmark));
    assert!(matches!(kinds[1], TokenType::QUESTION));
    assert!(matches!(kinds[2], TokenType::QmarkDot));
    assert!(matches!(kinds[3], TokenType::RANGE));
    assert!(matches!(kinds[4], TokenType::DOT));
}