From 4a4f1fffad29e7c8a84dc20f9d3e812c006775b9 Mon Sep 17 00:00:00 2001 From: Moe Charm Date: Sat, 16 Aug 2025 11:35:57 +0900 Subject: [PATCH] refactor(parser): Step 1 - Extract common utilities module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Successfully extracted utility methods from parser/mod.rs to common.rs: - Created ParserUtils trait with token manipulation methods - Extracted current_token, peek_token, advance, skip_newlines, etc. - All parser modules now use trait-based utilities - Reduced mod.rs by ~60 lines ✅ Build successful with no errors ✅ All functionality preserved - tested with using nyashstd 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/parser/common.rs | 122 ++++++++++++++++++++++++++++++++++++++ src/parser/expressions.rs | 34 +---------- src/parser/mod.rs | 101 ++++++++----------------------- src/parser/statements.rs | 1 + 4 files changed, 152 insertions(+), 106 deletions(-) create mode 100644 src/parser/common.rs diff --git a/src/parser/common.rs b/src/parser/common.rs new file mode 100644 index 00000000..b6aa1220 --- /dev/null +++ b/src/parser/common.rs @@ -0,0 +1,122 @@ +/*! + * Parser Common Utilities + * + * パーサーモジュール間で共有されるヘルパー関数や型定義 + * Extracted from parser/mod.rs as part of modularization + */ + +use crate::tokenizer::{Token, TokenType}; +use crate::ast::Span; +use super::ParseError; + +/// Parser utility methods +pub trait ParserUtils { + fn tokens(&self) -> &Vec; + fn current(&self) -> usize; + fn current_mut(&mut self) -> &mut usize; + + /// 現在のトークンを取得 + fn current_token(&self) -> &Token { + self.tokens().get(self.current()).unwrap_or(&Token { + token_type: TokenType::EOF, + line: 0, + column: 0, + }) + } + + /// 次のトークンを先読み（位置を進めない） + fn peek_token(&self) -> &TokenType { + if self.current() + 1 < self.tokens().len() { + &self.tokens()[self.current() + 1].token_type + } else { + &TokenType::EOF + } + } + + /// N個先のトークンを先読み + fn peek_nth_token(&self, n: usize) -> &TokenType { + if self.current() + n < self.tokens().len() { + &self.tokens()[self.current() + n].token_type + } else { + &TokenType::EOF + } + } + + /// 位置を1つ進める + fn advance(&mut self) { + if !self.is_at_end() { + *self.current_mut() += 1; + } + } + + /// NEWLINEトークンをスキップ + fn skip_newlines(&mut self) { + while matches!(self.current_token().token_type, TokenType::NEWLINE) && !self.is_at_end() { + self.advance(); + } + } + + /// 指定されたトークンタイプを消費 (期待通りでなければエラー) + fn consume(&mut self, expected: TokenType) -> Result { + if std::mem::discriminant(&self.current_token().token_type) == + std::mem::discriminant(&expected) { + let token = self.current_token().clone(); + self.advance(); + Ok(token) + } else { + let line = self.current_token().line; + Err(ParseError::UnexpectedToken { + found: self.current_token().token_type.clone(), + expected: format!("{:?}", expected), + line, + }) + } + } + + /// 現在のトークンが指定されたタイプかチェック + fn match_token(&self, token_type: &TokenType) -> bool { + std::mem::discriminant(&self.current_token().token_type) == + std::mem::discriminant(token_type) + } + + /// 複数のトークンタイプのいずれかにマッチするかチェック + fn match_any_token(&self, token_types: &[TokenType]) -> bool { + let current_discriminant = std::mem::discriminant(&self.current_token().token_type); + token_types.iter().any(|tt| { + std::mem::discriminant(tt) == current_discriminant + }) + } + + /// 終端に達したかチェック + fn is_at_end(&self) -> bool { + self.current() >= self.tokens().len() || + matches!(self.current_token().token_type, TokenType::EOF) + } + + /// 現在のトークンが行の終わり（NEWLINE or EOF）かチェック + fn is_line_end(&self) -> bool { + matches!(self.current_token().token_type, TokenType::NEWLINE | TokenType::EOF) + } + + /// エラー報告用の現在位置情報を取得 + fn current_position(&self) -> (usize, usize) { + let token = self.current_token(); + (token.line, token.column) + } + + /// 現在のトークンからSpanを作成 + fn current_span(&self) -> Span { + let token = self.current_token(); + Span { + start: 0, // Token doesn't have byte offset, so using 0 + end: 0, + line: token.line, + column: token.column, + } + } +} + +/// Helper function to create unknown span +pub fn unknown_span() -> Span { + Span::unknown() +} \ No newline at end of file diff --git a/src/parser/expressions.rs b/src/parser/expressions.rs index 8a6b8899..30ccfa0a 100644 --- a/src/parser/expressions.rs +++ b/src/parser/expressions.rs @@ -8,38 +8,10 @@ use crate::tokenizer::TokenType; use crate::ast::{ASTNode, BinaryOperator, LiteralValue, UnaryOperator, Span}; use super::{NyashParser, ParseError}; +use super::common::ParserUtils; -// ===== 🔥 Debug Macros (copied from parent module) ===== - -/// Infinite loop detection macro - must be called in every loop that advances tokens -/// Prevents parser from hanging due to token consumption bugs -/// Uses parser's debug_fuel field for centralized fuel management -macro_rules! must_advance { - ($parser:expr, $fuel:expr, $location:literal) => { - // デバッグ燃料がSomeの場合のみ制限チェック - if let Some(ref mut limit) = $parser.debug_fuel { - if *limit == 0 { - eprintln!("🚨 PARSER INFINITE LOOP DETECTED at {}", $location); - eprintln!("🔍 Current token: {:?} at line {}", $parser.current_token().token_type, $parser.current_token().line); - eprintln!("🔍 Parser position: {}/{}", $parser.current, $parser.tokens.len()); - return Err(ParseError::InfiniteLoop { - location: $location.to_string(), - token: $parser.current_token().token_type.clone(), - line: $parser.current_token().line, - }); - } - *limit -= 1; - } - // None の場合は無制限なのでチェックしない - }; -} - -/// Initialize debug fuel for loop monitoring -macro_rules! debug_fuel { - () => { - 100_000 // Default: 100k iterations should be enough for any reasonable program - }; -} +// Debug macros are now imported from the parent module via #[macro_export] +use crate::{must_advance, debug_fuel}; impl NyashParser { /// 式をパース (演算子優先順位あり) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 264abced..47fa17c7 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -12,11 +12,14 @@ */ // サブモジュール宣言 +mod common; mod expressions; mod statements; // mod declarations; // mod errors; +use common::ParserUtils; + use crate::tokenizer::{Token, TokenType, TokenizeError}; use crate::ast::{ASTNode, Span}; use std::collections::HashMap; @@ -27,6 +30,7 @@ use thiserror::Error; /// Infinite loop detection macro - must be called in every loop that advances tokens /// Prevents parser from hanging due to token consumption bugs /// Uses parser's debug_fuel field for centralized fuel management +#[macro_export] macro_rules! must_advance { ($parser:expr, $fuel:expr, $location:literal) => { // デバッグ燃料がSomeの場合のみ制限チェック @@ -35,7 +39,7 @@ macro_rules! must_advance { eprintln!("🚨 PARSER INFINITE LOOP DETECTED at {}", $location); eprintln!("🔍 Current token: {:?} at line {}", $parser.current_token().token_type, $parser.current_token().line); eprintln!("🔍 Parser position: {}/{}", $parser.current, $parser.tokens.len()); - return Err(ParseError::InfiniteLoop { + return Err($crate::parser::ParseError::InfiniteLoop { location: $location.to_string(), token: $parser.current_token().token_type.clone(), line: $parser.current_token().line, @@ -48,6 +52,7 @@ macro_rules! must_advance { } /// Initialize debug fuel for loop monitoring +#[macro_export] macro_rules! debug_fuel { () => { 100_000 // Default: 100k iterations should be enough for any reasonable program @@ -92,12 +97,27 @@ pub enum ParseError { /// Nyashパーサー - トークン列をASTに変換 pub struct NyashParser { - tokens: Vec, - current: usize, + pub(super) tokens: Vec, + pub(super) current: usize, /// 🔥 Static box依存関係追跡（循環依存検出用） - static_box_dependencies: std::collections::HashMap>, + pub(super) static_box_dependencies: std::collections::HashMap>, /// 🔥 デバッグ燃料：無限ループ検出用制限値 (None = 無制限) - debug_fuel: Option, + pub(super) debug_fuel: Option, +} + +// Implement ParserUtils trait +impl ParserUtils for NyashParser { + fn tokens(&self) -> &Vec { + &self.tokens + } + + fn current(&self) -> usize { + self.current + } + + fn current_mut(&mut self) -> &mut usize { + &mut self.current + } } impl NyashParser { @@ -1284,76 +1304,7 @@ impl NyashParser { } // Expression parsing methods are now in expressions.rs module - - // ===== ユーティリティメソッド ===== - - /// 現在のトークンを取得 - fn current_token(&self) -> &Token { - self.tokens.get(self.current).unwrap_or(&Token { - token_type: TokenType::EOF, - line: 0, - column: 0, - }) - } - - /// 次のトークンを先読み（位置を進めない） - fn peek_token(&self) -> &TokenType { - if self.current + 1 < self.tokens.len() { - &self.tokens[self.current + 1].token_type - } else { - &TokenType::EOF - } - } - - /// 位置を1つ進める - fn advance(&mut self) { - if !self.is_at_end() { - self.current += 1; - } - } - - /// NEWLINEトークンをスキップ - fn skip_newlines(&mut self) { - let mut skip_count = 0; - while matches!(self.current_token().token_type, TokenType::NEWLINE) && !self.is_at_end() { - self.advance(); - skip_count += 1; - } - if skip_count > 0 { - } - } - - /// 指定されたトークンタイプを消費 (期待通りでなければエラー) - fn consume(&mut self, expected: TokenType) -> Result { - - if std::mem::discriminant(&self.current_token().token_type) == - std::mem::discriminant(&expected) { - let token = self.current_token().clone(); - self.advance(); - Ok(token) - } else { - let line = self.current_token().line; - Err(ParseError::UnexpectedToken { - found: self.current_token().token_type.clone(), - expected: format!("{:?}", expected), - line, - }) - } - } - - /// 現在のトークンが指定されたタイプかチェック - fn match_token(&self, token_type: &TokenType) -> bool { - std::mem::discriminant(&self.current_token().token_type) == - std::mem::discriminant(token_type) - } - - /// 終端に達したかチェック - fn is_at_end(&self) -> bool { - self.current >= self.tokens.len() || - matches!(self.current_token().token_type, TokenType::EOF) - } - // Include, local, outbox, try/catch/throw parsing methods are now in statements.rs module - // Two-phase parser helper methods are no longer needed - simplified to direct parsing + // Utility methods are now in common.rs module via ParserUtils trait // ===== 🔥 Static Box循環依存検出 ===== diff --git a/src/parser/statements.rs b/src/parser/statements.rs index 28b0a103..52e39938 100644 --- a/src/parser/statements.rs +++ b/src/parser/statements.rs @@ -8,6 +8,7 @@ use crate::tokenizer::TokenType; use crate::ast::{ASTNode, CatchClause, Span}; use super::{NyashParser, ParseError}; +use super::common::ParserUtils; impl NyashParser { /// 文をパース