Created
May 6, 2025 00:52
-
-
Save arockwell/a6e450a1d60b99fb6c4494b6635645b9 to your computer and use it in GitHub Desktop.
Improved number parsing - balanced approach
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| fn number(&mut self, graphemes: &[&str]) { | |
| let token_col = self.column - 2; | |
| let mut is_float = false; | |
| let mut number = graphemes[self.column - 2].to_string(); | |
| // Part 1: Handle integer part (required) | |
| while self.consume_digits(&mut number, graphemes) {} | |
| // Part 2: Handle decimal part (optional) | |
| if self.column - 1 < graphemes.len() && graphemes[self.column - 1] == "." { | |
| is_float = true; | |
| number.push_str(graphemes[self.column - 1]); | |
| self.column += 1; | |
| // Must have at least one digit after decimal point | |
| if \!self.consume_digits(&mut number, graphemes) { | |
| self.errors.push(LexError::new( | |
| graphemes.concat(), | |
| self.line, | |
| self.column - 1, | |
| "expected at least one digit after decimal point".to_owned(), | |
| )); | |
| return; | |
| } | |
| // Continue consuming any remaining digits | |
| while self.consume_digits(&mut number, graphemes) {} | |
| } | |
| // Part 3: Handle scientific notation (optional) | |
| if self.column - 1 < graphemes.len() && | |
| (graphemes[self.column - 1] == "e" || graphemes[self.column - 1] == "E") { | |
| is_float = true; | |
| number.push_str(graphemes[self.column - 1]); | |
| self.column += 1; | |
| // Handle optional sign | |
| if self.column - 1 < graphemes.len() && | |
| (graphemes[self.column - 1] == "+" || graphemes[self.column - 1] == "-") { | |
| number.push_str(graphemes[self.column - 1]); | |
| self.column += 1; | |
| } | |
| // Must have at least one digit in exponent | |
| if \!self.consume_digits(&mut number, graphemes) { | |
| self.errors.push(LexError::new( | |
| graphemes.concat(), | |
| self.line, | |
| self.column - 1, | |
| "expected at least one digit in exponent".to_owned(), | |
| )); | |
| return; | |
| } | |
| // Continue consuming any remaining digits | |
| while self.consume_digits(&mut number, graphemes) {} | |
| } | |
| // Create the token with appropriate type | |
| self.tokens.push(Token::new( | |
| if is_float { TokenType::Float } else { TokenType::Integer }, | |
| number, | |
| self.line, | |
| token_col, | |
| )); | |
| } | |
| // Helper method to consume digits and underscores | |
| fn consume_digits(&mut self, number: &mut String, graphemes: &[&str]) -> bool { | |
| if self.column - 1 >= graphemes.len() { | |
| return false; | |
| } | |
| let grapheme = graphemes[self.column - 1]; | |
| // Handle digits | |
| if grapheme.chars().next().unwrap_or('\0').is_ascii_digit() { | |
| number.push_str(grapheme); | |
| self.column += 1; | |
| return true; | |
| } | |
| // Skip underscores as digit separators | |
| if grapheme == "_" { | |
| self.column += 1; | |
| return true; | |
| } | |
| return false; | |
| } | |
| EOF < /dev/null |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment