diff --git a/src/tokenizer.rs b/src/tokenizer.rs index a1e866c..a6e82fb 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -5,6 +5,7 @@ pub enum Token { LParen, RParen, Comma, + Period, NumLiteral(f64), StrLiteral(String), Identifier(String), @@ -34,6 +35,7 @@ pub fn tokenize(input: &str) -> Vec { ';' => true, '(' => true, ')' => true, + '.' => true, _ => false } } @@ -69,6 +71,8 @@ pub fn tokenize(input: &str) -> Vec { tokens.push(Token::RParen); } else if c == ',' { tokens.push(Token::Comma); + } else if c == '.' { + tokens.push(Token::Period); } else { let mut buffer = String::with_capacity(20); buffer.push(c);