greg
/
schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Tokenize periods separately

This commit is contained in:
greg 2015-07-26 01:51:15 -07:00
parent 71d2428e57
commit 9d6dc5a5f2
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokenizer.rs
View File

@ -5,6 +5,7 @@ pub enum Token {
LParen,
RParen,
Comma,
Period,
NumLiteral(f64),
StrLiteral(String),
Identifier(String),
@ -34,6 +35,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
';' => true,
'(' => true,
')' => true,
'.' => true,
_ => false
}
}
@ -69,6 +71,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
tokens.push(Token::RParen);
} else if c == ',' {
tokens.push(Token::Comma);
} else if c == '.' {
tokens.push(Token::Period);
} else {
let mut buffer = String::with_capacity(20);
buffer.push(c);