Add comma tokenization

This commit is contained in:
greg 2016-01-09 23:22:46 -08:00
parent 570650cbfa
commit 3e04cbfa29
1 changed files with 3 additions and 0 deletions

View File

@ -39,6 +39,7 @@ fn ends_identifier(c: &char) -> bool {
c == ')' ||
c == ',' ||
c == '.' ||
c == ',' ||
c == ':'
}
@ -67,6 +68,8 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
RParen
} else if c == ':' {
Colon
else if c == ',' {
Comma
} else if c == '"' {
let mut buffer = String::with_capacity(20);
loop {