greg
/
schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Finish tokenizing Op separately

This commit is contained in:
greg 2016-01-15 00:31:00 -08:00
parent f09a6e14ba
commit e1aa7ecb17
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokenizer.rs
View File

@ -98,11 +98,11 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
Ok(f) => NumLiteral(f),
Err(_) => return None
}
} else if !char::is_alphanumeric(c) { //TODO see if this what I want
} else if !char::is_alphanumeric(c) {
let mut buffer = String::with_capacity(20);
buffer.push(c);
loop {
if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') {
if iter.peek().map_or(false, |x| !char::is_alphanumeric(*x)) {
let n = iter.next().unwrap();
buffer.push(n);
} else {