From e1aa7ecb17cb431ac6a97f95bc35619dfad39e3f Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 15 Jan 2016 00:31:00 -0800 Subject: [PATCH] Finish tokenizing Op separately --- src/tokenizer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index d00f48b..e7d0469 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -98,11 +98,11 @@ pub fn tokenize(input: &str) -> Option> { Ok(f) => NumLiteral(f), Err(_) => return None } - } else if !char::is_alphanumeric(c) { //TODO see if this what I want + } else if !char::is_alphanumeric(c) { let mut buffer = String::with_capacity(20); buffer.push(c); loop { - if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') { + if iter.peek().map_or(false, |x| !char::is_alphanumeric(*x)) { let n = iter.next().unwrap(); buffer.push(n); } else {