diff --git a/src/tokenizer.rs b/src/tokenizer.rs index e7d0469..2c1fdd8 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -102,7 +102,7 @@ pub fn tokenize(input: &str) -> Option> { let mut buffer = String::with_capacity(20); buffer.push(c); loop { - if iter.peek().map_or(false, |x| !char::is_alphanumeric(*x)) { + if iter.peek().map_or(false, |x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)) { let n = iter.next().unwrap(); buffer.push(n); } else {