From f09a6e14baae897ed5e0191a7c4b816d5514e484 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 14 Jan 2016 20:52:29 -0800 Subject: [PATCH] Tokenizer work to support operators work in progress but committing to transfer --- src/tokenizer.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 0f42cd6..d00f48b 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -10,6 +10,7 @@ pub enum Token { NumLiteral(f64), StrLiteral(String), Identifier(String), + Op(String), Keyword(Kw) } @@ -97,6 +98,18 @@ pub fn tokenize(input: &str) -> Option> { Ok(f) => NumLiteral(f), Err(_) => return None } + } else if !char::is_alphanumeric(c) { //TODO see if this what I want + let mut buffer = String::with_capacity(20); + buffer.push(c); + loop { + if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') { + let n = iter.next().unwrap(); + buffer.push(n); + } else { + break; + } + } + Op(buffer) } else { let mut buffer = String::with_capacity(20); buffer.push(c);