From 8662a3ba0e7e55290ccec78adfccd1a8dad86a3a Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 6 Jan 2016 23:48:53 -0800 Subject: [PATCH] Make tokenize error-able --- src/main.rs | 8 ++++++-- src/tokenizer.rs | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9aff043..aeb31cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -55,15 +55,19 @@ impl ReplState for InterpreterState { } fn repl_handler(input: &str, state: &mut InterpreterState) -> String { + let tokens = match tokenize(input) { + None => return format!("Failure to tokenize"), + Some(t) => t + }; + if state.show_tokens { - println!("Tokens: {:?}", tokenize(input)); + println!("Tokens: {:?}", tokens); } if state.show_parse { println!("not implemented") } - let tokens = tokenize(input); let ast = parse(&tokens); format!("{:?}", tokens) } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 77e8df3..59418f5 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -31,7 +31,7 @@ fn is_digit(c: &char) -> bool { c.is_digit(10) } -pub fn tokenize(input: &str) -> Vec { +pub fn tokenize(input: &str) -> Option> { use self::Token::*; let mut tokens = Vec::new(); let mut iter = input.chars().peekable(); @@ -63,18 +63,20 @@ pub fn tokenize(input: &str) -> Vec { match iter.next() { Some(x) if x == '"' => break, Some(x) => buffer.push(x), - None => return tokens, + None => return None, } } StrLiteral(buffer) + } else if is_digit(&c) { + NumLiteral(45.0) } else { - StrLiteral("DUMMY".to_string()) + Identifier("DUMMY".to_string()) }; tokens.push(cur_tok); } - tokens + Some(tokens) } #[cfg(test)]