Change some func signatures around tokenizing and errors

This commit is contained in:
greg 2017-09-06 23:52:25 -07:00
parent 8bf5f40a2a
commit ac5bdd7bcb
2 changed files with 23 additions and 15 deletions

View File

@ -2,6 +2,8 @@ use language::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, ReplOut
mod parsing; mod parsing;
use self::parsing::TokenType;
pub struct Schala { pub struct Schala {
} }
@ -18,18 +20,16 @@ impl ProgrammingLanguageInterface for Schala {
fn evaluate_in_repl(&mut self, input: &str, options: &EvalOptions) -> ReplOutput { fn evaluate_in_repl(&mut self, input: &str, options: &EvalOptions) -> ReplOutput {
let mut output = ReplOutput::default(); let mut output = ReplOutput::default();
let tokens = match parsing::tokenize(input) { let tokens = parsing::tokenize(input);
Ok(tokens) => {
if options.debug_tokens { if options.debug_tokens {
output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", tokens))); output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", tokens)));
} }
tokens
}, let token_errors: Vec<&String> = tokens.iter().filter_map(|t| t.get_error()).collect();
Err(err) => { if token_errors.len() != 0 {
output.add_output(format!("Tokenization error: {:?}\n", err.msg)); output.add_output(format!("Tokenization error: {:?}\n", token_errors));
return output; return output;
} }
};
/* /*
let ast = match parsing::parse(tokens) { let ast = match parsing::parse(tokens) {

View File

@ -42,13 +42,22 @@ pub struct Token {
offset: usize, offset: usize,
} }
impl Token {
pub fn get_error(&self) -> Option<&String> {
match self.token_type {
TokenType::Error(ref s) => Some(s),
_ => None,
}
}
}
fn is_digit(c: &char) -> bool { fn is_digit(c: &char) -> bool {
c.is_digit(10) c.is_digit(10)
} }
type CharIter<'a> = Peekable<Enumerate<Chars<'a>>>; type CharIter<'a> = Peekable<Enumerate<Chars<'a>>>;
pub fn tokenize(input: &str) -> Result<Vec<Token>, TokenError> { pub fn tokenize(input: &str) -> Vec<Token> {
use self::TokenType::*; use self::TokenType::*;
let mut tokens: Vec<Token> = Vec::new(); let mut tokens: Vec<Token> = Vec::new();
@ -82,8 +91,7 @@ pub fn tokenize(input: &str) -> Result<Vec<Token>, TokenError> {
tokens.push(Token { token_type: cur_tok_type, offset: idx }); tokens.push(Token { token_type: cur_tok_type, offset: idx });
} }
tokens
Ok(tokens)
} }
fn handle_digit(c: char, input: &mut CharIter) -> TokenType { fn handle_digit(c: char, input: &mut CharIter) -> TokenType {