greg
/
schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert tokenizer to large match statement 

In the hopes of making it shorter
This commit is contained in:
greg 2017-01-01 18:18:55 -08:00
parent 9b62efc830
commit e5ee072b00
1 changed files with 72 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
src/tokenizer.rs
View File

@ -60,9 +60,7 @@ pub fn tokenize(input: &str) -> TokenizeResult {
let mut iter = input.chars().peekable(); let mut iter = input.chars().peekable();
while let Some(c) = iter.next() { while let Some(c) = iter.next() {
if char::is_whitespace(c) && c != '\n' { if c == '#' {
continue;
} else if c == '#' {
while let Some(c) = iter.next() { while let Some(c) = iter.next() {
if c == '\n' { if c == '\n' {
break; break;
@ -70,80 +68,81 @@ pub fn tokenize(input: &str) -> TokenizeResult {
} }
} }
let cur_tok = if c == '\n' { let cur_tok = match c {
Newline c if char::is_whitespace(c) && c != '\n' => continue,
} else if c == ';' { '\n' => Newline,
Semicolon ';' => Semicolon,
} else if c == '(' { '(' => LParen,
LParen ')' => RParen,
} else if c == ')' { ':' => Colon,
RParen ',' => Comma,
} else if c == ':' { '"' => {
Colon let mut buffer = String::with_capacity(20);
} else if c == ',' { loop {
Comma // TODO handle string escapes, interpolation
} else if c == '"' { match iter.next() {
let mut buffer = String::with_capacity(20); Some(x) if x == '"' => break,
loop { Some(x) => buffer.push(x),
// TODO handle string escapes, interpolation None => return Err(TokenizeError::new("Unclosed quote")),
match iter.next() { }
Some(x) if x == '"' => break,
Some(x) => buffer.push(x),
None => return Err(TokenizeError::new("Unclosed quote")),
} }
StrLiteral(buffer)
} }
StrLiteral(buffer) c if !char::is_alphanumeric(c) => {
} else if c == '.' && !iter.peek().map_or(false, |x| is_digit(x)) { let mut buffer = String::with_capacity(20);
Period buffer.push(c);
} else if is_digit(&c) || c == '.' { loop {
let mut buffer = String::with_capacity(20); if iter.peek().map_or(false,
buffer.push(c); |x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)) {
loop { let n = iter.next().unwrap();
if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') { buffer.push(n);
let n = iter.next().unwrap(); } else {
buffer.push(n); break;
}
}
Operator(Op { repr: buffer })
}
c => {
if c == '.' && !iter.peek().map_or(false, |x| is_digit(x)) {
Period
} else if is_digit(&c) || c == '.' {
let mut buffer = String::with_capacity(20);
buffer.push(c);
loop {
if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') {
let n = iter.next().unwrap();
buffer.push(n);
} else {
break;
}
}
match buffer.parse::<f64>() {
Ok(f) => NumLiteral(f),
Err(_) => return Err(TokenizeError::new("Failed to pase digit")),
}
} else { } else {
break; let mut buffer = String::with_capacity(20);
} buffer.push(c);
} loop {
match buffer.parse::<f64>() { if iter.peek().map_or(true, |x| ends_identifier(x)) {
Ok(f) => NumLiteral(f), break;
Err(_) => return Err(TokenizeError::new("Failed to pase digit")), } else {
} buffer.push(iter.next().unwrap());
} else if !char::is_alphanumeric(c) { }
let mut buffer = String::with_capacity(20); }
buffer.push(c);
loop {
if iter.peek().map_or(false,
|x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)) {
let n = iter.next().unwrap();
buffer.push(n);
} else {
break;
}
}
Operator(Op { repr: buffer })
} else {
let mut buffer = String::with_capacity(20);
buffer.push(c);
loop {
if iter.peek().map_or(true, |x| ends_identifier(x)) {
break;
} else {
buffer.push(iter.next().unwrap());
}
}
match &buffer[..] { match &buffer[..] {
"if" => Keyword(Kw::If), "if" => Keyword(Kw::If),
"then" => Keyword(Kw::Then), "then" => Keyword(Kw::Then),
"else" => Keyword(Kw::Else), "else" => Keyword(Kw::Else),
"while" => Keyword(Kw::While), "while" => Keyword(Kw::While),
"end" => Keyword(Kw::End), "end" => Keyword(Kw::End),
"let" => Keyword(Kw::Let), "let" => Keyword(Kw::Let),
"fn" => Keyword(Kw::Fn), "fn" => Keyword(Kw::Fn),
"null" => Keyword(Kw::Null), "null" => Keyword(Kw::Null),
b => Identifier(b.to_string()), b => Identifier(b.to_string()),
}
}
} }
}; };