greg
/
schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert tokenizer to large match statement 

In the hopes of making it shorter
This commit is contained in:
greg 2017-01-01 18:18:55 -08:00
parent 9b62efc830
commit e5ee072b00
1 changed files with 72 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/tokenizer.rs
View File

@ -60,9 +60,7 @@ pub fn tokenize(input: &str) -> TokenizeResult {
let mut iter = input.chars().peekable();
while let Some(c) = iter.next() {
if char::is_whitespace(c) && c != '\n' {
continue;
} else if c == '#' {
if c == '#' {
while let Some(c) = iter.next() {
if c == '\n' {
break;
@ -70,19 +68,15 @@ pub fn tokenize(input: &str) -> TokenizeResult {
}
}
let cur_tok = if c == '\n' {
Newline
} else if c == ';' {
Semicolon
} else if c == '(' {
LParen
} else if c == ')' {
RParen
} else if c == ':' {
Colon
} else if c == ',' {
Comma
} else if c == '"' {
let cur_tok = match c {
c if char::is_whitespace(c) && c != '\n' => continue,
'\n' => Newline,
';' => Semicolon,
'(' => LParen,
')' => RParen,
':' => Colon,
',' => Comma,
'"' => {
let mut buffer = String::with_capacity(20);
loop {
// TODO handle string escapes, interpolation
@ -93,7 +87,23 @@ pub fn tokenize(input: &str) -> TokenizeResult {
}
}
StrLiteral(buffer)
} else if c == '.' && !iter.peek().map_or(false, |x| is_digit(x)) {
}
c if !char::is_alphanumeric(c) => {
let mut buffer = String::with_capacity(20);
buffer.push(c);
loop {
if iter.peek().map_or(false,
|x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)) {
let n = iter.next().unwrap();
buffer.push(n);
} else {
break;
}
}
Operator(Op { repr: buffer })
}
c => {
if c == '.' && !iter.peek().map_or(false, |x| is_digit(x)) {
Period
} else if is_digit(&c) || c == '.' {
let mut buffer = String::with_capacity(20);
@ -110,19 +120,6 @@ pub fn tokenize(input: &str) -> TokenizeResult {
Ok(f) => NumLiteral(f),
Err(_) => return Err(TokenizeError::new("Failed to pase digit")),
}
} else if !char::is_alphanumeric(c) {
let mut buffer = String::with_capacity(20);
buffer.push(c);
loop {
if iter.peek().map_or(false,
|x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)) {
let n = iter.next().unwrap();
buffer.push(n);
} else {
break;
}
}
Operator(Op { repr: buffer })
} else {
let mut buffer = String::with_capacity(20);
buffer.push(c);
@ -145,6 +142,8 @@ pub fn tokenize(input: &str) -> TokenizeResult {
"null" => Keyword(Kw::Null),
b => Identifier(b.to_string()),
}
}
}
};
tokens.push(cur_tok);