greg
/
schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Filled out keyword tokenizing

This commit is contained in:
greg 2015-07-25 13:55:18 -07:00
parent edf100b583
commit 0e4469fa58
1 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/tokenizer.rs
View File

@ -17,7 +17,6 @@ pub enum Kw {
Then, Then,
Else, Else,
While, While,
Do,
End, End,
Let, Let,
Fn, Fn,
@ -92,9 +91,18 @@ pub fn tokenize(input: &str) -> Vec<Token> {
} }
fn handle_identifier(identifier: String) -> Token { fn handle_identifier(identifier: String) -> Token {
match &identifier[..] {
"let" => Token::Keyword(Kw::Let), let keyword = match &identifier[..] {
"if" => Token::Keyword(Kw::If), "let" => Kw::Let,
_ => Token::Identifier(identifier) "if" => Kw::If,
} "then" => Kw::Then,
"else" => Kw::Else,
"while" => Kw::While,
"end" => Kw::End,
"fn" => Kw::Fn,
"null" => Kw::Null,
_ => return Token::Identifier(identifier)
};
return Token::Keyword(keyword);
} }