More elaborate tokens

This commit is contained in:
greg 2017-09-04 12:17:20 -07:00
parent f3c8474c93
commit a53135a897

View File

@ -3,24 +3,44 @@ use std::rc::Rc;
#[allow(dead_code)]
#[derive(Debug)]
pub enum Token {
pub enum TokenType {
Newline,
Semicolon,
LParen,
RParen,
LSquareBracket,
RSquareBracket,
LCurlyBrace,
RCurlyBrace,
Comma,
Period,
Colon,
Digit(u8),
StrLiteral(Rc<String>),
Identifier(Rc<String>),
Keyword(Kw),
Operator(Rc<String>),
}
#[derive(Debug)]
pub enum Kw {
If,
Else,
Func,
Loop,
}
#[derive(Debug)]
pub struct Token {
token_type: TokenType,
line_number: u32,
char_number: u32,
}
pub fn tokenize(_input: &str) -> Result<Vec<Token>, TokenError> {
Ok(vec!())
}