More elaborate tokens
This commit is contained in:
parent
f3c8474c93
commit
a53135a897
@ -3,24 +3,44 @@ use std::rc::Rc;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
pub enum Token {
|
||||
pub enum TokenType {
|
||||
Newline,
|
||||
Semicolon,
|
||||
|
||||
LParen,
|
||||
RParen,
|
||||
|
||||
LSquareBracket,
|
||||
RSquareBracket,
|
||||
|
||||
LCurlyBrace,
|
||||
RCurlyBrace,
|
||||
|
||||
Comma,
|
||||
Period,
|
||||
Colon,
|
||||
Digit(u8),
|
||||
StrLiteral(Rc<String>),
|
||||
Identifier(Rc<String>),
|
||||
Keyword(Kw),
|
||||
Operator(Rc<String>),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Kw {
|
||||
If,
|
||||
Else,
|
||||
Func,
|
||||
Loop,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Token {
|
||||
token_type: TokenType,
|
||||
line_number: u32,
|
||||
char_number: u32,
|
||||
}
|
||||
|
||||
pub fn tokenize(_input: &str) -> Result<Vec<Token>, TokenError> {
|
||||
Ok(vec!())
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user