From a53135a8972315be5fdc1c8cca6dccb9090df221 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 4 Sep 2017 12:17:20 -0700 Subject: [PATCH] More elaborate tokens --- src/schala_lang/parsing.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/schala_lang/parsing.rs b/src/schala_lang/parsing.rs index 173bd9e..acf82e8 100644 --- a/src/schala_lang/parsing.rs +++ b/src/schala_lang/parsing.rs @@ -3,24 +3,44 @@ use std::rc::Rc; #[allow(dead_code)] #[derive(Debug)] -pub enum Token { +pub enum TokenType { Newline, Semicolon, + LParen, RParen, + LSquareBracket, RSquareBracket, + LCurlyBrace, RCurlyBrace, + Comma, Period, Colon, Digit(u8), StrLiteral(Rc), Identifier(Rc), + Keyword(Kw), Operator(Rc), } +#[derive(Debug)] +pub enum Kw { + If, + Else, + Func, + Loop, +} + +#[derive(Debug)] +pub struct Token { + token_type: TokenType, + line_number: u32, + char_number: u32, +} + pub fn tokenize(_input: &str) -> Result, TokenError> { Ok(vec!()) }