Some initial work on passing token metadata to AST

schala-lang/language/src/parsing.rs

@@ -64,6 +64,10 @@ impl TokenHandler {
   fn next(&mut self) -> TokenType {
     self.tokens.next().map(|ref t| { t.token_type.clone() }).unwrap_or(TokenType::EOF)
   }
+
+  fn next_with_metadata(&mut self) -> Token {
+    self.tokens.next().unwrap_or(Token { token_type: TokenType::EOF, offset: (0, 0) })
+  }
 }
 
 impl Parser {
@@ -86,6 +90,10 @@ impl Parser {
     self.token_handler.next()
   }
 
+  fn next_with_metadata(&mut self) -> Token {
+    self.token_handler.next_with_metadata()
+  }
+
   pub fn parse(&mut self) -> ParseResult<AST> {
     self.program()
   }
@@ -1015,7 +1023,7 @@ impl Parser {
   #[recursive_descent_method]
   fn int_literal(&mut self) -> ParseResult<Expression> {
     use self::ExpressionType::*;
-    match self.next() {
+    match self.next_with_metadata().get_token_type() {
       BinNumberSigil => {
         let digits = self.digits()?;
         let n = parse_binary(digits)?;

schala-lang/language/src/tokenizing.rs

@@ -91,6 +91,11 @@ pub struct Token {
   pub offset: (usize, usize),
 }
 
+#[derive(Debug, Clone)]
+pub struct TokenMetadata {
+  pub offset: (usize, usize)
+}
+
 impl Token {
   pub fn get_error(&self) -> Option<String> {
     match self.token_type {
@@ -101,6 +106,10 @@ impl Token {
   pub fn to_string_with_metadata(&self) -> String {
     format!("{}(L:{},c:{})", self.token_type, self.offset.0, self.offset.1)
   }
+
+  pub fn get_token_type(&self) -> TokenType {
+    self.token_type.clone()
+  }
 }
 
 const OPERATOR_CHARS: [char; 18] = ['!', '$', '%', '&', '*', '+', '-', '.', ':', '<', '>', '=', '?', '@', '^', '|', '~', '`'];