Some initial work on passing token metadata to AST

This commit is contained in:
greg 2018-11-16 03:51:03 -08:00
parent 5147e1a3eb
commit 2ec7bf3b9a
2 changed files with 18 additions and 1 deletions

View File

@ -64,6 +64,10 @@ impl TokenHandler {
fn next(&mut self) -> TokenType { fn next(&mut self) -> TokenType {
self.tokens.next().map(|ref t| { t.token_type.clone() }).unwrap_or(TokenType::EOF) self.tokens.next().map(|ref t| { t.token_type.clone() }).unwrap_or(TokenType::EOF)
} }
fn next_with_metadata(&mut self) -> Token {
self.tokens.next().unwrap_or(Token { token_type: TokenType::EOF, offset: (0, 0) })
}
} }
impl Parser { impl Parser {
@ -86,6 +90,10 @@ impl Parser {
self.token_handler.next() self.token_handler.next()
} }
fn next_with_metadata(&mut self) -> Token {
self.token_handler.next_with_metadata()
}
pub fn parse(&mut self) -> ParseResult<AST> { pub fn parse(&mut self) -> ParseResult<AST> {
self.program() self.program()
} }
@ -1015,7 +1023,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn int_literal(&mut self) -> ParseResult<Expression> { fn int_literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
match self.next() { match self.next_with_metadata().get_token_type() {
BinNumberSigil => { BinNumberSigil => {
let digits = self.digits()?; let digits = self.digits()?;
let n = parse_binary(digits)?; let n = parse_binary(digits)?;

View File

@ -91,6 +91,11 @@ pub struct Token {
pub offset: (usize, usize), pub offset: (usize, usize),
} }
#[derive(Debug, Clone)]
pub struct TokenMetadata {
pub offset: (usize, usize)
}
impl Token { impl Token {
pub fn get_error(&self) -> Option<String> { pub fn get_error(&self) -> Option<String> {
match self.token_type { match self.token_type {
@ -101,6 +106,10 @@ impl Token {
pub fn to_string_with_metadata(&self) -> String { pub fn to_string_with_metadata(&self) -> String {
format!("{}(L:{},c:{})", self.token_type, self.offset.0, self.offset.1) format!("{}(L:{},c:{})", self.token_type, self.offset.0, self.offset.1)
} }
pub fn get_token_type(&self) -> TokenType {
self.token_type.clone()
}
} }
const OPERATOR_CHARS: [char; 18] = ['!', '$', '%', '&', '*', '+', '-', '.', ':', '<', '>', '=', '?', '@', '^', '|', '~', '`']; const OPERATOR_CHARS: [char; 18] = ['!', '$', '%', '&', '*', '+', '-', '.', ':', '<', '>', '=', '?', '@', '^', '|', '~', '`'];