From 129d9ec6734b2cb6a1f3d85d9e760f9f6d3622e4 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 23 Oct 2019 00:48:59 -0700 Subject: [PATCH] A bunch of infrastructure for keeping track of AST node locations Plus a failing test to illustrate the reason we care --- schala-lang/codegen/src/lib.rs | 1 + schala-lang/language/src/lib.rs | 1 + schala-lang/language/src/parsing.rs | 17 +++++------- schala-lang/language/src/schala.rs | 4 +-- schala-lang/language/src/source_map.rs | 26 +++++++++++++++++++ schala-lang/language/src/symbol_table/test.rs | 5 ++-- schala-lang/language/src/tokenizing.rs | 10 ++++--- 7 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 schala-lang/language/src/source_map.rs diff --git a/schala-lang/codegen/src/lib.rs b/schala-lang/codegen/src/lib.rs index c752a9d..b46f940 100644 --- a/schala-lang/codegen/src/lib.rs +++ b/schala-lang/codegen/src/lib.rs @@ -32,6 +32,7 @@ impl Fold for RecursiveDescentFn { if self.parse_level != 0 { self.parse_level -= 1; } + result.map_err(|mut parse_error: ParseError| { parse_error.production_name = Some(stringify!(#ident).to_string()); parse_error diff --git a/schala-lang/language/src/lib.rs b/schala-lang/language/src/lib.rs index c32c464..bf0ce81 100644 --- a/schala-lang/language/src/lib.rs +++ b/schala-lang/language/src/lib.rs @@ -40,6 +40,7 @@ mod scope_resolution; mod builtin; mod reduced_ast; mod eval; +mod source_map; mod schala; diff --git a/schala-lang/language/src/parsing.rs b/schala-lang/language/src/parsing.rs index 79ba2e4..fbc8fcd 100644 --- a/schala-lang/language/src/parsing.rs +++ b/schala-lang/language/src/parsing.rs @@ -162,12 +162,12 @@ mod test; use std::rc::Rc; use std::str::FromStr; -use std::collections::HashMap; use crate::tokenizing::*; use crate::tokenizing::Kw::*; use crate::tokenizing::TokenKind::*; +use crate::source_map::{SourceMap, Location}; use crate::ast::*; /// Represents a parsing error @@ -204,9 +204,6 @@ pub struct Parser { source_map: SourceMap, } -struct SourceMap { - map: HashMap, idx: usize, - end_of_file: (usize, usize), + end_of_file: Location } impl TokenHandler { fn new(tokens: Vec) -> TokenHandler { let end_of_file = match tokens.last() { - None => (0, 0), - Some(t) => (t.line_num, t.char_num) + None => Location { line_num: 0, char_num : 0 }, + Some(t) => t.location, }; TokenHandler { idx: 0, tokens, end_of_file } } @@ -235,15 +232,15 @@ impl TokenHandler { self.peek_n(n).kind } fn peek(&mut self) -> Token { - self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) + self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file }) } /// calling peek_n(0) is the same thing as peek() fn peek_n(&mut self, n: usize) -> Token { - self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) + self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file }) } fn next(&mut self) -> Token { self.idx += 1; - self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) + self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file }) } } diff --git a/schala-lang/language/src/schala.rs b/schala-lang/language/src/schala.rs index 4eaefdf..4d879c6 100644 --- a/schala-lang/language/src/schala.rs +++ b/schala-lang/language/src/schala.rs @@ -126,8 +126,8 @@ fn parsing(input: Vec, handle: &mut Schala, comp: Option<&mut } fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String { - let line_num = error.token.line_num; - let ch = error.token.char_num; + let line_num = error.token.location.line_num; + let ch = error.token.location.char_num; let line_from_program = handle.source_reference.get_line(line_num); let location_pointer = format!("{}^", " ".repeat(ch)); diff --git a/schala-lang/language/src/source_map.rs b/schala-lang/language/src/source_map.rs new file mode 100644 index 0000000..64f38ba --- /dev/null +++ b/schala-lang/language/src/source_map.rs @@ -0,0 +1,26 @@ +use std::collections::HashMap; +use std::fmt; + +use crate::ast::ItemId; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Location { + pub line_num: usize, + pub char_num: usize +} + +impl fmt::Display for Location { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}:{}", self.line_num, self.char_num) + } +} + +pub struct SourceMap { + map: HashMap +} + +impl SourceMap { + pub fn new() -> SourceMap { + SourceMap { map: HashMap::new() } + } +} diff --git a/schala-lang/language/src/symbol_table/test.rs b/schala-lang/language/src/symbol_table/test.rs index 2dfdca4..8ca18ba 100644 --- a/schala-lang/language/src/symbol_table/test.rs +++ b/schala-lang/language/src/symbol_table/test.rs @@ -42,7 +42,7 @@ fn no_duplicates() { let ast = quick_ast(source); let output = symbol_table.add_top_level_symbols(&ast).unwrap_err(); println!("OUTPUT: {}", output); - assert!(output.contains("Duplicateq")) + assert!(output.contains("Duplicate")) } #[test] @@ -55,7 +55,8 @@ fn no_duplicates_2() { let mut symbol_table = SymbolTable::new(); let ast = quick_ast(source); let output = symbol_table.add_top_level_symbols(&ast).unwrap_err(); - assert!(output.contains("Duplicate")) + assert!(output.contains("Duplicate")); + assert!(output.contains("Line 3")); } #[test] diff --git a/schala-lang/language/src/tokenizing.rs b/schala-lang/language/src/tokenizing.rs index e4abbd0..75c54f4 100644 --- a/schala-lang/language/src/tokenizing.rs +++ b/schala-lang/language/src/tokenizing.rs @@ -4,6 +4,8 @@ use std::rc::Rc; use std::iter::{Iterator, Peekable}; use std::fmt; +use crate::source_map::Location; + #[derive(Debug, PartialEq, Clone)] pub enum TokenKind { Newline, Semicolon, @@ -89,8 +91,7 @@ lazy_static! { #[derive(Debug, Clone, PartialEq)] pub struct Token { pub kind: TokenKind, - pub line_num: usize, - pub char_num: usize + pub location: Location, } impl Token { @@ -101,7 +102,7 @@ impl Token { } } pub fn to_string_with_metadata(&self) -> String { - format!("{}(L:{},c:{})", self.kind, self.line_num, self.char_num) + format!("{}({})", self.kind, self.location) } pub fn get_kind(&self) -> TokenKind { @@ -169,7 +170,8 @@ pub fn tokenize(input: &str) -> Vec { c if is_operator(&c) => handle_operator(c, &mut input), unknown => Error(format!("Unexpected character: {}", unknown)), }; - tokens.push(Token { kind: cur_tok_kind, line_num, char_num }); + let location = Location { line_num, char_num }; + tokens.push(Token { kind: cur_tok_kind, location }); } tokens }