A bunch of infrastructure for keeping track of AST node locations

Plus a failing test to illustrate the reason we care

schala-lang/codegen/src/lib.rs

@@ -32,6 +32,7 @@ impl Fold for RecursiveDescentFn {
         if self.parse_level != 0 {
           self.parse_level -= 1;
         }
+
         result.map_err(|mut parse_error: ParseError| {
           parse_error.production_name = Some(stringify!(#ident).to_string());
           parse_error

schala-lang/language/src/lib.rs

@@ -40,6 +40,7 @@ mod scope_resolution;
 mod builtin;
 mod reduced_ast;
 mod eval;
+mod source_map;
 
 mod schala;
 

schala-lang/language/src/parsing.rs

@@ -162,12 +162,12 @@ mod test;
 
 use std::rc::Rc;
 use std::str::FromStr;
-use std::collections::HashMap;
 
 use crate::tokenizing::*;
 use crate::tokenizing::Kw::*;
 use crate::tokenizing::TokenKind::*;
 
+use crate::source_map::{SourceMap, Location};
 use crate::ast::*;
 
 /// Represents a parsing error
@@ -204,9 +204,6 @@ pub struct Parser {
   source_map: SourceMap,
 }
 
-struct SourceMap {
-  map: HashMap<ItemId, 
-}
 
 struct ParserRestrictions {
   no_struct_literal: bool
@@ -215,14 +212,14 @@ struct ParserRestrictions {
 struct TokenHandler {
   tokens: Vec<Token>,
   idx: usize,
-  end_of_file: (usize, usize),
+  end_of_file: Location
 }
 
 impl TokenHandler {
   fn new(tokens: Vec<Token>) -> TokenHandler {
     let end_of_file = match tokens.last() {
-      None => (0, 0),
+      None => Location { line_num: 0, char_num : 0 },
-      Some(t) => (t.line_num, t.char_num)
+      Some(t) => t.location,
     };
     TokenHandler { idx: 0, tokens, end_of_file }
   }
@@ -235,15 +232,15 @@ impl TokenHandler {
     self.peek_n(n).kind
   }
   fn peek(&mut self) -> Token {
-    self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
+    self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
   }
   /// calling peek_n(0) is the same thing as peek()
   fn peek_n(&mut self, n: usize) -> Token {
-    self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
+    self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
   }
   fn next(&mut self) -> Token {
     self.idx += 1;
-    self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
+    self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
   }
 }
 

schala-lang/language/src/schala.rs

@@ -126,8 +126,8 @@ fn parsing(input: Vec<tokenizing::Token>, handle: &mut Schala, comp: Option<&mut
 }
 
 fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String {
-  let line_num = error.token.line_num;
+  let line_num = error.token.location.line_num;
-  let ch = error.token.char_num;
+  let ch = error.token.location.char_num;
   let line_from_program = handle.source_reference.get_line(line_num);
   let location_pointer = format!("{}^", " ".repeat(ch));
 

schala-lang/language/src/source_map.rs

@@ -0,0 +1,26 @@
+use std::collections::HashMap;
+use std::fmt;
+
+use crate::ast::ItemId;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Location {
+  pub line_num: usize,
+  pub char_num: usize
+}
+
+impl fmt::Display for Location {
+  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    write!(f, "{}:{}", self.line_num, self.char_num)
+  }
+}
+
+pub struct SourceMap {
+  map: HashMap<ItemId, Location>
+}
+
+impl SourceMap {
+  pub fn new() -> SourceMap {
+    SourceMap { map: HashMap::new() }
+  }
+}

schala-lang/language/src/symbol_table/test.rs

@@ -42,7 +42,7 @@ fn no_duplicates() {
   let ast = quick_ast(source);
   let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
   println!("OUTPUT: {}", output);
-  assert!(output.contains("Duplicateq"))
+  assert!(output.contains("Duplicate"))
 }
 
 #[test]
@@ -55,7 +55,8 @@ fn no_duplicates_2() {
   let mut symbol_table = SymbolTable::new();
   let ast = quick_ast(source);
   let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
-  assert!(output.contains("Duplicate"))
+  assert!(output.contains("Duplicate"));
+  assert!(output.contains("Line 3"));
 }
 
 #[test]

schala-lang/language/src/tokenizing.rs

@@ -4,6 +4,8 @@ use std::rc::Rc;
 use std::iter::{Iterator, Peekable};
 use std::fmt;
 
+use crate::source_map::Location;
+
 #[derive(Debug, PartialEq, Clone)]
 pub enum TokenKind {
   Newline, Semicolon,
@@ -89,8 +91,7 @@ lazy_static! {
 #[derive(Debug, Clone, PartialEq)]
 pub struct Token {
   pub kind: TokenKind,
-  pub line_num: usize,
+  pub location: Location,
-  pub char_num: usize
 }
 
 impl Token {
@@ -101,7 +102,7 @@ impl Token {
     }
   }
   pub fn to_string_with_metadata(&self) -> String {
-    format!("{}(L:{},c:{})", self.kind, self.line_num, self.char_num)
+    format!("{}({})", self.kind, self.location)
   }
 
   pub fn get_kind(&self) -> TokenKind {
@@ -169,7 +170,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
       c if is_operator(&c) => handle_operator(c, &mut input),
       unknown => Error(format!("Unexpected character: {}", unknown)),
     };
-    tokens.push(Token { kind: cur_tok_kind, line_num, char_num });
+    let location = Location { line_num, char_num };
+    tokens.push(Token { kind: cur_tok_kind, location });
   }
   tokens
 }