Compiler bug!??!

Requires one unstable feature

TODO.md

@@ -1,5 +1,24 @@
+# Plan of attack
+
+1. modify visitor so it can handle scopes
+  -this is needed both to handle import scope correctly
+  -and also to support making FQSNs aware of function parameters
+
+2. Once FQSNs are aware of function parameters, most of the Rc<String> things in eval.rs can go away
+
 # TODO items
 
+-use 'let' sigil in patterns for variables :
+
+```
+  q is MyStruct(let a, Chrono::Trigga) then {
+
+  }
+```
+
+-idea: what if there was something like React jsx syntas built in? i.e. a way to automatically transform some kind of markup
+into a function call, cf. `<h1 prop="arg">` ->  h1(prop=arg)
+
 ## General code cleanup
 - I think I can restructure the parser to get rid of most instances of expect!, at least at the beginning of a rule
 DONE -experiment with storing metadata via ItemIds on AST nodes (cf. https://rust-lang.github.io/rustc-guide/hir.html, https://github.com/rust-lang/rust/blob/master/src/librustc/hir/mod.rs )
@@ -11,6 +30,9 @@ DONE -experiment with storing metadata via ItemIds on AST nodes (cf. https://rus
   -look at https://gitlab.haskell.org/ghc/ghc/wikis/pattern-synonyms
                                 2) the non-value-returning, default one like in rustc (cf. https://github.com/rust-unofficial/patterns/blob/master/patterns/visitor.md)
 
+-parser error - should report subset of AST parsed *so far*
+  - what if you used python 'def' syntax to define a function? what error message makes sense here?
+
 ## Reduction
  - make a good type for actual language builtins to avoid string comparisons
 

schala-lang/codegen/src/lib.rs

@@ -32,6 +32,7 @@ impl Fold for RecursiveDescentFn {
         if self.parse_level != 0 {
           self.parse_level -= 1;
         }
+
         result.map_err(|mut parse_error: ParseError| {
           parse_error.production_name = Some(stringify!(#ident).to_string());
           parse_error

schala-lang/language/src/ast.rs

@@ -65,6 +65,7 @@ pub enum StatementKind {
   Expression(Expression),
   Declaration(Declaration),
   Import(ImportSpecifier),
+  Module(ModuleSpecifier),
 }
 
 pub type Block = Vec<Statement>;
@@ -297,3 +298,10 @@ pub enum ImportedNames {
   List(Vec<Rc<String>>)
 }
 
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct ModuleSpecifier {
+  pub name: Rc<String>,
+  pub contents: Vec<Statement>,
+}
+

schala-lang/language/src/ast/operators.rs

@@ -68,10 +68,6 @@ impl BinOp {
     let s = token_kind_to_sigil(op_tok)?;
     Some(binop_precedences(s))
   }
-
-  pub fn get_precedence(&self) -> i32 {
-    binop_precedences(&self.sigil)
-  }
 }
 
 fn token_kind_to_sigil<'a>(tok: &'a TokenKind) -> Option<&'a str> {

schala-lang/language/src/ast/visitor.rs

@@ -4,9 +4,11 @@ use crate::ast::*;
 //TODO maybe these functions should take closures that return a KeepRecursing | StopHere type,
 //or a tuple of (T, <that type>)
 
-pub trait ASTVisitor: Sized {
+pub trait ASTVisitor<BlockHandler=()>: Sized {
+  type BlockHandler: BlockVisitor;
   fn ast(&mut self, _ast: &AST) {}
-  fn block(&mut self, _statements: &Vec<Statement>) {}
+  fn block(&mut self) -> Self::BlockHandler { Self::BlockHandler::new() }
+  fn block_finished(&mut self, handler: Self::BlockHandler) {}
   fn statement(&mut self, _statement: &Statement) {}
   fn declaration(&mut self, _declaration: &Declaration) {}
   fn signature(&mut self, _signature: &Signature) {}
@@ -29,6 +31,7 @@ pub trait ASTVisitor: Sized {
   fn invocation_argument(&mut self, _arg: &InvocationArgument) {}
   fn formal_param(&mut self, _param: &FormalParam) {}
   fn import(&mut self, _import: &ImportSpecifier) {}
+  fn module(&mut self, _module: &ModuleSpecifier) {}
   fn qualified_name(&mut self, _name: &QualifiedName) {}
   fn nat_literal(&mut self, _n: u64) {}
   fn float_literal(&mut self, _f: f64) {}
@@ -38,3 +41,15 @@ pub trait ASTVisitor: Sized {
   fn prefix_exp(&mut self, _op: &PrefixOp, _arg: &Expression) {}
   fn pattern(&mut self, _pat: &Pattern) {}
 }
+
+pub trait BlockVisitor {
+  fn new() -> Self;
+  fn pre_block(&mut self) {}
+  fn post_block(&mut self) {}
+}
+
+impl BlockVisitor for () {
+  fn new() -> () { () }
+}
+
+

schala-lang/language/src/ast/visitor_test.rs

@@ -22,7 +22,7 @@ impl ASTVisitor for Tester {
 #[test]
 fn foo() {
   let mut tester = Tester { count: 0, float_count: 0 };
-  let ast = quick_ast(r#"
+  let (ast, _) = quick_ast(r#"
 import gragh
 
 let a = 20 + 84

schala-lang/language/src/ast/walker.rs

@@ -1,7 +1,7 @@
 #![allow(dead_code)]
 use std::rc::Rc;
 use crate::ast::*;
-use crate::ast::visitor::ASTVisitor;
+use crate::ast::visitor::{ASTVisitor, BlockVisitor};
 use crate::util::deref_optional_box;
 
 pub fn walk_ast<V: ASTVisitor>(v: &mut V, ast: &AST) {
@@ -10,10 +10,14 @@ pub fn walk_ast<V: ASTVisitor>(v: &mut V, ast: &AST) {
 }
 
 fn walk_block<V: ASTVisitor>(v: &mut V, block: &Vec<Statement>) {
+  let mut block_handler = v.block();
+  block_handler.pre_block();
   for s in block {
     v.statement(s);
     statement(v, s);
   }
+  block_handler.post_block();
+  v.block_finished(block_handler);
 }
 
 fn statement<V: ASTVisitor>(v: &mut V, statement: &Statement) {
@@ -28,6 +32,10 @@ fn statement<V: ASTVisitor>(v: &mut V, statement: &Statement) {
       declaration(v, decl);
     },
     Import(ref import_spec) => v.import(import_spec),
+    Module(ref module_spec) => {
+      v.module(module_spec);
+      walk_block(v, &module_spec.contents);
+    }
   }
 }
 
@@ -40,7 +48,6 @@ fn declaration<V: ASTVisitor>(v: &mut V, decl: &Declaration) {
     },
     FuncDecl(sig, block) => {
       v.signature(&sig);
-      v.block(&block);
       walk_block(v, block);
     },
     TypeDecl { name, body, mutable  } => v.type_declaration(name, body, *mutable),
@@ -119,7 +126,6 @@ fn lambda<V: ASTVisitor>(v: &mut V, params: &Vec<FormalParam>, type_anno: Option
     formal_param(v, param);
   }
   v.type_annotation(type_anno);
-  v.block(body);
   walk_block(v, body);
 }
 
@@ -230,7 +236,6 @@ fn condition_arm<V: ASTVisitor>(v: &mut V, arm: &ConditionArm) {
     v.expression(guard);
     expression(v, guard);
   });
-  v.block(&arm.body);
   walk_block(v, &arm.body);
 }
 

schala-lang/language/src/eval.rs

@@ -1,27 +1,25 @@
-
-use std::cell::RefCell;
 use std::rc::Rc;
 use std::fmt::Write;
 use std::io;
 
 use itertools::Itertools;
 
+use crate::schala::SymbolTableHandle;
 use crate::util::ScopeStack;
 use crate::reduced_ast::{BoundVars, ReducedAST, Stmt, Expr, Lit, Func, Alternative, Subpattern};
-use crate::symbol_table::{SymbolSpec, Symbol, SymbolTable, ScopeSegment, FullyQualifiedSymbolName};
+use crate::symbol_table::{SymbolSpec, Symbol, SymbolTable, FullyQualifiedSymbolName};
 use crate::builtin::Builtin;
 
 mod test;
 
 pub struct State<'a> {
   values: ScopeStack<'a, Rc<String>, ValueEntry>,
-  symbol_table_handle: Rc<RefCell<SymbolTable>>,
 }
 
 impl<'a> State<'a> {
-  pub fn new(symbol_table_handle: Rc<RefCell<SymbolTable>>) -> State<'a> {
+  pub fn new() -> State<'a> {
     let values = ScopeStack::new(Some(format!("global")));
-    State { values, symbol_table_handle }
+    State { values }
   }
 
   pub fn debug_print(&self) -> String {
@@ -31,7 +29,6 @@ impl<'a> State<'a> {
   fn new_frame(&'a self, items: &'a Vec<Node>, bound_vars: &BoundVars) -> State<'a> {
     let mut inner_state = State {
       values: self.values.new_scope(None),
-      symbol_table_handle: self.symbol_table_handle.clone(),
     };
     for (bound_var, val) in bound_vars.iter().zip(items.iter()) {
       if let Some(bv) = bound_var.as_ref() {
@@ -70,12 +67,12 @@ fn paren_wrapped_vec(terms: impl Iterator<Item=String>) -> String {
 
 
 impl Node {
-  fn to_repl(&self, symbol_table: &SymbolTable) -> String {
+  fn to_repl(&self) -> String {
     match self {
-      Node::Expr(e) => e.to_repl(symbol_table),
+      Node::Expr(e) => e.to_repl(),
       Node::PrimObject { name, items, .. } if items.len() == 0 => format!("{}", name),
-      Node::PrimObject { name, items, .. } => format!("{}{}", name, paren_wrapped_vec(items.iter().map(|x| x.to_repl(symbol_table)))),
+      Node::PrimObject { name, items, .. } => format!("{}{}", name, paren_wrapped_vec(items.iter().map(|x| x.to_repl()))),
-      Node::PrimTuple { items } => format!("{}", paren_wrapped_vec(items.iter().map(|x| x.to_repl(symbol_table)))),
+      Node::PrimTuple { items } => format!("{}", paren_wrapped_vec(items.iter().map(|x| x.to_repl()))),
     }
   }
   fn is_true(&self) -> bool {
@@ -100,12 +97,10 @@ impl Expr {
   fn to_node(self) -> Node {
     Node::Expr(self)
   }
-  fn to_repl(&self, symbol_table: &SymbolTable) -> String {
+  fn to_repl(&self) -> String {
     use self::Lit::*;
     use self::Func::*;
 
-    let _ = symbol_table;
-
     match self {
       Expr::Lit(ref l) => match l {
         Nat(n) => format!("{}", n),
@@ -122,7 +117,7 @@ impl Expr {
       Expr::Constructor { type_name, arity, .. } => {
         format!("<constructor for `{}` arity {}>", type_name, arity)
       },
-      Expr::Tuple(exprs) => paren_wrapped_vec(exprs.iter().map(|x| x.to_repl(symbol_table))),
+      Expr::Tuple(exprs) => paren_wrapped_vec(exprs.iter().map(|x| x.to_repl())),
       _ => format!("{:?}", self),
     }
   }
@@ -157,8 +152,7 @@ impl<'a> State<'a> {
     for statement in ast.0 {
       match self.statement(statement) {
         Ok(Some(ref output)) if repl => {
-          let ref symbol_table = self.symbol_table_handle.borrow();
+          acc.push(Ok(output.to_repl()))
-          acc.push(Ok(output.to_repl(symbol_table)))
         },
         Ok(_) => (),
         Err(error) => {
@@ -212,7 +206,10 @@ impl<'a> State<'a> {
       Node::Expr(expr) => match expr {
         literal @ Lit(_) => Ok(Node::Expr(literal)),
         Call { box f, args } => self.call_expression(f, args),
-        Sym(v) => self.handle_sym(v),
+        Sym(name) => Ok(match self.values.lookup(&name) {
+          Some(ValueEntry::Binding { val, .. }) => val.clone(),
+          None => return Err(format!("Could not look up symbol {}", name))
+        }),
         Constructor { arity, ref name, tag, .. } if arity == 0 => Ok(Node::PrimObject { name: name.clone(), tag, items: vec![] }),
         constructor @ Constructor { .. } => Ok(Node::Expr(constructor)),
         func @ Func(_) => Ok(Node::Expr(func)),
@@ -264,7 +261,6 @@ impl<'a> State<'a> {
         }
         let mut func_state = State {
           values: self.values.new_scope(name.map(|n| format!("{}", n))),
-          symbol_table_handle: self.symbol_table_handle.clone(),
         };
         for (param, val) in params.into_iter().zip(args.into_iter()) {
           let val = func_state.expression(Node::Expr(val))?;
@@ -343,13 +339,11 @@ impl<'a> State<'a> {
 
       /* builtin functions */
       (IOPrint, &[ref anything]) => {
-        let ref symbol_table = self.symbol_table_handle.borrow();
+        print!("{}", anything.to_repl());
-        print!("{}", anything.to_repl(symbol_table));
         Expr::Unit.to_node()
       },
       (IOPrintLn, &[ref anything]) => {
-        let ref symbol_table = self.symbol_table_handle.borrow();
+        println!("{}", anything.to_repl());
-        println!("{}", anything.to_repl(symbol_table));
         Expr::Unit.to_node()
       },
       (IOGetLine, &[]) => {
@@ -458,46 +452,4 @@ impl<'a> State<'a> {
     }
     Err(format!("{:?} failed pattern match", cond))
   }
-
-  //TODO if I don't need to lookup by name here...
-  fn handle_sym(&mut self, name: Rc<String>) -> EvalResult<Node> {
-    use self::ValueEntry::*;
-    use self::Func::*;
-    //TODO add a layer of indirection here to talk to the symbol table first, and only then look up
-    //in the values table
-
-    let symbol_table = self.symbol_table_handle.borrow();
-    let value = symbol_table.lookup_by_fqsn(&fqsn!(name ; tr));
-    Ok(match value {
-      Some(Symbol { local_name, spec, .. }) => match spec {
-        //TODO I'll need this type_name later to do a table lookup
-        SymbolSpec::DataConstructor { type_name: _type_name, type_args, .. } => {
-          if type_args.len() == 0 {
-            Node::PrimObject { name: local_name.clone(), tag: 0, items: vec![] }
-          } else {
-            return Err(format!("This data constructor thing not done"))
-          }
-        },
-        SymbolSpec::Func(_) => match self.values.lookup(&name) {
-          Some(Binding { val: Node::Expr(Expr::Func(UserDefined { name, params, body })), .. }) => {
-            Node::Expr(Expr::Func(UserDefined { name: name.clone(), params: params.clone(), body: body.clone() }))
-          },
-          _ => unreachable!(),
-        },
-        SymbolSpec::RecordConstructor { .. } => return Err(format!("This shouldn't be a record!")),
-        SymbolSpec::Binding => match self.values.lookup(&name) {
-          Some(Binding { val, .. }) => val.clone(),
-          None => return Err(format!("Symbol {} exists in symbol table but not in evaluator table", name))
-        }
-        SymbolSpec::Type { name } => return Err(format!("Symbol {} not in scope", name)),
-      },
-      //TODO ideally this should be returning a runtime error if this is ever None, but it's not
-      //handling all bindings correctly yet
-      //None => return Err(format!("Couldn't find value {}", name)),
-      None => match self.values.lookup(&name) {
-        Some(Binding { val, .. }) => val.clone(),
-        None => return Err(format!("Couldn't find value {}", name)),
-      }
-    })
-  }
 }

schala-lang/language/src/eval/test.rs

@@ -9,17 +9,17 @@ use crate::reduced_ast::reduce;
 use crate::eval::State;
 
 fn evaluate_all_outputs(input: &str) -> Vec<Result<String, String>> {
-  let symbol_table = Rc::new(RefCell::new(SymbolTable::new()));
+  let (mut ast, source_map) = crate::util::quick_ast(input);
-  let mut state = State::new(symbol_table);
+  let source_map = Rc::new(RefCell::new(source_map));
-  let mut ast = crate::util::quick_ast(input);
+  let symbol_table = Rc::new(RefCell::new(SymbolTable::new(source_map)));
-  state.symbol_table_handle.borrow_mut().add_top_level_symbols(&ast).unwrap();
+  symbol_table.borrow_mut().add_top_level_symbols(&ast).unwrap();
   {
-  let mut t = &mut state.symbol_table_handle.borrow_mut();
+  let mut scope_resolver = ScopeResolver::new(symbol_table.clone());
-  let mut scope_resolver = ScopeResolver::new(&mut t);
   let _ = scope_resolver.resolve(&mut ast);
   }
 
-  let reduced = reduce(&ast, &state.symbol_table_handle.borrow());
+  let reduced = reduce(&ast, &symbol_table.borrow());
+  let mut state = State::new();
   let all_output = state.evaluate(reduced, true);
   all_output
 }

schala-lang/language/src/lib.rs

@@ -1,6 +1,5 @@
+#![feature(associated_type_defaults)] //needed for Visitor trait
 #![feature(trace_macros)]
-#![feature(custom_attribute)]
-//#![feature(unrestricted_attribute_tokens)]
 #![feature(slice_patterns, box_patterns, box_syntax)]
 
 //! `schala-lang` is where the Schala programming language is actually implemented. 
@@ -40,6 +39,7 @@ mod scope_resolution;
 mod builtin;
 mod reduced_ast;
 mod eval;
+mod source_map;
 
 mod schala;
 

schala-lang/language/src/parsing.rs

@@ -13,7 +13,7 @@
 //! ```text
 //! program := (statement delimiter)* EOF
 //! delimiter := NEWLINE | ";"
-//! statement := expression | declaration | import
+//! statement := expression | declaration | import | module
 //! block := "{" (statement delimiter)* "}"
 //! declaration := type_declaration | func_declaration | binding_declaration | impl_declaration
 //! ```
@@ -153,6 +153,10 @@
 //! ```text
 //! import := 'import' IDENTIFIER (:: IDENTIFIER)*  import_suffix
 //! import_suffix := ε | '::{' IDENTIFIER (, IDENTIFIER)* '}' | '*' //TODO add qualified, exclusions, etc.
+//!
+//! ## Modules
+//!
+//! module := 'module' IDENTIFIER '{' statement* '}'
 //! ```
 mod test;
 
@@ -163,7 +167,9 @@ use crate::tokenizing::*;
 use crate::tokenizing::Kw::*;
 use crate::tokenizing::TokenKind::*;
 
+use crate::source_map::Location;
 use crate::ast::*;
+use crate::schala::SourceMapHandle;
 
 /// Represents a parsing error
 #[derive(Debug)]
@@ -196,8 +202,10 @@ pub struct Parser {
   parse_level: u32,
   restrictions: ParserRestrictions,
   id_store: ItemIdStore,
+  source_map: SourceMapHandle
 }
 
+
 struct ParserRestrictions {
   no_struct_literal: bool
 }
@@ -205,14 +213,14 @@ struct ParserRestrictions {
 struct TokenHandler {
   tokens: Vec<Token>,
   idx: usize,
-  end_of_file: (usize, usize),
+  end_of_file: Location
 }
 
 impl TokenHandler {
   fn new(tokens: Vec<Token>) -> TokenHandler {
     let end_of_file = match tokens.last() {
-      None => (0, 0),
+      None => Location { line_num: 0, char_num : 0 },
-      Some(t) => (t.line_num, t.char_num)
+      Some(t) => t.location,
     };
     TokenHandler { idx: 0, tokens, end_of_file }
   }
@@ -225,49 +233,56 @@ impl TokenHandler {
     self.peek_n(n).kind
   }
   fn peek(&mut self) -> Token {
-    self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
+    self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
   }
   /// calling peek_n(0) is the same thing as peek()
   fn peek_n(&mut self, n: usize) -> Token {
-    self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
+    self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
   }
   fn next(&mut self) -> Token {
     self.idx += 1;
-    self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
+    self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
   }
 }
 
 impl Parser {
   /// Create a new parser initialized with some tokens.
-  pub fn new(initial_input: Vec<Token>) -> Parser {
+  pub fn new(source_map: SourceMapHandle) -> Parser {
     Parser {
-      token_handler: TokenHandler::new(initial_input),
+      token_handler: TokenHandler::new(vec![]),
       parse_record: vec![],
       parse_level: 0,
       restrictions: ParserRestrictions { no_struct_literal: false },
       id_store: ItemIdStore::new(),
+      source_map,
     }
   }
 
+  pub fn add_new_tokens(&mut self, new_tokens: Vec<Token>) {
+    self.token_handler = TokenHandler::new(new_tokens);
+  }
+
   /// Parse all loaded tokens up to this point.
   pub fn parse(&mut self) -> ParseResult<AST> {
     self.program()
   }
 
-  /*
-  pub fn parse_with_new_tokens(&mut self, new_tokens: Vec<Token>) -> ParseResult<AST> {
-
-  }
-  */
-
   pub fn format_parse_trace(&self) -> String {
     let mut buf = String::new();
+    buf.push_str("Parse productions:\n");
+    let mut next_token = None;
     for r in self.parse_record.iter() {
       let mut indent = String::new();
       for _ in 0..r.level {
-        indent.push(' ');
+        indent.push('.');
       }
-      buf.push_str(&format!("{}Production `{}`, token: {}\n", indent, r.production_name, r.next_token))
+      let effective_token = if next_token == Some(&r.next_token) {
+        "".to_string()
+      } else {
+        next_token = Some(&r.next_token);
+        format!(", next token: {}", r.next_token)
+      };
+      buf.push_str(&format!("{}`{}`{}\n", indent, r.production_name, effective_token));
     }
     buf
   }
@@ -355,16 +370,20 @@ impl Parser {
   #[recursive_descent_method]
   fn statement(&mut self) -> ParseResult<Statement> {
     //TODO handle error recovery here
-    let kind = match self.token_handler.peek().get_kind() {
+    let tok = self.token_handler.peek();
+    let kind = match tok.get_kind() {
       Keyword(Type) => self.type_declaration().map(|decl| { StatementKind::Declaration(decl) }),
       Keyword(Func)=> self.func_declaration().map(|func| { StatementKind::Declaration(func) }),
       Keyword(Let) => self.binding_declaration().map(|decl| StatementKind::Declaration(decl)),
       Keyword(Interface) => self.interface_declaration().map(|decl| StatementKind::Declaration(decl)),
       Keyword(Impl) => self.impl_declaration().map(|decl| StatementKind::Declaration(decl)),
       Keyword(Import) => self.import_declaration().map(|spec| StatementKind::Import(spec)),
+      Keyword(Module) => self.module_declaration().map(|spec| StatementKind::Module(spec)),
       _ => self.expression().map(|expr| { StatementKind::Expression(expr) } ),
     }?;
-    Ok(Statement { kind, id: self.id_store.fresh() })
+    let id = self.id_store.fresh();
+    self.source_map.borrow_mut().add_location(&id, tok.location); 
+    Ok(Statement { kind, id })
   }
 
   #[recursive_descent_method]
@@ -482,7 +501,6 @@ impl Parser {
     Ok(delimited!(self, LParen, formal_param, Comma, RParen))
   }
 
-  //TODO needs to support default values
   #[recursive_descent_method]
   fn formal_param(&mut self) -> ParseResult<FormalParam> {
     let name = self.identifier()?;
@@ -973,7 +991,7 @@ impl Parser {
         self.token_handler.next();
         Pattern::Literal(PatternLiteral::BoolPattern(false))
       },
-      StrLiteral(s) => {
+      StrLiteral { s, .. } => {
         self.token_handler.next();
         Pattern::Literal(PatternLiteral::StringPattern(s))
       },
@@ -1122,7 +1140,7 @@ impl Parser {
         let id = self.id_store.fresh();
         Ok(Expression::new(id, BoolLiteral(false)))
       },
-      StrLiteral(s) => {
+      StrLiteral {s, ..} => {
         self.token_handler.next();
         let id = self.id_store.fresh();
         Ok(Expression::new(id, StringLiteral(s.clone())))
@@ -1245,6 +1263,14 @@ impl Parser {
       _ => return ParseError::new_with_token("Expected '{{' or '*'", self.token_handler.peek()),
     })
   }
+
+  #[recursive_descent_method]
+  fn module_declaration(&mut self) -> ParseResult<ModuleSpecifier> {
+    expect!(self, Keyword(Kw::Module));
+    let name = self.identifier()?;
+    let contents = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict);
+    Ok(ModuleSpecifier { name, contents })
+  }
 }
 
 fn parse_binary(digits: String, tok: Token) -> ParseResult<u64> {

schala-lang/language/src/parsing/test.rs

@@ -1,10 +1,10 @@
 #![cfg(test)]
-use ::std::rc::Rc;
+use std::cell::RefCell;
+use std::rc::Rc;
 use std::str::FromStr;
 
-use super::tokenize;
+use super::{Parser, ParseResult, tokenize};
-use super::ParseResult;
+use crate::ast::*;
-use crate::ast::{ItemIdStore, AST, Expression, Statement, StatementKind, IfExpressionBody, Pattern, PatternLiteral, TypeBody, Enumerator, ForBody, InvocationArgument, FormalParam, PrefixOp, BinOp, QualifiedName, ImportSpecifier, ImportedNames, Condition, ConditionArm};
 use super::Declaration::*;
 use super::Signature;
 use super::TypeIdentifier::*;
@@ -13,9 +13,17 @@ use super::ExpressionKind::*;
 use super::Variant::*;
 use super::ForBody::*;
 
-fn parse(input: &str) -> ParseResult<AST> {
+fn make_parser(input: &str) -> Parser {
+  let source_map = crate::source_map::SourceMap::new();
+  let source_map_handle = Rc::new(RefCell::new(source_map));
   let tokens: Vec<crate::tokenizing::Token> = tokenize(input);
-  let mut parser = super::Parser::new(tokens);
+  let mut parser = super::Parser::new(source_map_handle);
+  parser.add_new_tokens(tokens);
+  parser
+}
+
+fn parse(input: &str) -> ParseResult<AST> {
+  let mut parser = make_parser(input);
   parser.parse()
 }
 
@@ -63,13 +71,18 @@ macro_rules! import {
   }
 }
 
+macro_rules! module {
+  ($module_spec:expr) => {
+    Statement { id: ItemIdStore::new_id(), kind: StatementKind::Module($module_spec) }
+  }
+}
+
 macro_rules! ex {
   ($expr_type:expr) => { Expression::new(ItemIdStore::new_id(), $expr_type) };
   ($expr_type:expr, $type_anno:expr) => { Expression::with_anno(ItemIdStore::new_id(), $expr_type, $type_anno) };
   (s $expr_text:expr) => {
     {
-      let tokens: Vec<crate::tokenizing::Token> = tokenize($expr_text);
+      let mut parser = make_parser($expr_text);
-      let mut parser = super::Parser::new(tokens);
       parser.expression().unwrap()
     }
   };
@@ -92,8 +105,7 @@ macro_rules! exst {
   };
   (s $statement_text:expr) => {
     {
-      let tokens: Vec<crate::tokenizing::Token> = tokenize($statement_text);
+      let mut parser = make_parser($statement_text);
-      let mut parser = super::Parser::new(tokens);
       parser.statement().unwrap()
     }
   }
@@ -796,3 +808,21 @@ fn if_expr() {
     }
   }
 }
+
+#[test]
+fn modules() {
+  parse_test_wrap_ast! {
+r#"
+  module ephraim {
+    let a = 10
+    fn nah() { 33 }
+  }
+"#,
+module!(
+  ModuleSpecifier { name: rc!(ephraim), contents: vec![
+    decl!(Binding { name: rc!(a), constant: true, type_anno: None, expr: ex!(s "10") }),
+    decl!(FuncDecl(Signature { name: rc!(nah), operator: false, params: vec![], type_anno: None }, vec![exst!(NatLiteral(33))])),
+  ] }
+)
+  }
+}

schala-lang/language/src/prelude.schala

@@ -6,8 +6,8 @@ type Ord = LT | EQ | GT
 
 fn map(input: Option<T>, func: Func): Option<T> {
   if input {
-    is Option::Some(x) -> Option::Some(func(x)),
+    is Option::Some(x) then Option::Some(func(x)),
-    is Option::None -> Option::None,
+    is Option::None then Option::None,
   }
 }
 

schala-lang/language/src/reduced_ast.rs

@@ -42,9 +42,9 @@ pub enum Stmt {
 pub enum Expr {
   Unit,
   Lit(Lit),
+  Sym(Rc<String>), //a Sym is anything that can be looked up by name at runtime - i.e. a function or variable address
   Tuple(Vec<Expr>),
   Func(Func),
-  Sym(Rc<String>),
   Constructor {
     type_name: Rc<String>,
     name: Rc<String>,
@@ -56,7 +56,7 @@ pub enum Expr {
     args: Vec<Expr>,
   },
   Assign {
-    val: Box<Expr>,
+    val: Box<Expr>, //TODO this probably can't be a val
     expr: Box<Expr>,
   },
   Conditional {
@@ -131,6 +131,12 @@ impl<'a> Reducer<'a> {
       StatementKind::Expression(expr) => Stmt::Expr(self.expression(&expr)),
       StatementKind::Declaration(decl) => self.declaration(&decl),
       StatementKind::Import(_) => Stmt::Noop,
+      StatementKind::Module(modspec) => {
+        for statement in modspec.contents.iter() {
+          self.statement(&statement);
+        }
+        Stmt::Noop
+      }
     }
   }
 
@@ -158,25 +164,7 @@ impl<'a> Reducer<'a> {
       BoolLiteral(b) => Expr::Lit(Lit::Bool(*b)),
       BinExp(binop, lhs, rhs) => self.binop(binop, lhs, rhs),
       PrefixExp(op, arg) => self.prefix(op, arg),
-      Value(qualified_name) => {
+      Value(qualified_name) => self.value(qualified_name),
-        let ref id = qualified_name.id;
-        let ref sym_name = match symbol_table.get_fqsn_from_id(id) {
-          Some(fqsn) => fqsn,
-          None => return Expr::ReductionError(format!("FQSN lookup for Value {:?} failed", qualified_name)),
-        };
-        //TODO this probably needs to change
-        let FullyQualifiedSymbolName(ref v) = sym_name;
-        let name = v.last().unwrap().name.clone();
-        match symbol_table.lookup_by_fqsn(&sym_name) {
-          Some(Symbol { spec: SymbolSpec::DataConstructor { index, type_args, type_name}, .. }) => Expr::Constructor {
-            type_name: type_name.clone(),
-            name: name.clone(),
-            tag: index.clone(),
-            arity: type_args.len(),
-          },
-          _ => Expr::Sym(name.clone()),
-        }
-      },
       Call { f, arguments } =>  self.reduce_call_expression(f, arguments),
       TupleLiteral(exprs) => Expr::Tuple(exprs.iter().map(|e| self.expression(e)).collect()),
       IfExpression { discriminator, body } => self.reduce_if_expression(deref_optional_box(discriminator), body),
@@ -189,6 +177,38 @@ impl<'a> Reducer<'a> {
     }
   }
 
+  fn value(&mut self, qualified_name: &QualifiedName) -> Expr {
+    let symbol_table = self.symbol_table;
+    let ref id = qualified_name.id;
+    let ref sym_name = match symbol_table.get_fqsn_from_id(id) {
+      Some(fqsn) => fqsn,
+      None => return Expr::ReductionError(format!("FQSN lookup for Value {:?} failed", qualified_name)),
+    };
+
+    //TODO this probably needs to change
+    let FullyQualifiedSymbolName(ref v) = sym_name;
+    let name = v.last().unwrap().name.clone();
+
+    let Symbol { local_name, spec, .. } = match symbol_table.lookup_by_fqsn(&sym_name) {
+      Some(s) => s,
+      //None => return Expr::ReductionError(format!("Symbol {:?} not found", sym_name)),
+      None => return Expr::Sym(name.clone())
+    };
+
+    match spec {
+      SymbolSpec::RecordConstructor { .. } => Expr::ReductionError(format!("AST reducer doesn't expect a RecordConstructor here")),
+      SymbolSpec::DataConstructor { index, type_args, type_name } => Expr::Constructor {
+        type_name: type_name.clone(),
+        name: name.clone(),
+        tag: index.clone(),
+        arity: type_args.len(),
+      },
+      SymbolSpec::Func(_) => Expr::Sym(local_name.clone()),
+      SymbolSpec::Binding => Expr::Sym(local_name.clone()), //TODO not sure if this is right, probably needs to eventually be fqsn
+      SymbolSpec::Type { .. } => Expr::ReductionError("AST reducer doesnt expect a type here".to_string())
+    }
+  }
+
   fn reduce_lambda(&mut self, params: &Vec<FormalParam>, body: &Block) -> Expr {
     Expr::Func(Func::UserDefined {
       name: None,
@@ -444,7 +464,6 @@ impl Pattern {
       VarOrName(QualifiedName { components, id  }) => {
         // if fqsn is Some, treat this as a symbol pattern. If it's None, treat it
         // as a variable.
-        println!("Calling VarOrName reduction with : {:?}", components);
         let fqsn = symbol_table.get_fqsn_from_id(&id);
         match fqsn.and_then(|fqsn| symbol_table.lookup_by_fqsn(&fqsn)) {
           Some(symbol) => handle_symbol(Some(symbol), &vec![], symbol_table),

schala-lang/language/src/schala.rs

@@ -10,17 +10,22 @@ use schala_repl::{ProgrammingLanguageInterface,
 ComputationRequest, ComputationResponse,
 LangMetaRequest, LangMetaResponse, GlobalOutputStats,
 DebugResponse, DebugAsk};
-use crate::{ast, reduced_ast, tokenizing, parsing, eval, typechecking, symbol_table};
+use crate::{ast, reduced_ast, tokenizing, parsing, eval, typechecking, symbol_table, source_map};
+
+pub type SymbolTableHandle = Rc<RefCell<symbol_table::SymbolTable>>;
+pub type SourceMapHandle = Rc<RefCell<source_map::SourceMap>>;
 
 /// All the state necessary to parse and execute a Schala program are stored in this struct.
 /// `state` represents the execution state for the AST-walking interpreter, the other fields
 /// should be self-explanatory.
 pub struct Schala {
   source_reference: SourceReference,
+  source_map: SourceMapHandle,
   state: eval::State<'static>,
-  symbol_table: Rc<RefCell<symbol_table::SymbolTable>>,
+  symbol_table: SymbolTableHandle,
+  resolver: crate::scope_resolution::ScopeResolver<'static>,
   type_context: typechecking::TypeContext<'static>,
-  active_parser: Option<parsing::Parser>,
+  active_parser: parsing::Parser,
 }
 
 impl Schala {
@@ -34,13 +39,17 @@ impl Schala {
 impl Schala {
   /// Creates a new Schala environment *without* any prelude.
   fn new_blank_env() -> Schala {
-    let symbols = Rc::new(RefCell::new(symbol_table::SymbolTable::new()));
+    let source_map = Rc::new(RefCell::new(source_map::SourceMap::new()));
+    let symbols = Rc::new(RefCell::new(symbol_table::SymbolTable::new(source_map.clone())));
     Schala {
+      //TODO maybe these can be the same structure
       source_reference: SourceReference::new(),
       symbol_table: symbols.clone(),
-      state: eval::State::new(symbols),
+      source_map: source_map.clone(),
+      resolver: crate::scope_resolution::ScopeResolver::new(symbols.clone()),
+      state: eval::State::new(),
       type_context: typechecking::TypeContext::new(),
-      active_parser: None,
+      active_parser: parsing::Parser::new(source_map)
     }
   }
 
@@ -51,7 +60,10 @@ impl Schala {
     let mut s = Schala::new_blank_env();
 
     let request = ComputationRequest { source: prelude, debug_requests: HashSet::default() };
-    s.run_computation(request);
+    let response = s.run_computation(request);
+    if let Err(msg) = response.main_output {
+      panic!("Error in prelude, panicking: {}", msg);
+    }
     s
   }
 
@@ -94,10 +106,10 @@ fn tokenizing(input: &str, _handle: &mut Schala, comp: Option<&mut PassDebugArti
 }
 
 fn parsing(input: Vec<tokenizing::Token>, handle: &mut Schala, comp: Option<&mut PassDebugArtifact>) -> Result<ast::AST, String> {
-  use crate::parsing::Parser;
   use ParsingDebugType::*;
 
-  let mut parser = handle.active_parser.take().unwrap_or_else(|| Parser::new(input));
+  let ref mut parser = handle.active_parser;
+  parser.add_new_tokens(input);
   let ast = parser.parse();
 
   comp.map(|comp| {
@@ -115,13 +127,13 @@ fn parsing(input: Vec<tokenizing::Token>, handle: &mut Schala, comp: Option<&mut
     };
     comp.add_artifact(debug_info);
   });
-  ast.map_err(|err| format_parse_error(err, handle))
+  ast.map_err(|err| format_parse_error(err, &handle.source_reference))
 }
 
-fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String {
+fn format_parse_error(error: parsing::ParseError, source_reference: &SourceReference) -> String {
-  let line_num = error.token.line_num;
+  let line_num = error.token.location.line_num;
-  let ch = error.token.char_num;
+  let ch = error.token.location.char_num;
-  let line_from_program = handle.source_reference.get_line(line_num);
+  let line_from_program = source_reference.get_line(line_num);
   let location_pointer = format!("{}^", " ".repeat(ch));
 
   let line_num_digits = format!("{}", line_num).chars().count();
@@ -151,9 +163,7 @@ fn symbol_table(input: ast::AST, handle: &mut Schala, comp: Option<&mut PassDebu
 }
 
 fn scope_resolution(mut input: ast::AST, handle: &mut Schala, _com: Option<&mut PassDebugArtifact>) -> Result<ast::AST, String> {
-  let mut symbol_table = handle.symbol_table.borrow_mut();
+  let () = handle.resolver.resolve(&mut input)?;
-  let mut resolver = crate::scope_resolution::ScopeResolver::new(&mut symbol_table);
-  let () = resolver.resolve(&mut input)?;
   Ok(input)
 }
 

schala-lang/language/src/scope_resolution.rs

@@ -1,26 +1,28 @@
 use std::rc::Rc;
 
-use crate::symbol_table::{SymbolTable, ScopeSegment, FullyQualifiedSymbolName};
+use crate::schala::SymbolTableHandle;
+use crate::symbol_table::{ScopeSegment, FullyQualifiedSymbolName};
 use crate::ast::*;
 use crate::util::ScopeStack;
 
 type FQSNPrefix = Vec<ScopeSegment>;
-type NameScopeStack<'t> = ScopeStack<'t, Rc<String>, FQSNPrefix>;
 
 pub struct ScopeResolver<'a> {
-  symbol_table: &'a mut SymbolTable,
+  symbol_table_handle: SymbolTableHandle,
   name_scope_stack: ScopeStack<'a, Rc<String>, FQSNPrefix>,
 }
 
 impl<'a> ASTVisitor for ScopeResolver<'a> {
+  //TODO need to un-insert these - maybe need to rethink visitor
   fn import(&mut self, import_spec: &ImportSpecifier) {
+    let ref symbol_table = self.symbol_table_handle.borrow();
     let ImportSpecifier { ref path_components, ref imported_names, .. } = &import_spec;
     match imported_names {
       ImportedNames::All => {
         let prefix = FullyQualifiedSymbolName(path_components.iter().map(|c| ScopeSegment {
           name: c.clone(),
         }).collect());
-        let members = self.symbol_table.lookup_children_of_fqsn(&prefix);
+        let members = symbol_table.lookup_children_of_fqsn(&prefix);
         for member in members.into_iter() {
           let local_name = member.0.last().unwrap().name.clone();
           self.name_scope_stack.insert(local_name.clone(), member.0);
@@ -45,15 +47,17 @@ impl<'a> ASTVisitor for ScopeResolver<'a> {
   }
 
   fn qualified_name(&mut self, qualified_name: &QualifiedName) {
+    let ref mut symbol_table = self.symbol_table_handle.borrow_mut();
     let fqsn = self.lookup_name_in_scope(&qualified_name);
     let ref id = qualified_name.id;
-    self.symbol_table.map_id_to_fqsn(id, fqsn);
+    symbol_table.map_id_to_fqsn(id, fqsn);
   }
 
   fn named_struct(&mut self, name: &QualifiedName, _fields: &Vec<(Rc<String>, Expression)>) {
+    let ref mut symbol_table = self.symbol_table_handle.borrow_mut();
     let ref id = name.id;
     let fqsn = self.lookup_name_in_scope(&name);
-    self.symbol_table.map_id_to_fqsn(id, fqsn);
+    symbol_table.map_id_to_fqsn(id, fqsn);
   }
 
   fn pattern(&mut self, pat: &Pattern) {
@@ -70,9 +74,9 @@ impl<'a> ASTVisitor for ScopeResolver<'a> {
 }
 
 impl<'a> ScopeResolver<'a> {
-  pub fn new(symbol_table: &'a mut SymbolTable) -> ScopeResolver {
+  pub fn new(symbol_table_handle: SymbolTableHandle) -> ScopeResolver<'static> {
     let name_scope_stack = ScopeStack::new(None);
-    ScopeResolver { symbol_table, name_scope_stack }
+    ScopeResolver { symbol_table_handle, name_scope_stack }
   }
   pub fn resolve(&mut self, ast: &mut AST) -> Result<(), String> {
     walk_ast(self, ast);
@@ -97,10 +101,11 @@ impl<'a> ScopeResolver<'a> {
 
   /// this might be a variable or a pattern. if a variable, set to none
   fn qualified_name_in_pattern(&mut self, qualified_name: &QualifiedName) {
+    let ref mut symbol_table = self.symbol_table_handle.borrow_mut();
     let ref id = qualified_name.id;
     let fqsn = self.lookup_name_in_scope(qualified_name);
-    if self.symbol_table.lookup_by_fqsn(&fqsn).is_some() {
+    if symbol_table.lookup_by_fqsn(&fqsn).is_some() {
-      self.symbol_table.map_id_to_fqsn(&id, fqsn);
+      symbol_table.map_id_to_fqsn(&id, fqsn);
     }
   }
 }

schala-lang/language/src/source_map.rs

@@ -0,0 +1,39 @@
+use std::collections::HashMap;
+use std::fmt;
+
+use crate::ast::ItemId;
+
+pub type LineNumber = usize;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Location {
+  pub line_num: LineNumber,
+  pub char_num: usize,
+}
+
+impl fmt::Display for Location {
+  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    write!(f, "{}:{}", self.line_num, self.char_num)
+  }
+}
+
+pub struct SourceMap {
+  map: HashMap<ItemId, Location>
+}
+
+impl SourceMap {
+  pub fn new() -> SourceMap {
+    SourceMap { map: HashMap::new() }
+  }
+
+  pub fn add_location(&mut self, id: &ItemId, loc: Location) {
+    self.map.insert(id.clone(), loc);
+  }
+
+  pub fn lookup(&self, id: &ItemId) -> Option<Location> {
+    match self.map.get(id) {
+      Some(loc) => Some(loc.clone()),
+      None => None
+    }
+  }
+}

schala-lang/language/src/symbol_table.rs

@@ -4,8 +4,10 @@ use std::rc::Rc;
 use std::fmt;
 use std::fmt::Write;
 
+use crate::schala::SourceMapHandle;
+use crate::source_map::{SourceMap, LineNumber};
 use crate::ast;
-use crate::ast::{ItemId, TypeBody, TypeSingletonName, Signature, Statement, StatementKind};
+use crate::ast::{ItemId, TypeBody, TypeSingletonName, Signature, Statement, StatementKind, ModuleSpecifier};
 use crate::typechecking::TypeName;
 
 
@@ -15,7 +17,7 @@ macro_rules! fqsn {
     {
       let mut vec = vec![];
       $(
-        vec.push(ScopeSegment::new(Rc::new($name.to_string())));
+        vec.push(crate::symbol_table::ScopeSegment::new(std::rc::Rc::new($name.to_string())));
       )*
       FullyQualifiedSymbolName(vec)
     }
@@ -26,8 +28,35 @@ mod symbol_trie;
 use symbol_trie::SymbolTrie;
 mod test;
 
-type LineNumber = u32;
+/// Keeps track of what names were used in a given namespace. Call try_register to add a name to
-type SymbolTrackTable = HashMap<Rc<String>, LineNumber>;
+/// the table, or report an error if a name already exists.
+struct DuplicateNameTrackTable {
+  table: HashMap<Rc<String>, LineNumber>,
+}
+
+impl DuplicateNameTrackTable {
+  fn new() -> DuplicateNameTrackTable {
+    DuplicateNameTrackTable { table: HashMap::new() }
+  }
+
+  fn try_register(&mut self, name: &Rc<String>, id: &ItemId, source_map: &SourceMap) -> Result<(), LineNumber> {
+    match self.table.entry(name.clone()) {
+      Entry::Occupied(o) => {
+        let line_number = o.get();
+        Err(*line_number)
+      },
+      Entry::Vacant(v) => {
+        let line_number = if let Some(loc) = source_map.lookup(id) {
+          loc.line_num
+        } else {
+          0
+        };
+        v.insert(line_number);
+        Ok(())
+      }
+    }
+  }
+}
 
 #[derive(PartialEq, Eq, Hash, Debug, Clone, PartialOrd, Ord)]
 pub struct FullyQualifiedSymbolName(pub Vec<ScopeSegment>);
@@ -63,15 +92,16 @@ impl ScopeSegment {
 
 //cf. p. 150 or so of Language Implementation Patterns
 pub struct SymbolTable {
+  source_map_handle: SourceMapHandle,
   symbol_path_to_symbol: HashMap<FullyQualifiedSymbolName, Symbol>,
   id_to_fqsn: HashMap<ItemId, FullyQualifiedSymbolName>,
   symbol_trie: SymbolTrie,
 }
 
-//TODO add various types of lookups here, maybe multiple hash tables internally?
 impl SymbolTable {
-  pub fn new() -> SymbolTable {
+  pub fn new(source_map_handle: SourceMapHandle) -> SymbolTable {
     SymbolTable {
+      source_map_handle,
       symbol_path_to_symbol: HashMap::new(),
       id_to_fqsn: HashMap::new(),
       symbol_trie: SymbolTrie::new()
@@ -161,49 +191,51 @@ impl SymbolTable {
   fn add_symbols_from_scope<'a>(&'a mut self, statements: &Vec<Statement>, scope_name_stack: &mut Vec<ScopeSegment>) -> Result<(), String> {
     use self::ast::Declaration::*;
 
-    fn insert_and_check_duplicate_symbol(table: &mut SymbolTrackTable, name: &Rc<String>) -> Result<(), String> {
+    let mut seen_identifiers = DuplicateNameTrackTable::new();
-      match table.entry(name.clone()) {
+    let mut seen_modules = DuplicateNameTrackTable::new();
-        Entry::Occupied(o) => {
-          let line_number = o.get(); //TODO make this actually work
-          Err(format!("Duplicate definition: {}. It's already defined at {}", name, line_number))
-        },
-        Entry::Vacant(v) => {
-          let line_number = 0; //TODO should work
-          v.insert(line_number);
-          Ok(())
-        }
-      }
-    }
-
-    let mut seen_identifiers: SymbolTrackTable = HashMap::new();
 
     for statement in statements.iter() {
-      if let Statement { kind: StatementKind::Declaration(decl), .. } = statement {
+      match statement {
-        match decl {
+        Statement { kind: StatementKind::Declaration(decl), id } => {
-          FuncSig(ref signature) => {
+          match decl {
-            insert_and_check_duplicate_symbol(&mut seen_identifiers, &signature.name)?;
+            FuncSig(ref signature) => {
-            self.add_function_signature(signature, scope_name_stack)?
+              seen_identifiers.try_register(&signature.name, &id, &self.source_map_handle.borrow())
+                .map_err(|line| format!("Duplicate function definition: {}. It's already defined at {}", signature.name, line))?;
+              self.add_function_signature(signature, scope_name_stack)?
+            }
+            FuncDecl(ref signature, ref body) => {
+              seen_identifiers.try_register(&signature.name, &id, &self.source_map_handle.borrow())
+                .map_err(|line| format!("Duplicate function definition: {}. It's already defined at {}", signature.name, line))?;
+              self.add_function_signature(signature, scope_name_stack)?;
+              scope_name_stack.push(ScopeSegment{
+                name: signature.name.clone(),
+              });
+              let output = self.add_symbols_from_scope(body, scope_name_stack);
+              scope_name_stack.pop();
+              output?
+            },
+            TypeDecl { name, body, mutable } => {
+              seen_identifiers.try_register(&name.name, &id, &self.source_map_handle.borrow())
+                .map_err(|line| format!("Duplicate type definition: {}. It's already defined at {}", name.name, line))?;
+              self.add_type_decl(name, body, mutable, scope_name_stack)?
+            },
+            Binding { name, .. } => {
+              seen_identifiers.try_register(&name, &id, &self.source_map_handle.borrow())
+                .map_err(|line| format!("Duplicate variable definition: {}. It's already defined at {}", name, line))?;
+              self.add_new_symbol(name, scope_name_stack, SymbolSpec::Binding);
+            }
+            _ => ()
           }
-          FuncDecl(ref signature, ref body) => {
+        },
-            insert_and_check_duplicate_symbol(&mut seen_identifiers, &signature.name)?;
+        Statement { kind: StatementKind::Module(ModuleSpecifier { name, contents}), id } => {
-            self.add_function_signature(signature, scope_name_stack)?;
+          seen_modules.try_register(&name, &id, &self.source_map_handle.borrow())
-            scope_name_stack.push(ScopeSegment{
+            .map_err(|line| format!("Duplicate module definition: {}. It's already defined at {}", name, line))?;
-              name: signature.name.clone(),
+          scope_name_stack.push(ScopeSegment { name: name.clone() });
-            });
+          let output = self.add_symbols_from_scope(contents, scope_name_stack);
-            let output = self.add_symbols_from_scope(body, scope_name_stack);
+          scope_name_stack.pop();
-            let _ = scope_name_stack.pop();
+          output?
-            output?
+        },
-          },
+        _ => ()
-          TypeDecl { name, body, mutable } => {
-            insert_and_check_duplicate_symbol(&mut seen_identifiers, &name.name)?;
-            self.add_type_decl(name, body, mutable, scope_name_stack)?
-          },
-          Binding { name, .. } => {
-            insert_and_check_duplicate_symbol(&mut seen_identifiers, name)?;
-            self.add_new_symbol(name, scope_name_stack, SymbolSpec::Binding);
-          }
-          _ => ()
-        }
       }
     }
     Ok(())

schala-lang/language/src/symbol_table/symbol_trie.rs

@@ -1,6 +1,5 @@
-use std::rc::Rc;
 use radix_trie::{Trie, TrieCommon, TrieKey};
-use super::{ScopeSegment, FullyQualifiedSymbolName};
+use super::FullyQualifiedSymbolName;
 use std::hash::{Hasher, Hash};
 use std::collections::hash_map::DefaultHasher;
 

schala-lang/language/src/symbol_table/test.rs

@@ -1,18 +1,25 @@
 #![cfg(test)]
+use std::cell::RefCell;
+use std::rc::Rc;
 
-#[macro_use]
 use super::*;
 use crate::util::quick_ast;
 
+fn add_symbols_from_source(src: &str) -> (SymbolTable, Result<(), String>) {
+  let (ast, source_map) = quick_ast(src);
+  let source_map = Rc::new(RefCell::new(source_map));
+  let mut symbol_table = SymbolTable::new(source_map);
+  let result = symbol_table.add_top_level_symbols(&ast);
+  (symbol_table, result)
+}
+
 macro_rules! values_in_table {
-  ($source:literal, $single_value:expr) => {
+  ($source:expr, $single_value:expr) => {
-    values_in_table!($source | $single_value);
+    values_in_table!($source => $single_value);
   };
-  ($source:literal | $( $value:expr ),* ) => {
+  ($source:expr => $( $value:expr ),* ) => {
     {
-    let mut symbol_table = SymbolTable::new();
+    let (symbol_table, _) = add_symbols_from_source($source);
-    let ast = quick_ast($source);
-    symbol_table.add_top_level_symbols(&ast).unwrap();
     $(
       match symbol_table.lookup_by_fqsn($value) {
         Some(_spec) => (),
@@ -26,40 +33,39 @@ macro_rules! values_in_table {
 #[test]
 fn basic_symbol_table() {
   values_in_table! { "let a = 10; fn b() { 20 }", &fqsn!("b"; tr) };
-  values_in_table! { "type Option<T> = Some(T) | None" |
+  values_in_table! { "type Option<T> = Some(T) | None" =>
     &fqsn!("Option"; tr),
     &fqsn!("Option"; ty, "Some"; tr),
     &fqsn!("Option"; ty, "None"; tr) };
 }
 
 #[test]
-fn no_duplicates() {
+fn no_function_definition_duplicates() {
   let source = r#"
   fn a() { 1 }
   fn b() { 2 }
   fn a() { 3 }
   "#;
-  let mut symbol_table = SymbolTable::new();
+  let (_, output) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
+  assert!(output.unwrap_err().contains("Duplicate function definition: a"))
-  let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
-  assert!(output.contains("Duplicate"))
 }
 
 #[test]
-fn no_duplicates_2() {
+fn no_variable_definition_duplicates() {
   let source = r#"
-  let a = 20;
+  let x = 9
-  let q = 39;
+  let a = 20
-  let a = 30;
+  let q = 39
+  let a = 30
   "#;
-  let mut symbol_table = SymbolTable::new();
+  let (_, output) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
+  let output = output.unwrap_err();
-  let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
+  assert!(output.contains("Duplicate variable definition: a"));
-  assert!(output.contains("Duplicate"))
+  assert!(output.contains("already defined at 2"));
 }
 
 #[test]
-fn no_duplicates_3() {
+fn no_variable_definition_duplicates_in_function() {
   let source = r#"
   fn a() {
     let a = 20
@@ -68,14 +74,13 @@ fn no_duplicates_3() {
   }
 
   fn q() {
+    let a = 29
     let x = 30
     let x = 33
   }
   "#;
-  let mut symbol_table = SymbolTable::new();
+  let (_, output) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
+  assert!(output.unwrap_err().contains("Duplicate variable definition: x"))
-  let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
-  assert!(output.contains("Duplicate"))
 }
 
 #[test]
@@ -88,9 +93,7 @@ fn dont_falsely_detect_duplicates() {
   }
   let q = 39;
   "#;
-  let mut symbol_table = SymbolTable::new();
+  let (symbol_table, _) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
-  symbol_table.add_top_level_symbols(&ast).unwrap();
   assert!(symbol_table.lookup_by_fqsn(&fqsn!["a"; tr]).is_some());
   assert!(symbol_table.lookup_by_fqsn(&fqsn!["some_func"; fn, "a";tr]).is_some());
 }
@@ -104,9 +107,7 @@ fn inner_func(arg) {
 }
 x + inner_func(x)
 }"#;
-  let mut symbol_table = SymbolTable::new();
+  let (symbol_table, _) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
-  symbol_table.add_top_level_symbols(&ast).unwrap();
   assert!(symbol_table.lookup_by_fqsn(&fqsn!("outer_func"; tr)).is_some());
   assert!(symbol_table.lookup_by_fqsn(&fqsn!("outer_func"; fn, "inner_func"; tr)).is_some());
 }
@@ -126,9 +127,7 @@ fn second_inner_func() {
 
 inner_func(x)
 }"#;
-  let mut symbol_table = SymbolTable::new();
+  let (symbol_table, _) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
-  symbol_table.add_top_level_symbols(&ast).unwrap();
   assert!(symbol_table.lookup_by_fqsn(&fqsn!("outer_func"; tr)).is_some());
   assert!(symbol_table.lookup_by_fqsn(&fqsn!("outer_func"; fn, "inner_func"; tr)).is_some());
   assert!(symbol_table.lookup_by_fqsn(&fqsn!("outer_func"; fn, "second_inner_func"; tr)).is_some());
@@ -152,8 +151,43 @@ fn second_inner_func() {
 
 inner_func(x)
 }"#;
-  let mut symbol_table = SymbolTable::new();
+  let (_, output) = add_symbols_from_source(source);
-  let ast = quick_ast(source);
+  assert!(output.unwrap_err().contains("Duplicate"))
-  let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
+}
-  assert!(output.contains("Duplicate"))
+
+#[test]
+fn modules() {
+  let source = r#"
+module stuff {
+  fn item() {
+  }
+}
+
+fn item()
+"#;
+  values_in_table! { source  =>
+    &fqsn!("item"; tr),
+    &fqsn!("stuff"; tr, "item"; tr)
+  };
+}
+
+#[test]
+fn duplicate_modules() {
+  let source = r#"
+    module q {
+      fn foo() { 4 }
+    }
+
+    module a {
+      fn foo() { 334 }
+    }
+
+    module a {
+      fn foo() { 256.1 }
+    }
+    "#;
+  let (_, output) = add_symbols_from_source(source);
+  let output = output.unwrap_err();
+  assert!(output.contains("Duplicate module"));
+  assert!(output.contains("already defined at 5"));
 }

schala-lang/language/src/tokenizing.rs

@@ -4,6 +4,8 @@ use std::rc::Rc;
 use std::iter::{Iterator, Peekable};
 use std::fmt;
 
+use crate::source_map::Location;
+
 #[derive(Debug, PartialEq, Clone)]
 pub enum TokenKind {
   Newline, Semicolon,
@@ -19,7 +21,10 @@ pub enum TokenKind {
 
   Operator(Rc<String>),
   DigitGroup(Rc<String>), HexLiteral(Rc<String>), BinNumberSigil,
-  StrLiteral(Rc<String>),
+  StrLiteral {
+    s: Rc<String>,
+    prefix: Option<Rc<String>>
+  },
   Identifier(Rc<String>),
   Keyword(Kw),
 
@@ -35,7 +40,7 @@ impl fmt::Display for TokenKind {
       &Operator(ref s) => write!(f, "Operator({})", **s),
       &DigitGroup(ref s) => write!(f, "DigitGroup({})", s),
       &HexLiteral(ref s) => write!(f, "HexLiteral({})", s),
-      &StrLiteral(ref s) => write!(f, "StrLiteral({})", s),
+      &StrLiteral {ref s, .. } => write!(f, "StrLiteral({})", s),
       &Identifier(ref s) => write!(f, "Identifier({})", s),
       &Error(ref s) => write!(f, "Error({})", s),
       other => write!(f, "{:?}", other),
@@ -86,11 +91,10 @@ lazy_static! {
     };
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Token {
   pub kind: TokenKind,
-  pub line_num: usize,
+  pub location: Location,
-  pub char_num: usize
 }
 
 impl Token {
@@ -101,7 +105,7 @@ impl Token {
     }
   }
   pub fn to_string_with_metadata(&self) -> String {
-    format!("{}(L:{},c:{})", self.kind, self.line_num, self.char_num)
+    format!("{}({})", self.kind, self.location)
   }
 
   pub fn get_kind(&self) -> TokenKind {
@@ -162,14 +166,15 @@ pub fn tokenize(input: &str) -> Vec<Token> {
       '(' => LParen, ')' => RParen,
       '{' => LCurlyBrace, '}' => RCurlyBrace,
       '[' => LSquareBracket, ']' => RSquareBracket,
-      '"' => handle_quote(&mut input),
+      '"' => handle_quote(&mut input, None),
       '\\' => Backslash,
       c if c.is_digit(10) => handle_digit(c, &mut input),
       c if c.is_alphabetic() || c == '_' => handle_alphabetic(c, &mut input),
       c if is_operator(&c) => handle_operator(c, &mut input),
       unknown => Error(format!("Unexpected character: {}", unknown)),
     };
-    tokens.push(Token { kind: cur_tok_kind, line_num, char_num });
+    let location = Location { line_num, char_num };
+    tokens.push(Token { kind: cur_tok_kind, location });
   }
   tokens
 }
@@ -189,7 +194,7 @@ fn handle_digit(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) ->
   }
 }
 
-fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
+fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>, quote_prefix: Option<&str>) -> TokenKind {
   let mut buf = String::new();
   loop {
     match input.next().map(|(_, _, c)| { c }) {
@@ -211,7 +216,7 @@ fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind
       None => return TokenKind::Error(format!("Unclosed string")),
     }
   }
-  TokenKind::StrLiteral(Rc::new(buf))
+  TokenKind::StrLiteral { s: Rc::new(buf), prefix: quote_prefix.map(|s| Rc::new(s.to_string())) }
 }
 
 fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
@@ -223,6 +228,10 @@ fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>
 
   loop {
     match input.peek().map(|&(_, _, c)| { c }) {
+      Some(c) if c == '"' => {
+        input.next();
+        return handle_quote(input, Some(&buf));
+      },
       Some(c) if c.is_alphanumeric() || c == '_' => {
         input.next();
         buf.push(c);
@@ -323,4 +332,13 @@ mod schala_tokenizer_tests {
     let token_kinds: Vec<TokenKind> = tokenize("1 `plus` 2").into_iter().map(move |t| t.kind).collect();
     assert_eq!(token_kinds, vec![digit!("1"), op!("plus"), digit!("2")]);
   }
+
+  #[test]
+  fn string_literals() {
+    let token_kinds: Vec<TokenKind> = tokenize(r#""some string""#).into_iter().map(move |t| t.kind).collect();
+    assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]);
+
+    let token_kinds: Vec<TokenKind> = tokenize(r#"b"some bytestring""#).into_iter().map(move |t| t.kind).collect();
+    assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]);
+  }
 }

schala-lang/language/src/typechecking.rs

@@ -14,6 +14,7 @@ pub struct TypeData {
 }
 
 impl TypeData {
+  #[allow(dead_code)]
   pub fn new() -> TypeData {
     TypeData { ty: None }
   }
@@ -276,6 +277,7 @@ impl<'a> TypeContext<'a> {
       StatementKind::Expression(e) => self.expr(e),
       StatementKind::Declaration(decl) => self.decl(&decl),
       StatementKind::Import(_) => Ok(ty!(Unit)),
+      StatementKind::Module(_) => Ok(ty!(Unit)),
     }
   }
 
@@ -460,7 +462,7 @@ mod typechecking_tests {
   macro_rules! assert_type_in_fresh_context {
     ($string:expr, $type:expr) => {
       let mut tc = TypeContext::new();
-      let ref ast = crate::util::quick_ast($string);
+      let (ref ast, _) = crate::util::quick_ast($string);
       let ty = tc.typecheck(ast).unwrap();
       assert_eq!(ty, $type)
     }

schala-lang/language/src/util.rs

@@ -48,10 +48,18 @@ impl<'a, T, V> ScopeStack<'a, T, V> where T: Hash + Eq {
 
 /// this is intended for use in tests, and does no error-handling whatsoever
 #[allow(dead_code)]
-pub fn quick_ast(input: &str) -> crate::ast::AST {
+pub fn quick_ast(input: &str) -> (crate::ast::AST, crate::source_map::SourceMap) {
+  use std::cell::RefCell;
+  use std::rc::Rc;
+
+  let source_map = crate::source_map::SourceMap::new();
+  let source_map_handle = Rc::new(RefCell::new(source_map));
   let tokens = crate::tokenizing::tokenize(input);
-  let mut parser = crate::parsing::Parser::new(tokens);
+  let mut parser = crate::parsing::Parser::new(source_map_handle.clone());
-  parser.parse().unwrap()
+  parser.add_new_tokens(tokens);
+  let output = parser.parse();
+  std::mem::drop(parser);
+  (output.unwrap(), Rc::try_unwrap(source_map_handle).map_err(|_| ()).unwrap().into_inner())
 }
 
 #[allow(unused_macros)]

schala-repl/src/repl/command_tree.rs

@@ -49,8 +49,8 @@ impl CommandTree {
   }
   pub fn get_help(&self) -> &str {
     match self {
-      CommandTree::Terminal { help_msg, ..} => help_msg.as_ref().map(|s| s.as_str()).unwrap_or(""),
+      CommandTree::Terminal { help_msg, ..} => help_msg.as_ref().map(|s| s.as_str()).unwrap_or("<no help text provided>"),
-      CommandTree::NonTerminal { help_msg, .. } => help_msg.as_ref().map(|s| s.as_str()).unwrap_or(""),
+      CommandTree::NonTerminal { help_msg, .. } => help_msg.as_ref().map(|s| s.as_str()).unwrap_or("<no help text provided>"),
       CommandTree::Top(_) => ""
     }
   }

schala-repl/src/repl/help.rs

@@ -13,7 +13,12 @@ pub fn help(repl: &mut Repl, arguments: &[&str]) -> InterpreterDirectiveOutput {
         None => format!("Directive `{}` not found", commands.last().unwrap()),
         Some(dir) => {
           let mut buf = String::new();
-          writeln!(buf, "`{}` - {}", dir.get_cmd(), dir.get_help()).unwrap();
+          let cmd = dir.get_cmd();
+          let children = dir.get_children();
+          writeln!(buf, "`{}` - {}", cmd, dir.get_help()).unwrap();
+          for sub in children.iter() {
+            writeln!(buf, "\t`{} {}` - {}", cmd, sub.get_cmd(), sub.get_help()).unwrap();
+          }
           buf
         }
       })