Swap over parser

schala-lang/language/source-files/prelude.schala

@@ -14,8 +14,8 @@ fn getline(arg) { }
 
 fn map(input: Option<T>, func: Func): Option<T> {
   if input {
-    is Option::Some(x) then Option::Some(func(x)),
-    is Option::None then Option::None,
+    is Option::Some(x) then Option::Some(func(x))
+    is Option::None then Option::None
   }
 }
 

schala-lang/language/src/schala.rs

@@ -19,7 +19,7 @@ pub struct Schala<'a> {
     /// Contains information for type-checking
     type_context: type_inference::TypeContext,
     /// Schala Parser
-    active_parser: parsing::Parser,
+    active_parser: parsing::new::Parser,
 
     /// Execution state for AST-walking interpreter
     eval_state: tree_walk_eval::State<'a>,
@@ -47,7 +47,7 @@ impl<'a> Schala<'a> {
             source_reference: SourceReference::new(),
             symbol_table: symbol_table::SymbolTable::new(),
             type_context: type_inference::TypeContext::new(),
-            active_parser: parsing::Parser::new(),
+            active_parser: parsing::new::Parser::new(),
             eval_state: tree_walk_eval::State::new(),
             timings: Vec::new(),
         }
@@ -74,18 +74,10 @@ impl<'a> Schala<'a> {
         self.timings = vec![];
         let sw = Stopwatch::start_new();
 
-        // 1st stage - tokenization
-        // TODO tokenize should return its own error type
-        let tokens = tokenizing::tokenize(source);
-        if let Some(err) = SchalaError::from_tokens(&tokens) {
-            return Err(err);
-        }
-
-        //2nd stage - parsing
-        self.active_parser.add_new_tokens(tokens);
+        self.source_reference.load_new_source(source);
         let ast = self
             .active_parser
-            .parse()
+            .parse(source)
             .map_err(|err| SchalaError::from_parse_error(err, &self.source_reference))?;
         self.timings.push(("parsing", sw.elapsed()));
 
@@ -133,6 +125,7 @@ impl SourceReference {
     }
 
     pub(crate) fn load_new_source(&mut self, source: &str) {
+        self.newline_offsets = vec![];
         for (offset, ch) in source.as_bytes().iter().enumerate() {
             if *ch == ('\n' as u8) {
                 self.newline_offsets.push(offset);
@@ -143,6 +136,11 @@ impl SourceReference {
 
     // (line_start, line_num, the string itself)
     pub fn get_line(&self, line: usize) -> (usize, usize, String) {
+
+        if self.newline_offsets.is_empty() {
+            return (0, 0, self.last_source.as_ref().cloned().unwrap());
+        }
+
         //TODO make sure this is utf8-safe
         let start_idx = match self.newline_offsets.binary_search(&line) {
             Ok(index) | Err(index) => index,
@@ -192,7 +190,6 @@ impl<'a> ProgrammingLanguageInterface for Schala<'a> {
 
     fn run_computation(&mut self, request: ComputationRequest<Self::Config>) -> ComputationResponse {
         let ComputationRequest { source, debug_requests: _, config: _ } = request;
-        self.source_reference.load_new_source(source);
         let sw = Stopwatch::start_new();
 
         let main_output =