From ecebbb2eaea4f4f434119a859bdb1d39a382f2b1 Mon Sep 17 00:00:00 2001 From: greg Date: Sat, 24 Mar 2018 18:38:28 -0700 Subject: [PATCH] Fix interspersing of newlines in tokenizer infra --- schala-lang/src/lib.rs | 1 - schala-lang/src/tokenizing.rs | 4 +++- source_files/schala/first.schala | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/schala-lang/src/lib.rs b/schala-lang/src/lib.rs index ca006a5..ca20e71 100644 --- a/schala-lang/src/lib.rs +++ b/schala-lang/src/lib.rs @@ -82,7 +82,6 @@ impl ProgrammingLanguageInterface for Schala { } }; - //symbol table match self.type_context.add_top_level_types(&ast) { Ok(()) => (), diff --git a/schala-lang/src/tokenizing.rs b/schala-lang/src/tokenizing.rs index e2a6c08..2692aeb 100644 --- a/schala-lang/src/tokenizing.rs +++ b/schala-lang/src/tokenizing.rs @@ -111,9 +111,11 @@ pub fn tokenize(input: &str) -> Vec { let mut tokens: Vec = Vec::new(); let mut input = input.lines().enumerate() + .intersperse((0, "\n")) .flat_map(|(line_idx, ref line)| { line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch)) - }).peekable(); + }) + .peekable(); while let Some((line_idx, ch_idx, c)) = input.next() { let cur_tok_type = match c { diff --git a/source_files/schala/first.schala b/source_files/schala/first.schala index a8cca94..25e958d 100644 --- a/source_files/schala/first.schala +++ b/source_files/schala/first.schala @@ -4,7 +4,7 @@ fn main() { a + b } -foo +//foo print(main())