From dd2b4893a4720b2403ae3072ed95bdbd75c96f49 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Thu, 31 Dec 2015 22:20:59 -0800
Subject: [PATCH] Get rid of Separator token

Have separate newline and semicolon tokens
---
 src/parser.rs    |  2 +-
 src/tokenizer.rs | 17 ++++++++---------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/parser.rs b/src/parser.rs
index 9ffd62a..e53a7f6 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -99,7 +99,7 @@ impl Parser {
 
     fn parse(&mut self) -> ParseResult<AST> {
         let r = self.expr();
-        try!(self.expect(Token::Separator));
+        try!(self.expect(Token::Newline));
         try!(self.expect(Token::EOF));
         r
     }
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 082b32f..e07f258 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1,7 +1,8 @@
 #[derive(Debug, Clone, PartialEq)]
 pub enum Token {
     EOF,
-    Separator,
+    Newline,
+    Semicolon,
     LParen,
     RParen,
     Comma,
@@ -62,12 +63,10 @@ pub fn tokenize(input: &str) -> Vec<Token> {
                     break;
                 }
             }
-        } else if c == ';' || c == '\n' {
-            if let Some(&Token::Separator) = tokens.last() {
-                //skip past multiple separators
-            } else {
-                tokens.push(Token::Separator);
-            }
+        } else if c == ';' {
+            tokens.push(Token::Semicolon);
+        } else if c == '\n' {
+            tokens.push(Token::Newline);
         } else if c == '(' {
             tokens.push(Token::LParen);
         } else if c == ')' {
@@ -123,12 +122,12 @@ mod tests {
     fn tokeniziation_tests() {
         let t1 = "let a = 3\n";
         assert_eq!(format!("{:?}", tokenize(t1)),
-            "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]");
+            "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Newline, EOF]");
 
         // this is intentional
         let t2 = "a + b*c\n";
         assert_eq!(format!("{:?}", tokenize(t2)),
-            "[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Separator, EOF]");
+            "[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Newline, EOF]");
 
     }
 }