Get rid of Separator token

Have separate newline and semicolon tokens

src/parser.rs

@@ -99,7 +99,7 @@ impl Parser {
 
     fn parse(&mut self) -> ParseResult<AST> {
         let r = self.expr();
-        try!(self.expect(Token::Separator));
+        try!(self.expect(Token::Newline));
         try!(self.expect(Token::EOF));
         r
     }

src/tokenizer.rs

@@ -1,7 +1,8 @@
 #[derive(Debug, Clone, PartialEq)]
 pub enum Token {
     EOF,
-    Separator,
+    Newline,
+    Semicolon,
     LParen,
     RParen,
     Comma,
@@ -62,12 +63,10 @@ pub fn tokenize(input: &str) -> Vec<Token> {
                     break;
                 }
             }
-        } else if c == ';' || c == '\n' {
+        } else if c == ';' {
-            if let Some(&Token::Separator) = tokens.last() {
+            tokens.push(Token::Semicolon);
-                //skip past multiple separators
+        } else if c == '\n' {
-            } else {
+            tokens.push(Token::Newline);
-                tokens.push(Token::Separator);
-            }
         } else if c == '(' {
             tokens.push(Token::LParen);
         } else if c == ')' {
@@ -123,12 +122,12 @@ mod tests {
     fn tokeniziation_tests() {
         let t1 = "let a = 3\n";
         assert_eq!(format!("{:?}", tokenize(t1)),
-            "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]");
+            "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Newline, EOF]");
 
         // this is intentional
         let t2 = "a + b*c\n";
         assert_eq!(format!("{:?}", tokenize(t2)),
-            "[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Separator, EOF]");
+            "[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Newline, EOF]");
 
     }
 }