From 169e662049d23ab7031ff3e31fc8db82b20446cf Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Fri, 24 Jul 2015 15:46:53 -0700
Subject: [PATCH] Collapse Separator tokens

only ever gonna be one in a row
---
 src/tokenizer.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index f8aa512..dd03a68 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -60,7 +60,10 @@ pub fn tokenize(input: &str) -> Vec<Token> {
                 }
             }
         } else if c == ';' || c == '\n' {
-            tokens.push(Token::Separator);
+            if let Some(&Token::Separator) = tokens.last() {
+            } else {
+                tokens.push(Token::Separator);
+            }
         } else if c == '(' {
             tokens.push(Token::LParen);
         } else if c == ')' {