From 9d6dc5a5f20b5c90da6906a30394f6e7c097064f Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sun, 26 Jul 2015 01:51:15 -0700
Subject: [PATCH] Tokenize periods separately

---
 src/tokenizer.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index a1e866c..a6e82fb 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -5,6 +5,7 @@ pub enum Token {
     LParen,
     RParen,
     Comma,
+    Period,
     NumLiteral(f64),
     StrLiteral(String),
     Identifier(String),
@@ -34,6 +35,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
             ';' => true,
             '(' => true,
             ')' => true,
+            '.' => true,
             _ => false
         }
     }
@@ -69,6 +71,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
             tokens.push(Token::RParen);
         } else if c == ',' {
             tokens.push(Token::Comma);
+        } else if c == '.' {
+            tokens.push(Token::Period);
         } else {
             let mut buffer = String::with_capacity(20);
             buffer.push(c);