From 2f7a1850dbd6a4c42088a84e09ac518e5c0ed5eb Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Thu, 7 Jan 2016 02:25:32 -0800
Subject: [PATCH] Finish tokenizing

---
 src/tokenizer.rs | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 5bb3aab..e0c3614 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -31,6 +31,18 @@ fn is_digit(c: &char) -> bool {
     c.is_digit(10)
 }
 
+fn ends_identifier(c: &char) -> bool {
+    let c = *c;
+    char::is_whitespace(c) ||
+    is_digit(&c) ||
+    c == ';' ||
+    c == '(' ||
+    c == ')' ||
+    c == ',' ||
+    c == '.' ||
+    c == ':'
+}
+
 pub fn tokenize(input: &str) -> Option<Vec<Token>> {
     use self::Token::*;
     let mut tokens = Vec::new();
@@ -67,7 +79,9 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
                 }
             }
             StrLiteral(buffer)
-        } else if is_digit(&c) {
+        } else if c == '.' && !iter.peek().map_or(false, |x| is_digit(x)) {
+            Period
+        } else if is_digit(&c) || c == '.' {
             let mut buffer = String::with_capacity(20);
             buffer.push(c);
             loop {
@@ -83,7 +97,21 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
                 Err(_) => return None
             }
         } else {
-            Identifier("DUMMY".to_string())
+            let mut buffer = String::with_capacity(20);
+            buffer.push(c);
+            loop {
+                if iter.peek().map_or(false, |x| ends_identifier(x)) {
+                    break;
+                } else {
+                    buffer.push(iter.next().unwrap());
+                }
+            }
+
+            match &buffer[..] {
+                "if" => Keyword(Kw::If),
+                "then" => Keyword(Kw::Then),
+                b => Identifier(b.to_string())
+            }
         };
 
         tokens.push(cur_tok);