Add tokenization test

src/tokenizer.rs

@@ -115,3 +115,15 @@ fn handle_identifier(identifier: String) -> Token {
 
     return Token::Keyword(keyword);
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tokeniziation_tests() {
+        let t1 = "let a = 3\n";
+        assert_eq!(format!("{:?}", tokenize(t1)),
+            "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]");
+    }
+}