diff --git a/src/tokenizer.rs b/src/tokenizer.rs index f5b19ab..b3f96ea 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -115,3 +115,15 @@ fn handle_identifier(identifier: String) -> Token { return Token::Keyword(keyword); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tokeniziation_tests() { + let t1 = "let a = 3\n"; + assert_eq!(format!("{:?}", tokenize(t1)), + "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]"); + } +}