greg
/
schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Tightened tokenization tests

This commit is contained in:
greg 2017-01-02 01:54:46 -08:00
parent 4f8ff35d0f
commit 48343d3fad
1 changed files with 32 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/tokenizer.rs
View File

@ -144,42 +144,53 @@ fn tokenize_identifier(c: char, iter: &mut Peekable<Chars>) -> Result<Token, Tok
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*;
use super::Token::*;
macro_rules! tokentest { macro_rules! token_test {
($input:expr, $output:expr) => { ($input: expr, $output: pat, $ifexpr: expr) => {
{
let tokens = tokenize($input).unwrap(); let tokens = tokenize($input).unwrap();
assert_eq!(format!("{:?}", tokens), $output); match tokens[..] {
$output if $ifexpr => (),
_ => panic!("Actual output: {:?}", tokens),
} }
} }
} }
use super::*;
#[test] #[test]
fn tokeniziation_tests() { fn basic_tokeniziation_tests() {
tokentest!("let a = 3\n", token_test!("let a = 3\n",
"[Keyword(Let), Identifier(\"a\"), Operator(Op(\"=\")), \ [Keyword(Kw::Let), Identifier(ref a), Operator(Op(ref b)), NumLiteral(3.0), Newline],
NumLiteral(3), Newline]"); a == "a" && b == "=");
tokentest!("2+1", token_test!("2+1",
"[NumLiteral(2), Operator(Op(\"+\")), NumLiteral(1)]"); [NumLiteral(2.0), Operator(Op(ref a)), NumLiteral(1.0)],
a == "+");
tokentest!("2 + 1", token_test!("2 + 1",
"[NumLiteral(2), Operator(Op(\"+\")), NumLiteral(1)]"); [NumLiteral(2.0), Operator(Op(ref a)), NumLiteral(1.0)],
a == "+");
tokentest!("2.3*49.2", token_test!("2.3*49.2",
"[NumLiteral(2.3), Operator(Op(\"*\")), NumLiteral(49.2)]"); [NumLiteral(2.3), Operator(Op(ref a)), NumLiteral(49.2)],
a == "*");
assert!(tokenize("2.4.5").is_err()); assert!(tokenize("2.4.5").is_err());
} }
#[test] #[test]
#[ignore] fn string_test() {
fn more_tokenization() { token_test!("null + \"a string\"",
// it would be nice to support complicated operators in a nice, haskell-ish way [Keyword(Kw::Null), Operator(Op(ref a)), StrLiteral(ref b)],
tokentest!("a *> b", a == "+" && b == "a string");
"[Identifier(\"a\"), Identifier(\"*>\"), Identifier(\"b\"), EOF]"); }
#[test]
fn operator_test() {
token_test!("a *> b",
[Identifier(ref a), Operator(Op(ref b)), Identifier(ref c)],
a == "a" && b == "*>" && c == "b");
} }
} }