diff --git a/src/schala_lang/tokenizing.rs b/src/schala_lang/tokenizing.rs index 38f9ded..e2a6c08 100644 --- a/src/schala_lang/tokenizing.rs +++ b/src/schala_lang/tokenizing.rs @@ -117,8 +117,8 @@ pub fn tokenize(input: &str) -> Vec { while let Some((line_idx, ch_idx, c)) = input.next() { let cur_tok_type = match c { - '/' => match input.peek() { - Some(&(_, _, '/')) => { + '/' => match input.peek().map(|t| t.2) { + Some('/') => { while let Some((_, _, c)) = input.next() { if c == '\n' { break; @@ -126,8 +126,22 @@ pub fn tokenize(input: &str) -> Vec { } continue; }, - Some(&(_, _, '*')) => { - continue + Some('*') => { + input.next(); + let mut comment_level = 1; + while let Some((_, _, c)) = input.next() { + if c == '*' && input.peek().map(|t| t.2) == Some('/') { + input.next(); + comment_level -= 1; + } else if c == '/' && input.peek().map(|t| t.2) == Some('*') { + input.next(); + comment_level += 1; + } + if comment_level == 0 { + break; + } + } + continue; }, _ => Slash }, @@ -251,7 +265,6 @@ mod schala_tokenizer_tests { macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } } macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } } - #[test] fn tokens() { let a = tokenize("let a: A = c ++ d"); @@ -265,4 +278,10 @@ mod schala_tokenizer_tests { let token_types: Vec = tokenize("4_8").into_iter().map(move |t| t.token_type).collect(); assert_eq!(token_types, vec![digit!("4"), Underscore, digit!("8")]); } + + #[test] + fn comments() { + let token_types: Vec = tokenize("1 + /* hella /* bro */ */ 2").into_iter().map(move |t| t.token_type).collect(); + assert_eq!(token_types, vec![digit!("1"), op!("+"), digit!("2")]); + } }