From 1d0aa178f7b62a063b9a8d53955444dc4325597c Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Fri, 5 Jan 2018 11:03:58 +0100 Subject: [PATCH] Lex CRLF line endings properly (#292) --- src/lexer.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 6841bb7..cd98163 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -135,8 +135,8 @@ impl<'a> Lexer<'a> { static ref PAREN_R: Regex = token(r"[)]" ); static ref AT: Regex = token(r"@" ); static ref COMMA: Regex = token(r"," ); - static ref COMMENT: Regex = token(r"#([^!\n\r].*)?$" ); - static ref EOF: Regex = token(r"(?-m)$" ); + static ref COMMENT: Regex = token(r"#([^!\n\r]*)?\r?$" ); + static ref EOF: Regex = token(r"\z" ); static ref EOL: Regex = token(r"\n|\r\n" ); static ref EQUALS: Regex = token(r"=" ); static ref INTERPOLATION_END: Regex = token(r"[}][}]" ); @@ -526,6 +526,12 @@ c: b "((())))N(+.", } + summary_test! { + crlf_newline, + "#\r\n#asdf\r\n", + "#$#$.", + } + error_test! { name: tokenize_space_then_tab, input: "a: @@ -616,8 +622,8 @@ c: b } error_test! { - name: mixed_leading_whitespace, - input: "a:\n\t echo hello", + name: mixed_leading_whitespace, + input: "a:\n\t echo hello", index: 3, line: 1, column: 0,