just/src/lexer.rs

use common::*;

use CompilationErrorKind::*;
use TokenKind::*;

fn re(pattern: &str) -> Regex {
  Regex::new(pattern).unwrap()
}

fn token(pattern: &str) -> Regex {
  let mut s = String::new();
  s += r"^(?m)([ \t]*)(";
  s += pattern;
  s += ")";
  re(&s)
}

fn mixed_whitespace(text: &str) -> bool {
  !(text.chars().all(|c| c == ' ') || text.chars().all(|c| c == '\t'))
}

pub struct Lexer<'a> {
  tokens: Vec<Token<'a>>,
  text: &'a str,
  rest: &'a str,
  index: usize,
  column: usize,
  line: usize,
  state: Vec<State<'a>>,
}

#[derive(PartialEq)]
enum State<'a> {
  Start,
  Indent(&'a str),
  Text,
  Interpolation,
}

impl<'a> Lexer<'a> {
  pub fn lex(text: &'a str) -> CompilationResult<Vec<Token<'a>>> {
    let lexer = Lexer {
      tokens: vec![],
      rest: text,
      index: 0,
      line: 0,
      column: 0,
      state: vec![State::Start],
      text,
    };

    lexer.inner()
  }

  fn error(&self, kind: CompilationErrorKind<'a>) -> CompilationError<'a> {
    CompilationError {
      text: self.text,
      index: self.index,
      line: self.line,
      column: self.column,
      width: None,
      kind,
    }
  }

  fn token(&self, prefix: &'a str, lexeme: &'a str, kind: TokenKind) -> Token<'a> {
    Token {
      index: self.index,
      line: self.line,
      column: self.column,
      text: self.text,
      prefix,
      lexeme,
      kind,
    }
  }

  fn lex_indent(&mut self) -> CompilationResult<'a, Option<Token<'a>>> {
    lazy_static! {
      static ref INDENT: Regex = re(r"^([ \t]*)[^ \t\n\r]");
    }

    let indentation = INDENT
      .captures(self.rest)
      .map(|captures| captures.get(1).unwrap().as_str());

    if self.column == 0 {
      if let Some(kind) = match (self.state.last().unwrap(), indentation) {
        // ignore: was no indentation and there still isn't
        //         or current line is blank
        (&State::Start, Some("")) | (_, None) => None,
        // indent: was no indentation, now there is
        (&State::Start, Some(current)) => {
          if mixed_whitespace(current) {
            return Err(self.error(MixedLeadingWhitespace {
              whitespace: current,
            }));
          }
          //indent = Some(current);
          self.state.push(State::Indent(current));
          Some(Indent)
        }
        // dedent: there was indentation and now there isn't
        (&State::Indent(_), Some("")) => {
          // indent = None;
          self.state.pop();
          Some(Dedent)
        }
        // was indentation and still is, check if the new indentation matches
        (&State::Indent(previous), Some(current)) => {
          if !current.starts_with(previous) {
            return Err(self.error(InconsistentLeadingWhitespace {
              expected: previous,
              found: current,
            }));
          }
          None
        }
        // at column 0 in some other state: this should never happen
        (&State::Text, _) | (&State::Interpolation, _) => {
          return Err(self.error(Internal {
            message: "unexpected state at column 0".to_string(),
          }));
        }
      } {
        return Ok(Some(self.token("", "", kind)));
      }
    }
    Ok(None)
  }

  pub fn inner(mut self) -> CompilationResult<'a, Vec<Token<'a>>> {
    lazy_static! {
      static ref AT: Regex = token(r"@");
      static ref BACKTICK: Regex = token(r"`[^`\n\r]*`");
      static ref COLON: Regex = token(r":");
      static ref COMMA: Regex = token(r",");
      static ref COMMENT: Regex = token(r"#([^\n\r][^\n\r]*)?\r?$");
      static ref EOF: Regex = token(r"\z");
      static ref EOL: Regex = token(r"\n|\r\n");
      static ref EQUALS: Regex = token(r"=");
      static ref INTERPOLATION_END: Regex = token(r"[}][}]");
      static ref INTERPOLATION_START_TOKEN: Regex = token(r"[{][{]");
      static ref NAME: Regex = token(r"([a-zA-Z_][a-zA-Z0-9_-]*)");
      static ref PAREN_L: Regex = token(r"[(]");
      static ref PAREN_R: Regex = token(r"[)]");
      static ref PLUS: Regex = token(r"[+]");
      static ref RAW_STRING: Regex = token(r#"'[^']*'"#);
      static ref STRING: Regex = token(r#"["]"#);
      static ref UNTERMINATED_RAW_STRING: Regex = token(r#"'[^']*"#);
      static ref INTERPOLATION_START: Regex = re(r"^[{][{]");
      static ref LEADING_TEXT: Regex = re(r"^(?m)(.+?)[{][{]");
      static ref LINE: Regex = re(r"^(?m)[ \t]+[^ \t\n\r].*$");
      static ref TEXT: Regex = re(r"^(?m)(.+)");
    }

    loop {
      if let Some(token) = self.lex_indent()? {
        self.tokens.push(token);
      }

      // insert a dedent if we're indented and we hit the end of the file
      if &State::Start != self.state.last().unwrap() && EOF.is_match(self.rest) {
        let token = self.token("", "", Dedent);
        self.tokens.push(token);
      }

      let (prefix, lexeme, kind) = if let (0, &State::Indent(indent), Some(captures)) = (
        self.column,
        self.state.last().unwrap(),
        LINE.captures(self.rest),
      ) {
        let line = captures.get(0).unwrap().as_str();
        if !line.starts_with(indent) {
          return Err(self.error(Internal {
            message: "unexpected indent".to_string(),
          }));
        }
        self.state.push(State::Text);
        (&line[0..indent.len()], "", Line)
      } else if let Some(captures) = EOF.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Eof,
        )
      } else if let State::Text = *self.state.last().unwrap() {
        if let Some(captures) = INTERPOLATION_START.captures(self.rest) {
          self.state.push(State::Interpolation);
          ("", captures.get(0).unwrap().as_str(), InterpolationStart)
        } else if let Some(captures) = LEADING_TEXT.captures(self.rest) {
          ("", captures.get(1).unwrap().as_str(), Text)
        } else if let Some(captures) = TEXT.captures(self.rest) {
          ("", captures.get(1).unwrap().as_str(), Text)
        } else if let Some(captures) = EOL.captures(self.rest) {
          self.state.pop();
          (
            captures.get(1).unwrap().as_str(),
            captures.get(2).unwrap().as_str(),
            Eol,
          )
        } else {
          return Err(self.error(Internal {
            message: format!("Could not match token in text state: \"{}\"", self.rest),
          }));
        }
      } else if let Some(captures) = INTERPOLATION_START_TOKEN.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          InterpolationStart,
        )
      } else if let Some(captures) = INTERPOLATION_END.captures(self.rest) {
        if self.state.last().unwrap() == &State::Interpolation {
          self.state.pop();
        }
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          InterpolationEnd,
        )
      } else if let Some(captures) = NAME.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Name,
        )
      } else if let Some(captures) = EOL.captures(self.rest) {
        if self.state.last().unwrap() == &State::Interpolation {
          return Err(self.error(UnterminatedInterpolation));
        }
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Eol,
        )
      } else if let Some(captures) = BACKTICK.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Backtick,
        )
      } else if let Some(captures) = COLON.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Colon,
        )
      } else if let Some(captures) = AT.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          At,
        )
      } else if let Some(captures) = COMMA.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Comma,
        )
      } else if let Some(captures) = PAREN_L.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          ParenL,
        )
      } else if let Some(captures) = PAREN_R.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          ParenR,
        )
      } else if let Some(captures) = PLUS.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Plus,
        )
      } else if let Some(captures) = EQUALS.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Equals,
        )
      } else if let Some(captures) = COMMENT.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          Comment,
        )
      } else if let Some(captures) = RAW_STRING.captures(self.rest) {
        (
          captures.get(1).unwrap().as_str(),
          captures.get(2).unwrap().as_str(),
          RawString,
        )
      } else if UNTERMINATED_RAW_STRING.is_match(self.rest) {
        return Err(self.error(UnterminatedString));
      } else if let Some(captures) = STRING.captures(self.rest) {
        let prefix = captures.get(1).unwrap().as_str();
        let contents = &self.rest[prefix.len() + 1..];
        if contents.is_empty() {
          return Err(self.error(UnterminatedString));
        }
        let mut len = 0;
        let mut escape = false;
        for c in contents.chars() {
          if c == '\n' || c == '\r' {
            return Err(self.error(UnterminatedString));
          } else if !escape && c == '"' {
            break;
          } else if !escape && c == '\\' {
            escape = true;
          } else if escape {
            escape = false;
          }
          len += c.len_utf8();
        }
        let start = prefix.len();
        let content_end = start + len + 1;
        if escape || content_end >= self.rest.len() {
          return Err(self.error(UnterminatedString));
        }
        (prefix, &self.rest[start..=content_end], StringToken)
      } else {
        return Err(self.error(UnknownStartOfToken));
      };

      let token = self.token(prefix, lexeme, kind);
      self.tokens.push(token);

      let len = prefix.len() + lexeme.len();

      if len == 0 {
        let last = self.tokens.last().unwrap();
        match last.kind {
          Eof => {}
          _ => {
            return Err(last.error(Internal {
              message: format!("zero length token: {:?}", last),
            }));
          }
        }
      }

      match self.tokens.last().unwrap().kind {
        Eol => {
          self.line += 1;
          self.column = 0;
        }
        Eof => {
          break;
        }
        RawString => {
          let lexeme_lines = lexeme.lines().count();
          self.line += lexeme_lines - 1;
          if lexeme_lines == 1 {
            self.column += len;
          } else {
            self.column = lexeme.lines().last().unwrap().len();
          }
        }
        _ => {
          self.column += len;
        }
      }

      self.rest = &self.rest[len..];
      self.index += len;
    }

    Ok(self.tokens)
  }
}

#[cfg(test)]
mod test {
  use super::*;

  macro_rules! summary_test {
    ($name:ident, $input:expr, $expected:expr $(,)*) => {
      #[test]
      fn $name() {
        let input = $input;
        let expected = $expected;
        let tokens = ::Lexer::lex(input).unwrap();
        let roundtrip = tokens
          .iter()
          .map(|t| {
            let mut s = String::new();
            s += t.prefix;
            s += t.lexeme;
            s
          })
          .collect::<Vec<_>>()
          .join("");
        let actual = token_summary(&tokens);
        if actual != expected {
          panic!(
            "token summary mismatch:\nexpected: {}\ngot:      {}\n",
            expected, actual
          );
        }
        assert_eq!(input, roundtrip);
      }
    };
  }

  fn token_summary(tokens: &[Token]) -> String {
    tokens
      .iter()
      .map(|t| match t.kind {
        At => "@",
        Backtick => "`",
        Colon => ":",
        Comma => ",",
        Comment { .. } => "#",
        Dedent => "<",
        Eof => ".",
        Eol => "$",
        Equals => "=",
        Indent { .. } => ">",
        InterpolationEnd => "}",
        InterpolationStart => "{",
        Line { .. } => "^",
        Name => "N",
        ParenL => "(",
        ParenR => ")",
        Plus => "+",
        RawString => "'",
        StringToken => "\"",
        Text => "_",
      })
      .collect::<Vec<_>>()
      .join("")
  }

  macro_rules! error_test {
    (
      name:     $name:ident,
      input:    $input:expr,
      index:    $index:expr,
      line:     $line:expr,
      column:   $column:expr,
      width:    $width:expr,
      kind:     $kind:expr,
    ) => {
      #[test]
      fn $name() {
        let input = $input;

        let expected = CompilationError {
          text: input,
          index: $index,
          line: $line,
          column: $column,
          width: $width,
          kind: $kind,
        };

        if let Err(error) = Lexer::lex(input) {
          assert_eq!(error.text, expected.text);
          assert_eq!(error.index, expected.index);
          assert_eq!(error.line, expected.line);
          assert_eq!(error.column, expected.column);
          assert_eq!(error.kind, expected.kind);
          assert_eq!(error, expected);
        } else {
          panic!("tokenize succeeded but expected: {}\n{}", expected, input);
        }
      }
    };
  }

  summary_test! {
    tokenize_strings,
    r#"a = "'a'" + '"b"' + "'c'" + '"d"'#echo hello"#,
    r#"N="+'+"+'#."#,
  }

  summary_test! {
    tokenize_recipe_interpolation_eol,
    "foo: # some comment
 {{hello}}
",
    "N:#$>^{N}$<.",
  }

  summary_test! {
    tokenize_recipe_interpolation_eof,
    "foo: # more comments
 {{hello}}
# another comment
",
    "N:#$>^{N}$<#$.",
  }

  summary_test! {
    tokenize_recipe_complex_interpolation_expression,
    "foo: #lol\n {{a + b + \"z\" + blarg}}",
    "N:#$>^{N+N+\"+N}<.",
  }

  summary_test! {
    tokenize_recipe_multiple_interpolations,
    "foo:,#ok\n {{a}}0{{b}}1{{c}}",
    "N:,#$>^{N}_{N}_{N}<.",
  }

  summary_test! {
    tokenize_junk,
    "bob

hello blah blah blah : a b c #whatever
    ",
    "N$$NNNN:NNN#$.",
  }

  summary_test! {
    tokenize_empty_lines,
    "
# this does something
hello:
  asdf
  bsdf

  csdf

  dsdf # whatever

# yolo
  ",
    "$#$N:$>^_$^_$$^_$$^_$$<#$.",
  }

  summary_test! {
    tokenize_comment_before_variable,
    "
#
A='1'
echo:
  echo {{A}}
  ",
    "$#$N='$N:$>^_{N}$<.",
  }

  summary_test! {
    tokenize_interpolation_backticks,
    "hello:\n echo {{`echo hello` + `echo goodbye`}}",
    "N:$>^_{`+`}<.",
  }

  summary_test! {
    tokenize_assignment_backticks,
    "a = `echo hello` + `echo goodbye`",
    "N=`+`.",
  }

  summary_test! {
    tokenize_multiple,
    "
hello:
  a
  b

  c

  d

# hello
bob:
  frank
  ",

    "$N:$>^_$^_$$^_$$^_$$<#$N:$>^_$<.",
  }

  summary_test! {
    tokenize_comment,
    "a:=#",
    "N:=#."
  }

  summary_test! {
    tokenize_comment_with_bang,
    "a:=#foo!",
    "N:=#."
  }

  summary_test! {
    tokenize_order,
    r"
b: a
  @mv a b

a:
  @touch F
  @touch a

d: c
  @rm c

c: b
  @mv b c",
    "$N:N$>^_$$<N:$>^_$^_$$<N:N$>^_$$<N:N$>^_<.",
  }

  summary_test! {
    tokenize_parens,
    r"((())) )abc(+",
    "((())))N(+.",
  }

  summary_test! {
    crlf_newline,
    "#\r\n#asdf\r\n",
    "#$#$.",
  }

  error_test! {
    name:  tokenize_space_then_tab,
    input: "a:
 0
 1
\t2
",
    index:  9,
    line:   3,
    column: 0,
    width:  None,
    kind:   InconsistentLeadingWhitespace{expected: " ", found: "\t"},
  }

  error_test! {
    name:  tokenize_tabs_then_tab_space,
    input: "a:
\t\t0
\t\t 1
\t  2
",
    index:  12,
    line:   3,
    column: 0,
    width:  None,
    kind:   InconsistentLeadingWhitespace{expected: "\t\t", found: "\t  "},
  }

  error_test! {
    name: tokenize_unknown,
    input: "~",
    index:  0,
    line:   0,
    column: 0,
    width:  None,
    kind:   UnknownStartOfToken,
  }

  error_test! {
    name: unterminated_string,
    input: r#"a = ""#,
    index:  3,
    line:   0,
    column: 3,
    width:  None,
    kind:   UnterminatedString,
  }

  error_test! {
    name: unterminated_string_with_escapes,
    input: r#"a = "\n\t\r\"\\"#,
    index:  3,
    line:   0,
    column: 3,
    width:  None,
    kind:   UnterminatedString,
  }

  error_test! {
    name:  unterminated_raw_string,
    input: "r a='asdf",
    index:  4,
    line:   0,
    column: 4,
    width:  None,
    kind:   UnterminatedString,
  }

  error_test! {
    name:  unterminated_interpolation,
    input: "foo:\n echo {{
",
    index:  13,
    line:   1,
    column: 8,
    width:  None,
    kind:   UnterminatedInterpolation,
  }

  error_test! {
    name:   mixed_leading_whitespace,
    input:  "a:\n\t echo hello",
    index:  3,
    line:   1,
    column: 0,
    width:  None,
    kind:   MixedLeadingWhitespace{whitespace: "\t "},
  }
}