Parsing and tokenizing tests are now mostly passsing, not running

recipes though.
This commit is contained in:
Casey Rodarmor 2016-10-27 00:13:10 -07:00
parent aae665a4e9
commit d5f81dc0b4
3 changed files with 198 additions and 148 deletions

2
notes
View File

@ -37,6 +37,8 @@ notes
or should non-slash recipes still run in this directory?
will need to change things a great deal
- indentation is line continuation
- should i disallow a shebang recipe where the shebang isn't on the first line?
- add insane borrow checker issue to issue tracker
- add context to unexpected_token error
"while parsing a recipe"
"while parsing an expression"

View File

@ -13,7 +13,7 @@ extern crate tempdir;
use std::io::prelude::*;
use std::{fs, fmt, process, io};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::{BTreeMap, HashSet};
use std::fmt::Display;
use regex::Regex;
@ -54,10 +54,11 @@ fn re(pattern: &str) -> Regex {
struct Recipe<'a> {
line_number: usize,
name: &'a str,
lines: Vec<&'a str>,
fragments: Vec<Vec<Fragment<'a>>>,
variables: BTreeSet<&'a str>,
variable_tokens: Vec<Token<'a>>,
lines: Vec<String>,
// fragments: Vec<Vec<Fragment<'a>>>,
// variables: BTreeSet<&'a str>,
// variable_tokens: Vec<Token<'a>>,
new_lines: Vec<Vec<Fragmant<'a>>>,
dependencies: Vec<&'a str>,
dependency_tokens: Vec<Token<'a>>,
arguments: Vec<&'a str>,
@ -66,17 +67,47 @@ struct Recipe<'a> {
}
#[derive(PartialEq, Debug)]
enum Fragment<'a> {
Text{text: &'a str},
Variable{name: &'a str},
enum Fragmant<'a> {
Text{text: Token<'a>},
Expression{expression: Expression<'a>},
}
#[derive(PartialEq, Debug)]
enum Expression<'a> {
Variable{name: &'a str, token: Token<'a>},
String{contents: &'a str},
Concatination{lhs: Box<Expression<'a>>, rhs: Box<Expression<'a>>},
}
impl<'a> Expression<'a> {
fn variables(&'a self) -> Variables<'a> {
Variables {
stack: vec![self],
}
}
}
struct Variables<'a> {
stack: Vec<&'a Expression<'a>>,
}
impl<'a> Iterator for Variables<'a> {
type Item = &'a Token<'a>;
fn next(&mut self) -> Option<&'a Token<'a>> {
match self.stack.pop() {
None => None,
Some(&Expression::Variable{ref token,..}) => Some(token),
Some(&Expression::String{..}) => None,
Some(&Expression::Concatination{ref lhs, ref rhs}) => {
self.stack.push(lhs);
self.stack.push(rhs);
self.next()
}
}
}
}
impl<'a> Display for Expression<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
match *self {
@ -118,7 +149,7 @@ impl<'a> Recipe<'a> {
);
let mut text = String::new();
// add the shebang
text += self.lines[0];
text += &self.lines[0];
text += "\n";
// add blank lines so that lines in the generated script
// have the same line number as the corresponding lines
@ -127,7 +158,7 @@ impl<'a> Recipe<'a> {
text += "\n"
}
for line in &self.lines[1..] {
text += line;
text += &line;
text += "\n";
}
try!(
@ -163,7 +194,7 @@ impl<'a> Recipe<'a> {
});
} else {
for command in &self.lines {
let mut command = *command;
let mut command = &command[0..];
if !command.starts_with('@') {
warn!("{}", command);
} else {
@ -202,21 +233,20 @@ impl<'a> Display for Recipe<'a> {
try!(write!(f, " {}", dependency))
}
for (i, fragments) in self.fragments.iter().enumerate() {
for (i, pieces) in self.new_lines.iter().enumerate() {
if i == 0 {
try!(writeln!(f, ""));
}
for (j, fragment) in fragments.iter().enumerate() {
for (j, piece) in pieces.iter().enumerate() {
if j == 0 {
try!(write!(f, " "));
}
match *fragment {
Fragment::Text{text} => try!(write!(f, "{}", text)),
Fragment::Variable{name} => try!(write!(f, "{}{}{}", "{{", name, "}}")),
match piece {
&Fragmant::Text{ref text} => try!(write!(f, "{}", text.lexeme)),
&Fragmant::Expression{ref expression} => try!(write!(f, "{}{}{}", "{{", expression, "}}")),
}
}
if i + 1 < self.fragments.len() {
if i + 1 < self.new_lines.len() {
try!(write!(f, "\n"));
}
}
@ -378,8 +408,6 @@ enum ErrorKind<'a> {
DuplicateVariable{variable: &'a str},
ArgumentShadowsVariable{argument: &'a str},
MixedLeadingWhitespace{whitespace: &'a str},
UnclosedInterpolationDelimiter,
BadInterpolationVariableName{recipe: &'a str, text: &'a str},
ExtraLeadingWhitespace,
InconsistentLeadingWhitespace{expected: &'a str, found: &'a str},
OuterShebang,
@ -478,12 +506,6 @@ impl<'a> Display for Error<'a> {
ErrorKind::OuterShebang => {
try!(writeln!(f, "a shebang \"#!\" is reserved syntax outside of recipes"))
}
ErrorKind::UnclosedInterpolationDelimiter => {
try!(writeln!(f, "unmatched {}", "{{"))
}
ErrorKind::BadInterpolationVariableName{recipe, text} => {
try!(writeln!(f, "recipe {} contains a bad variable interpolation: {}", recipe, text))
}
ErrorKind::UnknownDependency{recipe, unknown} => {
try!(writeln!(f, "recipe {} has unknown dependency {}", recipe, unknown));
}
@ -660,22 +682,6 @@ impl<'a> Token<'a> {
kind: kind,
}
}
/*
fn split(
self,
leading_prefix_len: usize,
lexeme_len: usize,
trailing_prefix_len: usize,
) -> (Token<'a>, Token<'a>) {
let len = self.prefix.len() + self.lexeme.len();
// let length = self.prefix.len() + self.lexeme.len();
// if lexeme_start > lexeme_end || lexeme_end > length {
// }
// panic!("Tried to split toke
}
*/
}
#[derive(Debug, PartialEq, Clone, Copy)]
@ -1071,29 +1077,37 @@ impl<'a> Parser<'a> {
return Err(self.unexpected_token(&token, &[Name, Eol, Eof]));
}
enum Piece<'a> {
Text{text: Token<'a>},
Expression{expression: Expression<'a>},
}
let mut new_lines = vec![];
let mut shebang = false;
if self.accepted(Indent) {
while !self.accepted(Dedent) {
if self.accepted(Eol) {
continue;
}
if let Some(token) = self.expect(Line) {
return Err(token.error(ErrorKind::InternalError{
message: format!("Expected a dedent but got {}", token.class)
message: format!("Expected a line but got {}", token.class)
}))
}
let mut pieces = vec![];
while !self.accepted(Eol) {
while !(self.accepted(Eol) || self.peek(Dedent)) {
if let Some(token) = self.accept(Text) {
pieces.push(Piece::Text{text: token});
if pieces.is_empty() {
if new_lines.is_empty() {
if token.lexeme.starts_with("#!") {
shebang = true;
}
} else if !shebang && token.lexeme.starts_with(" ") || token.lexeme.starts_with("\t") {
return Err(token.error(ErrorKind::ExtraLeadingWhitespace));
}
}
pieces.push(Fragmant::Text{text: token});
} else if let Some(token) = self.expect(InterpolationStart) {
return Err(self.unexpected_token(&token, &[Text, InterpolationStart, Eol]));
} else {
pieces.push(Piece::Expression{expression: try!(self.expression(true))});
pieces.push(Fragmant::Expression{expression: try!(self.expression(true))});
if let Some(token) = self.expect(InterpolationEnd) {
return Err(self.unexpected_token(&token, &[InterpolationEnd]));
}
@ -1104,8 +1118,7 @@ impl<'a> Parser<'a> {
}
}
panic!("done!");
/*
let mut lines = vec![];
let mut line_tokens = vec![];
let mut shebang = false;
@ -1195,6 +1208,7 @@ impl<'a> Parser<'a> {
}
fragments.push(line_fragments);
}
*/
Ok(Recipe {
line_number: line_number,
@ -1203,10 +1217,12 @@ impl<'a> Parser<'a> {
dependency_tokens: dependency_tokens,
arguments: arguments,
argument_tokens: argument_tokens,
fragments: fragments,
variables: variables,
variable_tokens: variable_tokens,
lines: lines,
// fragments: fragments,
// variables: variables,
// variable_tokens: variable_tokens,
lines: vec![],
new_lines: new_lines,
// lines: lines,
shebang: shebang,
})
}
@ -1226,7 +1242,7 @@ impl<'a> Parser<'a> {
Ok(lhs)
} else if let Some(token) = self.expect_eol() {
if interpolation {
Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd]))
return Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd]))
} else {
Err(self.unexpected_token(&token, &[Plus, Eol]))
}
@ -1298,11 +1314,36 @@ impl<'a> Parser<'a> {
}));
}
}
for variable in &recipe.variable_tokens {
let name = variable.lexeme;
if !(assignments.contains_key(&name) || recipe.arguments.contains(&name)) {
return Err(variable.error(ErrorKind::UnknownVariable{variable: name}));
for line in &recipe.new_lines {
for piece in line {
if let &Fragmant::Expression{ref expression} = piece {
for variable in expression.variables() {
let name = variable.lexeme;
if !(assignments.contains_key(&name) || recipe.arguments.contains(&name)) {
// There's a borrow issue here that seems to difficult to solve.
// The error derived from the variable token has too short a lifetime,
// so we create a new error from its contents, which do live long
// enough.
//
// I suspect the solution here is to give recipes, pieces, and expressions
// two lifetime parameters instead of one, with one being the lifetime
// of the struct, and the second being the lifetime of the tokens
// that it contains
let error = variable.error(ErrorKind::UnknownVariable{variable: name});
return Err(Error {
text: self.text,
index: error.index,
line: error.line,
column: error.column,
width: error.width,
kind: ErrorKind::UnknownVariable {
variable: &self.text[error.index..error.index + error.width.unwrap()],
}
});
}
}
}
}
}
}

View File

@ -53,7 +53,6 @@ fn token_summary(tokens: &[Token]) -> String {
}).collect::<Vec<_>>().join("")
}
/*
fn parse_success(text: &str) -> Justfile {
match super::parse(text) {
Ok(justfile) => justfile,
@ -84,7 +83,6 @@ fn parse_error(text: &str, expected: Error) {
panic!("Expected {:?} but parse succeeded", expected.kind);
}
}
*/
#[test]
fn tokenize_recipe_interpolation_eol() {
@ -196,7 +194,7 @@ fn tokenize_tabs_then_tab_space() {
}
#[test]
fn outer_shebang() {
fn tokenize_outer_shebang() {
let text = "#!/usr/bin/env bash";
tokenize_error(text, Error {
text: text,
@ -209,7 +207,7 @@ fn outer_shebang() {
}
#[test]
fn unknown_start_of_token() {
fn tokenize_unknown() {
let text = "~";
tokenize_error(text, Error {
text: text,
@ -221,7 +219,6 @@ fn unknown_start_of_token() {
});
}
/*
#[test]
fn parse_empty() {
parse_summary("
@ -239,20 +236,22 @@ x:
y:
z:
foo = \"x\"
bar = foo
goodbye = \"y\"
hello a b c : x y z #hello
#! blah
#blarg
{{ foo }}abc{{ goodbye\t }}xyz
{{ foo + bar}}abc{{ goodbye\t + \"x\" }}xyz
1
2
3
", "foo = \"x\" # \"x\"
", "bar = foo # \"x\"
foo = \"x\" # \"x\"
goodbye = \"y\" # \"y\"
hello a b c: x y z
#! blah
#blarg
{{foo}}abc{{goodbye}}xyz
{{foo + bar}}abc{{goodbye + \"x\"}}xyz
1
2
3
@ -456,54 +455,6 @@ fn write_or() {
assert_eq!("1, 2, 3, or 4", super::Or(&[1,2,3,4]).to_string());
}
#[test]
fn run_shebang() {
// this test exists to make sure that shebang recipes
// run correctly. although this script is still
// executed by sh its behavior depends on the value of a
// variable and continuing even though a command fails,
// whereas in plain recipes variables are not available
// in subsequent lines and execution stops when a line
// fails
let text = "
a:
#!/usr/bin/env sh
code=200
function x { return $code; }
x
x
";
match parse_success(text).run(&["a"]).unwrap_err() {
super::RunError::Code{recipe, code} => {
assert_eq!(recipe, "a");
assert_eq!(code, 200);
},
other => panic!("expected an code run error, but got: {}", other),
}
}
#[test]
fn run_order() {
let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err));
let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned();
let text = r"
b: a
@mv a b
a:
@touch a
d: c
@rm c
c: b
@mv b c
";
super::std::env::set_current_dir(path).expect("failed to set current directory");
parse_success(text).run(&["a", "d"]).unwrap();
}
#[test]
fn unknown_recipes() {
match parse_success("a:\nb:\nc:").run(&["a", "x", "y", "z"]).unwrap_err() {
@ -512,17 +463,6 @@ fn unknown_recipes() {
}
}
#[test]
fn code_error() {
match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() {
super::RunError::Code{recipe, code} => {
assert_eq!(recipe, "fail");
assert_eq!(code, 100);
},
other @ _ => panic!("expected a code run error, but got: {}", other),
}
}
#[test]
fn extra_whitespace() {
// we might want to make extra leading whitespace a line continuation in the future,
@ -576,24 +516,24 @@ fn bad_interpolation_variable_name() {
let text = "a:\n echo {{hello--hello}}";
parse_error(text, Error {
text: text,
index: 4,
index: 11,
line: 1,
column: 1,
width: Some(21),
kind: ErrorKind::BadInterpolationVariableName{recipe: "a", text: "hello--hello"}
column: 8,
width: Some(12),
kind: ErrorKind::BadName{name: "hello--hello"}
});
}
#[test]
fn unclosed_interpolation_delimiter() {
let text = "a:\n echo {{";
let text = "a:\n echo {{ foo";
parse_error(text, Error {
text: text,
index: 4,
index: 15,
line: 1,
column: 1,
width: Some(7),
kind: ErrorKind::UnclosedInterpolationDelimiter,
column: 12,
width: Some(0),
kind: ErrorKind::UnexpectedToken{expected: vec![Plus, Eol, InterpolationEnd], found: Dedent},
});
}
@ -621,15 +561,82 @@ fn unknown_interpolation_variable() {
width: Some(5),
kind: ErrorKind::UnknownVariable{variable: "hello"},
});
}
// let text = "x:\n echo\n {{ lol }}";
// parse_error(text, Error {
// text: text,
// index: 11,
// line: 2,
// column: 2,
// width: Some(3),
// kind: ErrorKind::UnknownVariable{variable: "lol"},
// });
#[test]
fn unknown_second_interpolation_variable() {
let text = "wtf=\"x\"\nx:\n echo\n foo {{wtf}} {{ lol }}";
parse_error(text, Error {
text: text,
index: 33,
line: 3,
column: 16,
width: Some(3),
kind: ErrorKind::UnknownVariable{variable: "lol"},
});
}
#[test]
fn run_order() {
let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err));
let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned();
let text = r"
b: a
@mv a b
a:
@touch F
@touch a
d: c
@rm c
c: b
@mv b c";
tokenize_success(text, "$N:N$>^_$$<N:$>^_$^_$$<N:N$>^_$$<N:N$>^_<.");
super::std::env::set_current_dir(path).expect("failed to set current directory");
parse_success(text).run(&["a", "d"]).unwrap();
if let Err(_) = super::std::fs::metadata("F") {
panic!("recipes did not run");
}
}
/*
#[test]
fn run_shebang() {
// this test exists to make sure that shebang recipes
// run correctly. although this script is still
// executed by sh its behavior depends on the value of a
// variable and continuing even though a command fails,
// whereas in plain recipes variables are not available
// in subsequent lines and execution stops when a line
// fails
let text = "
a:
#!/usr/bin/env sh
code=200
function x { return $code; }
x
x
";
match parse_success(text).run(&["a"]).unwrap_err() {
super::RunError::Code{recipe, code} => {
assert_eq!(recipe, "a");
assert_eq!(code, 200);
},
other => panic!("expected an code run error, but got: {}", other),
}
}
#[test]
fn code_error() {
match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() {
super::RunError::Code{recipe, code} => {
assert_eq!(recipe, "fail");
assert_eq!(code, 100);
},
other @ _ => panic!("expected a code run error, but got: {}", other),
}
}
*/