Add shell-expanded strings (#2055)

Cargo.lock

@@ -544,6 +544,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
+ "shellexpand",
  "similar",
  "snafu",
  "strum",
@@ -908,6 +909,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "shellexpand"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b"
+dependencies = [
+ "dirs",
+]
+
 [[package]]
 name = "similar"
 version = "2.5.0"

Cargo.toml

@@ -43,6 +43,7 @@ semver = "1.0.20"
 serde = { version = "1.0.130", features = ["derive", "rc"] }
 serde_json = "1.0.68"
 sha2 = "0.10"
+shellexpand = "3.1.0"
 similar = { version = "2.1.0", features = ["unicode"] }
 snafu = "0.8.0"
 strum = { version = "0.26.0", features = ["derive"] }

README.md

@@ -1242,8 +1242,8 @@ escapes := "\t\n\r\"\\"
 ```
 
 Indented versions of both single- and double-quoted strings, delimited by
-triple single- or triple double-quotes, are supported. Indented string lines
+triple single- or double-quotes, are supported. Indented string lines are
-are stripped of a leading line break, and leading whitespace common to all
+stripped of a leading line break, and leading whitespace common to all
 non-blank lines:
 
 ```just
@@ -1267,6 +1267,24 @@ sequence processing takes place after unindentation. The unindentation
 algorithm does not take escape-sequence produced whitespace or newlines into
 account.
 
+Strings prefixed with `x` are shell expanded<sup>master</sup>:
+
+```justfile
+foobar := x'~/$FOO/${BAR}'
+```
+
+| Value | Replacement |
+|------|-------------|
+| `$VAR` | value of environment variable `VAR` |
+| `${VAR}` | value of environment variable `VAR` |
+| Leading `~` | path to current user's home directory |
+| Leading `~USER` | path to `USER`'s home directory |
+
+This expansion is performed at compile time, so variables from `.env` files and
+exported `just` variables cannot be used. However, this allows shell expanded
+strings to be used in places like settings and import paths, which cannot
+depend on `just` variables and `.env` files.
+
 ### Ignoring Errors
 
 Normally, if a command returns a non-zero exit status, execution will stop. To

src/compile_error.rs

@@ -206,6 +206,7 @@ impl Display for CompileError<'_> {
           )
         }
       }
+      ShellExpansion { err } => write!(f, "Shell expansion failed: {err}"),
       RequiredParameterFollowsDefaultParameter { parameter } => write!(
         f,
         "Non-default parameter `{parameter}` follows default parameter"

src/compile_error_kind.rs

@@ -82,6 +82,9 @@ pub(crate) enum CompileErrorKind<'src> {
   RequiredParameterFollowsDefaultParameter {
     parameter: &'src str,
   },
+  ShellExpansion {
+    err: shellexpand::LookupError<env::VarError>,
+  },
   UndefinedVariable {
     variable: &'src str,
   },

src/keyword.rs

@@ -26,6 +26,7 @@ pub(crate) enum Keyword {
   True,
   WindowsPowershell,
   WindowsShell,
+  X,
 }
 
 impl Keyword {
@@ -43,3 +44,14 @@ impl<'a> PartialEq<&'a str> for Keyword {
     self.lexeme() == *other
   }
 }
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn keyword_case() {
+    assert_eq!(Keyword::X.lexeme(), "x");
+    assert_eq!(Keyword::IgnoreComments.lexeme(), "ignore-comments");
+  }
+}

src/parser.rs

@@ -550,7 +550,7 @@ impl<'run, 'src> Parser<'run, 'src> {
 
   /// Parse a value, e.g. `(bar)`
   fn parse_value(&mut self) -> CompileResult<'src, Expression<'src>> {
-    if self.next_is(StringToken) {
+    if self.next_is(StringToken) || self.next_are(&[Identifier, StringToken]) {
       Ok(Expression::StringLiteral {
         string_literal: self.parse_string_literal()?,
       })
@@ -604,6 +604,8 @@ impl<'run, 'src> Parser<'run, 'src> {
   fn parse_string_literal_token(
     &mut self,
   ) -> CompileResult<'src, (Token<'src>, StringLiteral<'src>)> {
+    let expand = self.accepted_keyword(Keyword::X)?;
+
     let token = self.expect(StringToken)?;
 
     let kind = StringKind::from_string_or_backtick(token)?;
@@ -648,7 +650,23 @@ impl<'run, 'src> Parser<'run, 'src> {
       unindented
     };
 
-    Ok((token, StringLiteral { kind, raw, cooked }))
+    let cooked = if expand {
+      shellexpand::full(&cooked)
+        .map_err(|err| token.error(CompileErrorKind::ShellExpansion { err }))?
+        .into_owned()
+    } else {
+      cooked
+    };
+
+    Ok((
+      token,
+      StringLiteral {
+        cooked,
+        expand,
+        kind,
+        raw,
+      },
+    ))
   }
 
   /// Parse a string literal, e.g. `"FOO"`

src/settings.rs

@@ -201,11 +201,13 @@ mod tests {
           kind: StringKind::from_token_start("\"").unwrap(),
           raw: "asdf.exe",
           cooked: "asdf.exe".to_string(),
+          expand: false,
         },
         arguments: vec![StringLiteral {
           kind: StringKind::from_token_start("\"").unwrap(),
           raw: "-nope",
           cooked: "-nope".to_string(),
+          expand: false,
         }],
       }),
       ..Default::default()

src/string_literal.rs

@@ -2,13 +2,18 @@ use super::*;
 
 #[derive(PartialEq, Debug, Clone, Ord, Eq, PartialOrd)]
 pub(crate) struct StringLiteral<'src> {
+  pub(crate) cooked: String,
+  pub(crate) expand: bool,
   pub(crate) kind: StringKind,
   pub(crate) raw: &'src str,
-  pub(crate) cooked: String,
 }
 
 impl Display for StringLiteral<'_> {
   fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+    if self.expand {
+      write!(f, "x")?;
+    }
+
     write!(
       f,
       "{}{}{}",

tests/lib.rs

@@ -91,6 +91,7 @@ mod search_arguments;
 mod shadowing_parameters;
 mod shebang;
 mod shell;
+mod shell_expansion;
 mod show;
 mod slash_operator;
 mod string;

tests/shell_expansion.rs

@@ -0,0 +1,67 @@
+use super::*;
+
+#[test]
+fn strings_are_shell_expanded() {
+  Test::new()
+    .justfile(
+      "
+        x := x'$JUST_TEST_VARIABLE'
+      ",
+    )
+    .env("JUST_TEST_VARIABLE", "FOO")
+    .args(["--evaluate", "x"])
+    .stdout("FOO")
+    .run();
+}
+
+#[test]
+fn shell_expanded_error_messages_highlight_string_token() {
+  Test::new()
+    .justfile(
+      "
+        x := x'$FOOOOOOOOOOOOOOOOOOOOOOOOOOOOO'
+      ",
+    )
+    .env("JUST_TEST_VARIABLE", "FOO")
+    .args(["--evaluate", "x"])
+    .status(1)
+    .stderr(
+    "
+      error: Shell expansion failed: error looking key 'FOOOOOOOOOOOOOOOOOOOOOOOOOOOOO' up: environment variable not found
+       ——▶ justfile:1:7
+        │
+      1 │ x := x'$FOOOOOOOOOOOOOOOOOOOOOOOOOOOOO'
+        │       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      ")
+    .run();
+}
+
+#[test]
+fn shell_expanded_strings_are_dumped_correctly() {
+  Test::new()
+    .justfile(
+      "
+        x := x'$JUST_TEST_VARIABLE'
+      ",
+    )
+    .env("JUST_TEST_VARIABLE", "FOO")
+    .args(["--dump", "--unstable"])
+    .stdout("x := x'$JUST_TEST_VARIABLE'\n")
+    .run();
+}
+
+#[test]
+fn shell_expanded_strings_can_be_used_in_settings() {
+  Test::new()
+    .justfile(
+      "
+        set dotenv-filename := x'$JUST_TEST_VARIABLE'
+
+        @foo:
+          echo $DOTENV_KEY
+      ",
+    )
+    .env("JUST_TEST_VARIABLE", ".env")
+    .stdout("dotenv-value\n")
+    .run();
+}

tests/test.rs

@@ -199,7 +199,7 @@ impl Test {
     let stdout = if self.unindent_stdout {
       unindent(&self.stdout)
     } else {
-      self.stdout
+      self.stdout.clone()
     };
     let stderr = unindent(&self.stderr);
 
@@ -212,9 +212,9 @@ impl Test {
     }
 
     let mut child = command
-      .args(self.args)
+      .args(&self.args)
       .envs(&self.env)
-      .current_dir(self.tempdir.path().join(self.current_dir))
+      .current_dir(self.tempdir.path().join(&self.current_dir))
       .stdin(Stdio::piped())
       .stdout(Stdio::piped())
       .stderr(Stdio::piped())
@@ -266,7 +266,7 @@ impl Test {
     }
 
     if self.test_round_trip && self.status == EXIT_SUCCESS {
-      test_round_trip(self.tempdir.path());
+      self.round_trip();
     }
 
     Output {
@@ -275,42 +275,44 @@ impl Test {
       tempdir: self.tempdir,
     }
   }
-}
 
-fn test_round_trip(tmpdir: &Path) {
+  fn round_trip(&self) {
-  println!("Reparsing...");
+    println!("Reparsing...");
 
-  let output = Command::new(executable_path("just"))
+    let output = Command::new(executable_path("just"))
-    .current_dir(tmpdir)
+      .current_dir(self.tempdir.path())
-    .arg("--dump")
+      .arg("--dump")
-    .output()
+      .envs(&self.env)
-    .expect("just invocation failed");
+      .output()
+      .expect("just invocation failed");
 
-  if !output.status.success() {
+    if !output.status.success() {
-    panic!("dump failed: {}", output.status);
+      panic!("dump failed: {} {:?}", output.status, output);
+    }
+
+    let dumped = String::from_utf8(output.stdout).unwrap();
+
+    let reparsed_path = self.tempdir.path().join("reparsed.just");
+
+    fs::write(&reparsed_path, &dumped).unwrap();
+
+    let output = Command::new(executable_path("just"))
+      .current_dir(self.tempdir.path())
+      .arg("--justfile")
+      .arg(&reparsed_path)
+      .arg("--dump")
+      .envs(&self.env)
+      .output()
+      .expect("just invocation failed");
+
+    if !output.status.success() {
+      panic!("reparse failed: {}", output.status);
+    }
+
+    let reparsed = String::from_utf8(output.stdout).unwrap();
+
+    assert_eq!(reparsed, dumped, "reparse mismatch");
   }
-
-  let dumped = String::from_utf8(output.stdout).unwrap();
-
-  let reparsed_path = tmpdir.join("reparsed.just");
-
-  fs::write(&reparsed_path, &dumped).unwrap();
-
-  let output = Command::new(executable_path("just"))
-    .current_dir(tmpdir)
-    .arg("--justfile")
-    .arg(&reparsed_path)
-    .arg("--dump")
-    .output()
-    .expect("just invocation failed");
-
-  if !output.status.success() {
-    panic!("reparse failed: {}", output.status);
-  }
-
-  let reparsed = String::from_utf8(output.stdout).unwrap();
-
-  assert_eq!(reparsed, dumped, "reparse mismatch");
 }
 
 pub fn assert_eval_eq(expression: &str, result: &str) {