Merge fort-nix/nix-bitcoin#559: Define tests via flake

edbaeb9813 tests: define tests via flake (Erik Arvstedt)
90e942e5ae nodeinfo: rename `nodeinfoLib` -> `lib` (Erik Arvstedt)
8eaa4cce30 tests: move `mkIfTest` to `nix-bitcoin.lib` (Erik Arvstedt)
47a09ec214 flake: expose `supportedSystems` (Erik Arvstedt)
b0dfa69e84 nixos-search/flake: formatting (Erik Arvstedt)
d428755399 flake: rename input `nixpkgsUnstable` -> `nixpkgs-unstable` (Erik Arvstedt)
a12b701e75 tests/container: don't require `services.clightning` to be defined (Erik Arvstedt)
450de19803 tests/run-tests.sh: print examples before running (Erik Arvstedt)
5f1bb2a8fc tests/copy-src: always copy .git dir (Erik Arvstedt)
a87a59a86b make-container.sh: improve root handling (Erik Arvstedt)
b616d7ac1b profiles/hardened: support pure eval mode (Erik Arvstedt)
73d2fbb448 add compatibility with Nix PR #6530 (`Source tree abstraction`) (Erik Arvstedt)
3c816b862c tests/vmWithoutTests: poweroff on shell exit (Erik Arvstedt)
1d3f49f8da tests, example: avoid lengthy documentation build (Erik Arvstedt)
b840548d40 test/shellcheck-services: add configurable source prefix (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK edbaeb9813

Tree-SHA512: 824c028917816725fb12cd6808947994b13646514ae4dca092e11e6237314ac13157adbba7e79110820d54657eca4f5f4c80946216fa3cb4c7801aec2d0b517d

.cirrus.yml

@@ -30,7 +30,7 @@ task:
       # This script is run as root
       build_script:
         - echo "sandbox = true" >> /etc/nix/nix.conf
-        - nix shell --inputs-from . nixpkgs#{bash,coreutils,gawk,cachix} -c ./test/ci/build.sh
+        - nix shell --inputs-from . nixpkgs#{bash,coreutils,cachix} -c ./test/ci/build.sh $scenario
 
     - name: flake
       build_script:

examples/README.md

@@ -55,6 +55,8 @@ The internal test suite is also useful for exploring features.\
 The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on
 the host system.
 
+`run-tests.sh` requires Nix >= 2.10.
+
 ```bash
 git clone https://github.com/fort-nix/nix-bitcoin
 cd nix-bitcoin/test
@@ -83,6 +85,27 @@ c systemctl status bitcoind
 ```
 See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.
 
+#### Flakes
+Tests can also be directly accessed via Flakes:
+```bash
+# Build test
+nix build --no-link ..#tests.default
+
+# Run a node in a VM. No tests are executed.
+nix run ..#tests.default.vm
+
+# Run a Python test shell inside a VM node
+nix run ..#tests.default.run -- --debug
+
+# Run a node in a container. Requires extra-container, systemd and root privileges
+nix run ..#tests.default.container
+nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05
+
+# Run a command in a container
+nix run ..#tests.default.container -- --run c nodeinfo
+nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05
+```
+
 ### Real-world example
 Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org
 to see the configuration of a nix-bitcoin node that's used in production.

examples/flakes/flake.nix

@@ -14,7 +14,7 @@
         # Import the secure-node preset, an opinionated config to enhance security
         # and privacy.
         #
-        # "${nix-bitcoin}/modules/presets/secure-node.nix"
+        # (nix-bitcoin + "/modules/presets/secure-node.nix")
 
         {
           # Automatically generate all secrets required by services.

examples/qemu-vm/minimal-vm.nix

@@ -13,13 +13,13 @@ rec {
     QEMU_OPTS="-smp $(nproc) -m 1500" ${vm}/bin/run-*-vm
   '';
 
-  vm = (import "${nixpkgs}/nixos" {
+  vm = (import (nixpkgs + "/nixos") {
     inherit system;
     configuration = { config, lib, modulesPath, ... }: {
       imports = [
         nix-bitcoin.nixosModules.default
-        "${nix-bitcoin}/modules/presets/secure-node.nix"
+        (nix-bitcoin + "/modules/presets/secure-node.nix")
-        "${modulesPath}/virtualisation/qemu-vm.nix"
+        (modulesPath + "/virtualisation/qemu-vm.nix")
       ];
 
       virtualisation.graphics = false;
@@ -29,6 +29,9 @@ rec {
       # For faster startup in offline VMs
       services.clightning.extraConfig = "disable-dns";
 
+      # Avoid lengthy build of the nixos manual
+      documentation.nixos.enable = false;
+
       nixpkgs.pkgs = pkgs;
       services.getty.autologinUser = "root";
       nix.nixPath = [ "nixpkgs=${nixpkgs}" ];

examples/qemu-vm/vm-config.nix

@@ -3,7 +3,7 @@
   # Disable the hardened preset to improve VM performance
   disabledModules = [ <nix-bitcoin/modules/presets/hardened.nix> ];
 
-  imports = [ "${modulesPath}/virtualisation/qemu-vm.nix" ];
+  imports = [ (modulesPath + "/virtualisation/qemu-vm.nix" ];
 
   config = {
     virtualisation.graphics = false;

flake.lock

@@ -1,5 +1,27 @@
 {
   "nodes": {
+    "extra-container": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1666443795,
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "rev": "3b69ecfd363983cdee4db7f5d118b0ca099d23ed",
+        "type": "github"
+      },
+      "original": {
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -31,7 +53,7 @@
         "type": "github"
       }
     },
-    "nixpkgsUnstable": {
+    "nixpkgs-unstable": {
       "locked": {
         "lastModified": 1666570118,
         "narHash": "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=",
@@ -49,9 +71,10 @@
     },
     "root": {
       "inputs": {
+        "extra-container": "extra-container",
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
-        "nixpkgsUnstable": "nixpkgsUnstable"
+        "nixpkgs-unstable": "nixpkgs-unstable"
       }
     }
   },

flake.nix

@@ -6,11 +6,16 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05";
-    nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
+    extra-container = {
+      url = "github:erikarvstedt/extra-container";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
   };
 
-  outputs = { self, nixpkgs, nixpkgsUnstable, flake-utils }:
+  outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, ... }:
     let
       supportedSystems = [
         "x86_64-linux"
@@ -18,14 +23,22 @@
         "aarch64-linux"
         "armv7l-linux"
       ];
+
+      test = import ./test/tests.nix nixpkgs.lib;
     in {
       lib = {
         mkNbPkgs = {
           system
           , pkgs ? nixpkgs.legacyPackages.${system}
-          , pkgsUnstable ? nixpkgsUnstable.legacyPackages.${system}
+          , pkgsUnstable ? nixpkgs-unstable.legacyPackages.${system}
         }:
           import ./pkgs { inherit pkgs pkgsUnstable; };
+
+        test = {
+          inherit (test) scenarios;
+        };
+
+        inherit supportedSystems;
       };
 
       overlays.default = final: prev: let
@@ -91,7 +104,12 @@
         # Allow accessing the whole nested `nbPkgs` attrset (including `modulesPkgs`)
         # via this flake.
         # `packages` is not allowed to contain nested pkgs attrsets.
-        legacyPackages = nbPkgs;
+        legacyPackages =
+          nbPkgs //
+          (test.pkgs self pkgs) //
+          {
+            extra-container = self.inputs.extra-container.packages.${system}.default;
+          };
 
         apps = rec {
           default = vm;

modules/nix-bitcoin.nix

@@ -12,7 +12,7 @@ with lib;
 
       lib = mkOption {
         readOnly = true;
-        default = import ../pkgs/lib.nix lib pkgs;
+        default = import ../pkgs/lib.nix lib pkgs config;
         defaultText = "nix-bitcoin/pkgs/lib.nix";
       };
 

modules/nodeinfo.nix

@@ -22,7 +22,7 @@ let
         '';
       };
 
-      nodeinfoLib = mkOption {
+      lib = mkOption {
         internal = true;
         readOnly = true;
         default = nodeinfoLib;

modules/presets/hardened.nix

@@ -1,7 +1,8 @@
-{
+{ modulesPath, ... }: {
   imports = [
-    # Source: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix
+    # Source:
-    <nixpkgs/nixos/modules/profiles/hardened.nix>
+    # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix
+    (modulesPath + "/profiles/hardened.nix")
   ];
 
   ## Reset some options set by the hardened profile

pkgs/lib.nix

@@ -1,4 +1,4 @@
-lib: pkgs:
+lib: pkgs: config:
 
 with lib;
 
@@ -115,4 +115,8 @@ let self = {
       (map (ip: "IP:${ip}") cert.extraIPs)
     );
 
+  test = {
+    mkIfTest = test: mkIf (config.tests.${test} or false);
+  };
+
 }; in self

pkgs/nixpkgs-pinned.nix

@@ -16,5 +16,5 @@ let
 in
 {
   nixpkgs = fetch lockedInputs.nixpkgs;
-  nixpkgs-unstable = fetch lockedInputs.nixpkgsUnstable;
+  nixpkgs-unstable = fetch lockedInputs.nixpkgs-unstable;
 }

pkgs/spark-wallet/composition.nix

@@ -5,7 +5,7 @@
   }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}:
 
 let
-  nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
+  nodeEnv = import (pkgs.path + "/pkgs/development/node-packages/node-env.nix") {
     inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
     inherit pkgs nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;

test/ci/build.sh

@@ -1,12 +1,14 @@
 #!/usr/bin/env bash
 
 # This script can also be run locally for testing:
-#   scenario=default ./build.sh
+#   ./build.sh <scenario>
 #
 # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system.
 
 set -euo pipefail
 
+scenario=$1
+
 if [[ -v CIRRUS_CI ]]; then
     if [[ ! -e /dev/kvm ]]; then
         >&2 echo "No KVM available on VM host."
@@ -16,5 +18,5 @@ if [[ -v CIRRUS_CI ]]; then
     chmod o+rw /dev/kvm
 fi
 
-# shellcheck disable=SC2154
+cd "${BASH_SOURCE[0]%/*}"
-"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario"
+exec ./build-to-cachix.sh --expr "(builtins.getFlake (toString ../..)).legacyPackages.\${builtins.currentSystem}.tests.$scenario"

test/clightning-replication.nix

@@ -1,17 +1,14 @@
 # You can run this test via `run-tests.sh -s clightningReplication`
 
-let
+makeTestVM: pkgs:
-  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
-in
-import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }:
 with pkgs.lib;
 let
-  keyDir = "${nixpkgs}/nixos/tests/initrd-network-ssh";
+  keyDir = pkgs.path + "/nixos/tests/initrd-network-ssh";
   keys = {
-    server = "${keyDir}/ssh_host_ed25519_key";
+    server = keyDir + "/ssh_host_ed25519_key";
-    client = "${keyDir}/id_ed25519";
+    client = keyDir + "/id_ed25519";
-    serverPub = readFile "${keys.server}.pub";
+    serverPub = readFile (keys.server + ".pub");
-    clientPub = readFile "${keys.client}.pub";
+    clientPub = readFile (keys.client + ".pub");
   };
 
   clientBaseConfig = {
@@ -29,7 +26,7 @@ let
     };
   };
 in
-{
+makeTestVM {
   name = "clightning-replication";
 
   nodes = let nodes = {
@@ -150,4 +147,4 @@ in
           # A gocryptfs has been created on the server
           server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf")
   '';
-})
+}

test/lib/copy-src.sh

@@ -14,7 +14,7 @@ atExit() {
 trap "atExit" EXIT
 
 # shellcheck disable=SC2154
-rsync -a --delete --exclude='.git*' "$scriptDir/../" "$tmp/src"
+rsync -a --delete "$scriptDir/../" "$tmp/src"
 echo "Copied src"
 
 # shellcheck disable=SC2154

test/lib/create-git-repo.sh

@@ -1,15 +0,0 @@
-# Create and maintain a minimal git repo at the root of the copied src
-(
-  # shellcheck disable=SC2154,SC2164
-  cd "$scriptDir/.."
-  amend=(--amend)
-
-  if [[ ! -e .git ]] || ! git rev-parse HEAD 2>/dev/null; then
-    git init
-    amend=()
-  fi
-  git add .
-  if ! git diff --quiet --cached; then
-    git commit -a "${amend[@]}" -m -
-  fi
-) >/dev/null

test/lib/make-container.sh

@@ -53,17 +53,10 @@
 
 set -euo pipefail
 
-if [[ $EUID != 0 ]]; then
+# These vars are set by ../run-tests.sh
-    # NixOS containers require root permissions.
+: "${container:=}"
-    # By using sudo here and not at the user's call-site extra-container can detect if it is running
+: "${scriptDir:=}"
-    # inside an existing shell session (by checking an internal environment variable).
-    #
-    # shellcheck disable=SC2154
-    exec sudo scenario="$scenario" scriptDir="$scriptDir" NIX_PATH="$NIX_PATH" PATH="$PATH" \
-         scenarioOverridesFile="${scenarioOverridesFile:-}" "$scriptDir/lib/make-container.sh" "$@"
-fi
 
-export containerName=nb-test
 containerCommand=shell
 
 while [[ $# -gt 0 ]]; do
@@ -79,14 +72,17 @@ while [[ $# -gt 0 ]]; do
 done
 
 containerBin=$(type -P extra-container) || true
-if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) ]]; then
+if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.11*) ]]; then
-    echo "Building extra-container. Skip this step by adding extra-container 0.10 to PATH."
+    echo
-    nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \
+    echo "Building extra-container. Skip this step by adding extra-container 0.11 to PATH."
-      -A pinned.extra-container >/dev/null
+    nix build --out-link /tmp/extra-container "$scriptDir"/..#extra-container
+    # When this script is run as root, e.g. when run in an extra-container shell,
+    # chown the gcroot symlink to the regular (login) user so that the symlink can be
+    # overwritten when this script is run without root.
+    if [[ $EUID == 0 ]]; then
+        chown "$(logname):" --no-dereference /tmp/extra-container
+    fi
     export PATH="/tmp/extra-container/bin${PATH:+:}$PATH"
 fi
 
-read -rd '' src <<EOF || true
+exec "$container"/bin/container "$containerCommand" "$@"
-((import "$scriptDir/tests.nix" {}).getTest "$scenario").container
-EOF
-exec extra-container "$containerCommand" -E "$src" "$@"

test/lib/make-test-vm.nix

@@ -1,6 +1,6 @@
 pkgs:
 let
-  pythonTesting = import "${toString pkgs.path}/nixos/lib/testing-python.nix" {
+  pythonTesting = import (pkgs.path + "/nixos/lib/testing-python.nix") {
     system = pkgs.stdenv.hostPlatform.system;
     inherit pkgs;
   };

test/lib/make-test.nix

@@ -1,29 +1,25 @@
-pkgs:
+flake: pkgs: makeTestVM:
 let
-  makeVM = import ./make-test-vm.nix pkgs;
+  inherit (flake.inputs) extra-container;
-  inherit (pkgs) lib;
+  inherit (pkgs.stdenv.hostPlatform) system;
 in
 
-name: testConfig:
+{ name ? "nix-bitcoin-test", config }:
-{
+let
-  vm = makeVM {
+  inherit (pkgs) lib;
-    name = "nix-bitcoin-${name}";
+
+  testConfig = config;
+
+  test = makeTestVM {
+    inherit name;
 
     nodes.machine = { config, ... }: {
-      imports = [ testConfig ];
+      imports = [
+        testConfig
+        commonVmConfig
+      ];
 
-      virtualisation = {
+      test.shellcheckServices.enable = true;
-        # Needed because duplicity requires 270 MB of free temp space, regardless of backup size
-        diskSize = 1024;
-
-        # Min. 800 MiB needed to avoid 'out of memory' errors
-        memorySize = lib.mkDefault 2048;
-
-        cores = lib.mkDefault 2;
-      };
-
-      # Run shellcheck on all nix-bitcoin services during machine build time
-      system.extraDependencies = [ config.test.shellcheckServices ];
     };
 
     testScript = nodes: let
@@ -48,7 +44,7 @@ name: testConfig:
         (builtins.readFile ./../tests.py)
         cfg.test.extraTestScript
         # Don't run tests in interactive mode.
-        # is_interactive is set in ../run-tests.sh
+        # is_interactive is set in ./run-vm.sh
         ''
           if not "is_interactive" in vars():
               run_tests()
@@ -56,7 +52,15 @@ name: testConfig:
       ];
   };
 
-  container = {
+  # A VM runner for interactive use
+  run = pkgs.writers.writeBashBin "run-vm" ''
+    . ${./run-vm.sh} ${test.driver} "$@"
+  '';
+
+  mkContainer = legacyInstallDirs:
+    extra-container.lib.buildContainers {
+      inherit system legacyInstallDirs;
+      config = {
         # The container name has a 11 char length limit
         containers.nb-test = { config, ... }: {
           imports = [
@@ -74,9 +78,9 @@ name: testConfig:
             # has been resolved. This will also improve security.
             (
               let
-            clightning = config.config.services.clightning;
+                s = config.config.services;
               in
-            lib.mkIf (clightning.enable && clightning.replication.enable) {
+                lib.mkIf (s ? clightning && s.clightning.enable && s.clightning.replication.enable) {
                   bindMounts."/dev/fuse" = { hostPath = "/dev/fuse"; };
                   allowedDevices = [ { node = "/dev/fuse"; modifier = "rw"; } ];
                 }
@@ -84,26 +88,62 @@ name: testConfig:
           ];
         };
       };
+    };
+
+  container = mkContainer false;
+  containerLegacy = mkContainer true;
 
   # This allows running a test scenario in a regular NixOS VM.
   # No tests are executed.
-  vmWithoutTests = (pkgs.nixos ({ config, ... }: {
+  vm = (pkgs.nixos ({ config, ... }: {
     imports = [
       testConfig
-      "${toString pkgs.path}/nixos/modules/virtualisation/qemu-vm.nix"
+      commonVmConfig
+      (pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix")
     ];
     virtualisation.graphics = false;
     services.getty.autologinUser = "root";
 
-    # Provide a shortcut for instant poweroff from within the machine
+    # Avoid lengthy build of the nixos manual
-    environment.systemPackages = with pkgs; [
+    documentation.nixos.enable = false;
-      (lowPrio (writeScriptBin "q" ''
+
+    # Power off VM when the user exits the shell
+    systemd.services."serial-getty@".preStop = ''
       echo o >/proc/sysrq-trigger
-       ''))
+    '';
-    ];
 
     system.stateVersion = lib.mkDefault config.system.nixos.release;
-  })).config.system.build.vm;
+  })).config.system.build.vm.overrideAttrs (old: {
+    meta = old.meta // { mainProgram = "run-vm-in-tmpdir"; };
+    buildCommand =  old.buildCommand + "\n" + ''
+      install -m 700 ${./run-vm-without-tests.sh} $out/bin/run-vm-in-tmpdir
+      patchShebangs $out/bin/run-vm-in-tmpdir
+    '';
+  });
+
+  commonVmConfig = {
+    virtualisation = {
+      # Needed because duplicity requires 270 MB of free temp space, regardless of backup size
+      diskSize = 1024;
+
+      # Min. 800 MiB needed to avoid 'out of memory' errors
+      memorySize = lib.mkDefault 2048;
+
+      # There are no perf gains beyond 3 cores.
+      # Benchmark: Ryzen 7 2700 (8 cores), VM test `default` as of 34f6eb90.
+      # Num. Cores    | 1   | 2  | 3  | 4  | 6
+      # Runtime (sec) | 125 | 95 | 89 | 89 | 90
+      cores = lib.mkDefault 3;
+    };
+  };
+in
+test // {
+  inherit
+    run
+    vm
+    container
+    # For NixOS with `system.stateVersion` <22.05
+    containerLegacy;
 
   config = testConfig;
 }

test/lib/run-vm-without-tests.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script uses the following env vars:
+# NIX_BITCOIN_VM_ENABLE_NETWORK
+# NIX_BITCOIN_VM_DATADIR
+# QEMU_OPTS
+# QEMU_NET_OPTS
+
+if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then
+    dataDir=$NIX_BITCOIN_VM_DATADIR
+else
+    dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+    trap 'rm -rf "$dataDir"' EXIT
+fi
+
+if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then
+    QEMU_NET_OPTS='restrict=on'
+fi
+
+# TODO-EXTERNAL:
+# Pass PATH because run-*-vm is impure (requires coreutils from PATH)
+env -i \
+    PATH="$PATH" \
+    USE_TMPDIR=1 \
+    TMPDIR="$dataDir" \
+    NIX_DISK_IMAGE="$dataDir/img.qcow2" \
+    QEMU_OPTS="${QEMU_OPTS:-}" \
+    QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \
+    "${BASH_SOURCE[0]%/*}"/run-*-vm

test/lib/run-vm.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script uses the following env vars:
+# NIX_BITCOIN_VM_ENABLE_NETWORK
+# NIX_BITCOIN_VM_DATADIR
+# QEMU_OPTS
+# QEMU_NET_OPTS
+
+if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then
+    dataDir=$NIX_BITCOIN_VM_DATADIR
+else
+    dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+    trap 'rm -rf "$dataDir"' EXIT
+fi
+
+testDriver=$1
+shift
+
+# Variable 'tests' contains the Python code that is executed by the driver on startup
+if [[ ${1:-} == --debug ]]; then
+    shift
+    echo "Running interactive testing environment"
+    # Start REPL.
+    # Use `code.interact` for the REPL instead of the builtin test driver REPL
+    # because it supports low featured terminals like Emacs' shell-mode.
+    tests='
+is_interactive = True
+exec(open(os.environ["testScript"]).read())
+if "machine" in vars(): machine.start()
+import code
+code.interact(local=globals())
+'
+    echo
+    echo "Starting VM, data dir: $dataDir"
+else
+    tests='exec(open(os.environ["testScript"]).read())'
+fi
+
+if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then
+    QEMU_NET_OPTS='restrict=on'
+fi
+
+# The VM creates a VDE control socket in $PWD
+env --chdir "$dataDir" -i \
+    USE_TMPDIR=1 \
+    TMPDIR="$dataDir" \
+    QEMU_OPTS="-nographic ${QEMU_OPTS:-}"  \
+    QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \
+    "$testDriver/bin/nixos-test-driver" <(echo "$tests") "$@"

test/lib/shellcheck-services.nix

@@ -1,8 +1,24 @@
 { config, pkgs, lib, extendModules, ... }@args:
 with lib;
 let
-  options = {
+  options.test.shellcheckServices = {
-    test.shellcheckServices = mkOption {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to shellcheck services during system build time.
+      '';
+    };
+
+    sourcePrefix = mkOption {
+      type = with types; nullOr str;
+      default = null;
+      description = ''
+        The definition source path prefix of services to include in the shellcheck.
+      '';
+    };
+
+    runShellcheck = mkOption {
       readOnly = true;
       description = ''
         A derivation that runs shellcheck on all bash scripts included
@@ -12,26 +28,29 @@ let
     };
   };
 
-  # A list of all service names that are defined by nix-bitcoin.
+  cfg = config.test.shellcheckServices;
+
+  # A list of all service names that are defined in source paths prefixed by
+  # `sourcePrefix`.
   # [ "bitcoind", "clightning", ... ]
   #
   # Algorithm: Parse defintions of `systemd.services` and return all services
-  # that only have definitions located in the nix-bitcoin source.
+  # that only have definitions located within `sourcePrefix`.
-  nix-bitcoin-services = let
+  servicesToCheck = let
+    inherit (cfg) sourcePrefix;
     systemdServices = args.options.systemd.services;
     configSystemdServices = args.config.systemd.services;
-    nix-bitcoin-source = toString ../..;
+    matchingServices = collectServices true;
-    nbServices = collectServices true;
+    nonMatchingServices = collectServices false;
-    nonNbServices = collectServices false;
     # Return set of services ({ service1 = true; service2 = true; ... })
-    # which are either defined or not defined by nix-bitcoin, depending
+    # which are either defined or not defined within `sourcePrefix`, depending
-    # on `fromNixBitcoin`.
+    # on `shouldMatch`.
-    collectServices = fromNixBitcoin: lib.listToAttrs (builtins.concatLists (zipListsWith (services: file:
+    collectServices = shouldMatch: lib.listToAttrs (builtins.concatLists (zipListsWith (services: file:
       let
-        isNbSource = lib.hasPrefix nix-bitcoin-source file;
+        isMatching = lib.hasPrefix sourcePrefix file;
       in
         # Nix has no boolean XOR, so use `if`
-        lib.optionals (if fromNixBitcoin then isNbSource else !isNbSource) (
+        lib.optionals (if shouldMatch then isMatching else !isMatching) (
           (map (service: { name = service; value = true; }) (builtins.attrNames services))
         )
     # TODO-EXTERNAL:
@@ -39,13 +58,13 @@ let
     # is included in nixpkgs stable.
     ) systemdServices.definitions systemdServices.files));
   in
-    # Calculate set difference: nbServices - nonNbServices
+    # Calculate set difference: matchingServices - nonMatchingServices
     # and exclude unavailable services (defined via `mkIf false ...`) by checking `configSystemdServices`.
-    builtins.filter (nbService:
+    builtins.filter (prefixedService:
-      configSystemdServices ? ${nbService} && (! nonNbServices ? ${nbService})
+      configSystemdServices ? ${prefixedService} && (! nonMatchingServices ? ${prefixedService})
-    ) (builtins.attrNames nbServices);
+    ) (builtins.attrNames matchingServices);
 
-  # The concatenated list of values of ExecStart, ExecStop, ... (`scriptAttrs`) of all `nix-bitcoin-services`.
+  # The concatenated list of values of ExecStart, ExecStop, ... (`scriptAttrs`) of all `servicesToCheck`.
   serviceCmds = let
     scriptAttrs = [
       "ExecStartPre"
@@ -69,7 +88,7 @@ let
             if builtins.typeOf cmd == "list" then cmd else [ cmd ]
         )
       ) scriptAttrs
-    ) nix-bitcoin-services;
+    ) servicesToCheck;
 
   # A list of all binaries included in `serviceCmds`
   serviceBinaries = map (cmd: builtins.head (
@@ -95,4 +114,15 @@ let
 in
 {
   inherit options;
+
+  config = mkIf (cfg.enable && cfg.sourcePrefix != null) {
+    assertions = [
+      {
+        assertion = builtins.length servicesToCheck > 0;
+        message = "test.shellcheckServices: No services found with source prefix `${cfg.sourcePrefix}`";
+      }
+    ];
+
+    system.extraDependencies = [ shellcheckServices ];
+  };
 }

test/nixos-search/flake.nix

@@ -3,6 +3,7 @@
 # it to the main flake.
 {
   inputs.nixos-search.url = "github:nixos/nixos-search";
+
   outputs = { self, nixos-search }: {
     inherit (nixos-search) packages;
 

test/run-tests.sh

@@ -66,7 +66,6 @@ scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
 args=("$@")
 scenario=
 outLinkPrefix=
-ciBuild=
 while :; do
     case $1 in
         --scenario|-s)
@@ -89,10 +88,6 @@ while :; do
                 exit 1
             fi
             ;;
-        --ci)
-            shift
-            ciBuild=1
-            ;;
         --copy-src|-c)
             shift
             if [[ ! $_nixBitcoinInCopiedSrc ]]; then
@@ -105,183 +100,142 @@ while :; do
     esac
 done
 
-numCPUs=${numCPUs:-$(nproc)}
+tmpDir=
-# Min. 800 MiB needed to avoid 'out of memory' errors
+# Sets global var `tmpDir`
-memoryMiB=${memoryMiB:-2048}
+makeTmpDir() {
-
+    if [[ ! $tmpDir ]]; then
-NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..")
+        tmpDir=$(mktemp -d /tmp/nix-bitcoin-tests.XXX)
-export NIX_PATH
+        # shellcheck disable=SC2064
-
+        trap "rm -rf '$tmpDir'" EXIT
-runAtExit=
+    fi
-trap 'eval "$runAtExit"' EXIT
+}
 
 # Support explicit scenario definitions
 if [[ $scenario = *' '* ]]; then
-    scenarioOverridesFile=$(mktemp "${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX")
+    makeTmpDir
-    export scenarioOverridesFile
+    export scenarioOverridesFile=$tmpDir/scenario-overrides.nix
-
+    echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
-    # shellcheck disable=SC2016
-    runAtExit+='rm -f "$scenarioOverridesFile";'
-    echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
     scenario=tmp
 fi
 
 # Run the test. No temporary files are left on the host system.
 run() {
-    # TMPDIR is also used by the test driver for VM tmp files
+    makeTmpDir
-    TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
+    buildTestAttr .run --out-link "$tmpDir/run-vm"
-    export TMPDIR
+    NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/run-vm/bin/run-vm" "$@"
-    runAtExit+="rm -rf ${TMPDIR};"
-
-    nix-build --out-link "$TMPDIR/driver" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver
-
-    # Variable 'tests' contains the Python code that is executed by the driver on startup
-    if [[ $1 == --interactive ]]; then
-        echo "Running interactive testing environment"
-        tests=$(
-            echo 'is_interactive = True'
-            echo 'exec(open(os.environ["testScript"]).read())'
-            # Start VM
-            echo 'if "machine" in vars(): machine.start()'
-            # Start REPL.
-            # Use `code.interact` for the REPL instead of the builtin test driver REPL
-            # because it supports low featured terminals like Emacs' shell-mode.
-            echo 'import code'
-            echo 'code.interact(local=globals())'
-        )
-    else
-        tests='exec(open(os.environ["testScript"]).read())'
-    fi
-
-    echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-    [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on'
-    cd "$TMPDIR" # The VM creates a VDE control socket in $PWD
-    env -i \
-        NIX_PATH="$NIX_PATH" \
-        TMPDIR="$TMPDIR" \
-        USE_TMPDIR=1 \
-        QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS"  \
-        QEMU_NET_OPTS="$QEMU_NET_OPTS" \
-        "$TMPDIR/driver/bin/nixos-test-driver" <(echo "$tests")
 }
 
 debug() {
-    run --interactive
+    run --debug
-}
-
-evalTest() {
-    nix-instantiate --eval -E "($(vmTestNixExpr)).outPath"
-}
-
-instantiate() {
-    nix-instantiate -E "$(vmTestNixExpr)" "$@"
 }
 
 container() {
-    export scriptDir scenario
+    local nixosContainer
+    if ! nixosContainer=$(type -p nixos-container) \
+       || grep -q '"/etc/nixos-containers"' "$nixosContainer"; then
+        local attr=container
+    else
+        # NixOS with `system.stateVersion` <22.05
+        local attr=containerLegacy
+    fi
+    echo "Building container"
+    makeTmpDir
+    export container=$tmpDir/container
+    buildTestAttr ".$attr" --out-link "$container"
+    export scriptDir
     "$scriptDir/lib/make-container.sh" "$@"
 }
 
 # Run a regular NixOS VM
 vm() {
-    TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+    makeTmpDir
-    export TMPDIR
+    buildTestAttr .vm --out-link "$tmpDir/vm"
-    runAtExit+="rm -rf $TMPDIR;"
+    NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/vm/bin/run-vm-in-tmpdir"
-
-    nix-build --out-link "$TMPDIR/vm" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vmWithoutTests"
-
-    echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-    [[ $NB_TEST_ENABLE_NETWORK ]] || export QEMU_NET_OPTS="restrict=on,$QEMU_NET_OPTS"
-
-    # shellcheck disable=SC2211
-    USE_TMPDIR=1 \
-    NIX_DISK_IMAGE=$TMPDIR/img.qcow2 \
-    QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS"  \
-      "$TMPDIR"/vm/bin/run-*-vm
-}
-
-doBuild() {
-    name=$1
-    shift
-    if [[ $ciBuild ]]; then
-        "$scriptDir/ci/build-to-cachix.sh" "$@"
-    else
-        if [[ $outLinkPrefix ]]; then
-            outLink="--out-link $outLinkPrefix-$name"
-        else
-            outLink=--no-out-link
-        fi
-        nix-build $outLink "$@"
-    fi
 }
 
 # Run the test by building the test derivation
 buildTest() {
-    vmTestNixExpr | doBuild $scenario "$@" -
+    buildTestAttr "" "$@"
 }
 
-vmTestNixExpr() {
+evalTest() {
-    extraQEMUOpts=
+    nixInstantiateTest "" "$@"
-
+    # Print out path
-    if [[ $ciBuild ]]; then
+    nix-store -q "$drv"
-        # On continuous integration nodes there are few other processes running alongside the
+    # Print drv path
-        # test, so use more memory here for maximum performance.
+    realpath "$drv"
-        memoryMiB=4096
-        memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
-        memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
-        # Round down to nearest multiple of 50 MiB for improved test build caching
-        # shellcheck disable=SC2017
-        ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50))
-        ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB
-        >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-        >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB"
-
-        # VMX is usually not available on CI nodes due to recursive virtualisation.
-        # Explicitly disable VMX, otherwise QEMU 4.20 fails with message
-        # "error: failed to set MSR 0x48b to 0x159ff00000000"
-        extraQEMUOpts="-cpu host,-vmx"
-    fi
-
-    cat <<EOF
-    ((import "$scriptDir/tests.nix" {}).getTest "$scenario").vm.overrideAttrs (old: rec {
-      buildCommand = ''
-        export QEMU_OPTS="-smp $numCPUs -m $memoryMiB $extraQEMUOpts"
-        echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-      '' + old.buildCommand;
-    })
-EOF
 }
 
-checkFlakeSupport() {
+buildTestAttr() {
-    testName=$1
+    local attr=$1
-    if [[ ! -v hasFlakes ]]; then
+    shift
-        if [[ $(nix flake 2>&1) == *"requires a sub-command"* ]]; then
+    # TODO-EXTERNAL:
-            hasFlakes=1
+    # Simplify and switch to pure build when `nix build` can build flake function outputs
+    nixInstantiateTest "$attr"
+    nixBuild "$scenario" "$drv" "$@"
+}
+
+buildTests() {
+    local -n tests=$1
+    shift
+    # TODO-EXTERNAL:
+    # Simplify and switch to pure build when `nix build` can instantiate flake function outputs
+    # shellcheck disable=SC2207
+    drvs=($(nixInstantiate "pkgs.instantiateTests \"${tests[*]}\""))
+    for i in "${!tests[@]}"; do
+        testName=${tests[$i]}
+        drv=${drvs[$i]}
+        echo
+        echo "Building test '$testName'"
+        nixBuild "$testName" "$drv" "$@"
+    done
+}
+
+# Instantiate an attr of the test defined in global var `scenario`
+nixInstantiateTest() {
+    local attr=$1
+    shift
+    if [[ ${scenarioOverridesFile:-} ]]; then
+        local file="extraScenariosFile = \"$scenarioOverridesFile\";"
     else
-            hasFlakes=
+        local file=
     fi
+    nixInstantiate "(pkgs.getTest { name = \"$scenario\"; $file })$attr" "$@" >/dev/null
+}
+
+drv=
+# Sets global var `drv` to the gcroot link of the instantiated derivation
+nixInstantiate() {
+    local expr=$1
+    shift
+    makeTmpDir
+    drv="$tmpDir/drv"
+    nix-instantiate --add-root "$drv" -E "
+      let
+        pkgs = (builtins.getFlake \"git+file://$scriptDir/..\").legacyPackages.\${builtins.currentSystem};
+      in
+        $expr
+    " "$@"
+}
+
+nixBuild() {
+    local outLinkName=$1
+    shift
+    args=(--print-out-paths -L)
+    if [[ $outLinkPrefix ]]; then
+        args+=(--out-link "$outLinkPrefix-$outLinkName")
+    else
+        args+=(--no-link)
     fi
-    if [[ ! $hasFlakes ]]; then
+    nix build "${args[@]}" "$@"
-        echo "Skipping test '$testName'. Nix flake support is not enabled."
-        return 1
-    fi
 }
 
 flake() {
-    if ! checkFlakeSupport "flake"; then return; fi
-
     nix flake check "$scriptDir/.."
 }
 
 # Test generating module documentation for search.nixos.org
 nixosSearch() {
-    if ! checkFlakeSupport "nixosSearch"; then return; fi
-
-    if [[ $_nixBitcoinInCopiedSrc ]]; then
-      # flake-info requires that its target flake is under version control
-      . "$scriptDir/lib/create-git-repo.sh"
-    fi
-
     if [[ $outLinkPrefix ]]; then
         # Add gcroots for flake-info
         nix build "$scriptDir/nixos-search#flake-info" -o "$outLinkPrefix-flake-info"
@@ -290,38 +244,41 @@ nixosSearch() {
 }
 
 # A basic subset of tests to keep the total runtime within
-# manageable bounds (<4 min on desktop systems).
+# manageable bounds.
 # These are also run on the CI server.
-basic() {
+basic=(
-    scenario=default buildTest "$@"
+    default
-    scenario=netns buildTest "$@"
+    netns
-    scenario=netnsRegtest buildTest "$@"
+    netnsRegtest
-}
+)
+basic() { buildTests basic "$@"; }
 
 # All tests that only consist of building a nix derivation.
-# Their output is cached in /nix/store.
+# shellcheck disable=2034
-buildable() {
+buildable=(
-    basic "$@"
+    "${basic[@]}"
-    scenario=full buildTest "$@"
+    full
-    scenario=regtest buildTest "$@"
+    regtest
-    scenario=hardened buildTest "$@"
+    hardened
-    scenario=clightningReplication buildTest "$@"
+    clightningReplication
-    scenario=lndPruned buildTest "$@"
+    lndPruned
-}
+)
+buildable() { buildTests buildable "$@"; }
 
 examples() {
-    script="
+    # shellcheck disable=SC2016
+    script='
       set -e
-      ./deploy-container.sh
+      runExample() { echo; echo Running example $1; ./$1; }
-      ./deploy-container-minimal.sh
+      runExample deploy-container.sh
-      ./deploy-qemu-vm.sh
+      runExample deploy-container-minimal.sh
-      ./deploy-krops.sh
+      runExample deploy-qemu-vm.sh
-    "
+      runExample deploy-krops.sh
+    '
     (cd "$scriptDir/../examples" && nix-shell --run "$script")
 }
 
 shellcheck() {
-    if ! checkFlakeSupport "shellcheck"; then return; fi
     "$scriptDir/shellcheck.sh"
 }
 

test/tests.nix

@@ -1,19 +1,11 @@
 # Integration tests, can be run without internet access.
 
+lib:
 let
-  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+  # Included in all scenarios
-in
+  baseConfig = { config, pkgs, ... }: with lib; let
-
-{ extraScenarios ? { ... }: {}
-, pkgs ? import nixpkgs { config = {}; overlays = []; }
-}:
-with pkgs.lib;
-let
-  globalPkgs = pkgs;
-
-  baseConfig = { pkgs, config, ... }: let
     cfg = config.services;
-    mkIfTest = test: mkIf (config.tests.${test} or false);
+    inherit (config.nix-bitcoin.lib.test) mkIfTest;
   in {
     imports = [
       ./lib/test-lib.nix
@@ -32,9 +24,6 @@ let
     };
 
     config = mkMerge [{
-      # Share the same pkgs instance among tests
-      nixpkgs.pkgs = mkDefault globalPkgs;
-
       environment.systemPackages = mkMerge (with pkgs; [
         # Needed to test macaroon creation
         (mkIfTest "btcpayserver" [ openssl xxd ])
@@ -176,8 +165,9 @@ let
     ];
   };
 
-  scenarios = {
+  scenarios = with lib; {
-    base = baseConfig; # Included in all scenarios
+    # Included in all scenarios by ./lib/make-test.nix
+    base = baseConfig;
 
     default = scenarios.secureNode;
 
@@ -273,7 +263,7 @@ let
       environment.systemPackages = [ pkgs.fping ];
     };
 
-    regtestBase = { config, ... }: {
+    regtestBase = { config, pkgs, ... }: {
       tests.regtest = true;
       test.data.num_blocks = 100;
 
@@ -323,9 +313,10 @@ let
       services.lnd.enable = true;
       services.bitcoind.prune = 1000;
     };
+  };
 
-    ## Examples / debug helper
+  ## Example scenarios that showcase extra features
-
+  exampleScenarios = with lib; {
     # Run a selection of tests in scenario 'netns'
     selectedTests = {
       imports = [ scenarios.netns ];
@@ -342,40 +333,82 @@ let
       # See ./lib/test-lib.nix for a description
       test.container.exposeLocalhost = true;
     };
-
-    adhoc = {
-      # <Add your config here>
-      # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios.
-    };
   };
+in {
+  inherit scenarios;
 
-  overrides = builtins.getEnv "scenarioOverridesFile";
+  pkgs = flake: pkgs: rec {
-  extraScenarios' = (if (overrides != "") then import overrides else extraScenarios) {
+    # A basic test using the nix-bitcoin test framework
-    inherit scenarios pkgs;
+    makeTestBasic = import ./lib/make-test.nix flake pkgs makeTestVM;
-    inherit (pkgs) lib;
-  };
-  allScenarios = scenarios // extraScenarios';
 
-  makeTest = name: config:
+    # Wraps `makeTest` in NixOS' testing-python.nix so that the drv includes the
-    makeTest' name {
+    # log output and the test driver
+    makeTestVM = import ./lib/make-test-vm.nix pkgs;
+
+    # A test using the nix-bitcoin test framework, with some helpful defaults
+    makeTest = { name ? "nix-bitcoin-test", config }:
+      makeTestBasic {
+        inherit name;
+        config = {
           imports = [
-        allScenarios.base
+            scenarios.base
             config
           ];
-    };
+          # Share the same pkgs instance among tests
-  makeTest' = import ./lib/make-test.nix pkgs;
+          nixpkgs.pkgs = pkgs.lib.mkDefault pkgs;
-
-  tests = builtins.mapAttrs makeTest allScenarios // {
-    clightningReplication.vm = import ./clightning-replication.nix {
-      inherit pkgs;
-      inherit (pkgs.stdenv) system;
         };
       };
 
-  getTest = name: tests.${name} or (makeTest name {
+    # A test using the nix-bitcoin test framework, with defaults specific to nix-bitcoin
-    services.${name}.enable = true;
+    makeTestNixBitcoin = { name, config }:
-  });
+      makeTest {
+        name = "nix-bitcoin-${name}";
+        config = {
+          imports = [ config ];
+          test.shellcheckServices.sourcePrefix = toString ./..;
+        };
+      };
+
+    makeTests = scenarios: let
+      mainTests = builtins.mapAttrs (name: config:
+        makeTestNixBitcoin { inherit name config; }
+      ) scenarios;
     in
-  tests // {
+      {
-    inherit getTest;
+        clightningReplication = import ./clightning-replication.nix makeTestVM pkgs;
+      } // mainTests;
+
+    tests = makeTests scenarios;
+
+    ## Helper for ./run-tests.sh
+
+    getTest = { name, extraScenariosFile ? null }:
+      let
+        tests = makeTests (scenarios // (
+          lib.optionalAttrs (extraScenariosFile != null)
+            (import extraScenariosFile {
+              inherit scenarios lib pkgs;
+              nix-bitcoin = flake;
+            })
+        ));
+      in
+        tests.${name} or (makeTestNixBitcoin {
+          inherit name;
+          config = {
+            services.${name}.enable = true;
+          };
+        });
+
+    instantiateTests = testNames:
+      let
+        testNames' = lib.splitString " " testNames;
+      in
+        map (name:
+          let
+            test = tests.${name};
+          in
+            builtins.seq (builtins.trace "Evaluating test '${name}'" test.outPath)
+              test
+        ) testNames';
+  };
 }