diff --git a/.cirrus.yml b/.cirrus.yml index 430078f..ccbe158 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,7 +30,7 @@ task: # This script is run as root build_script: - echo "sandbox = true" >> /etc/nix/nix.conf - - nix shell --inputs-from . nixpkgs#{bash,coreutils,gawk,cachix} -c ./test/ci/build.sh + - nix shell --inputs-from . nixpkgs#{bash,coreutils,cachix} -c ./test/ci/build.sh $scenario - name: flake build_script: diff --git a/examples/README.md b/examples/README.md index 64a54df..0ede270 100644 --- a/examples/README.md +++ b/examples/README.md @@ -55,6 +55,8 @@ The internal test suite is also useful for exploring features.\ The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on the host system. +`run-tests.sh` requires Nix >= 2.10. + ```bash git clone https://github.com/fort-nix/nix-bitcoin cd nix-bitcoin/test @@ -83,6 +85,27 @@ c systemctl status bitcoind ``` See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation. +#### Flakes +Tests can also be directly accessed via Flakes: +```bash +# Build test +nix build --no-link ..#tests.default + +# Run a node in a VM. No tests are executed. +nix run ..#tests.default.vm + +# Run a Python test shell inside a VM node +nix run ..#tests.default.run -- --debug + +# Run a node in a container. Requires extra-container, systemd and root privileges +nix run ..#tests.default.container +nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05 + +# Run a command in a container +nix run ..#tests.default.container -- --run c nodeinfo +nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05 +``` + ### Real-world example Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org to see the configuration of a nix-bitcoin node that's used in production. diff --git a/examples/flakes/flake.nix b/examples/flakes/flake.nix index 24b7d53..4b23845 100644 --- a/examples/flakes/flake.nix +++ b/examples/flakes/flake.nix @@ -14,7 +14,7 @@ # Import the secure-node preset, an opinionated config to enhance security # and privacy. # - # "${nix-bitcoin}/modules/presets/secure-node.nix" + # (nix-bitcoin + "/modules/presets/secure-node.nix") { # Automatically generate all secrets required by services. diff --git a/examples/qemu-vm/minimal-vm.nix b/examples/qemu-vm/minimal-vm.nix index c04f27e..25a66c7 100644 --- a/examples/qemu-vm/minimal-vm.nix +++ b/examples/qemu-vm/minimal-vm.nix @@ -13,13 +13,13 @@ rec { QEMU_OPTS="-smp $(nproc) -m 1500" ${vm}/bin/run-*-vm ''; - vm = (import "${nixpkgs}/nixos" { + vm = (import (nixpkgs + "/nixos") { inherit system; configuration = { config, lib, modulesPath, ... }: { imports = [ nix-bitcoin.nixosModules.default - "${nix-bitcoin}/modules/presets/secure-node.nix" - "${modulesPath}/virtualisation/qemu-vm.nix" + (nix-bitcoin + "/modules/presets/secure-node.nix") + (modulesPath + "/virtualisation/qemu-vm.nix") ]; virtualisation.graphics = false; @@ -29,6 +29,9 @@ rec { # For faster startup in offline VMs services.clightning.extraConfig = "disable-dns"; + # Avoid lengthy build of the nixos manual + documentation.nixos.enable = false; + nixpkgs.pkgs = pkgs; services.getty.autologinUser = "root"; nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; diff --git a/examples/qemu-vm/vm-config.nix b/examples/qemu-vm/vm-config.nix index 28f8cd3..b3cd0d3 100644 --- a/examples/qemu-vm/vm-config.nix +++ b/examples/qemu-vm/vm-config.nix @@ -3,7 +3,7 @@ # Disable the hardened preset to improve VM performance disabledModules = [ ]; - imports = [ "${modulesPath}/virtualisation/qemu-vm.nix" ]; + imports = [ (modulesPath + "/virtualisation/qemu-vm.nix" ]; config = { virtualisation.graphics = false; diff --git a/flake.lock b/flake.lock index 24b5cbc..14b7d68 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,27 @@ { "nodes": { + "extra-container": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1666443795, + "owner": "erikarvstedt", + "repo": "extra-container", + "rev": "3b69ecfd363983cdee4db7f5d118b0ca099d23ed", + "type": "github" + }, + "original": { + "owner": "erikarvstedt", + "repo": "extra-container", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1659877975, @@ -31,7 +53,7 @@ "type": "github" } }, - "nixpkgsUnstable": { + "nixpkgs-unstable": { "locked": { "lastModified": 1666570118, "narHash": "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=", @@ -49,9 +71,10 @@ }, "root": { "inputs": { + "extra-container": "extra-container", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", - "nixpkgsUnstable": "nixpkgsUnstable" + "nixpkgs-unstable": "nixpkgs-unstable" } } }, diff --git a/flake.nix b/flake.nix index 2ad95c0..3be5d2a 100644 --- a/flake.nix +++ b/flake.nix @@ -6,11 +6,16 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05"; - nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; + extra-container = { + url = "github:erikarvstedt/extra-container"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; }; - outputs = { self, nixpkgs, nixpkgsUnstable, flake-utils }: + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, ... }: let supportedSystems = [ "x86_64-linux" @@ -18,14 +23,22 @@ "aarch64-linux" "armv7l-linux" ]; + + test = import ./test/tests.nix nixpkgs.lib; in { lib = { mkNbPkgs = { system , pkgs ? nixpkgs.legacyPackages.${system} - , pkgsUnstable ? nixpkgsUnstable.legacyPackages.${system} + , pkgsUnstable ? nixpkgs-unstable.legacyPackages.${system} }: import ./pkgs { inherit pkgs pkgsUnstable; }; + + test = { + inherit (test) scenarios; + }; + + inherit supportedSystems; }; overlays.default = final: prev: let @@ -91,7 +104,12 @@ # Allow accessing the whole nested `nbPkgs` attrset (including `modulesPkgs`) # via this flake. # `packages` is not allowed to contain nested pkgs attrsets. - legacyPackages = nbPkgs; + legacyPackages = + nbPkgs // + (test.pkgs self pkgs) // + { + extra-container = self.inputs.extra-container.packages.${system}.default; + }; apps = rec { default = vm; diff --git a/modules/nix-bitcoin.nix b/modules/nix-bitcoin.nix index 0ad9eb8..e00d497 100644 --- a/modules/nix-bitcoin.nix +++ b/modules/nix-bitcoin.nix @@ -12,7 +12,7 @@ with lib; lib = mkOption { readOnly = true; - default = import ../pkgs/lib.nix lib pkgs; + default = import ../pkgs/lib.nix lib pkgs config; defaultText = "nix-bitcoin/pkgs/lib.nix"; }; diff --git a/modules/nodeinfo.nix b/modules/nodeinfo.nix index e6adba2..5c0cd14 100644 --- a/modules/nodeinfo.nix +++ b/modules/nodeinfo.nix @@ -22,7 +22,7 @@ let ''; }; - nodeinfoLib = mkOption { + lib = mkOption { internal = true; readOnly = true; default = nodeinfoLib; diff --git a/modules/presets/hardened.nix b/modules/presets/hardened.nix index f7521d8..123b0b9 100644 --- a/modules/presets/hardened.nix +++ b/modules/presets/hardened.nix @@ -1,7 +1,8 @@ -{ +{ modulesPath, ... }: { imports = [ - # Source: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix - + # Source: + # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix + (modulesPath + "/profiles/hardened.nix") ]; ## Reset some options set by the hardened profile diff --git a/pkgs/lib.nix b/pkgs/lib.nix index dd0a803..4a6970d 100644 --- a/pkgs/lib.nix +++ b/pkgs/lib.nix @@ -1,4 +1,4 @@ -lib: pkgs: +lib: pkgs: config: with lib; @@ -115,4 +115,8 @@ let self = { (map (ip: "IP:${ip}") cert.extraIPs) ); + test = { + mkIfTest = test: mkIf (config.tests.${test} or false); + }; + }; in self diff --git a/pkgs/nixpkgs-pinned.nix b/pkgs/nixpkgs-pinned.nix index d3f3acc..41b0983 100644 --- a/pkgs/nixpkgs-pinned.nix +++ b/pkgs/nixpkgs-pinned.nix @@ -16,5 +16,5 @@ let in { nixpkgs = fetch lockedInputs.nixpkgs; - nixpkgs-unstable = fetch lockedInputs.nixpkgsUnstable; + nixpkgs-unstable = fetch lockedInputs.nixpkgs-unstable; } diff --git a/pkgs/spark-wallet/composition.nix b/pkgs/spark-wallet/composition.nix index 087380e..7eb0a0d 100644 --- a/pkgs/spark-wallet/composition.nix +++ b/pkgs/spark-wallet/composition.nix @@ -5,7 +5,7 @@ }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}: let - nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" { + nodeEnv = import (pkgs.path + "/pkgs/development/node-packages/node-env.nix") { inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript; inherit pkgs nodejs; libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; diff --git a/test/ci/build.sh b/test/ci/build.sh index af80ffe..90d6b4f 100755 --- a/test/ci/build.sh +++ b/test/ci/build.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash # This script can also be run locally for testing: -# scenario=default ./build.sh +# ./build.sh # # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system. set -euo pipefail +scenario=$1 + if [[ -v CIRRUS_CI ]]; then if [[ ! -e /dev/kvm ]]; then >&2 echo "No KVM available on VM host." @@ -16,5 +18,5 @@ if [[ -v CIRRUS_CI ]]; then chmod o+rw /dev/kvm fi -# shellcheck disable=SC2154 -"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario" +cd "${BASH_SOURCE[0]%/*}" +exec ./build-to-cachix.sh --expr "(builtins.getFlake (toString ../..)).legacyPackages.\${builtins.currentSystem}.tests.$scenario" diff --git a/test/clightning-replication.nix b/test/clightning-replication.nix index 1a4c3bc..70d8102 100644 --- a/test/clightning-replication.nix +++ b/test/clightning-replication.nix @@ -1,17 +1,14 @@ # You can run this test via `run-tests.sh -s clightningReplication` -let - nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs; -in -import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }: +makeTestVM: pkgs: with pkgs.lib; let - keyDir = "${nixpkgs}/nixos/tests/initrd-network-ssh"; + keyDir = pkgs.path + "/nixos/tests/initrd-network-ssh"; keys = { - server = "${keyDir}/ssh_host_ed25519_key"; - client = "${keyDir}/id_ed25519"; - serverPub = readFile "${keys.server}.pub"; - clientPub = readFile "${keys.client}.pub"; + server = keyDir + "/ssh_host_ed25519_key"; + client = keyDir + "/id_ed25519"; + serverPub = readFile (keys.server + ".pub"); + clientPub = readFile (keys.client + ".pub"); }; clientBaseConfig = { @@ -29,7 +26,7 @@ let }; }; in -{ +makeTestVM { name = "clightning-replication"; nodes = let nodes = { @@ -150,4 +147,4 @@ in # A gocryptfs has been created on the server server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf") ''; -}) +} diff --git a/test/lib/copy-src.sh b/test/lib/copy-src.sh index e19134c..b1b720e 100644 --- a/test/lib/copy-src.sh +++ b/test/lib/copy-src.sh @@ -14,7 +14,7 @@ atExit() { trap "atExit" EXIT # shellcheck disable=SC2154 -rsync -a --delete --exclude='.git*' "$scriptDir/../" "$tmp/src" +rsync -a --delete "$scriptDir/../" "$tmp/src" echo "Copied src" # shellcheck disable=SC2154 diff --git a/test/lib/create-git-repo.sh b/test/lib/create-git-repo.sh deleted file mode 100644 index 6721b4b..0000000 --- a/test/lib/create-git-repo.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Create and maintain a minimal git repo at the root of the copied src -( - # shellcheck disable=SC2154,SC2164 - cd "$scriptDir/.." - amend=(--amend) - - if [[ ! -e .git ]] || ! git rev-parse HEAD 2>/dev/null; then - git init - amend=() - fi - git add . - if ! git diff --quiet --cached; then - git commit -a "${amend[@]}" -m - - fi -) >/dev/null diff --git a/test/lib/make-container.sh b/test/lib/make-container.sh index 19d009f..b500b59 100755 --- a/test/lib/make-container.sh +++ b/test/lib/make-container.sh @@ -53,17 +53,10 @@ set -euo pipefail -if [[ $EUID != 0 ]]; then - # NixOS containers require root permissions. - # By using sudo here and not at the user's call-site extra-container can detect if it is running - # inside an existing shell session (by checking an internal environment variable). - # - # shellcheck disable=SC2154 - exec sudo scenario="$scenario" scriptDir="$scriptDir" NIX_PATH="$NIX_PATH" PATH="$PATH" \ - scenarioOverridesFile="${scenarioOverridesFile:-}" "$scriptDir/lib/make-container.sh" "$@" -fi +# These vars are set by ../run-tests.sh +: "${container:=}" +: "${scriptDir:=}" -export containerName=nb-test containerCommand=shell while [[ $# -gt 0 ]]; do @@ -79,14 +72,17 @@ while [[ $# -gt 0 ]]; do done containerBin=$(type -P extra-container) || true -if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) ]]; then - echo "Building extra-container. Skip this step by adding extra-container 0.10 to PATH." - nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \ - -A pinned.extra-container >/dev/null +if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.11*) ]]; then + echo + echo "Building extra-container. Skip this step by adding extra-container 0.11 to PATH." + nix build --out-link /tmp/extra-container "$scriptDir"/..#extra-container + # When this script is run as root, e.g. when run in an extra-container shell, + # chown the gcroot symlink to the regular (login) user so that the symlink can be + # overwritten when this script is run without root. + if [[ $EUID == 0 ]]; then + chown "$(logname):" --no-dereference /tmp/extra-container + fi export PATH="/tmp/extra-container/bin${PATH:+:}$PATH" fi -read -rd '' src </proc/sysrq-trigger - '')) - ]; + # Avoid lengthy build of the nixos manual + documentation.nixos.enable = false; + + # Power off VM when the user exits the shell + systemd.services."serial-getty@".preStop = '' + echo o >/proc/sysrq-trigger + ''; system.stateVersion = lib.mkDefault config.system.nixos.release; - })).config.system.build.vm; + })).config.system.build.vm.overrideAttrs (old: { + meta = old.meta // { mainProgram = "run-vm-in-tmpdir"; }; + buildCommand = old.buildCommand + "\n" + '' + install -m 700 ${./run-vm-without-tests.sh} $out/bin/run-vm-in-tmpdir + patchShebangs $out/bin/run-vm-in-tmpdir + ''; + }); + + commonVmConfig = { + virtualisation = { + # Needed because duplicity requires 270 MB of free temp space, regardless of backup size + diskSize = 1024; + + # Min. 800 MiB needed to avoid 'out of memory' errors + memorySize = lib.mkDefault 2048; + + # There are no perf gains beyond 3 cores. + # Benchmark: Ryzen 7 2700 (8 cores), VM test `default` as of 34f6eb90. + # Num. Cores | 1 | 2 | 3 | 4 | 6 + # Runtime (sec) | 125 | 95 | 89 | 89 | 90 + cores = lib.mkDefault 3; + }; + }; +in +test // { + inherit + run + vm + container + # For NixOS with `system.stateVersion` <22.05 + containerLegacy; config = testConfig; } diff --git a/test/lib/run-vm-without-tests.sh b/test/lib/run-vm-without-tests.sh new file mode 100644 index 0000000..8d57586 --- /dev/null +++ b/test/lib/run-vm-without-tests.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script uses the following env vars: +# NIX_BITCOIN_VM_ENABLE_NETWORK +# NIX_BITCOIN_VM_DATADIR +# QEMU_OPTS +# QEMU_NET_OPTS + +if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then + dataDir=$NIX_BITCOIN_VM_DATADIR +else + dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) + trap 'rm -rf "$dataDir"' EXIT +fi + +if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then + QEMU_NET_OPTS='restrict=on' +fi + +# TODO-EXTERNAL: +# Pass PATH because run-*-vm is impure (requires coreutils from PATH) +env -i \ + PATH="$PATH" \ + USE_TMPDIR=1 \ + TMPDIR="$dataDir" \ + NIX_DISK_IMAGE="$dataDir/img.qcow2" \ + QEMU_OPTS="${QEMU_OPTS:-}" \ + QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \ + "${BASH_SOURCE[0]%/*}"/run-*-vm diff --git a/test/lib/run-vm.sh b/test/lib/run-vm.sh new file mode 100644 index 0000000..f44f0a1 --- /dev/null +++ b/test/lib/run-vm.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script uses the following env vars: +# NIX_BITCOIN_VM_ENABLE_NETWORK +# NIX_BITCOIN_VM_DATADIR +# QEMU_OPTS +# QEMU_NET_OPTS + +if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then + dataDir=$NIX_BITCOIN_VM_DATADIR +else + dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) + trap 'rm -rf "$dataDir"' EXIT +fi + +testDriver=$1 +shift + +# Variable 'tests' contains the Python code that is executed by the driver on startup +if [[ ${1:-} == --debug ]]; then + shift + echo "Running interactive testing environment" + # Start REPL. + # Use `code.interact` for the REPL instead of the builtin test driver REPL + # because it supports low featured terminals like Emacs' shell-mode. + tests=' +is_interactive = True +exec(open(os.environ["testScript"]).read()) +if "machine" in vars(): machine.start() +import code +code.interact(local=globals()) +' + echo + echo "Starting VM, data dir: $dataDir" +else + tests='exec(open(os.environ["testScript"]).read())' +fi + +if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then + QEMU_NET_OPTS='restrict=on' +fi + +# The VM creates a VDE control socket in $PWD +env --chdir "$dataDir" -i \ + USE_TMPDIR=1 \ + TMPDIR="$dataDir" \ + QEMU_OPTS="-nographic ${QEMU_OPTS:-}" \ + QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \ + "$testDriver/bin/nixos-test-driver" <(echo "$tests") "$@" diff --git a/test/lib/shellcheck-services.nix b/test/lib/shellcheck-services.nix index 8d0b209..2bad44f 100644 --- a/test/lib/shellcheck-services.nix +++ b/test/lib/shellcheck-services.nix @@ -1,8 +1,24 @@ { config, pkgs, lib, extendModules, ... }@args: with lib; let - options = { - test.shellcheckServices = mkOption { + options.test.shellcheckServices = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to shellcheck services during system build time. + ''; + }; + + sourcePrefix = mkOption { + type = with types; nullOr str; + default = null; + description = '' + The definition source path prefix of services to include in the shellcheck. + ''; + }; + + runShellcheck = mkOption { readOnly = true; description = '' A derivation that runs shellcheck on all bash scripts included @@ -12,26 +28,29 @@ let }; }; - # A list of all service names that are defined by nix-bitcoin. + cfg = config.test.shellcheckServices; + + # A list of all service names that are defined in source paths prefixed by + # `sourcePrefix`. # [ "bitcoind", "clightning", ... ] # # Algorithm: Parse defintions of `systemd.services` and return all services - # that only have definitions located in the nix-bitcoin source. - nix-bitcoin-services = let + # that only have definitions located within `sourcePrefix`. + servicesToCheck = let + inherit (cfg) sourcePrefix; systemdServices = args.options.systemd.services; configSystemdServices = args.config.systemd.services; - nix-bitcoin-source = toString ../..; - nbServices = collectServices true; - nonNbServices = collectServices false; + matchingServices = collectServices true; + nonMatchingServices = collectServices false; # Return set of services ({ service1 = true; service2 = true; ... }) - # which are either defined or not defined by nix-bitcoin, depending - # on `fromNixBitcoin`. - collectServices = fromNixBitcoin: lib.listToAttrs (builtins.concatLists (zipListsWith (services: file: + # which are either defined or not defined within `sourcePrefix`, depending + # on `shouldMatch`. + collectServices = shouldMatch: lib.listToAttrs (builtins.concatLists (zipListsWith (services: file: let - isNbSource = lib.hasPrefix nix-bitcoin-source file; + isMatching = lib.hasPrefix sourcePrefix file; in # Nix has no boolean XOR, so use `if` - lib.optionals (if fromNixBitcoin then isNbSource else !isNbSource) ( + lib.optionals (if shouldMatch then isMatching else !isMatching) ( (map (service: { name = service; value = true; }) (builtins.attrNames services)) ) # TODO-EXTERNAL: @@ -39,13 +58,13 @@ let # is included in nixpkgs stable. ) systemdServices.definitions systemdServices.files)); in - # Calculate set difference: nbServices - nonNbServices + # Calculate set difference: matchingServices - nonMatchingServices # and exclude unavailable services (defined via `mkIf false ...`) by checking `configSystemdServices`. - builtins.filter (nbService: - configSystemdServices ? ${nbService} && (! nonNbServices ? ${nbService}) - ) (builtins.attrNames nbServices); + builtins.filter (prefixedService: + configSystemdServices ? ${prefixedService} && (! nonMatchingServices ? ${prefixedService}) + ) (builtins.attrNames matchingServices); - # The concatenated list of values of ExecStart, ExecStop, ... (`scriptAttrs`) of all `nix-bitcoin-services`. + # The concatenated list of values of ExecStart, ExecStop, ... (`scriptAttrs`) of all `servicesToCheck`. serviceCmds = let scriptAttrs = [ "ExecStartPre" @@ -69,7 +88,7 @@ let if builtins.typeOf cmd == "list" then cmd else [ cmd ] ) ) scriptAttrs - ) nix-bitcoin-services; + ) servicesToCheck; # A list of all binaries included in `serviceCmds` serviceBinaries = map (cmd: builtins.head ( @@ -95,4 +114,15 @@ let in { inherit options; + + config = mkIf (cfg.enable && cfg.sourcePrefix != null) { + assertions = [ + { + assertion = builtins.length servicesToCheck > 0; + message = "test.shellcheckServices: No services found with source prefix `${cfg.sourcePrefix}`"; + } + ]; + + system.extraDependencies = [ shellcheckServices ]; + }; } diff --git a/test/nixos-search/flake.nix b/test/nixos-search/flake.nix index 113a71a..8c462bc 100644 --- a/test/nixos-search/flake.nix +++ b/test/nixos-search/flake.nix @@ -3,6 +3,7 @@ # it to the main flake. { inputs.nixos-search.url = "github:nixos/nixos-search"; + outputs = { self, nixos-search }: { inherit (nixos-search) packages; diff --git a/test/run-tests.sh b/test/run-tests.sh index 0fa227e..13eb647 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -66,7 +66,6 @@ scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) args=("$@") scenario= outLinkPrefix= -ciBuild= while :; do case $1 in --scenario|-s) @@ -89,10 +88,6 @@ while :; do exit 1 fi ;; - --ci) - shift - ciBuild=1 - ;; --copy-src|-c) shift if [[ ! $_nixBitcoinInCopiedSrc ]]; then @@ -105,183 +100,142 @@ while :; do esac done -numCPUs=${numCPUs:-$(nproc)} -# Min. 800 MiB needed to avoid 'out of memory' errors -memoryMiB=${memoryMiB:-2048} - -NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..") -export NIX_PATH - -runAtExit= -trap 'eval "$runAtExit"' EXIT +tmpDir= +# Sets global var `tmpDir` +makeTmpDir() { + if [[ ! $tmpDir ]]; then + tmpDir=$(mktemp -d /tmp/nix-bitcoin-tests.XXX) + # shellcheck disable=SC2064 + trap "rm -rf '$tmpDir'" EXIT + fi +} # Support explicit scenario definitions if [[ $scenario = *' '* ]]; then - scenarioOverridesFile=$(mktemp "${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX") - export scenarioOverridesFile - - # shellcheck disable=SC2016 - runAtExit+='rm -f "$scenarioOverridesFile";' - echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" + makeTmpDir + export scenarioOverridesFile=$tmpDir/scenario-overrides.nix + echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" scenario=tmp fi # Run the test. No temporary files are left on the host system. run() { - # TMPDIR is also used by the test driver for VM tmp files - TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) - export TMPDIR - runAtExit+="rm -rf ${TMPDIR};" - - nix-build --out-link "$TMPDIR/driver" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver - - # Variable 'tests' contains the Python code that is executed by the driver on startup - if [[ $1 == --interactive ]]; then - echo "Running interactive testing environment" - tests=$( - echo 'is_interactive = True' - echo 'exec(open(os.environ["testScript"]).read())' - # Start VM - echo 'if "machine" in vars(): machine.start()' - # Start REPL. - # Use `code.interact` for the REPL instead of the builtin test driver REPL - # because it supports low featured terminals like Emacs' shell-mode. - echo 'import code' - echo 'code.interact(local=globals())' - ) - else - tests='exec(open(os.environ["testScript"]).read())' - fi - - echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on' - cd "$TMPDIR" # The VM creates a VDE control socket in $PWD - env -i \ - NIX_PATH="$NIX_PATH" \ - TMPDIR="$TMPDIR" \ - USE_TMPDIR=1 \ - QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ - QEMU_NET_OPTS="$QEMU_NET_OPTS" \ - "$TMPDIR/driver/bin/nixos-test-driver" <(echo "$tests") + makeTmpDir + buildTestAttr .run --out-link "$tmpDir/run-vm" + NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/run-vm/bin/run-vm" "$@" } debug() { - run --interactive -} - -evalTest() { - nix-instantiate --eval -E "($(vmTestNixExpr)).outPath" -} - -instantiate() { - nix-instantiate -E "$(vmTestNixExpr)" "$@" + run --debug } container() { - export scriptDir scenario + local nixosContainer + if ! nixosContainer=$(type -p nixos-container) \ + || grep -q '"/etc/nixos-containers"' "$nixosContainer"; then + local attr=container + else + # NixOS with `system.stateVersion` <22.05 + local attr=containerLegacy + fi + echo "Building container" + makeTmpDir + export container=$tmpDir/container + buildTestAttr ".$attr" --out-link "$container" + export scriptDir "$scriptDir/lib/make-container.sh" "$@" } # Run a regular NixOS VM vm() { - TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) - export TMPDIR - runAtExit+="rm -rf $TMPDIR;" - - nix-build --out-link "$TMPDIR/vm" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vmWithoutTests" - - echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - [[ $NB_TEST_ENABLE_NETWORK ]] || export QEMU_NET_OPTS="restrict=on,$QEMU_NET_OPTS" - - # shellcheck disable=SC2211 - USE_TMPDIR=1 \ - NIX_DISK_IMAGE=$TMPDIR/img.qcow2 \ - QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ - "$TMPDIR"/vm/bin/run-*-vm -} - -doBuild() { - name=$1 - shift - if [[ $ciBuild ]]; then - "$scriptDir/ci/build-to-cachix.sh" "$@" - else - if [[ $outLinkPrefix ]]; then - outLink="--out-link $outLinkPrefix-$name" - else - outLink=--no-out-link - fi - nix-build $outLink "$@" - fi + makeTmpDir + buildTestAttr .vm --out-link "$tmpDir/vm" + NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/vm/bin/run-vm-in-tmpdir" } # Run the test by building the test derivation buildTest() { - vmTestNixExpr | doBuild $scenario "$@" - + buildTestAttr "" "$@" } -vmTestNixExpr() { - extraQEMUOpts= - - if [[ $ciBuild ]]; then - # On continuous integration nodes there are few other processes running alongside the - # test, so use more memory here for maximum performance. - memoryMiB=4096 - memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo) - memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo) - # Round down to nearest multiple of 50 MiB for improved test build caching - # shellcheck disable=SC2017 - ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50)) - ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB - >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB" - - # VMX is usually not available on CI nodes due to recursive virtualisation. - # Explicitly disable VMX, otherwise QEMU 4.20 fails with message - # "error: failed to set MSR 0x48b to 0x159ff00000000" - extraQEMUOpts="-cpu host,-vmx" - fi - - cat <&1) == *"requires a sub-command"* ]]; then - hasFlakes=1 - else - hasFlakes= - fi +buildTestAttr() { + local attr=$1 + shift + # TODO-EXTERNAL: + # Simplify and switch to pure build when `nix build` can build flake function outputs + nixInstantiateTest "$attr" + nixBuild "$scenario" "$drv" "$@" +} + +buildTests() { + local -n tests=$1 + shift + # TODO-EXTERNAL: + # Simplify and switch to pure build when `nix build` can instantiate flake function outputs + # shellcheck disable=SC2207 + drvs=($(nixInstantiate "pkgs.instantiateTests \"${tests[*]}\"")) + for i in "${!tests[@]}"; do + testName=${tests[$i]} + drv=${drvs[$i]} + echo + echo "Building test '$testName'" + nixBuild "$testName" "$drv" "$@" + done +} + +# Instantiate an attr of the test defined in global var `scenario` +nixInstantiateTest() { + local attr=$1 + shift + if [[ ${scenarioOverridesFile:-} ]]; then + local file="extraScenariosFile = \"$scenarioOverridesFile\";" + else + local file= fi - if [[ ! $hasFlakes ]]; then - echo "Skipping test '$testName'. Nix flake support is not enabled." - return 1 + nixInstantiate "(pkgs.getTest { name = \"$scenario\"; $file })$attr" "$@" >/dev/null +} + +drv= +# Sets global var `drv` to the gcroot link of the instantiated derivation +nixInstantiate() { + local expr=$1 + shift + makeTmpDir + drv="$tmpDir/drv" + nix-instantiate --add-root "$drv" -E " + let + pkgs = (builtins.getFlake \"git+file://$scriptDir/..\").legacyPackages.\${builtins.currentSystem}; + in + $expr + " "$@" +} + +nixBuild() { + local outLinkName=$1 + shift + args=(--print-out-paths -L) + if [[ $outLinkPrefix ]]; then + args+=(--out-link "$outLinkPrefix-$outLinkName") + else + args+=(--no-link) fi + nix build "${args[@]}" "$@" } flake() { - if ! checkFlakeSupport "flake"; then return; fi - nix flake check "$scriptDir/.." } # Test generating module documentation for search.nixos.org nixosSearch() { - if ! checkFlakeSupport "nixosSearch"; then return; fi - - if [[ $_nixBitcoinInCopiedSrc ]]; then - # flake-info requires that its target flake is under version control - . "$scriptDir/lib/create-git-repo.sh" - fi - if [[ $outLinkPrefix ]]; then # Add gcroots for flake-info nix build "$scriptDir/nixos-search#flake-info" -o "$outLinkPrefix-flake-info" @@ -290,38 +244,41 @@ nixosSearch() { } # A basic subset of tests to keep the total runtime within -# manageable bounds (<4 min on desktop systems). +# manageable bounds. # These are also run on the CI server. -basic() { - scenario=default buildTest "$@" - scenario=netns buildTest "$@" - scenario=netnsRegtest buildTest "$@" -} +basic=( + default + netns + netnsRegtest +) +basic() { buildTests basic "$@"; } # All tests that only consist of building a nix derivation. -# Their output is cached in /nix/store. -buildable() { - basic "$@" - scenario=full buildTest "$@" - scenario=regtest buildTest "$@" - scenario=hardened buildTest "$@" - scenario=clightningReplication buildTest "$@" - scenario=lndPruned buildTest "$@" -} +# shellcheck disable=2034 +buildable=( + "${basic[@]}" + full + regtest + hardened + clightningReplication + lndPruned +) +buildable() { buildTests buildable "$@"; } examples() { - script=" + # shellcheck disable=SC2016 + script=' set -e - ./deploy-container.sh - ./deploy-container-minimal.sh - ./deploy-qemu-vm.sh - ./deploy-krops.sh - " + runExample() { echo; echo Running example $1; ./$1; } + runExample deploy-container.sh + runExample deploy-container-minimal.sh + runExample deploy-qemu-vm.sh + runExample deploy-krops.sh + ' (cd "$scriptDir/../examples" && nix-shell --run "$script") } shellcheck() { - if ! checkFlakeSupport "shellcheck"; then return; fi "$scriptDir/shellcheck.sh" } diff --git a/test/tests.nix b/test/tests.nix index cd30357..b0f9c69 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -1,19 +1,11 @@ # Integration tests, can be run without internet access. +lib: let - nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs; -in - -{ extraScenarios ? { ... }: {} -, pkgs ? import nixpkgs { config = {}; overlays = []; } -}: -with pkgs.lib; -let - globalPkgs = pkgs; - - baseConfig = { pkgs, config, ... }: let + # Included in all scenarios + baseConfig = { config, pkgs, ... }: with lib; let cfg = config.services; - mkIfTest = test: mkIf (config.tests.${test} or false); + inherit (config.nix-bitcoin.lib.test) mkIfTest; in { imports = [ ./lib/test-lib.nix @@ -32,9 +24,6 @@ let }; config = mkMerge [{ - # Share the same pkgs instance among tests - nixpkgs.pkgs = mkDefault globalPkgs; - environment.systemPackages = mkMerge (with pkgs; [ # Needed to test macaroon creation (mkIfTest "btcpayserver" [ openssl xxd ]) @@ -176,8 +165,9 @@ let ]; }; - scenarios = { - base = baseConfig; # Included in all scenarios + scenarios = with lib; { + # Included in all scenarios by ./lib/make-test.nix + base = baseConfig; default = scenarios.secureNode; @@ -273,7 +263,7 @@ let environment.systemPackages = [ pkgs.fping ]; }; - regtestBase = { config, ... }: { + regtestBase = { config, pkgs, ... }: { tests.regtest = true; test.data.num_blocks = 100; @@ -323,9 +313,10 @@ let services.lnd.enable = true; services.bitcoind.prune = 1000; }; + }; - ## Examples / debug helper - + ## Example scenarios that showcase extra features + exampleScenarios = with lib; { # Run a selection of tests in scenario 'netns' selectedTests = { imports = [ scenarios.netns ]; @@ -342,40 +333,82 @@ let # See ./lib/test-lib.nix for a description test.container.exposeLocalhost = true; }; - - adhoc = { - # - # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios. - }; }; +in { + inherit scenarios; - overrides = builtins.getEnv "scenarioOverridesFile"; - extraScenarios' = (if (overrides != "") then import overrides else extraScenarios) { - inherit scenarios pkgs; - inherit (pkgs) lib; + pkgs = flake: pkgs: rec { + # A basic test using the nix-bitcoin test framework + makeTestBasic = import ./lib/make-test.nix flake pkgs makeTestVM; + + # Wraps `makeTest` in NixOS' testing-python.nix so that the drv includes the + # log output and the test driver + makeTestVM = import ./lib/make-test-vm.nix pkgs; + + # A test using the nix-bitcoin test framework, with some helpful defaults + makeTest = { name ? "nix-bitcoin-test", config }: + makeTestBasic { + inherit name; + config = { + imports = [ + scenarios.base + config + ]; + # Share the same pkgs instance among tests + nixpkgs.pkgs = pkgs.lib.mkDefault pkgs; + }; + }; + + # A test using the nix-bitcoin test framework, with defaults specific to nix-bitcoin + makeTestNixBitcoin = { name, config }: + makeTest { + name = "nix-bitcoin-${name}"; + config = { + imports = [ config ]; + test.shellcheckServices.sourcePrefix = toString ./..; + }; + }; + + makeTests = scenarios: let + mainTests = builtins.mapAttrs (name: config: + makeTestNixBitcoin { inherit name config; } + ) scenarios; + in + { + clightningReplication = import ./clightning-replication.nix makeTestVM pkgs; + } // mainTests; + + tests = makeTests scenarios; + + ## Helper for ./run-tests.sh + + getTest = { name, extraScenariosFile ? null }: + let + tests = makeTests (scenarios // ( + lib.optionalAttrs (extraScenariosFile != null) + (import extraScenariosFile { + inherit scenarios lib pkgs; + nix-bitcoin = flake; + }) + )); + in + tests.${name} or (makeTestNixBitcoin { + inherit name; + config = { + services.${name}.enable = true; + }; + }); + + instantiateTests = testNames: + let + testNames' = lib.splitString " " testNames; + in + map (name: + let + test = tests.${name}; + in + builtins.seq (builtins.trace "Evaluating test '${name}'" test.outPath) + test + ) testNames'; }; - allScenarios = scenarios // extraScenarios'; - - makeTest = name: config: - makeTest' name { - imports = [ - allScenarios.base - config - ]; - }; - makeTest' = import ./lib/make-test.nix pkgs; - - tests = builtins.mapAttrs makeTest allScenarios // { - clightningReplication.vm = import ./clightning-replication.nix { - inherit pkgs; - inherit (pkgs.stdenv) system; - }; - }; - - getTest = name: tests.${name} or (makeTest name { - services.${name}.enable = true; - }); -in - tests // { - inherit getTest; - } +}