Merge #339: Misc improvements

4cddf284e9 treewide: remove use of deprecated `stdenv.lib` (Erik Arvstedt)
6ba5880b2c test.py: improve composability of test 'banlist-and-restart' (Erik Arvstedt)
44439e2a81 tests: optimize building multiple tests at once (Erik Arvstedt)
9ca52af523 tests: improve make-test-vm.nix (Erik Arvstedt)
08fe9ba84a services: add finer-grained address family restrictions (Erik Arvstedt)
020433cec6 services: add helper fn setAllowedIPAddresses (Erik Arvstedt)
cdf27d9d0c bitcoind: improve service timeouts (Erik Arvstedt)
09cd3ce5e4 lnd: show curl error messages (Erik Arvstedt)
d214605b32 spark-wallet: add flakes compatibility (Erik Arvstedt)
81db927f66 spark-wallet/generate: remove supplement.json (Erik Arvstedt)
84b3217c3d fetch-release: minor improvements (Erik Arvstedt)
45d0964e27 examples/shell.nix: minor improvements (Erik Arvstedt)
cc7149eb78 examples: improve robustness of deploy scripts (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  nixbitcoin:
    ACK 4cddf284e9

Tree-SHA512: 5f863a406a56b4b916ea662c411dce6884a7633a49e862015ddf68c20e772a14108095782753c2e33061d8d19cbf9053239f98644cb12fd883ef79e68d8a977b

examples/deploy-container.sh

@@ -8,10 +8,12 @@ set -euo pipefail
 # Run with option `--interactive` or `-i` to start a shell for interacting with
 # the node.
 
-if [[ ! -v IN_NIX_SHELL ]]; then
+if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then
     echo "Running script in nix shell env..."
     cd "${BASH_SOURCE[0]%/*}"
     exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*"
+else
+    cd "$NIX_BITCOIN_EXAMPLES_DIR"
 fi
 
 if [[ $(sysctl -n net.ipv4.ip_forward || sudo sysctl -n net.ipv4.ip_forward) != 1 ]]; then
@@ -22,7 +24,7 @@ fi
 
 if [[ $EUID != 0 ]]; then
     # NixOS containers require root permissions
-    exec sudo "PATH=$PATH" "NIX_PATH=$NIX_PATH" "IN_NIX_SHELL=$IN_NIX_SHELL" "${BASH_SOURCE[0]}" "$@"
+    exec sudo "PATH=$PATH" "NIX_PATH=$NIX_PATH" "NIX_BITCOIN_EXAMPLES_DIR=$NIX_BITCOIN_EXAMPLES_DIR" "${BASH_SOURCE[0]}" "$@"
 fi
 
 interactive=

examples/deploy-krops.sh

@@ -12,10 +12,12 @@ set -euo pipefail
 # MAKE SURE TO REPLACE the SSH identity file if you use this script for
 # anything serious.
 
-if [[ ! -v IN_NIX_SHELL ]]; then
+if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then
     echo "Running script in nix shell env..."
     cd "${BASH_SOURCE[0]%/*}"
     exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*"
+else
+    cd "$NIX_BITCOIN_EXAMPLES_DIR"
 fi
 
 source qemu-vm/run-vm.sh

examples/deploy-qemu-vm.sh

@@ -11,10 +11,12 @@ set -euo pipefail
 # MAKE SURE TO REPLACE the SSH identity file if you use this script for
 # anything serious.
 
-if [[ ! -v IN_NIX_SHELL ]]; then
+if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then
     echo "Running script in nix shell env..."
     cd "${BASH_SOURCE[0]%/*}"
     exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*"
+else
+    cd "$NIX_BITCOIN_EXAMPLES_DIR"
 fi
 
 source qemu-vm/run-vm.sh

examples/shell.nix

@@ -8,25 +8,28 @@ let
     else nix-bitcoin-release;
 
   nixpkgs-path = (import "${toString nix-bitcoin-path}/pkgs/nixpkgs-pinned.nix").nixpkgs;
-  nixpkgs = import nixpkgs-path {};
-  nix-bitcoin = nixpkgs.callPackage nix-bitcoin-path {};
+  pkgs = import nixpkgs-path {};
+  nix-bitcoin = pkgs.callPackage nix-bitcoin-path {};
 
   nix-bitcoin-unpacked = (import <nixpkgs> {}).runCommand "nix-bitcoin-src" {} ''
     mkdir $out; tar xf ${builtins.fetchurl nix-bitcoin-release} -C $out
   '';
 in
-with nixpkgs;
-
+with pkgs;
 stdenv.mkDerivation rec {
   name = "nix-bitcoin-environment";
 
-  path = lib.makeBinPath [ nix-bitcoin.extra-container figlet ];
+  path = lib.makeBinPath [ nix-bitcoin.extra-container ];
 
   shellHook = ''
     export NIX_PATH="nixpkgs=${nixpkgs-path}:nix-bitcoin=${toString nix-bitcoin-path}:."
     export PATH="${path}''${PATH:+:}$PATH"
 
-    alias fetch-release="${toString nix-bitcoin-path}/helper/fetch-release"
+    export NIX_BITCOIN_EXAMPLES_DIR="${toString ./.}"
+
+    fetch-release() {
+      ${toString nix-bitcoin-path}/helper/fetch-release
+    }
 
     krops-deploy() {
       # Ensure strict permissions on secrets/ directory before rsyncing it to
@@ -35,7 +38,13 @@ stdenv.mkDerivation rec {
       $(nix-build --no-out-link ${toString ./krops/deploy.nix})
     }
 
-    figlet "nix-bitcoin"
+    # Print logo if
+    # 1. stdout is a TTY, i.e. we're not piping the output
+    # 2. the shell is interactive
+    if [[ -t 1 && $- == *i* ]]; then
+      ${figlet}/bin/figlet "nix-bitcoin"
+    fi
+
     (mkdir -p secrets; cd secrets; env -i ${nix-bitcoin.generate-secrets})
 
     # Don't run this hook when another nix-shell is run inside this shell

helper/fetch-release

@@ -1,36 +1,40 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i bash -p bash coreutils curl jq gnugrep gnupg
+#!nix-shell -i bash -p bash coreutils curl jq gnupg
 set -euo pipefail
 
 scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
 
-REPO=fort-nix/nix-bitcoin
-if [[ ! -v VERSION ]]; then
-    VERSION=$(curl --silent "https://api.github.com/repos/$REPO/releases/latest" | jq -r '.tag_name' | tail -c +2)
+repo=fort-nix/nix-bitcoin
+if [[ ! -v version ]]; then
+    version=$(curl --silent "https://api.github.com/repos/$repo/releases/latest" | jq -r '.tag_name' | tail -c +2)
 fi
 
 TMPDIR=$(mktemp -d)
-GPG_HOME=$(mktemp -d)
-trap "rm -rf $TMPDIR $GPG_HOME" EXIT
+trap "rm -rf $TMPDIR" EXIT
+
+GPG_HOME=$TMPDIR/gpg-home
+mkdir -p -m 700 "$GPG_HOME"
 
 cd $TMPDIR
-BASEURL=https://github.com/$REPO/releases/download/v$VERSION
-curl --silent -L -O $BASEURL/SHA256SUMS.txt
-curl --silent -L -O $BASEURL/SHA256SUMS.txt.asc
+baseUrl=https://github.com/$repo/releases/download/v$version
+curl --silent -L -O $baseUrl/SHA256SUMS.txt
+curl --silent -L -O $baseUrl/SHA256SUMS.txt.asc
 
-# Import key and verify fingerprint
+# Import key
 gpg --homedir $GPG_HOME --import "$scriptDir/key-jonasnick.bin" &> /dev/null
+# Verify key fingerprint
 gpg --homedir $GPG_HOME --list-keys 36C71A37C9D988BDE82508D9B1A70E4F8DCD0366 > /dev/null
 
+# Verify signature for SHA256SUMS.txt
 gpg --homedir $GPG_HOME --verify SHA256SUMS.txt.asc &> /dev/null || {
-    echo "ERROR: Signature verification failed. Please open an issue in the project repository."
+    echo "Error: Signature verification failed. Please open an issue in the project repository."
     exit 1
 }
 
-SHA256=$(cat SHA256SUMS.txt | grep -Eo '^[^ ]+')
+sha256=$(cat SHA256SUMS.txt | cut -d\  -f1)
 cat <<EOF
 {
-  url = "$BASEURL/nix-bitcoin-$VERSION.tar.gz";
-  sha256 = "$SHA256";
+  url = "$baseUrl/nix-bitcoin-$version.tar.gz";
+  sha256 = "$sha256";
 }
 EOF

modules/bitcoind.nix

@@ -55,6 +55,8 @@ let
     # Extra options
     ${cfg.extraConfig}
   '';
+
+  zmqServerEnabled = (cfg.zmqpubrawblock != null) || (cfg.zmqpubrawtx != null);
 in {
   options = {
     services.bitcoind = {
@@ -351,15 +353,14 @@ in {
         NotifyAccess = "all";
         User = cfg.user;
         Group = cfg.group;
-        TimeoutStartSec = 300;
+        TimeoutStartSec = "5min";
+        TimeoutStopSec = "10min";
         ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'";
         Restart = "on-failure";
         UMask = mkIf cfg.dataDirReadableByGroup "0027";
         ReadWritePaths = cfg.dataDir;
-      } // (if cfg.enforceTor
-            then nbLib.allowTor
-            else nbLib.allowAnyIP)
-        // optionalAttrs (cfg.zmqpubrawblock != null || cfg.zmqpubrawtx != null) nbLib.allowAnyProtocol;
+      } // nbLib.allowedIPAddresses cfg.enforceTor
+        // optionalAttrs zmqServerEnabled nbLib.allowNetlink;
     };
 
     # Use this to update the banlist:
@@ -384,7 +385,7 @@ in {
         User = cfg.user;
         Group = cfg.group;
         ReadWritePaths = cfg.dataDir;
-      } // nbLib.allowTor;
+      } // nbLib.allowLocalIPAddresses;
     };
 
     users.users.${cfg.user}.group = cfg.group;

modules/btcpayserver.nix

@@ -155,10 +155,7 @@ in {
         RestartSec = "10s";
         ReadWritePaths = cfg.nbxplorer.dataDir;
         MemoryDenyWriteExecute = "false";
-      } // (if cfg.nbxplorer.enforceTor
-        then nbLib.allowTor
-        else nbLib.allowAnyIP
-      );
+      } // nbLib.allowedIPAddresses cfg.nbxplorer.enforceTor;
     };
 
     systemd.services.btcpayserver = let
@@ -204,10 +201,7 @@ in {
         RestartSec = "10s";
         ReadWritePaths = cfg.btcpayserver.dataDir;
         MemoryDenyWriteExecute = "false";
-      } // (if cfg.btcpayserver.enforceTor
-        then nbLib.allowTor
-        else nbLib.allowAnyIP
-      );
+      } // nbLib.allowedIPAddresses cfg.btcpayserver.enforceTor;
     }; in self;
 
     users.users.${cfg.nbxplorer.user} = {

modules/clightning-plugins/zmq.nix

@@ -4,6 +4,8 @@ with lib;
 let
   cfg = config.services.clightning.plugins.zmq;
 
+  nbLib = config.nix-bitcoin.lib;
+
   endpoints = [
     "channel-opened"
     "connect"
@@ -38,5 +40,9 @@ in
       plugin=${config.nix-bitcoin.pkgs.clightning-plugins.zmq.path}
       ${concatStrings (map setEndpoint endpoints)}
     '';
+
+    # The zmq server requires AF_NETLINK
+    systemd.services.clightning.serviceConfig.RestrictAddressFamilies =
+      mkForce nbLib.allowNetlink.RestrictAddressFamilies;
   };
 }

modules/clightning.nix

@@ -128,10 +128,7 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
-      } // (if cfg.enforceTor
-          then nbLib.allowTor
-          else nbLib.allowAnyIP
-        );
+      } // nbLib.allowedIPAddresses cfg.enforceTor;
       # Wait until the rpc socket appears
       postStart = ''
         while [[ ! -e ${cfg.networkDir}/lightning-rpc ]]; do

modules/electrs.nix

@@ -102,10 +102,7 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = "${cfg.dataDir} ${if cfg.high-memory then "${bitcoind.dataDir}" else ""}";
-      } // (if cfg.enforceTor
-          then nbLib.allowTor
-          else nbLib.allowAnyIP
-        );
+      } // nbLib.allowedIPAddresses cfg.enforceTor;
     };
 
     users.users.${cfg.user} = {

modules/lightning-loop.nix

@@ -102,9 +102,7 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
-      } // (if cfg.enforceTor
-            then nbLib.allowTor
-            else nbLib.allowAnyIP);
+      } // nbLib.allowedIPAddresses cfg.enforceTor;
     };
 
      nix-bitcoin.secrets = {

modules/lightning-pool.nix

@@ -100,9 +100,8 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
-      } // (if cfg.enforceTor
-            then nbLib.allowTor
-            else nbLib.allowAnyIP);
+      } // (nbLib.allowedIPAddresses cfg.enforceTor)
+        // nbLib.allowNetlink; # required by gRPC-Go
     };
   };
 }

modules/liquid.nix

@@ -239,10 +239,7 @@ in {
         PIDFile = pidFile;
         Restart = "on-failure";
         ReadWritePaths = cfg.dataDir;
-      } // (if cfg.enforceTor
-          then nbLib.allowTor
-          else nbLib.allowAnyIP
-        );
+      } // nbLib.allowedIPAddresses cfg.enforceTor;
     };
 
     users.users.${cfg.user} = {

modules/lnd.nix

@@ -198,6 +198,7 @@ in {
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
         ExecStartPost = let
+          curl = "${pkgs.curl}/bin/curl -s --show-error";
           restUrl = "https://${cfg.restAddress}:${toString cfg.restPort}/v1";
         in [
           (nbLib.script "lnd-create-wallet" ''
@@ -212,13 +213,13 @@ in {
               if [[ ! -f "$mnemonic" ]]; then
                 echo Create lnd seed
                 umask u=r,go=
-                ${pkgs.curl}/bin/curl -s \
+                ${curl} \
                   --cacert ${secretsDir}/lnd-cert \
                   -X GET ${restUrl}/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > "$mnemonic"
               fi
 
               echo Create lnd wallet
-              ${pkgs.curl}/bin/curl -s --output /dev/null --show-error \
+              ${curl} --output /dev/null \
                 --cacert ${secretsDir}/lnd-cert \
                 -X POST -d "{\"wallet_password\": \"$(cat ${secretsDir}/lnd-wallet-password | tr -d '\n' | base64 -w0)\", \
                 \"cipher_seed_mnemonic\": $(cat "$mnemonic" | tr -d '\n')}" \
@@ -231,8 +232,7 @@ in {
               done
             else
               echo Unlock lnd wallet
-
-              ${pkgs.curl}/bin/curl -s \
+              ${curl} \
                 -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${networkDir}/admin.macaroon')" \
                 --cacert ${secretsDir}/lnd-cert \
                 -X POST \
@@ -251,7 +251,7 @@ in {
             ${lib.concatMapStrings (macaroon: ''
               echo "Create custom macaroon ${macaroon}"
               macaroonPath="$RUNTIME_DIRECTORY/${macaroon}.macaroon"
-              ${pkgs.curl}/bin/curl -s \
+              ${curl} \
                 -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${networkDir}/admin.macaroon')" \
                 --cacert ${secretsDir}/lnd-cert \
                 -X POST \
@@ -262,10 +262,7 @@ in {
             '') (attrNames cfg.macaroons)}
           '')
         ];
-      } // (if cfg.enforceTor
-          then nbLib.allowTor
-          else nbLib.allowAnyIP
-        ) // nbLib.allowAnyProtocol;  # For ZMQ
+      } // nbLib.allowedIPAddresses cfg.enforceTor;
     };
 
     users.users.${cfg.user} = {

modules/recurring-donations.nix

@@ -83,9 +83,7 @@ in {
         ExecStart = "${pkgs.bash}/bin/bash ${recurring-donations-script}";
         User = "recurring-donations";
         Type = "oneshot";
-      } // (if cfg.enforceTor
-            then nbLib.allowTor
-            else nbLib.allowAnyIP);
+      } // nbLib.allowedIPAddresses cfg.enforceTor;
     };
     systemd.timers.recurring-donations = {
       requires = [ "clightning.service" ];

modules/spark-wallet.nix

@@ -79,9 +79,7 @@ in {
         User = cfg.user;
         Restart = "on-failure";
         RestartSec = "10s";
-      } // (if cfg.enforceTor
-            then nbLib.allowTor
-            else nbLib.allowAnyIP)
+      } // nbLib.allowedIPAddresses cfg.enforceTor
         // nbLib.nodejs;
     };
     nix-bitcoin.secrets.spark-wallet-login.user = cfg.user;

pkgs/clboss/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchurl, pkgconfig, curl, libev, sqlite }:
+{ lib, stdenv, fetchurl, pkgconfig, curl, libev, sqlite }:
 
 let
   curlWithGnuTLS = curl.override { gnutlsSupport = true; sslSupport = false; };
@@ -16,7 +16,7 @@ stdenv.mkDerivation rec {
 
   enableParallelBuilding = true;
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     description = "Automated C-Lightning Node Manager";
     homepage = "https://github.com/ZmnSCPxj/clboss";
     maintainers = with maintainers; [ nixbitcoin ];

pkgs/elementsd/default.nix

@@ -1,8 +1,8 @@
-{ stdenv, fetchurl, pkgconfig, autoreconfHook, openssl, db48, boost, zeromq, rapidcheck
+{ lib, stdenv, fetchurl, pkgconfig, autoreconfHook, openssl, db48, boost, zeromq, rapidcheck
 , zlib, miniupnpc, qtbase ? null, qttools ? null, wrapQtAppsHook ? null, utillinux, protobuf, python3, qrencode, libevent
 , withGui }:
 
-with stdenv.lib;
+with lib;
 stdenv.mkDerivation rec {
   pname = "elements${optionalString (!withGui) "d"}";
   version = "0.18.1.9";

pkgs/hwi/default.nix

@@ -1,6 +1,4 @@
-{ pkgs, stdenv, fetchurl, fetchFromGitHub, python3 }:
-
-with stdenv.lib;
+{ lib, pkgs, stdenv, fetchurl, fetchFromGitHub, python3 }:
 
 let
   python = python3.override {

pkgs/hwi/ecdsa/default.nix

@@ -1,4 +1,5 @@
-{ stdenv
+{ lib
+, stdenv
 , buildPythonPackage
 , fetchPypi
 , pkgs
@@ -18,7 +19,7 @@ buildPythonPackage rec {
   # Only needed for tests
   checkInputs = [ pkgs.openssl ];
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     description = "ECDSA cryptographic signature library";
     homepage = "https://github.com/warner/python-ecdsa";
     license = licenses.mit;

pkgs/hwi/hidapi/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, libusb1, udev, darwin, fetchPypi, buildPythonPackage, cython }:
+{ lib, stdenv, libusb1, udev, darwin, fetchPypi, buildPythonPackage, cython }:
 
 buildPythonPackage rec {
   pname = "hidapi";
@@ -10,18 +10,18 @@ buildPythonPackage rec {
   };
 
   propagatedBuildInputs =
-    stdenv.lib.optionals stdenv.isLinux [ libusb1 udev ] ++
-    stdenv.lib.optionals stdenv.isDarwin [ darwin.IOKit darwin.apple_sdk.frameworks.CoreFoundation ] ++
+    lib.optionals stdenv.isLinux [ libusb1 udev ] ++
+    lib.optionals stdenv.isDarwin [ darwin.IOKit darwin.apple_sdk.frameworks.CoreFoundation ] ++
     [ cython ];
 
   # Fix the USB backend library lookup
-  postPatch = stdenv.lib.optionalString stdenv.isLinux ''
+  postPatch = lib.optionalString stdenv.isLinux ''
     libusb=${libusb1.dev}/include/libusb-1.0
     test -d $libusb || { echo "ERROR: $libusb doesn't exist, please update/fix this build expression."; exit 1; }
     sed -i -e "s|/usr/include/libusb-1.0|$libusb|" setup.py
   '';
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     description = "A Cython interface to the hidapi from https://github.com/signal11/hidapi";
     homepage = "https://github.com/trezor/cython-hidapi";
     # license can actually be either bsd3 or gpl3

pkgs/lib.nix

@@ -33,15 +33,23 @@ let self = {
       SystemCallArchitectures = "native";
   };
 
+  allowNetlink = {
+    RestrictAddressFamilies = self.defaultHardening.RestrictAddressFamilies + " AF_NETLINK";
+  };
+
   # nodejs applications apparently rely on memory write execute
   nodejs = { MemoryDenyWriteExecute = "false"; };
-  # Allow tor traffic. Allow takes precedence over Deny.
-  allowTor = {
+
+  # Allow takes precedence over Deny.
+  allowLocalIPAddresses = {
     IPAddressAllow = "127.0.0.1/32 ::1/128 169.254.0.0/16";
   };
-  # Allow any traffic
-  allowAnyIP = { IPAddressAllow = "any"; };
-  allowAnyProtocol = { RestrictAddressFamilies = "~"; };
+  allowAllIPAddresses = { IPAddressAllow = "any"; };
+  allowTor = self.allowLocalIPAddresses;
+  allowedIPAddresses = onlyLocal:
+    if onlyLocal
+    then self.allowLocalIPAddresses
+    else self.allowAllIPAddresses;
 
   enforceTor = mkOption {
     type = types.bool;

pkgs/secp256k1/default.nix

@@ -1,6 +1,4 @@
-{ stdenv, fetchFromGitHub, autoreconfHook }:
-
-let inherit (stdenv.lib) optionals; in
+{ lib, stdenv, fetchFromGitHub, autoreconfHook }:
 
 stdenv.mkDerivation {
   pname = "secp256k1";
@@ -18,7 +16,7 @@ stdenv.mkDerivation {
 
   configureFlags = ["--enable-module-recovery" "--disable-jni" "--enable-experimental" "--enable-module-ecdh" "--enable-benchmark=no" ];
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     description = "Optimized C library for EC operations on curve secp256k1";
     homepage = "https://github.com/bitcoin-core/secp256k1";
     license = with licenses; [ mit ];

pkgs/spark-wallet/composition.nix

@@ -5,11 +5,7 @@
   }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-10_x"}:
 
 let
-  globalBuildInputs = pkgs.lib.attrValues (import ./supplement.nix {
-    inherit nodeEnv;
-    inherit (pkgs) fetchurl fetchgit;
-  });
-  nodeEnv = import ./node-env.nix {
+  nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
     inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
     inherit nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
@@ -17,5 +13,5 @@ let
 in
 import ./node-packages.nix {
   inherit (pkgs) fetchurl fetchgit;
-  inherit nodeEnv globalBuildInputs;
+  inherit nodeEnv;
 }

pkgs/spark-wallet/default.nix

@@ -1,5 +1,5 @@
 { stdenv, pkgs, lib }:
-lib.head (lib.attrValues (import ./composition.nix {
+lib.head (builtins.attrValues (import ./composition.nix {
     inherit pkgs;
     inherit (stdenv.hostPlatform) system;
 }))

pkgs/spark-wallet/generate.sh

@@ -20,7 +20,14 @@ jq '.dependencies["qrcode-terminal"] = .optionalDependencies["qrcode-terminal"]'
 
 # Run node2nix
 cp pkg.json $TMPDIR/pkg.json
-node2nix --nodejs-10 -i $TMPDIR/pkg.json -c composition.nix --no-copy-node-env --supplement-input supplement.json
+node2nix --nodejs-10 -i $TMPDIR/pkg.json -c composition.nix --no-copy-node-env
+
+# Set node env import.
+# The reason for not providing a custom node-env.nix file is the following:
+# To be flakes-compatible, we have to locate the nixpgs source via `pkgs.path` instead of `<nixpkgs>`.
+# This requires the `pkgs` variable which is available only in composition.nix, not in node-env.nix.
+nodeEnvImport='import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix"'
+sed -i "s|import ./node-env.nix|$nodeEnvImport|" composition.nix
 
 # Use verified source in node-packages.nix
 url="https://github.com/shesek/spark-wallet/releases/download/v$version/spark-wallet-$version-npm.tgz"

pkgs/spark-wallet/node-env.nix

@@ -1 +0,0 @@
-import <nixpkgs/pkgs/development/node-packages/node-env.nix>

pkgs/spark-wallet/supplement.json

@@ -1,2 +0,0 @@
-[
-]

test/lib/make-container.sh

@@ -84,6 +84,6 @@ if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.6*) ]]
 fi
 
 read -d '' src <<EOF || true
-(import "$scriptDir/tests.nix" { scenario = "$scenario"; }).container
+((import "$scriptDir/tests.nix" {}).getTest "$scenario").container
 EOF
 exec extra-container $containerCommand -E "$src" "$@"

test/lib/make-test-vm.nix

@@ -1,37 +1,45 @@
-testArgs:
-
+pkgs:
 let
-  pkgs = import <nixpkgs> { config = {}; overlays = []; };
-
-  test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs);
-
-  fixedTest = { ... }@args:
-    let
-       pkgsFixed = pkgs // {
-         # Fix the black Python code formatter that's used in the test to allow the test
-         # script to have longer lines. The default width of 88 chars is too restrictive for
-         # our script.
-         python3Packages = pkgs.python3Packages // {
-           black = pkgs.writeScriptBin "black" ''
-             fileToCheck=''${@:$#}
-             [[ $fileToCheck = *test-script ]] && extraArgs='--line-length 100'
-             exec ${pkgs.python3Packages.black}/bin/black $extraArgs "$@"
-           '';
+  pythonTesting = import "${toString pkgs.path}/nixos/lib/testing-python.nix" {
+    system = pkgs.stdenv.hostPlatform.system;
+    inherit pkgs;
   };
-       };
-       test' = test (args // { pkgs = pkgsFixed; });
 in
-      # See nixpkgs/nixos/lib/testing-python.nix for the original definition
-      test'.overrideAttrs (_: {
-        # 1. Save test output
-        # 2. Add link to driver so that a gcroot to a test prevents the driver from
+
+args:
+let
+  test = pythonTesting.makeTest args;
+
+  fixedDriver = test.driver.overrideAttrs (old: let
+    # Allow the test script to have longer lines by fixing the call to the 'black'
+    # code formatter.
+    # The default width of 88 chars is too restrictive for our script.
+    parts = builtins.split ''/nix/store/[^ ]+/black '' old.buildCommand;
+    preMatch = builtins.elemAt parts 0;
+    postMatch = builtins.elemAt parts 2;
+  in {
+    # See `mkDriver` in nixpkgs/nixos/lib/testing-python.nix for the original definition of `buildCommand`
+    buildCommand = ''
+      ${preMatch}${pkgs.python3Packages.black}/bin/black --line-length 100 ${postMatch}
+    '';
+    # Keep reference to the `testDriver` derivation, required by `buildCommand`
+    testDriverReference = old.buildCommand;
+  });
+
+  # 1. Use fixed driver
+  # 2. Save test logging output
+  # 3. Add link to driver so that a gcroot to a test prevents the driver from
   #    being garbage-collected
+  fixedTest = test.overrideAttrs (_: {
+    # See `runTests` in nixpkgs/nixos/lib/testing-python.nix for the original definition of `buildCommand`
     buildCommand = ''
       mkdir $out
-          LOGFILE=$out/output.xml tests='exec(os.environ["testScript"])' ${test'.driver}/bin/nixos-test-driver
-          ln -s ${test'.driver} $out/driver
+      LOGFILE=$out/output.xml tests='exec(os.environ["testScript"])' ${fixedDriver}/bin/nixos-test-driver
+      ln -s ${fixedDriver} $out/driver
     '';
-      }) // { inherit (test') nodes driver; } ;
-
+  }) // {
+    driver = fixedDriver;
+    inherit (test) nodes;
+  };
 in
   fixedTest

test/lib/make-test.nix

@@ -1,13 +1,25 @@
-scenario: testConfig:
+pkgs:
+let
+  makeVM = import ./make-test-vm.nix pkgs;
+  inherit (pkgs) lib;
+in
 
+name: testConfig:
 {
-  vm = import ./make-test-vm.nix (pkgs: {
-    name = "nix-bitcoin-${scenario}";
+  vm = makeVM {
+    name = "nix-bitcoin-${name}";
 
     machine = {
       imports = [ testConfig ];
+      virtualisation = {
         # Needed because duplicity requires 270 MB of free temp space, regardless of backup size
-      virtualisation.diskSize = 1024;
+        diskSize = 1024;
+
+        # Min. 800 MiB needed to avoid 'out of memory' errors
+        memorySize = lib.mkDefault 2048;
+
+        cores = lib.mkDefault 2;
+      };
     };
 
     testScript = nodes: let
@@ -37,7 +49,7 @@ scenario: testConfig:
               run_tests()
         ''
       ];
-  });
+  };
 
   container = {
     # The container name has a 11 char length limit

test/run-tests.sh

@@ -110,7 +110,7 @@ trap 'eval "$runAtExit"' EXIT
 if [[ $scenario = *' '* ]]; then
     export scenarioOverridesFile=$(mktemp ${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX)
     runAtExit+='rm -f "$scenarioOverridesFile";'
-    echo "{ testEnv, config, pkgs, lib }: with testEnv; with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
+    echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
     scenario=tmp
 fi
 
@@ -120,7 +120,7 @@ run() {
     export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
     runAtExit+="rm -rf $TMPDIR;"
 
-    nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
+    nix-build --out-link $TMPDIR/driver -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver
 
     # Variable 'tests' contains the Python code that is executed by the driver on startup
     if [[ $1 == --interactive ]]; then
@@ -212,7 +212,7 @@ vmTestNixExpr() {
     fi
 
     cat <<EOF
-    ((import "$scriptDir/tests.nix" { scenario = "$scenario"; }).vm {}).overrideAttrs (old: rec {
+    ((import "$scriptDir/tests.nix" {}).getTest "$scenario").vm.overrideAttrs (old: rec {
       buildCommand = ''
         export QEMU_OPTS="-smp $numCPUs -m $memoryMiB $extraQEMUOpts"
         echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"

test/tests.nix

@@ -1,14 +1,20 @@
 # Integration tests, can be run without internet access.
 
-{ scenario ? "default" }:
+let
+  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+in
 
-import ./lib/make-test.nix scenario (
-{ config, pkgs, lib, ... }: with lib;
-let testEnv = rec {
+{ extraScenarios ? { ... }: {}
+, pkgs ? import nixpkgs { config = {}; overlays = []; }
+}:
+with pkgs.lib;
+let
+  globalPkgs = pkgs;
+
+  baseConfig = { pkgs, config, ... }: let
     cfg = config.services;
     mkIfTest = test: mkIf (config.tests.${test} or false);
-
-  baseConfig = {
+  in {
     imports = [
       ./lib/test-lib.nix
       ../modules/modules.nix
@@ -26,6 +32,9 @@ let testEnv = rec {
     };
 
     config = mkMerge [{
+      # Share the same pkgs instance among tests
+      nixpkgs.pkgs = mkDefault globalPkgs;
+
       tests.bitcoind = cfg.bitcoind.enable;
       services.bitcoind = {
         enable = true;
@@ -183,14 +192,14 @@ let testEnv = rec {
       ];
     };
 
-    netnsBase = {
+    netnsBase = { config, pkgs, ... }: {
       nix-bitcoin.netns-isolation.enable = true;
       test.data.netns = config.nix-bitcoin.netns-isolation.netns;
       tests.netns-isolation = true;
       environment.systemPackages = [ pkgs.fping ];
     };
 
-    regtestBase = {
+    regtestBase = { config, ... }: {
       tests.regtest = true;
 
       services.bitcoind.regtest = true;
@@ -241,20 +250,29 @@ let testEnv = rec {
       # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios.
     };
   };
-};
-in
-  let
+
   overrides = builtins.getEnv "scenarioOverridesFile";
-    scenarios = testEnv.scenarios // (optionalAttrs (overrides != "") (import overrides {
-      inherit testEnv config pkgs lib;
-    }));
-    autoScenario = {
-      services.${scenario}.enable = true;
+  extraScenarios' = (if (overrides != "") then import overrides else extraScenarios) {
+    inherit scenarios pkgs;
+    inherit (pkgs) lib;
   };
-  in {
+  allScenarios = scenarios // extraScenarios';
+
+  makeTest = name: config:
+    makeTest' name {
       imports = [
-      scenarios.base
-      (scenarios.${scenario} or autoScenario)
+        allScenarios.base
+        config
       ];
+    };
+  makeTest' = import ./lib/make-test.nix pkgs;
+
+  tests = builtins.mapAttrs makeTest allScenarios;
+
+  getTest = name: tests.${name} or (makeTest name {
+    services.${name}.enable = true;
+  });
+in
+  tests // {
+    inherit getTest;
   }
-)

test/tests.py

@@ -352,10 +352,9 @@ def _():
     # Current time in µs
     pre_restart = succeed("date +%s.%6N").rstrip()
 
-    # Sanity-check system by restarting all services
-    succeed(
-        "systemctl restart bitcoind clightning lnd lightning-loop lightning-pool spark-wallet liquidd"
-    )
+    # Sanity-check system by restarting bitcoind.
+    # This also restarts all services depending on bitcoind.
+    succeed("systemctl restart bitcoind")
 
     # Now that the bitcoind restart triggered a banlist import restart, check that
     # re-importing already banned addresses works
@@ -367,27 +366,37 @@ def _():
 
 @test("regtest")
 def _():
-    if "electrs" in enabled_tests:
+    def enabled(unit):
+        if unit in enabled_tests:
+            # Wait because the unit might have been restarted in the preceding
+            # 'banlist-and-restart' test
+            machine.wait_for_unit(unit)
+            return True
+        else:
+            return False
+
+    if enabled("electrs"):
+        machine.wait_for_unit("onion-addresses")
         machine.wait_until_succeeds(log_has_string("electrs", "BlockchainInfo"))
         get_block_height_cmd = (
             """echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'"""
             f" | nc -N {ip('electrs')} 50001 | jq -M .result.height"
         )
         assert_full_match(get_block_height_cmd, "10\n")
-    if "clightning" in enabled_tests:
+    if enabled("clightning"):
         machine.wait_until_succeeds(
             "[[ $(runuser -u operator -- lightning-cli getinfo | jq -M .blockheight) == 10 ]]"
         )
-    if "lnd" in enabled_tests:
+    if enabled("lnd"):
         machine.wait_until_succeeds(
             "[[ $(runuser -u operator -- lncli getinfo | jq -M .block_height) == 10 ]]"
         )
-    if "lightning-loop" in enabled_tests:
+    if enabled("lightning-loop"):
         machine.wait_until_succeeds(
             log_has_string("lightning-loop", "Starting event loop at height 10")
         )
         succeed("runuser -u operator -- loop getparams")
-    if "lightning-pool" in enabled_tests:
+    if enabled("lightning-pool"):
         machine.wait_until_succeeds(
             log_has_string("lightning-pool", "lnd is now fully synced to its chain backend")
         )