Merge fort-nix/nix-bitcoin#374: Update to NixOS 21.05

a2454975a5 doas: fix recursive calls to doas (Erik Arvstedt)
7c876664b1 netns test: update matching of 'capsh' output (Erik Arvstedt)
308a11f22b tests: avoid postgresql timeout failures on CI nodes (Erik Arvstedt)
01804e6dfb tests: improve test script formatting (Erik Arvstedt)
1be924529d tests: adapt to new linter (Erik Arvstedt)
c1c663d0a9 tests: fix formatting (Erik Arvstedt)
c4c2b03e19 extra-container: 0.6 -> 0.7 (Erik Arvstedt)
161baa7e68 joinmarket-ob-watcher: allow required 'mbind' system call (Erik Arvstedt)
ca64a4a64f clightning-plugins.prometheus: use current nixpkgs version of prometheus-client (Erik Arvstedt)
3aab1fc267 spark-wallet: update to new node-env (Erik Arvstedt)
a0e5894f1f backups: remove illegal option definition (Erik Arvstedt)
35fe939cf8 security: update /proc restriction mechanism (Erik Arvstedt)
178a0dcf8f services: use new 'tor' options (Erik Arvstedt)
e44f78ebb8 services: set isSystemUser for service users (Erik Arvstedt)
0ef66c920b treewide: use services.getty option (Erik Arvstedt)
a25ceecca5 update to NixOS 21.05 (Erik Arvstedt)
b758150c9e pinned: expose nixpkgsStable, nixpkgsUnstable (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK a2454975a5

Tree-SHA512: a8a25c25d835662ec63c3a042eb237d29b857b2030d9023a8b4ead94e03a4f9dffe2d6616e2a286800e40288985e5db3a55056d6b45d8984161b9a19aba28a60

examples/configuration.nix

@@ -240,7 +240,7 @@
   # compatible, in order to avoid breaking some software such as database
   # servers. You should change this only after NixOS release notes say you
   # should.
-  system.stateVersion = "18.09"; # Did you read the comment?
+  system.stateVersion = "21.05"; # Did you read the comment?
 
   # The nix-bitcoin release version that your config is compatible with.
   # When upgrading to a backwards-incompatible release, nix-bitcoin will display an

examples/qemu-vm/vm-config.nix

@@ -4,7 +4,7 @@
 
   config = {
     virtualisation.graphics = false;
-    services.mingetty.autologinUser = "root";
+    services.getty.autologinUser = "root";
     users.users.root = {
       openssh.authorizedKeys.keyFiles = [ ./id-vm.pub ];
     };

modules/backups.nix

@@ -82,7 +82,6 @@ in {
       services.postgresqlBackup = {
         enable = true;
         databases = [ "btcpaydb" ];
-        startAt = [];
       };
       systemd.services.duplicity = rec {
         wants = [ "postgresqlBackup-btcpaydb.service" ];

modules/bitcoind.nix

@@ -193,7 +193,7 @@ in {
       };
       proxy = mkOption {
         type = types.nullOr types.str;
-        default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+        default = if cfg.enforceTor then config.nix-bitcoin.torClientAddressWithPort else null;
         description = "Connect through SOCKS5 proxy";
       };
       listen = mkOption {
@@ -388,7 +388,10 @@ in {
       } // nbLib.allowLocalIPAddresses;
     };
 
-    users.users.${cfg.user}.group = cfg.group;
+    users.users.${cfg.user} = {
+      isSystemUser = true;
+      group = cfg.group;
+    };
     users.groups.${cfg.group} = {};
     users.groups.bitcoinrpc-public = {};
     nix-bitcoin.operator.groups = [ cfg.group ];

modules/btcpayserver.nix

@@ -184,7 +184,7 @@ in {
         network=${config.services.bitcoind.network}
         bind=${cfg.btcpayserver.address}
         port=${toString cfg.btcpayserver.port}
-        socksendpoint=${cfg.tor.client.socksListenAddress}
+        socksendpoint=${config.nix-bitcoin.torClientAddressWithPort}
         btcexplorerurl=${nbExplorerUrl}
         btcexplorercookiefile=${nbExplorerCookie}
         postgres=User ID=${cfg.btcpayserver.user};Host=/run/postgresql;Database=btcpaydb
@@ -230,6 +230,7 @@ in {
     }; in self;
 
     users.users.${cfg.nbxplorer.user} = {
+      isSystemUser = true;
       group = cfg.nbxplorer.group;
       extraGroups = [ "bitcoinrpc-public" ]
                     ++ optional cfg.btcpayserver.lbtc cfg.liquidd.group;
@@ -237,6 +238,7 @@ in {
     };
     users.groups.${cfg.nbxplorer.group} = {};
     users.users.${cfg.btcpayserver.user} = {
+      isSystemUser = true;
       group = cfg.btcpayserver.group;
       extraGroups = [ cfg.nbxplorer.group ]
                     ++ optional (cfg.btcpayserver.lightningBackend == "clightning") cfg.clightning.user;

modules/charge-lnd.nix

@@ -133,8 +133,8 @@ in
     };
 
     users.users.${user} = {
-      group = group;
       isSystemUser = true;
+      group = group;
     };
     users.groups.${group} = {};
   };

modules/clightning.nix

@@ -34,7 +34,7 @@ in {
     };
     proxy = mkOption {
       type = types.nullOr types.str;
-      default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+      default = if cfg.enforceTor then config.nix-bitcoin.torClientAddressWithPort else null;
       description = ''
         Socks proxy for connecting to Tor nodes (or for all connections if option always-use-proxy is set).
       '';
@@ -140,6 +140,7 @@ in {
     };
 
     users.users.${cfg.user} = {
+      isSystemUser = true;
       group = cfg.group;
       extraGroups = [ "bitcoinrpc-public" ];
     };

modules/electrs.nix

@@ -106,6 +106,7 @@ in {
     };
 
     users.users.${cfg.user} = {
+      isSystemUser = true;
       group = cfg.group;
       extraGroups = [ "bitcoinrpc-public" ] ++ optionals cfg.high-memory [ bitcoind.user ];
     };

modules/joinmarket-ob-watcher.nix

@@ -5,7 +5,13 @@ let
   cfg = config.services.joinmarket-ob-watcher;
   nbLib = config.nix-bitcoin.lib;
   nbPkgs = config.nix-bitcoin.pkgs;
-  torAddress = builtins.head (builtins.split ":" config.services.tor.client.socksListenAddress);
+
+  socks5Settings = with config.services.tor.client.socksListenAddress; ''
+    socks5 = true
+    socks5_host = ${addr}
+    socks5_port = ${toString port}
+  '';
+
   configFile = builtins.toFile "config" ''
     [BLOCKCHAIN]
     blockchain_source = no-blockchain
@@ -15,18 +21,14 @@ let
     channel = joinmarket-pit
     port = 6697
     usessl = true
-    socks5 = true
+    ${socks5Settings}
-    socks5_host = ${torAddress}
-    socks5_port = 9050
 
     [MESSAGING:server2]
     host = ncwkrwxpq2ikcngxq3dy2xctuheniggtqeibvgofixpzvrwpa77tozqd.onion
     channel = joinmarket-pit
     port = 6667
     usessl = false
-    socks5 = true
+    ${socks5Settings}
-    socks5_host = ${torAddress}
-    socks5_port = 9050
   '';
 in {
   options.services.joinmarket-ob-watcher = {
@@ -78,6 +80,7 @@ in {
           ${nbPkgs.joinmarket}/bin/ob-watcher --datadir=${cfg.dataDir} \
             --host=${cfg.address} --port=${toString cfg.port}
         '';
+        SystemCallFilter = nbLib.defaultHardening.SystemCallFilter ++ [ "mbind" ] ;
         Restart = "on-failure";
         RestartSec = "10s";
       } // nbLib.allowTor;

modules/joinmarket.nix

@@ -10,7 +10,14 @@ let
   runAsUser = config.nix-bitcoin.runAsUserCmd;
 
   inherit (config.services) bitcoind;
-  torAddress = builtins.head (builtins.split ":" config.services.tor.client.socksListenAddress);
+
+  torAddress = config.services.tor.client.socksListenAddress;
+  socks5Settings = ''
+    socks5 = true
+    socks5_host = ${torAddress.addr}
+    socks5_port = ${toString torAddress.port}
+  '';
+
   # Based on https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/jmclient/jmclient/configure.py
   yg = cfg.yieldgenerator;
   configFile = builtins.toFile "config" ''
@@ -34,18 +41,14 @@ let
     channel = joinmarket-pit
     port = 6697
     usessl = true
-    socks5 = true
+    ${socks5Settings}
-    socks5_host = ${torAddress}
-    socks5_port = 9050
 
     [MESSAGING:server2]
     host = ncwkrwxpq2ikcngxq3dy2xctuheniggtqeibvgofixpzvrwpa77tozqd.onion
     channel = joinmarket-pit
     port = 6667
     usessl = false
-    socks5 = true
+    ${socks5Settings}
-    socks5_host = ${torAddress}
-    socks5_port = 9050
 
     [LOGGING]
     console_log_level = INFO
@@ -72,8 +75,8 @@ let
     disable_output_substitution = 0
     max_additional_fee_contribution = default
     min_fee_rate = 1.1
-    onion_socks5_host = ${torAddress}
+    onion_socks5_host = ${torAddress.addr}
-    onion_socks5_port = 9050
+    onion_socks5_port = ${toString torAddress.port}
     tor_control_host = unix:/run/tor/control
     hidden_service_ssl = false
 
@@ -270,6 +273,7 @@ in {
     };
 
     users.users.${cfg.user} = {
+      isSystemUser = true;
       group = cfg.group;
       home = cfg.dataDir;
       # Allow access to the tor control socket, needed for payjoin onion service creation

modules/lightning-loop.nix

@@ -60,7 +60,7 @@ in {
     };
     proxy = mkOption {
       type = types.nullOr types.str;
-      default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+      default = if cfg.enforceTor then config.nix-bitcoin.torClientAddressWithPort else null;
       description = "host:port of SOCKS5 proxy for connnecting to the loop server.";
     };
     extraConfig = mkOption {

modules/lightning-pool.nix

@@ -54,7 +54,7 @@ in {
     };
     proxy = mkOption {
       type = types.nullOr types.str;
-      default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+      default = if cfg.enforceTor then config.nix-bitcoin.torClientAddressWithPort else null;
       description = "host:port of SOCKS5 proxy for connnecting to the pool auction server.";
     };
     extraConfig = mkOption {

modules/liquid.nix

@@ -144,7 +144,7 @@ in {
       };
       proxy = mkOption {
         type = types.nullOr types.str;
-        default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+        default = if cfg.enforceTor then config.nix-bitcoin.torClientAddressWithPort else null;
         description = "Connect through SOCKS5 proxy";
       };
       listen = mkOption {
@@ -240,6 +240,7 @@ in {
     };
 
     users.users.${cfg.user} = {
+      isSystemUser = true;
       group = cfg.group;
       extraGroups = [ "bitcoinrpc-public" ];
     };

modules/lnd-rest-onion-service.nix

@@ -40,8 +40,9 @@ in {
   config = mkIf cfg.enable {
     services.tor = {
       enable = true;
-      hiddenServices.lnd-rest = nbLib.mkHiddenService {
+      relay.onionServices.lnd-rest = nbLib.mkOnionService {
-        toHost = lnd.restAddress;
+        target.addr = lnd.restAddress;
+        target.port = lnd.restPort;
         port = lnd.restPort;
       };
     };

modules/lnd.nix

@@ -83,7 +83,7 @@ in {
     };
     tor-socks = mkOption {
       type = types.nullOr types.str;
-      default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+      default = if cfg.enforceTor then config.nix-bitcoin.torClientAddressWithPort else null;
       description = "Socks proxy for connecting to Tor nodes";
     };
     macaroons = mkOption {
@@ -263,6 +263,7 @@ in {
     };
 
     users.users.${cfg.user} = {
+      isSystemUser = true;
       group = cfg.group;
       extraGroups = [ "bitcoinrpc-public" ];
       home = cfg.dataDir; # lnd creates .lnd dir in HOME

modules/modules.nix

@@ -49,13 +49,19 @@ with lib;
         default = import ../pkgs/lib.nix lib pkgs;
       };
 
+      torClientAddressWithPort = mkOption {
+        readOnly = true;
+        default =  with config.services.tor.client.socksListenAddress;
+          "${addr}:${toString port}";
+      };
+
       # Torify binary that works with custom Tor SOCKS addresses
       # Related issue: https://github.com/NixOS/nixpkgs/issues/94236
       torify = mkOption {
         readOnly = true;
         default = pkgs.writeScriptBin "torify" ''
           ${pkgs.tor}/bin/torify \
-            --address ${head (splitString ":" config.services.tor.client.socksListenAddress)} \
+            --address ${config.services.tor.client.socksListenAddress.addr} \
             "$@"
         '';
       };
@@ -64,7 +70,8 @@ with lib;
       runAsUserCmd = mkOption {
         readOnly = true;
         default = if config.security.doas.enable
-                  then "doas -u"
+                  # TODO: Use absolute path until https://github.com/NixOS/nixpkgs/pull/133622 is available.
+                  then "/run/wrappers/bin/doas -u"
                   else "sudo -u";
       };
     };

modules/netns-isolation.nix

@@ -97,8 +97,13 @@ in {
   # Base infrastructure
   {
     networking.dhcpcd.denyInterfaces = [ "nb-br" "nb-veth*" ];
-    services.tor.client.socksListenAddress = "${bridgeIp}:9050";
+    services.tor.client.socksListenAddress = {
-    networking.firewall.interfaces.nb-br.allowedTCPPorts = [ 9050 ];
+      addr = bridgeIp;
+      # Default NixOS values. These must be repeated when redefining this option.
+      port = 9050;
+      IsolateDestAddr = true;
+    };
+    networking.firewall.interfaces.nb-br.allowedTCPPorts = [ config.services.tor.client.socksListenAddress.port ];
     boot.kernel.sysctl."net.ipv4.ip_forward" = true;
 
     security.wrappers.netns-exec = {

modules/nodeinfo.nix

@@ -95,12 +95,12 @@ let
     '';
 
    mkIfOnionPort = name: fn:
-     if hiddenServices ? ${name} then
+     if onionServices ? ${name} then
-       fn (toString (builtins.elemAt hiddenServices.${name}.map 0).port)
+       fn (toString (builtins.elemAt onionServices.${name}.map 0).port)
      else
        "";
 
-  inherit (config.services.tor) hiddenServices;
+   inherit (config.services.tor.relay) onionServices;
 in {
   options = {
     nix-bitcoin.nodeinfo = {

modules/onion-services.nix

@@ -57,14 +57,14 @@ in {
       # Define hidden services
       services.tor = {
         enable = true;
-        hiddenServices = genAttrs activeServices (name:
+        relay.onionServices = genAttrs activeServices (name:
           let
             service = config.services.${name};
             inherit (cfg.${name}) externalPort;
-          in nbLib.mkHiddenService {
+          in nbLib.mkOnionService {
             port = if externalPort != null then externalPort else service.port;
-            toPort = service.port;
+            target.port = service.port;
-            toHost = if service.address == "0.0.0.0" then "127.0.0.1" else service.address;
+            target.addr = if service.address == "0.0.0.0" then "127.0.0.1" else service.address;
           }
         );
       };

modules/presets/hardened.nix

@@ -9,6 +9,6 @@
   # Needed for sandboxed builds and services
   security.allowUserNamespaces = true;
 
-  # The "scudo" allocator is broken on NixOS 20.09
+  # The "scudo" allocator is broken on NixOS >= 20.09
   environment.memoryAllocator.provider = "libc";
 }

modules/presets/secure-node.nix

@@ -18,7 +18,7 @@ in {
 
     networking.firewall.enable = true;
 
-    nix-bitcoin.security.hideProcessInformation = true;
+    nix-bitcoin.security.dbusHideProcessInformation = true;
 
     # Use doas instead of sudo
     security.doas.enable = true;
@@ -29,7 +29,7 @@ in {
     ];
 
     # sshd
-    services.tor.hiddenServices.sshd = nbLib.mkHiddenService { port = 22; };
+    services.tor.relay.onionServices.sshd = nbLib.mkOnionService { port = 22; };
     nix-bitcoin.onionAddresses.access.${operatorName} = [ "sshd" ];
 
     services.bitcoind = {

modules/recurring-donations.nix

@@ -11,7 +11,7 @@ let
       NAME=$1
       AMOUNT=$2
       echo Attempting to pay $AMOUNT sat to $NAME
-      INVOICE=$(curl --socks5-hostname ${config.services.tor.client.socksListenAddress} -d "satoshi_amount=$AMOUNT&payment_method=ln&id=$NAME&type=profile" -X POST https://api.tallyco.in/v1/payment/request/ | jq -r '.lightning_pay_request') 2> /dev/null
+      INVOICE=$(curl --socks5-hostname ${config.nix-bitcoin.torClientAddressWithPort} -d "satoshi_amount=$AMOUNT&payment_method=ln&id=$NAME&type=profile" -X POST https://api.tallyco.in/v1/payment/request/ | jq -r '.lightning_pay_request') 2> /dev/null
       if [ -z "$INVOICE" ] || [ "$INVOICE" = "null" ]; then
         echo "ERROR: did not get invoice from tallycoin"
         return
@@ -97,6 +97,7 @@ in {
     };
 
     users.users.recurring-donations = {
+      isSystemUser = true;
       group = "recurring-donations";
       extraGroups = [ config.services.clightning.group ];
     };

modules/security.nix

@@ -1,20 +1,29 @@
-{ config, lib, pkgs, options, ... }:
+{ config, lib, pkgs, ... }:
 
+with lib;
 {
   options = {
-    nix-bitcoin.security.hideProcessInformation = options.security.hideProcessInformation;
+    nix-bitcoin.security.dbusHideProcessInformation = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Only allow users with group 'proc' to retrieve systemd unit information like
+        cgroup paths (i.e. (sub)process command lines) via D-Bus.
+
+        This mitigates a systemd security issue where (sub)process command lines can
+        be retrieved by services even when their access to /proc is restricted
+        (via ProtectProc).
+
+        This option works by restricting the D-Bus method 'GetUnitProcesses', which
+        is also used internally by `systemctl status`.
+      '';
+    };
   };
 
-  config = lib.mkIf config.nix-bitcoin.security.hideProcessInformation {
+  config = mkIf config.nix-bitcoin.security.dbusHideProcessInformation {
-    # Only show the current user's processes in /proc.
+    users.groups.proc = {};
-    # Users with group 'proc' can still access all processes.
+    nix-bitcoin.operator.groups = [ "proc" ]; # Enable operator access to systemd-status
-    security.hideProcessInformation = true;
 
-    # This mitigates a systemd security issue leaking (sub)process
-    # command lines.
-    # Only allow users with group 'proc' to retrieve systemd unit information like
-    # cgroup paths (i.e. (sub)process command lines) via D-Bus.
-    # This D-Bus call is used by `systemctl status`.
     services.dbus.packages = lib.mkAfter [ # Apply at the end to override the default policy
       (pkgs.writeTextDir "etc/dbus-1/system.d/dbus.conf" ''
         <busconfig>

modules/spark-wallet.nix

@@ -8,7 +8,7 @@ let
 
   # Use wasabi rate provider because the default (bitstamp) doesn't accept
   # connections through Tor
-  torRateProvider = "--rate-provider wasabi --proxy socks5h://${config.services.tor.client.socksListenAddress}";
+  torRateProvider = "--rate-provider wasabi --proxy socks5h://${config.nix-bitcoin.torClientAddressWithPort}";
   startScript = ''
     ${optionalString (cfg.getPublicAddressCmd != "") ''
       publicURL="--public-url http://$(${cfg.getPublicAddressCmd})"
@@ -65,6 +65,7 @@ in {
     services.clightning.enable = true;
 
     users.users.${cfg.user} = {
+      isSystemUser = true;
       group = cfg.group;
       extraGroups = [ config.services.clightning.group ];
     };

pkgs/clightning-plugins/default.nix

@@ -17,7 +17,7 @@ let
     monitor = {};
     prometheus = {
       extraPkgs = [ prometheus_client ];
-      patchRequirements = "--replace prometheus-client==0.6.0 prometheus-client==0.8.0";
+      patchRequirements = "--replace prometheus-client==0.6.0 prometheus-client==0.9.0";
     };
     rebalance = {};
     summary = {

pkgs/extra-container/default.nix

@@ -4,11 +4,11 @@
 
 stdenv.mkDerivation rec {
   pname = "extra-container";
-  version = "0.6";
+  version = "0.7";
 
   src = builtins.fetchTarball {
     url = "https://github.com/erikarvstedt/extra-container/archive/${version}.tar.gz";
-    sha256 = "0hm4xfjbqjrrq7n1pkbs33lpw9k5q3ms3psprqhfsxkkwzy78zlm";
+    sha256 = "1hcbi611vm0kn8rl7q974wcjkihpddan6m3p7hx8l8jnv18ydng8";
   };
 
   buildCommand = ''

pkgs/lib.nix

@@ -17,9 +17,8 @@ let self = {
       ProtectKernelModules = "true";
       ProtectKernelLogs = "true";
       ProtectClock = "true";
-      # Test and enable these when systemd v247 is available
+      ProtectProc = "invisible";
-      # ProtectProc = "invisible";
+      ProcSubset = "pid";
-      # ProcSubset = "pid";
       ProtectControlGroups = "true";
       RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
       RestrictNamespaces = "true";
@@ -80,7 +79,7 @@ let self = {
     default = "exec";
   };
 
-  mkHiddenService = map: {
+  mkOnionService = map: {
     map = [ map ];
     version = 3;
   };

pkgs/nixpkgs-pinned.nix

@@ -8,8 +8,9 @@ in
 {
   # To update, run ../helper/fetch-channel REV
   nixpkgs = fetch {
-    rev = "359e6542e1d41eb18df55c82bdb08bf738fae2cf";
+    # nixos-21.05 (2021-08-03)
-    sha256 = "05v28njaas9l26ibc6vy6imvy7grbkli32bmv0n32x6x9cf68gf9";
+    rev = "d4590d21006387dcb190c516724cb1e41c0f8fdf";
+    sha256 = "17q39hlx1x87xf2rdygyimj8whdbx33nzszf4rxkc6b85wz0l38n";
   };
   nixpkgs-unstable = fetch {
     rev = "16105403bdd843540cbef9c63fc0f16c1c6eaa70";

pkgs/pinned.nix

@@ -21,6 +21,8 @@ in
     lightning-loop
     lightning-pool;
 
+  inherit nixpkgsStable nixpkgsUnstable;
+
   stable = nixBitcoinPkgsStable;
   unstable = nixBitcoinPkgsUnstable;
 }

pkgs/spark-wallet/composition.nix

@@ -6,7 +6,8 @@
 
 let
   nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
-    inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
+    inherit pkgs;
+    inherit (pkgs) lib stdenv python2 runCommand writeTextFile;
     inherit nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };

test/lib/make-test-vm.nix

@@ -10,36 +10,16 @@ args:
 let
   test = pythonTesting.makeTest args;
 
-  fixedDriver = test.driver.overrideAttrs (old: let
+  # 1. Save test logging output
-    # Allow the test script to have longer lines by fixing the call to the 'black'
+  # 2. Add link to driver so that a gcroot to a test prevents the driver from
-    # code formatter.
-    # The default width of 88 chars is too restrictive for our script.
-    parts = builtins.split ''/nix/store/[^ ]+/black '' old.buildCommand;
-    preMatch = builtins.elemAt parts 0;
-    postMatch = builtins.elemAt parts 2;
-  in {
-    # See `mkDriver` in nixpkgs/nixos/lib/testing-python.nix for the original definition of `buildCommand`
-    buildCommand = ''
-      ${preMatch}${pkgs.python3Packages.black}/bin/black --line-length 100 ${postMatch}
-    '';
-    # Keep reference to the `testDriver` derivation, required by `buildCommand`
-    testDriverReference = old.buildCommand;
-  });
-
-  # 1. Use fixed driver
-  # 2. Save test logging output
-  # 3. Add link to driver so that a gcroot to a test prevents the driver from
   #    being garbage-collected
   fixedTest = test.overrideAttrs (_: {
     # See `runTests` in nixpkgs/nixos/lib/testing-python.nix for the original definition of `buildCommand`
     buildCommand = ''
       mkdir $out
-      LOGFILE=$out/output.xml tests='exec(os.environ["testScript"])' ${fixedDriver}/bin/nixos-test-driver
+      LOGFILE=$out/output.xml tests='exec(os.environ["testScript"])' ${test.driver}/bin/nixos-test-driver
-      ln -s ${fixedDriver} $out/driver
+      ln -s ${test.driver} $out/driver
     '';
-  }) // {
+  });
-    driver = fixedDriver;
-    inherit (test) nodes;
-  };
 in
   fixedTest

test/lib/make-test.nix

@@ -69,7 +69,7 @@ name: testConfig:
       "${toString pkgs.path}/nixos/modules/virtualisation/qemu-vm.nix"
     ];
     virtualisation.graphics = false;
-    services.mingetty.autologinUser = "root";
+    services.getty.autologinUser = "root";
 
     # Provide a shortcut for instant poweroff from within the machine
     environment.systemPackages = with pkgs; [

test/tests.nix

@@ -106,6 +106,9 @@ let
       systemd.services.setup-secrets.preStart = mkIfTest "security" ''
         install -D -o nobody -g nogroup -m777 <(:) /secrets/dummy
       '';
+
+      # Avoid timeout failures on slow CI nodes
+      systemd.services.postgresql.serviceConfig.TimeoutStartSec = "3min";
     }
     (mkIf config.test.features.clightningPlugins {
       services.clightning.plugins = {

test/tests.py

@@ -1,45 +1,40 @@
 from collections import OrderedDict
 import json
 
+logger = machine.logger
 
 def succeed(*cmds):
     """Returns the concatenated output of all cmds"""
     return machine.succeed(*cmds)
 
-
 def assert_matches(cmd, regexp):
     out = succeed(cmd)
     if not re.search(regexp, out):
         raise Exception(f"Pattern '{regexp}' not found in '{out}'")
 
-
 def assert_full_match(cmd, regexp):
     out = succeed(cmd)
     if not re.fullmatch(regexp, out):
         raise Exception(f"Pattern '{regexp}' doesn't match '{out}'")
 
-
 def log_has_string(unit, str):
     return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
 
-
 def assert_no_failure(unit):
     """Unit should not have failed since the system is running"""
     machine.fail(log_has_string(unit, "Failed with result"))
 
-
 def assert_running(unit):
     with machine.nested(f"waiting for unit: {unit}"):
         machine.wait_for_unit(unit)
     assert_no_failure(unit)
 
-
 def wait_for_open_port(address, port):
     def is_port_open(_):
         status, _ = machine.execute(f"nc -z {address} {port}")
         return status == 0
 
-    with log.nested(f"Waiting for TCP port {address}:{port}"):
+    with logger.nested(f"Waiting for TCP port {address}:{port}"):
         retry(is_port_open)
 
 
@@ -47,14 +42,11 @@ def wait_for_open_port(address, port):
 
 tests = OrderedDict()
 
-
 def test(name):
     def x(fn):
         tests[name] = fn
-
     return x
 
-
 def run_tests():
     enabled = enabled_tests.copy()
     to_run = []
@@ -66,10 +58,9 @@ def run_tests():
         raise RuntimeError(f"The following tests are enabled but not defined: {enabled}")
     machine.connect()  # Visually separate boot output from the test output
     for test in to_run:
-        with log.nested(f"test: {test}"):
+        with logger.nested(f"test: {test}"):
             tests[test]()
 
-
 def run_test(test):
     tests[test]()
 
@@ -77,7 +68,6 @@ def run_test(test):
 ### Tests
 # All tests are executed in the order they are defined here
 
-
 @test("security")
 def _():
     assert_running("setup-secrets")
@@ -85,9 +75,6 @@ def _():
     succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
 
     if "secure-node" in enabled_tests:
-        # Access to '/proc' should be restricted
-        machine.succeed("grep -Fq hidepid=2 /proc/mounts")
-
         machine.wait_for_unit("bitcoind")
         # `systemctl status` run by unprivileged users shouldn't leak cgroup info
         assert_matches(
@@ -97,7 +84,6 @@ def _():
         # The 'operator' with group 'proc' has full access
         assert_full_match("runuser -u operator -- systemctl status bitcoind 2>&1 >/dev/null", "")
 
-
 @test("bitcoind")
 def _():
     assert_running("bitcoind")
@@ -115,7 +101,6 @@ def _():
         log_has_string("bitcoind", "RPC User public not allowed to call method stop")
     )
 
-
 @test("electrs")
 def _():
     assert_running("electrs")
@@ -123,14 +108,12 @@ def _():
     # Check RPC connection to bitcoind
     machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
 
-
 # Impure: Stops electrs
 # Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages
 @test("stop-electrs")
 def _():
     succeed("systemctl stop electrs")
 
-
 @test("liquidd")
 def _():
     assert_running("liquidd")
@@ -138,7 +121,6 @@ def _():
     assert_matches("runuser -u operator -- elements-cli getnetworkinfo | jq", '"version"')
     succeed("runuser -u operator -- liquidswap-cli --help")
 
-
 @test("clightning")
 def _():
     assert_running("clightning")
@@ -154,10 +136,9 @@ def _():
                 f"Output of 'lightning-cli plugin list':\n{plugin_list}"
             )
         else:
-            log.log("Active clightning plugins:")
+            logger.log("Active clightning plugins:")
             for p in test_data["clightning-plugins"]:
-                log.log(os.path.basename(p))
+                logger.log(os.path.basename(p))
-
 
 @test("lnd")
 def _():
@@ -165,12 +146,10 @@ def _():
     assert_matches("runuser -u operator -- lncli getinfo | jq", '"version"')
     assert_no_failure("lnd")
 
-
 @test("lnd-rest-onion-service")
 def _():
     assert_matches("runuser -u operator -- lndconnect-rest-onion -j", ".onion")
 
-
 @test("lightning-loop")
 def _():
     assert_running("lightning-loop")
@@ -184,7 +163,6 @@ def _():
         )
     )
 
-
 @test("lightning-pool")
 def _():
     assert_running("lightning-pool")
@@ -198,14 +176,12 @@ def _():
         )
     )
 
-
 @test("charge-lnd")
 def _():
     # charge-lnd is a oneshot service that is started by a timer under regular operation
     succeed("systemctl start charge-lnd")
     assert_no_failure("charge-lnd")
 
-
 @test("btcpayserver")
 def _():
     assert_running("nbxplorer")
@@ -223,7 +199,6 @@ def _():
         '"version"',
     )
 
-
 @test("spark-wallet")
 def _():
     assert_running("spark-wallet")
@@ -231,7 +206,6 @@ def _():
     spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
     assert_matches(f"curl -s {spark_auth}@{ip('spark-wallet')}:9737", "Spark")
 
-
 @test("joinmarket")
 def _():
     assert_running("joinmarket")
@@ -239,20 +213,17 @@ def _():
         log_has_string("joinmarket", "JMDaemonServerProtocolFactory starting on 27183")
     )
 
-
 @test("joinmarket-yieldgenerator")
 def _():
     machine.wait_until_succeeds(
-        log_has_string("joinmarket-yieldgenerator", "Critical error updating blockheight.",)
+        log_has_string("joinmarket-yieldgenerator", "Critical error updating blockheight.")
     )
 
-
 @test("joinmarket-ob-watcher")
 def _():
     assert_running("joinmarket-ob-watcher")
     machine.wait_until_succeeds(log_has_string("joinmarket-ob-watcher", "Starting ob-watcher"))
 
-
 @test("nodeinfo")
 def _():
     status, _ = machine.execute("systemctl is-enabled --quiet onion-addresses 2> /dev/null")
@@ -262,12 +233,10 @@ def _():
     info = json.loads(json_info)
     assert info["bitcoind"]["local_address"]
 
-
 @test("secure-node")
 def _():
     assert_running("onion-addresses")
 
-
 # Run this test before the following tests that shut down services
 # (and their corresponding network namespaces).
 @test("netns-isolation")
@@ -302,9 +271,9 @@ def _():
 
     if "joinmarket" in enabled_tests:
         # netns-exec should drop capabilities
-        assert_full_match(
+        assert_matches(
             "runuser -u operator -- netns-exec nb-joinmarket capsh --print | grep Current",
-            "Current: =\n",
+            re.compile("^Current: =$", re.MULTILINE),
         )
 
     if "clightning" in enabled_tests:
@@ -350,7 +319,6 @@ def _():
 
     assert_file_exists("secrets/lnd-wallet-password")
 
-
 # Impure: restarts services
 @test("banlist-and-restart")
 def _():
@@ -371,7 +339,6 @@ def _():
     )
     assert_no_failure("bitcoind-import-banlist")
 
-
 @test("regtest")
 def _():
     def enabled(unit):
@@ -410,14 +377,9 @@ def _():
         )
         succeed("runuser -u operator -- pool orders list")
 
-
 if "netns-isolation" in enabled_tests:
-
     def ip(name):
         return test_data["netns"][name]["address"]
-
-
 else:
-
     def ip(_):
         return "127.0.0.1"