Merge #308: Update nixpkgs-unstable, bitcoind

b114d0c3b1 bitcoind: use systemd startup notification (Erik Arvstedt)
332d0e70c8 bitcoind: support onion address announcing (Erik Arvstedt)
9662c19ab1 onionServices: use actual user name of services (Erik Arvstedt)
5c09845e6f bitcoind: tag incoming connections as onion on enforceTor (Erik Arvstedt)
8f9ea61d6e update nixpkgs-unstable (Erik Arvstedt)
05e5ec99ec modules packages: build electrs, lightning-loop with nixpkgs stable (Erik Arvstedt)
44546561fc run-tests: allow defining scenarios via cmdline args (Erik Arvstedt)
fc40776689 improve backup test (Erik Arvstedt)
9a67a32779 fix build-to-cachix (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK b114d0c3b1
  nixbitcoin:
    ACK b114d0c3b1

Tree-SHA512: fbf4810fb0e1aeb46618c53cea3d6bde582eb92837afabe77e5bbf5c4add82277af4eba6bfeae6331f8992902e5dbe5081f2a016121a02dab3e797d53f232dd2

.cirrus.yml

@@ -39,4 +39,4 @@ task:
   build_script:
     - echo "sandbox = true" >> /etc/nix/nix.conf
     - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)"
-    - nix run -f '<nixpkgs>' bash cachix -c ./ci/build.sh
+    - nix run -f '<nixpkgs>' bash coreutils cachix -c ./ci/build.sh

ci/build-to-cachix.sh

@@ -13,11 +13,17 @@ trap 'echo Error at line $LINENO' ERR
 
 atExit() {
     rm -rf $tmpDir
-    if [[ -v cachixPid ]]; then kill $cachixPid; fi
+    if [[ -v cachixPid ]]; then stopCachix; fi
 }
 tmpDir=$(mktemp -d -p /tmp)
 trap atExit EXIT
 
+stopCachix() {
+    kill $cachixPid 2>/dev/null || true
+    # Wait for cachix to finish
+    tail --pid=$cachixPid -f /dev/null
+}
+
 ## Instantiate
 
 time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null
@@ -44,6 +50,7 @@ fi
 nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
 
 if [[ $CACHIX_SIGNING_KEY ]]; then
+    stopCachix
     cachix push $cachixCache $outPath
 fi
 

examples/README.md

@@ -55,5 +55,11 @@ c systemctl status bitcoind
 
 # Explore a single feature
 ./run-tests.sh --scenario electrs container
+
+# Run a command in a container
+./run-tests.sh --scenario '{
+  services.clightning.enable = true;
+  nix-bitcoin.nodeinfo.enable = true;
+}' container --run c nodeinfo
 ```
 See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.

examples/configuration.nix

@@ -28,6 +28,10 @@
   # LND and electrs are not compatible with pruning.
   # services.bitcoind.prune = 100000;
   #
+  # Set this to accounce the onion service address to peers.
+  # The onion service allows accepting incoming connections via Tor.
+  # nix-bitcoin.onionServices.bitcoind.public = true;
+  #
   # You can add options that are not defined in modules/bitcoind.nix as follows
   # services.bitcoind.extraConfig = ''
   #   maxorphantx=110

modules/bitcoind.nix

@@ -11,6 +11,8 @@ let
     # We're already logging via journald
     nodebuglogfile=1
 
+    startupnotify=/run/current-system/systemd/bin/systemd-notify --ready
+
     ${optionalString cfg.regtest ''
       regtest=1
       [regtest]
@@ -22,7 +24,7 @@ let
     ${optionalString (cfg.assumevalid != null) "assumevalid=${cfg.assumevalid}"}
 
     # Connection options
-    ${optionalString cfg.listen "bind=${cfg.address}"}
+    ${optionalString cfg.listen "bind=${cfg.address}${optionalString cfg.enforceTor "=onion"}"}
     port=${toString cfg.port}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     listen=${if cfg.listen then "1" else "0"}
@@ -67,6 +69,14 @@ in {
         default = 8333;
         description = "Port to listen for peer connections.";
       };
+      getPublicAddressCmd = mkOption {
+        type = types.str;
+        default = "";
+        description = ''
+          Bash expression which outputs the public service address to announce to peers.
+          If left empty, no address is announced.
+        '';
+      };
       package = mkOption {
         type = types.package;
         default = config.nix-bitcoin.pkgs.bitcoind;
@@ -328,21 +338,22 @@ in {
           ${extraRpcauth}
           ${/* Enable bitcoin-cli for group 'bitcoin' */ ""}
           printf "rpcuser=${cfg.rpc.users.privileged.name}\nrpcpassword="; cat "${secretsDir}/bitcoin-rpcpassword-privileged";
+          echo
+          ${optionalString (cfg.getPublicAddressCmd != "") ''
+            echo "externalip=$(${cfg.getPublicAddressCmd})"
+          ''}
         )
         confFile='${cfg.dataDir}/bitcoin.conf'
         if [[ ! -e $confFile || $cfg != $(cat $confFile) ]]; then
           install -o '${cfg.user}' -g '${cfg.group}' -m 640 <(echo "$cfg") $confFile
         fi
       '';
-      postStart = ''
-        # Poll until bitcoind accepts commands. This can take a long time.
-        while ! ${cfg.cli}/bin/bitcoin-cli getnetworkinfo &> /dev/null; do
-          sleep 1
-        done
-      '';
       serviceConfig = nix-bitcoin-services.defaultHardening // {
+        Type = "notify";
+        NotifyAccess = "all";
         User = "${cfg.user}";
         Group = "${cfg.group}";
+        TimeoutStartSec = 300;
         ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'";
         Restart = "on-failure";
         UMask = mkIf cfg.dataDirReadableByGroup "0027";

modules/onion-addresses.nix

@@ -27,13 +27,22 @@ in {
         /var/lib/onion-addresses/myuser.
       '';
     };
+    services = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = ''
+        Services that can access their onion address via file
+        /var/lib/onion-addresses/<service>
+        The file is readable only by the service user.
+      '';
+    };
     dataDir = mkOption {
       readOnly = true;
       default = "/var/lib/onion-addresses";
     };
   };
 
-  config = mkIf (cfg.access != {}) {
+  config = mkIf (cfg.access != {} || cfg.services != []) {
     systemd.services.onion-addresses = {
       wantedBy = [ "tor.service" ];
       bindsTo = [ "tor.service" ];
@@ -42,6 +51,7 @@ in {
         Type = "oneshot";
         RemainAfterExit = true;
         StateDirectory = "onion-addresses";
+        StateDirectoryMode = "771";
         PrivateNetwork = "true"; # This service needs no network access
         PrivateUsers = "false";
         CapabilityBoundingSet = "CAP_CHOWN CAP_FSETID CAP_SETFCAP CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_IPC_OWNER";
@@ -70,6 +80,13 @@ in {
           '')
           (builtins.attrNames cfg.access)
         }
+
+        ${concatMapStrings (service: ''
+          onionFile=/var/lib/tor/onion/${service}/hostname
+          if [[ -e $onionFile ]]; then
+            install -o ${config.systemd.services.${service}.serviceConfig.User} -m 400 $onionFile ${service}
+          fi
+        '') cfg.services}
       '';
     };
   };

modules/onion-services.nix

@@ -71,12 +71,12 @@ in {
         );
       };
 
-      # Enable public services to access their own onion addresses
+      nix-bitcoin.onionAddresses = {
-      nix-bitcoin.onionAddresses.access = (
+        # Enable public services to access their own onion addresses
-        genAttrs publicServices singleton
+        services = publicServices;
-      ) // {
+
         # Allow the operator user to access onion addresses for all active services
-        ${config.nix-bitcoin.operator.name} = mkIf config.nix-bitcoin.operator.enable activeServices;
+        access.${config.nix-bitcoin.operator.name} = mkIf config.nix-bitcoin.operator.enable activeServices;
       };
       systemd.services = let
         onionAddresses = [ "onion-addresses.service" ];
@@ -96,7 +96,7 @@ in {
           in srv.public && srv.enable
         ) services;
       in genAttrs publicServices' (service: {
-        getPublicAddressCmd = "cat ${config.nix-bitcoin.onionAddresses.dataDir}/${service}/${service}";
+        getPublicAddressCmd = "cat ${config.nix-bitcoin.onionAddresses.dataDir}/${service}";
       });
     }
 

pkgs/nixpkgs-pinned.nix

@@ -12,7 +12,7 @@ in
     sha256 = "1vjh0np1rlirbhhj9b2d0zhrqdmiji5svxh9baqq7r3680af1iif";
   };
   nixpkgs-unstable = fetch {
-    rev = "296793637b22bdb4d23b479879eba0a71c132a66";
+    rev = "891f607d5301d6730cb1f9dcf3618bcb1ab7f10e";
-    sha256 = "0j09yih9693w5vjx64ikfxyja1ha7pisygrwrpg3wfz3sssglg69";
+    sha256 = "1cr39f0sbr0h5d83dv1q34mcpwnkwwbdk5fqlyqp2mnxghzwssng";
   };
 }

pkgs/pinned.nix

@@ -13,9 +13,6 @@ in
     lnd
     nbxplorer
     btcpayserver;
-  inherit (nixBitcoinPkgsUnstable)
-    electrs
-    lightning-loop;
 
   stable = nixBitcoinPkgsStable;
   unstable = nixBitcoinPkgsUnstable;

test/pkgs-unstable.nix

@@ -4,10 +4,15 @@ let
   nbPkgs = import ../pkgs { inherit pkgs; };
   pkgsUnstable = with nbPkgs; [
     electrs
-    elementsd
     hwi
     joinmarket
     lightning-loop
+
+    ## elementsd fails with error
+    # test/key_properties.cpp:16:10: fatal error: rapidcheck/boost_test.h: No such file or directory
+    # 16 | #include <rapidcheck/boost_test.h>
+    #    |          ^~~~~~~~~~~~~~~~~~~~~~~~~
+    # elementsd
   ];
 in
 pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable)

test/run-tests.sh

@@ -10,7 +10,13 @@
 #   Test specific scenario
 #   ./run-tests.sh --scenario|-s <scenario>
 #
-#     When <scenario> is undefined, the test is run with an adhoc scenario
+#   - When <scenario> contains a space, <scenario> is treated as nix code defining
+#     a scenario. It is evaluated in the same context as other scenarios in ./tests.nix
+#
+#     Example:
+#     ./run-tests.sh -s "{ nix-bitcoin.nodeinfo.enable = true; }" container --run c nodeinfo
+#
+#   - When <scenario> does not name a scenario, the test is run with an adhoc scenario
 #     where services.<scenario> is enabled.
 #
 #     Example:
@@ -80,13 +86,24 @@ numCPUs=${numCPUs:-$(nproc)}
 # Min. 800 MiB needed to avoid 'out of memory' errors
 memoryMiB=${memoryMiB:-2048}
 
-export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs)
+export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..")
+
+runAtExit=
+trap 'eval "$runAtExit"' EXIT
+
+# Support explicit scenario definitions
+if [[ $scenario = *' '* ]]; then
+    export scenarioOverridesFile=$(mktemp ${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX)
+    runAtExit+='rm -f "$scenarioOverridesFile";'
+    echo "{ testEnv, config, pkgs, lib }: with testEnv; with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
+    scenario=tmp
+fi
 
 # Run the test. No temporary files are left on the host system.
 run() {
     # TMPDIR is also used by the test driver for VM tmp files
     export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
-    trap "rm -rf $TMPDIR" EXIT
+    runAtExit+="rm -rf $TMPDIR;"
 
     nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
 
@@ -134,7 +151,8 @@ instantiate() {
 }
 
 container() {
-  . "$scriptDir/lib/make-container.sh" "$@"
+    export scriptDir scenario
+    "$scriptDir/lib/make-container.sh" "$@"
 }
 
 doBuild() {

test/tests.nix

@@ -187,6 +187,7 @@ let testEnv = rec {
       services.bitcoind.regtest = true;
       systemd.services.bitcoind.postStart = mkAfter ''
         cli=${config.services.bitcoind.cli}/bin/bitcoin-cli
+        $cli createwallet "test"
         address=$($cli getnewaddress)
         $cli generatetoaddress 10 $address
       '';

test/tests.py

@@ -286,6 +286,9 @@ def _():
 # Impure: stops bitcoind (and dependent services)
 @test("backups")
 def _():
+    # For testing that bitcoind wallets are backed up
+    succeed("bitcoin-cli -named createwallet wallet_name=test blank=true >/dev/null")
+
     succeed("systemctl stop bitcoind")
     succeed("systemctl start duplicity")
     machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded."))
@@ -296,12 +299,18 @@ def _():
         "0 differences found",
     )
     # Backup should include important files
-    files = succeed(f"{run_duplicity} list-current-files file:///var/lib/localBackups")
+    files = {
-    assert "var/lib/clightning/bitcoin/hsm_secret" in files
+        "bitcoind": "var/lib/bitcoind/test/wallet.dat",
-    assert "secrets/lnd-seed-mnemonic" in files
+        "clightning": "var/lib/clightning/bitcoin/hsm_secret",
-    assert "secrets/jm-wallet-seed" in files
+        "lnd": "secrets/lnd-seed-mnemonic",
-    assert "var/lib/bitcoind/wallet.dat" in files
+        "joinmarket": "secrets/jm-wallet-seed",
-    assert "var/backup/postgresql/btcpaydb.sql.gz" in files
+        "btcpayserver": "var/backup/postgresql/btcpaydb.sql.gz",
+    }
+    actual_files = succeed(f"{run_duplicity} list-current-files file:///var/lib/localBackups")
+
+    for test, file in files.items():
+        if test in enabled_tests and file not in actual_files:
+            raise Exception(f"Backup file '{file}' is missing.")
 
 
 # Impure: restarts services