Merge fort-nix/nix-bitcoin#586: Misc. improvements

addfa8ec6bab320de0b808fd76809635947377e8 test: support `run`, `debug` commands in basic NixOS tests (Erik Arvstedt)
ae733d887eae83ec589af53aa11638d711135a18 tests/clightning-replication: reuse `pkgs` instance (Erik Arvstedt)
6cbd0d93ae80176a853c31ca8cf5891bd6fb0646 tests: rename `clightningReplication` -> `clightning-replication` (Erik Arvstedt)
85310b533a00f1f5dfbc535a91356bab3692d4c6 secrets: use type `lines` for `generateSecretsCmds` (Erik Arvstedt)
bc2f66d4f122dbd37679159002275f87ed87deb4 bitcoind, liquid: increase start/stop timeouts (Erik Arvstedt)
519ae31202a7e878fe7fe3be83f321df741acf43 netns-isolation: improve formatting (Erik Arvstedt)
a1023696e69fbe871cfadf18d054d87eede9bba0 netns-isolation: reserve netns id for mempool (Erik Arvstedt)
34fe8675bd6e0396e3986e0d4b262e166708d8b0 add option `nix-bitcoin.pkgOverlays` (Erik Arvstedt)
a3bdecb10bd6d27d040d10ac9255d99b86d630df helper: add start-bash-session.sh (Erik Arvstedt)
690a8f6256bfed9b9ed074d89e734364b87681f1 nodeinfo: extract fn `mkInfoLong` (Erik Arvstedt)
2af642f56ad0856d5a07d0bb27722b3861719b75 improve comments (Erik Arvstedt)
5634f0887391419b10a3e8c43c6377214b79b115 rtl: make `extraConfig` recursively mergeable (Erik Arvstedt)
b76728a1ec187700fd303933510f0b5a82a2996a treewide: use bool literals for systemd (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK addfa8ec6bab320de0b808fd76809635947377e8

Tree-SHA512: 46f779f8477b566ffc6d0dfb024f2098757f509b2b3e0cbb509cf3308de7029e913f6e6c3d6d3d226cc72f8a5031fd5586b2efdf7c2d9d15f4bdd7ed08b27425

examples/README.md

@@ -55,7 +55,7 @@ The [nix-bitcoin test suite](../test/README.md) is also useful for exploring fea
 
 ### Real-world example
 Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org
-to see the configuration of a nix-bitcoin node that's used in production.
+to see the configuration of a Flakes-based nix-bitcoin node that's used in production.
 
 The commands in `shell.nix` allow you to locally run the node in a VM or container.
 

helper/start-bash-session.sh

@@ -0,0 +1,12 @@
+# Start an interactive bash session in the current bash environment.
+# This is helpful for debugging bash scripts like pkg update scripts,
+# by adding `source <path-to>/start-bash-session.sh` at the location to
+# be inspected.
+
+
+# BASH_ENVIRONMENT contains definitions of read-only variables like 'BASHOPTS' that
+# cause warnings on evaluation. Suppress these warnings while sourcing.
+#
+# shellcheck disable=SC2016
+BASH_ENVIRONMENT=<(declare -p; declare -pf) \
+  bash --rcfile <(echo 'source $BASH_ENVIRONMENT 2>/dev/null')

modules/bitcoind.nix

@@ -421,8 +421,8 @@ in {
         NotifyAccess = "all";
         User = cfg.user;
         Group = cfg.group;
-        TimeoutStartSec = "10min";
-        TimeoutStopSec = "10min";
+        TimeoutStartSec = "30min";
+        TimeoutStopSec = "30min";
         ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'";
         Restart = "on-failure";
         UMask = mkIf cfg.dataDirReadableByGroup "0027";

modules/btcpayserver.nix

@@ -193,7 +193,7 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = [ cfg.nbxplorer.dataDir ];
-        MemoryDenyWriteExecute = "false";
+        MemoryDenyWriteExecute = false;
       } // nbLib.allowedIPAddresses cfg.nbxplorer.tor.enforce;
     };
 
@@ -239,7 +239,7 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = [ cfg.btcpayserver.dataDir ];
-        MemoryDenyWriteExecute = "false";
+        MemoryDenyWriteExecute = false;
       } // nbLib.allowedIPAddresses cfg.btcpayserver.tor.enforce;
     }; in self;
 

modules/clightning-plugins/clboss.nix

@@ -17,8 +17,8 @@ let cfg = config.services.clightning.plugins.clboss; in
       default = 30000;
       description = mdDoc ''
         Target amount (in satoshi) that CLBOSS will leave on-chain.
-        clboss will only open new channels if this amount is smaller than
-        the funds in your clightning wallet.
+        clboss will only open new channels if the funds in your clightning wallet are
+        larger than this amount.
       '';
     };
     min-channel = mkOption {

modules/liquid.nix

@@ -270,8 +270,8 @@ in {
         NotifyAccess = "all";
         User = cfg.user;
         Group = cfg.group;
-        TimeoutStartSec = "10min";
-        TimeoutStopSec = "10min";
+        TimeoutStartSec = "2h";
+        TimeoutStopSec = "2h";
         ExecStart = "${nbPkgs.elementsd}/bin/elementsd -datadir='${cfg.dataDir}'";
         Restart = "on-failure";
         ReadWritePaths = [ cfg.dataDir ];

modules/netns-isolation.nix

@@ -215,9 +215,11 @@ in {
 
         };
       };
-    in foldl (services: n:
-      services // (makeNetnsServices n netns.${n})
-    ) {} (builtins.attrNames netns));
+    in
+      foldl (services: n:
+        services // (makeNetnsServices n netns.${n})
+      ) {} (builtins.attrNames netns)
+    );
   }
 
   # Service-specific config
@@ -297,6 +299,7 @@ in {
         id = 31;
         connections = [ "bitcoind" ];
       };
+      # id = 32 reserved for the upcoming mempool module
     };
 
     services.bitcoind = {

modules/nix-bitcoin.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 with lib;
 {
@@ -8,6 +8,18 @@ with lib;
         type = types.attrs;
         default = (import ../pkgs { inherit pkgs; }).modulesPkgs;
         defaultText = "nix-bitcoin/pkgs.modulesPkgs";
+        apply = base:
+          let
+            final = foldl (prev: overlay:
+              prev // (overlay prev final)
+            ) base options.nix-bitcoin.pkgOverlays.definitions;
+          in
+            final;
+      };
+
+      pkgOverlays = mkOption {
+        internal = true;
+        type = with types; functionTo attrs;
       };
 
       lib = mkOption {

modules/nodeinfo.nix

@@ -73,9 +73,10 @@ let
             return
         info["onion_address"] = f"{onion_address}:{port}"
 
-    def add_service(service, make_info):
-        if not is_active(service):
-            infos[service] = "service is not running"
+    def add_service(service, make_info, systemd_service = None):
+        systemd_service = systemd_service or service
+        if not is_active(systemd_service):
+            infos[service] = f"'{systemd_service}.service' is not running"
         else:
             info = OrderedDict()
             exec(make_info, globals(), locals())
@@ -96,14 +97,19 @@ let
   ) (builtins.attrNames cfg.services);
 
   nodeinfoLib = rec {
-    mkInfo = extraCode: name: cfg: ''
+    mkInfo = extraCode: name: cfg:
+      mkInfoLong {
+        inherit extraCode name cfg;
+      };
+
+    mkInfoLong = { extraCode ? "", name, cfg, systemdServiceName ? name }: ''
       add_service("${name}", """
       info["local_address"] = "${nbLib.addressWithPort cfg.address cfg.port}"
     '' + mkIfOnionPort name (onionPort: ''
       set_onion_address(info, "${name}", ${onionPort})
     '') + extraCode + ''
 
-      """)
+      """, "${systemdServiceName}")
     '';
 
     mkIfOnionPort = name: fn:

modules/onion-addresses.nix

@@ -55,8 +55,8 @@ in {
         RemainAfterExit = true;
         StateDirectory = "onion-addresses";
         StateDirectoryMode = "771";
-        PrivateNetwork = "true"; # This service needs no network access
-        PrivateUsers = "false";
+        PrivateNetwork = true; # This service needs no network access
+        PrivateUsers = false;
         CapabilityBoundingSet = "CAP_CHOWN CAP_FSETID CAP_SETFCAP CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_IPC_OWNER";
       };
       script = ''

modules/presets/bitcoind-remote.nix

@@ -5,7 +5,9 @@ let
   secretsDir = config.nix-bitcoin.secretsDir;
 in {
   services.bitcoind = {
-    # Make the local bitcoin-cli work with the remote node
+    # Make the local bitcoin-cli work with the remote node.
+    # Without this, bitcoin-cli would try to use the .cookie file in the local
+    # bitcoind data dir for authorization, which doesn't exist.
     extraConfig = ''
       rpcuser=${cfg.rpc.users.privileged.name}
     '';

modules/rtl.nix

@@ -27,7 +27,7 @@ let
           description = mdDoc "Enable the clightning node interface.";
         };
         extraConfig = mkOption {
-          type = types.attrs;
+          type = with types; attrsOf anything;
           default = {};
           example = {
             Settings.userPersona = "MERCHANT";
@@ -52,7 +52,7 @@ let
           description = mdDoc "Enable swaps with lightning-loop.";
         };
         extraConfig = mkOption {
-          type = types.attrs;
+          type = with types; attrsOf anything;
           default = {};
           example = {
             Settings.userPersona = "MERCHANT";

modules/secrets/secrets.nix

@@ -28,7 +28,7 @@ let
     };
 
     generateSecretsCmds = mkOption {
-      type = types.attrsOf types.str;
+      type = types.attrsOf types.lines;
       default = {};
       description = mdDoc ''
         Bash expressions for generating secrets.

pkgs/lib.nix

@@ -7,28 +7,28 @@ with lib;
 let self = {
   # These settings roughly follow systemd's "strict" security profile
   defaultHardening = {
-      PrivateTmp = "true";
+      PrivateTmp = true;
       ProtectSystem = "strict";
-      ProtectHome = "true";
-      NoNewPrivileges = "true";
-      PrivateDevices = "true";
-      MemoryDenyWriteExecute = "true";
-      ProtectKernelTunables = "true";
-      ProtectKernelModules = "true";
-      ProtectKernelLogs = "true";
-      ProtectClock = "true";
+      ProtectHome = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      MemoryDenyWriteExecute = true;
+      ProtectKernelTunables = true;
+      ProtectKernelModules = true;
+      ProtectKernelLogs = true;
+      ProtectClock = true;
       ProtectProc = "invisible";
       ProcSubset = "pid";
-      ProtectControlGroups = "true";
+      ProtectControlGroups = true;
       RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
-      RestrictNamespaces = "true";
-      LockPersonality = "true";
+      RestrictNamespaces = true;
+      LockPersonality = true;
       IPAddressDeny = "any";
-      PrivateUsers = "true";
-      RestrictSUIDSGID = "true";
-      RemoveIPC = "true";
-      RestrictRealtime = "true";
-      ProtectHostname = "true";
+      PrivateUsers = true;
+      RestrictSUIDSGID = true;
+      RemoveIPC = true;
+      RestrictRealtime = true;
+      ProtectHostname = true;
       CapabilityBoundingSet = "";
       # @system-service whitelist and docker seccomp blacklist (except for "clone"
       # which is a core requirement for systemd services)
@@ -42,7 +42,7 @@ let self = {
   };
 
   # nodejs applications require memory write execute for JIT compilation
-  nodejs = { MemoryDenyWriteExecute = "false"; };
+  nodejs = { MemoryDenyWriteExecute = false; };
 
   # Allow takes precedence over Deny.
   allowLocalIPAddresses = {

pkgs/python-packages/default.nix

@@ -39,7 +39,6 @@ rec {
       ## Specific versions of packages that already exist in nixpkgs
 
       # cryptography 3.3.2, required by joinmarketdaemon
-      # Used in the private python package set for joinmarket (../joinmarket/default.nix)
       cryptography = callPackage ./specific-versions/cryptography {
         openssl = super.pkgs.openssl_1_1;
         cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {};

test/clightning-replication.nix

@@ -1,4 +1,4 @@
-# You can run this test via `run-tests.sh -s clightningReplication`
+# You can run this test via `run-tests.sh -s clightning-replication`
 
 makeTestVM: pkgs:
 with pkgs.lib;
@@ -14,6 +14,8 @@ let
   clientBaseConfig = {
     imports = [ ../modules/modules.nix ];
 
+    nixpkgs.pkgs = pkgs;
+
     nix-bitcoin.generateSecrets = true;
 
     services.clightning = {
@@ -54,7 +56,9 @@ makeTestVM {
       services.clightning.replication.encrypt = true;
     };
 
-    server = { ... }: {
+    server = {
+      nixpkgs.pkgs = pkgs;
+
       environment.etc."ssh-host-key" = {
         source = keys.server;
         mode = "400";

test/lib/make-test-vm.nix

@@ -27,4 +27,9 @@ let
     inherit (test) meta passthru;
   } // test;
 in
-  runTest
+  runTest // {
+    # A VM runner for interactive use
+    run = pkgs.writers.writeBashBin "run-vm" ''
+      . ${./run-vm.sh} ${runTest.driver} "$@"
+    '';
+  }

test/lib/make-test.nix

@@ -52,11 +52,6 @@ let
       ];
   };
 
-  # A VM runner for interactive use
-  run = pkgs.writers.writeBashBin "run-vm" ''
-    . ${./run-vm.sh} ${test.driver} "$@"
-  '';
-
   mkContainer = legacyInstallDirs:
     extra-container.lib.buildContainers {
       inherit system legacyInstallDirs;
@@ -139,7 +134,6 @@ let
 in
 test // {
   inherit
-    run
     vm
     container
     # For NixOS with `system.stateVersion` <22.05

test/run-tests.sh

@@ -272,7 +272,7 @@ buildable=(
     full
     regtest
     hardened
-    clightningReplication
+    clightning-replication
     lndPruned
 )
 buildable() { buildTests buildable "$@"; }

test/tests.nix

@@ -404,7 +404,7 @@ in {
       ) scenarios;
     in
       {
-        clightningReplication = import ./clightning-replication.nix makeTestVM pkgs;
+        clightning-replication = import ./clightning-replication.nix makeTestVM pkgs;
       } // mainTests;
 
     tests = makeTests scenarios;