Merge fort-nix/nix-bitcoin#385: Misc. improvments

fdcb68e96e examples/shell.nix: add new commands (Erik Arvstedt)
a2466b1127 secrets: allow extending generate-secrets (Erik Arvstedt)
24fd1e9bdc improve examples/shell.nix (Erik Arvstedt)
8a757e0486 push-release.sh: improve --dry-run mode (Erik Arvstedt)
82a2b148d8 secrets: minor fixes (Erik Arvstedt)
e1e3d8a92b secrets: simplify cert generation (Erik Arvstedt)
2c8e29b35b lnd: extract option `certPath` (Erik Arvstedt)
be12a49933 lightning-pool/loop: extract lnd variable (Erik Arvstedt)
955b44404c delete helper/fetch-channel (Erik Arvstedt)
5087ce245f minor cleanups (Erik Arvstedt)
0d2db4e79f backups: add option `postgresqlDatabases` (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  nixbitcoin:
    ACK fdcb68e96e
  jonasnick:
    ACK  fdcb68e96e

Tree-SHA512: a0fef5b6f8704a445b0e381a1713c14d1447e16798e7035bb005d2b61c4cde208f96fc6f152238b6ea2e9080c04fffe7f841073fa41a5c1e0597204e9ed805c2

examples/shell.nix

@@ -1,53 +1,8 @@
 let
-  # This is either a path to a local nix-bitcoin source or an attribute set to
+  nix-bitcoin = toString (import ./nix-bitcoin-release.nix);
-  # be used as the fetchurl argument.
-  nix-bitcoin-release = import ./nix-bitcoin-release.nix;
-
-  nix-bitcoin-path =
-    if builtins.isAttrs nix-bitcoin-release then nix-bitcoin-unpacked
-    else nix-bitcoin-release;
-
-  nixpkgs-path = (import "${toString nix-bitcoin-path}/pkgs/nixpkgs-pinned.nix").nixpkgs;
-  pkgs = import nixpkgs-path {};
-  nix-bitcoin = pkgs.callPackage nix-bitcoin-path {};
-
-  nix-bitcoin-unpacked = (import <nixpkgs> {}).runCommand "nix-bitcoin-src" {} ''
-    mkdir $out; tar xf ${builtins.fetchurl nix-bitcoin-release} -C $out
-  '';
 in
-with pkgs;
+  import "${nix-bitcoin}/helper/makeShell.nix" {
-stdenv.mkDerivation rec {
+    configDir = ./.;
-  name = "nix-bitcoin-environment";
+    # Set this to modify your shell
-
+    # extraShellInitCmds = (pkgs: ''<my bash code>'');
-  path = lib.makeBinPath [ nix-bitcoin.extra-container ];
+  }
-
-  shellHook = ''
-    export NIX_PATH="nixpkgs=${nixpkgs-path}:nix-bitcoin=${toString nix-bitcoin-path}:."
-    export PATH="${path}''${PATH:+:}$PATH"
-
-    export NIX_BITCOIN_EXAMPLES_DIR="${toString ./.}"
-
-    fetch-release() {
-      ${toString nix-bitcoin-path}/helper/fetch-release
-    }
-
-    krops-deploy() {
-      # Ensure strict permissions on secrets/ directory before rsyncing it to
-      # the target machine
-      chmod 700 ${toString ./secrets}
-      $(nix-build --no-out-link ${toString ./krops/deploy.nix})
-    }
-
-    # Print logo if
-    # 1. stdout is a TTY, i.e. we're not piping the output
-    # 2. the shell is interactive
-    if [[ -t 1 && $- == *i* ]]; then
-      ${figlet}/bin/figlet "nix-bitcoin"
-    fi
-
-    (mkdir -p secrets; cd secrets; env -i ${nix-bitcoin.generate-secrets})
-
-    # Don't run this hook when another nix-shell is run inside this shell
-    unset shellHook
-  '';
-}

flake.nix

@@ -62,7 +62,7 @@
         nbPkgs = self.mkNbPkgs { inherit system pkgs; };
 
         packages = flake-utils.lib.flattenTree (removeAttrs nbPkgs [
-          "pinned" "modulesPkgs" "nixops19_09" "krops"
+          "pinned" "modulesPkgs" "nixops19_09" "krops" "generate-secrets"
         ]) // {
           runVM = mkVMScript packages.vm;
 

helper/fetch-channel

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-rev=$1
-sha256=$(nix-prefetch-url --unpack https://github.com/nixos/nixpkgs/archive/$rev.tar.gz)
-echo "rev = \"$rev\";"
-echo "sha256 = \"$sha256\";"

helper/fetch-release

@@ -15,26 +15,28 @@ trap "rm -rf $TMPDIR" EXIT
 GPG_HOME=$TMPDIR/gpg-home
 mkdir -p -m 700 "$GPG_HOME"
 
-cd $TMPDIR
-baseUrl=https://github.com/$repo/releases/download/v$version
-curl --silent -L -O $baseUrl/SHA256SUMS.txt
-curl --silent -L -O $baseUrl/SHA256SUMS.txt.asc
-
 # Import key
 gpg --homedir $GPG_HOME --import "$scriptDir/key-jonasnick.bin" &> /dev/null
 # Verify key fingerprint
 gpg --homedir $GPG_HOME --list-keys 36C71A37C9D988BDE82508D9B1A70E4F8DCD0366 > /dev/null
 
-# Verify signature for SHA256SUMS.txt
+# Fetch nar-hash of release
-gpg --homedir $GPG_HOME --verify SHA256SUMS.txt.asc &> /dev/null || {
+cd $TMPDIR
+baseUrl=https://github.com/$repo/releases/download/v$version
+curl --silent -L -O $baseUrl/nar-hash.txt
+curl --silent -L -O $baseUrl/nar-hash.txt.asc
+
+# Verify signature for nar-hash
+gpg --homedir $GPG_HOME --verify nar-hash.txt.asc &> /dev/null || {
     echo "Error: Signature verification failed. Please open an issue in the project repository."
     exit 1
 }
 
-sha256=$(cat SHA256SUMS.txt | cut -d\  -f1)
+>&2 echo "Fetched and verified release $version"
+
 cat <<EOF
-{
+builtins.fetchTarball {
-  url = "$baseUrl/nix-bitcoin-$version.tar.gz";
+  url = "https://github.com/$repo/archive/v$version.tar.gz";
-  sha256 = "$sha256";
+  sha256 = "$(cat nar-hash.txt)";
 }
 EOF

helper/makeShell.nix

@@ -0,0 +1,110 @@
+{ configDir, extraShellInitCmds ? (pkgs: "") }:
+let
+  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+  pkgs = import nixpkgs {};
+  nbPkgs = import ../pkgs { inherit pkgs; };
+  cfgDir = toString configDir;
+in
+with pkgs;
+stdenv.mkDerivation rec {
+  name = "nix-bitcoin-environment";
+
+  path = lib.makeBinPath [ nbPkgs.extra-container ];
+
+  shellHook = ''
+    export NIX_PATH="nixpkgs=${nixpkgs}:nix-bitcoin=${toString ../.}:."
+    export PATH="${path}''${PATH:+:}$PATH"
+
+    export NIX_BITCOIN_EXAMPLES_DIR="${cfgDir}"
+
+    help() {
+        echo "nix-bitcoin path: ${toString ../.}"
+        echo
+        echo "Available commands"
+        echo "=================="
+        echo "deploy"
+        echo "  Run krops-deploy and eval-config in parallel."
+        echo "  This ensures that eval failures appear quickly when deploying."
+        echo "  In this case, deployment is stopped."
+        echo
+        echo "krops-deploy"
+        echo "  Deploy your node via krops"
+        echo
+        echo "eval-config"
+        echo "  Evaluate your node system configuration"
+        echo
+        echo "generate-secrets"
+        echo "  Create secrets required by your node configuration."
+        echo "  Secrets are written to ./secrets/"
+        echo "  This function is automatically called by krops-deploy."
+        echo
+        echo "update-nix-bitcoin"
+        echo "  Fetch and use the latest version of nix-bitcoin"
+    }
+    h() { help; }
+
+    fetch-release() {
+      ${toString ./fetch-release}
+    }
+
+    update-nix-bitcoin() {
+      fetch-release > "${cfgDir}/nix-bitcoin-release.nix"
+      exec nix-shell
+    }
+
+    generate-secrets() {(
+      set -euo pipefail
+      genSecrets=$(nix-build --no-out-link -I nixos-config="${cfgDir}/configuration.nix" \
+                   '<nixpkgs/nixos>' -A config.nix-bitcoin.generateSecretsScript)
+      mkdir -p "${cfgDir}/secrets"
+      (cd "${cfgDir}/secrets"; $genSecrets)
+    )}
+
+    deploy() {(
+      set -euo pipefail
+      krops-deploy &
+      kropsPid=$!
+      if eval-config; then
+        wait $kropsPid
+      else
+        # Kill all subprocesses
+        kill $(pidClosure $kropsPid)
+        return 1
+      fi
+    )}
+
+    krops-deploy() {(
+      set -euo pipefail
+      generate-secrets
+      # Ensure strict permissions on secrets/ directory before rsyncing it to
+      # the target machine
+      chmod 700 "${cfgDir}/secrets"
+      $(nix-build --no-out-link "${cfgDir}/krops/deploy.nix")
+    )}
+
+    eval-config() {
+      NIXOS_CONFIG="${cfgDir}/krops/krops-configuration.nix" nix eval --raw -f ${nixpkgs}/nixos system.outPath
+      echo
+    }
+
+    pidClosure() {
+      echo "$1"
+      for pid in $(ps -o pid= --ppid "$1"); do
+        pidClosure "$pid"
+      done
+    }
+
+    # Print welcome message if
+    # 1. stdout is a TTY, i.e. we're not piping the output
+    # 2. the shell is interactive
+    if [[ -t 1 && $- == *i* ]]; then
+      ${figlet}/bin/figlet "nix-bitcoin"
+      echo 'Enter "h" or "help" for documentation.'
+    fi
+
+    # Don't run this hook when another nix-shell is run inside this shell
+    unset shellHook
+
+    ${extraShellInitCmds pkgs}
+  '';
+}

helper/push-release.sh

@@ -3,14 +3,10 @@ set -euo pipefail
 
 REPO=fort-nix/nix-bitcoin
 BRANCH=master
-OAUTH_TOKEN=$(pass show nix-bitcoin/github/oauth-token)
+OAUTH_TOKEN=
 DRY_RUN=
 TAG_NAME=
 
-if [[ ! $OAUTH_TOKEN ]]; then
-    echo "Please set OAUTH_TOKEN variable"
-fi
-
 for arg in "$@"; do
     case $arg in
         --dry-run|-n)
@@ -26,18 +22,28 @@ if [[ ! $TAG_NAME ]]; then
     echo "$0 [--dry-run|-n] <tag_name>"
     exit
 fi
-if [[ $DRY_RUN ]]; then echo "Dry run"; fi
+if [[ $DRY_RUN ]]; then
+    echo "Dry run"
+else
+    OAUTH_TOKEN=$(pass show nix-bitcoin/github/oauth-token)
+    if [[ ! $OAUTH_TOKEN ]]; then
+        echo "Please set OAUTH_TOKEN variable"
+    fi
+fi
 
 RESPONSE=$(curl https://api.github.com/repos/$REPO/releases/latest 2> /dev/null)
 echo "Latest release" $(echo $RESPONSE | jq -r '.tag_name' | tail -c +2)
-while true; do
+
-    read -p "Create release $TAG_NAME? [yn] " yn
+if [[ ! $DRY_RUN ]]; then
-    case $yn in
+   while true; do
-        [Yy]* ) break;;
+       read -p "Create release $TAG_NAME? [yn] " yn
-        [Nn]* ) exit;;
+       case $yn in
-        * ) echo "Please answer y or n.";;
+           [Yy]* ) break;;
-    esac
+           [Nn]* ) exit;;
-done
+           * ) echo "Please answer y or n.";;
+       esac
+   done
+fi
 
 TMPDIR=$(mktemp -d)
 if [[ ! $DRY_RUN ]]; then trap "rm -rf $TMPDIR" EXIT; fi
@@ -53,6 +59,10 @@ SHA256SUMS=$TMPDIR/SHA256SUMS.txt
 (cd $TMPDIR; sha256sum $ARCHIVE_NAME > $SHA256SUMS)
 gpg -o $SHA256SUMS.asc -a --detach-sig $SHA256SUMS
 
+cd $TMPDIR
+nix hash to-sri --type sha256 $(nix-prefetch-url --unpack file://$ARCHIVE 2> /dev/null) > nar-hash.txt
+gpg -o nar-hash.txt.asc -a --detach-sig nar-hash.txt
+
 if [[ $DRY_RUN ]]; then
     echo "Created v$TAG_NAME in $TMPDIR"
     exit 0
@@ -71,6 +81,10 @@ post_asset() {
     curl -H "Authorization: token $OAUTH_TOKEN" --data-binary "@$1" -H "Content-Type: application/octet-stream" \
          $GH_ASSET/$(basename $1) &> /dev/null
 }
+post_asset nar-hash.txt
+post_asset nar-hash.txt.asc
+# Post additional assets for backwards compatibility.
+# This allows older nix-bitcoin installations to upgrade via `fetch-release`.
 post_asset $ARCHIVE
 post_asset $SHA256SUMS
 post_asset $SHA256SUMS.asc

modules/backups.nix

@@ -18,17 +18,22 @@ let
     ${config.services.nbxplorer.dataDir}
     ${config.services.btcpayserver.dataDir}
     ${config.services.joinmarket.dataDir}
-    ${config.services.postgresqlBackup.location}/btcpaydb.sql.gz
     ${optionalString config.nix-bitcoin.generateSecrets "${config.nix-bitcoin.secretsDir}"}
     /var/lib/tor
     /var/lib/nixos
 
+    ${builtins.concatStringsSep "\n" postgresqlBackupPaths}
+
     # Extra files
     ${cfg.extraFiles}
 
     # Exclude all unspecified files and directories
     - /
   '';
+
+  postgresqlBackupDir = config.services.postgresqlBackup.location;
+  postgresqlBackupPaths = map (db: "${postgresqlBackupDir}/${db}.sql.gz") cfg.postgresqlDatabases;
+  postgresqlBackupServices = map (db: "postgresqlBackup-${db}.service") cfg.postgresqlDatabases;
 in {
   options.services.backups = {
     enable = mkEnableOption "Backups service";
@@ -53,6 +58,11 @@ in {
         Run backup with the given frequency. If null, do not run automatically.
       '';
     };
+    postgresqlDatabases = mkOption {
+      type = types.listOf types.str;
+      default = [];
+      description = "List of database names to backup.";
+    };
     extraFiles = mkOption {
       type = types.lines;
       default = "";
@@ -63,8 +73,7 @@ in {
     };
   };
 
-  config = mkIf cfg.enable (mkMerge [
+  config = mkIf cfg.enable {
-    {
       environment.systemPackages = [ pkgs.duplicity ];
 
       services.duplicity = {
@@ -78,17 +87,24 @@ in {
         secretFile = "${config.nix-bitcoin.secretsDir}/backup-encryption-env";
       };
 
-      nix-bitcoin.secrets.backup-encryption-env.user = "root";
+      systemd.services.duplicity = {
-    }
+        wants = postgresqlBackupServices;
-    (mkIf config.services.btcpayserver.enable {
+        after = postgresqlBackupServices;
+      };
+
       services.postgresqlBackup = {
-        enable = true;
+        enable = mkIf (cfg.postgresqlDatabases != []) true;
-        databases = [ "btcpaydb" ];
+        databases = cfg.postgresqlDatabases;
       };
-      systemd.services.duplicity = rec {
+
-        wants = [ "postgresqlBackup-btcpaydb.service" ];
+      nix-bitcoin.secrets.backup-encryption-env.user = "root";
-        after = wants;
+      nix-bitcoin.generateSecretsCmds.backups = ''
-      };
+        makePasswordSecret backup-encryption-password
-    })
+        if [[ backup-encryption-password -nt backup-encryption-env ]]; then
-  ]);
+           echo "PASSPHRASE=$(cat backup-encryption-password)" > backup-encryption-env
+        fi
+      '';
+
+      services.backups.postgresqlDatabases = mkIf config.services.btcpayserver.enable [ "btcpaydb" ];
+    };
 }

modules/bitcoind.nix

@@ -394,15 +394,22 @@ in {
     };
     users.groups.${cfg.group} = {};
     users.groups.bitcoinrpc-public = {};
+
     nix-bitcoin.operator.groups = [ cfg.group ];
 
-    nix-bitcoin.secrets.bitcoin-rpcpassword-privileged.user = cfg.user;
+    nix-bitcoin.secrets = {
-    nix-bitcoin.secrets.bitcoin-rpcpassword-public = {
+      bitcoin-rpcpassword-privileged.user = cfg.user;
-      user = cfg.user;
+      bitcoin-rpcpassword-public = {
-      group = "bitcoinrpc-public";
+        user = cfg.user;
-    };
+        group = "bitcoinrpc-public";
+      };
 
-    nix-bitcoin.secrets.bitcoin-HMAC-privileged.user = cfg.user;
+      bitcoin-HMAC-privileged.user = cfg.user;
-    nix-bitcoin.secrets.bitcoin-HMAC-public.user = cfg.user;
+      bitcoin-HMAC-public.user = cfg.user;
+    };
+    nix-bitcoin.generateSecretsCmds.bitcoind = ''
+      makeBitcoinRPCPassword privileged
+      makeBitcoinRPCPassword public
+    '';
   };
 }

modules/btcpayserver.nix

@@ -210,9 +210,9 @@ in {
         install -m 600 ${configFile} '${cfg.btcpayserver.dataDir}/settings.config'
         ${optionalString (cfg.btcpayserver.lightningBackend == "lnd") ''
           {
-            echo -n "${lndConfig}";
+            echo -n "${lndConfig}"
-            ${pkgs.openssl}/bin/openssl x509 -noout -fingerprint -sha256 -in ${config.nix-bitcoin.secretsDir}/lnd-cert \
+            ${pkgs.openssl}/bin/openssl x509 -noout -fingerprint -sha256 -in ${config.services.lnd.certPath} \
-              | sed -e 's/.*=//;s/://g';
+              | sed -e 's/.*=//;s/://g'
           } >> '${cfg.btcpayserver.dataDir}/settings.config'
         ''}
       '';
@@ -253,5 +253,8 @@ in {
       };
       bitcoin-HMAC-btcpayserver.user = cfg.bitcoind.user;
     };
+    nix-bitcoin.generateSecretsCmds.btcpayserver = ''
+      makeBitcoinRPCPassword btcpayserver
+    '';
   };
 }

modules/charge-lnd.nix

@@ -105,7 +105,7 @@ in
         macaroonDir=${dataDir}/lnddir-proxy/data/chain/bitcoin/mainnet
         mkdir -p $macaroonDir
         ln -sf /run/lnd/charge-lnd.macaroon $macaroonDir
-        ln -sf ${config.nix-bitcoin.secretsDir}/lnd-cert ${dataDir}/lnddir-proxy/tls.cert
+        ln -sf ${lnd.certPath} ${dataDir}/lnddir-proxy/tls.cert
       '';
       serviceConfig = nbLib.defaultHardening // {
         ExecStart = ''

modules/joinmarket-ob-watcher.nix

@@ -131,5 +131,8 @@ in {
       bitcoin-rpcpassword-joinmarket-ob-watcher.user = cfg.user;
       bitcoin-HMAC-joinmarket-ob-watcher.user = bitcoind.user;
     };
+    nix-bitcoin.generateSecretsCmds.joinmarket-ob-watcher = ''
+      makeBitcoinRPCPassword joinmarket-ob-watcher
+    '';
   };
 }

modules/joinmarket.nix

@@ -305,6 +305,9 @@ in {
     };
 
     nix-bitcoin.secrets.jm-wallet-password.user = cfg.user;
+    nix-bitcoin.generateSecretsCmds.joinmarket = ''
+      makePasswordSecret jm-wallet-password
+    '';
   }
 
   (mkIf cfg.yieldgenerator.enable {

modules/lightning-loop.nix

@@ -6,6 +6,9 @@ let
   cfg = config.services.lightning-loop;
   nbLib = config.nix-bitcoin.lib;
   secretsDir = config.nix-bitcoin.secretsDir;
+
+  lnd = config.services.lnd;
+
   network = config.services.bitcoind.network;
   rpclisten = "${cfg.rpcAddress}:${toString cfg.rpcPort}";
   configFile = builtins.toFile "loop.conf" ''
@@ -17,9 +20,9 @@ let
     tlscertpath=${secretsDir}/loop-cert
     tlskeypath=${secretsDir}/loop-key
 
-    lnd.host=${config.services.lnd.rpcAddress}:${toString config.services.lnd.rpcPort}
+    lnd.host=${lnd.rpcAddress}:${toString lnd.rpcPort}
-    lnd.macaroonpath=${config.services.lnd.networkDir}/admin.macaroon
+    lnd.macaroonpath=${lnd.networkDir}/admin.macaroon
-    lnd.tlspath=${secretsDir}/lnd-cert
+    lnd.tlspath=${lnd.certPath}
 
     ${optionalString (cfg.proxy != null) "server.proxy=${cfg.proxy}"}
 
@@ -89,7 +92,7 @@ in {
     environment.systemPackages = [ cfg.package (hiPrio cfg.cli) ];
 
     systemd.tmpfiles.rules = [
-      "d '${cfg.dataDir}' 0770 ${config.services.lnd.user} ${config.services.lnd.group} - -"
+      "d '${cfg.dataDir}' 0770 ${lnd.user} ${lnd.group} - -"
     ];
 
     systemd.services.lightning-loop = {
@@ -98,7 +101,7 @@ in {
       after = [ "lnd.service" ];
       serviceConfig = nbLib.defaultHardening // {
         ExecStart = "${cfg.package}/bin/loopd --configfile=${configFile}";
-        User = config.services.lnd.user;
+        User = lnd.user;
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
@@ -106,8 +109,11 @@ in {
     };
 
      nix-bitcoin.secrets = {
-       loop-key.user = config.services.lnd.user;
+       loop-key.user = lnd.user;
-       loop-cert.user = config.services.lnd.user;
+       loop-cert.user = lnd.user;
      };
+     nix-bitcoin.generateSecretsCmds.lightning-loop = ''
+       makeCert loop '${optionalString (cfg.rpcAddress != "localhost") "IP:${cfg.rpcAddress}"}'
+    '';
   };
 }

modules/lightning-pool.nix

@@ -5,7 +5,9 @@ with lib;
 let
   cfg = config.services.lightning-pool;
   nbLib = config.nix-bitcoin.lib;
-  secretsDir = config.nix-bitcoin.secretsDir;
+
+  lnd = config.services.lnd;
+
   network = config.services.bitcoind.network;
   rpclisten = "${cfg.rpcAddress}:${toString cfg.rpcPort}";
   configFile = builtins.toFile "pool.conf" ''
@@ -13,9 +15,9 @@ let
     restlisten=${cfg.restAddress}:${toString cfg.restPort}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
 
-    lnd.host=${config.services.lnd.rpcAddress}:${toString config.services.lnd.rpcPort}
+    lnd.host=${lnd.rpcAddress}:${toString lnd.rpcPort}
-    lnd.macaroondir=${config.services.lnd.networkDir}
+    lnd.macaroondir=${lnd.networkDir}
-    lnd.tlspath=${secretsDir}/lnd-cert
+    lnd.tlspath=${lnd.certPath}
 
     ${cfg.extraConfig}
   '';

modules/liquid.nix

@@ -247,5 +247,8 @@ in {
     nix-bitcoin.operator.groups = [ cfg.group ];
 
     nix-bitcoin.secrets.liquid-rpcpassword.user = cfg.user;
+    nix-bitcoin.generateSecretsCmds.liquid = ''
+      makePasswordSecret liquid-rpcpassword
+    '';
   };
 }

modules/lnd-rest-onion-service.nix

@@ -5,7 +5,6 @@ with lib;
 let
   cfg = config.services.lnd.restOnionService;
   nbLib = config.nix-bitcoin.lib;
-  secretsDir = config.nix-bitcoin.secretsDir;
   runAsUser = config.nix-bitcoin.runAsUserCmd;
 
   lnd = config.services.lnd;
@@ -17,7 +16,7 @@ let
      --host=$(cat ${config.nix-bitcoin.onionAddresses.dataDir}/lnd/lnd-rest) \
      --port=${toString lnd.restPort} \
      --lnddir=${lnd.dataDir} \
-     --tlscertpath=${secretsDir}/lnd-cert "$@"
+     --tlscertpath=${lnd.certPath} "$@"
   '';
 in {
   options.services.lnd.restOnionService = {

modules/lnd.nix

@@ -14,7 +14,7 @@ let
   configFile = pkgs.writeText "lnd.conf" ''
     datadir=${cfg.dataDir}
     logdir=${cfg.dataDir}/logs
-    tlscertpath=${secretsDir}/lnd-cert
+    tlscertpath=${cfg.certPath}
     tlskeypath=${secretsDir}/lnd-key
 
     listen=${toString cfg.address}:${toString cfg.port}
@@ -126,7 +126,7 @@ in {
       ''
         ${runAsUser} ${cfg.user} ${cfg.package}/bin/lncli \
           --rpcserver ${cfg.rpcAddress}:${toString cfg.rpcPort} \
-          --tlscertpath '${secretsDir}/lnd-cert' \
+          --tlscertpath '${cfg.certPath}' \
           --macaroonpath '${networkDir}/admin.macaroon' "$@"
       '';
       description = "Binary to connect with the lnd instance.";
@@ -149,6 +149,11 @@ in {
       default = cfg.user;
       description = "The group as which to run LND.";
     };
+    certPath = mkOption {
+      readOnly = true;
+      default = "${secretsDir}/lnd-cert";
+      description = "LND TLS certificate path.";
+    };
     inherit (nbLib) enforceTor;
   };
 
@@ -211,7 +216,7 @@ in {
           # Retrying is necessary because it can happen that the lnd socket is
           # existing, but the RPC service isn't yet, which results in error
           # "waiting to start, RPC services not available".
-          curl = "${pkgs.curl}/bin/curl -s --show-error --retry 10 --cacert ${secretsDir}/lnd-cert";
+          curl = "${pkgs.curl}/bin/curl -s --show-error --retry 10 --cacert ${cfg.certPath}";
           restUrl = "https://${cfg.restAddress}:${toString cfg.restPort}/v1";
         in [
           (nbLib.script "lnd-create-wallet" ''
@@ -288,7 +293,14 @@ in {
       lnd-wallet-password.user = cfg.user;
       lnd-key.user = cfg.user;
       lnd-cert.user = cfg.user;
-      lnd-cert.permissions = "0444"; # world readable
+      lnd-cert.permissions = "444"; # world readable
     };
+    # Advantages of manually pre-generating certs:
+    # - Reduces dynamic state
+    # - Enables deployment of a mesh of server plus client nodes with predefined certs
+    nix-bitcoin.generateSecretsCmds.lnd = ''
+      makePasswordSecret lnd-wallet-password
+      makeCert lnd '${optionalString (cfg.rpcAddress != "localhost") "IP:${cfg.rpcAddress}"}'
+    '';
   };
 }

modules/secrets/secrets.nix

@@ -2,9 +2,6 @@
 
 with lib;
 let
-  cfg = config.nix-bitcoin;
-in
-{
   options.nix-bitcoin = {
     secretsDir = mkOption {
       type = types.path;
@@ -24,8 +21,16 @@ in
       type = types.bool;
       default = false;
       description = ''
-        Automatically generate all required secrets.
+        Automatically generate all required secrets at system startup.
-        Make sure to create a backup of the generated secrets.
+        Note: Make sure to create a backup of the generated secrets.
+      '';
+    };
+
+    generateSecretsCmds = mkOption {
+      type = types.attrsOf types.str;
+      default = {};
+      description = ''
+        Bash expressions for generating secrets.
       '';
     };
 
@@ -60,7 +65,6 @@ in
     };
 
     secretsSetupMethod = mkOption {
-      internal = true;
       type = types.str;
       default = throw  ''
         Error: No secrets setup method has been defined.
@@ -73,8 +77,68 @@ in
          - Set `nix-bitcoin.secretsSetupMethod = "manual"` if you want to manually setup secrets
       '';
     };
+
+    generateSecretsScript = mkOption {
+      internal = true;
+      default = let
+        rpcauthSrc = builtins.fetchurl {
+          url = "https://raw.githubusercontent.com/bitcoin/bitcoin/d6cde007db9d3e6ee93bd98a9bbfdce9bfa9b15b/share/rpcauth/rpcauth.py";
+          sha256 = "189mpplam6yzizssrgiyv70c9899ggh8cac76j4n7v0xqzfip07n";
+        };
+        rpcauth = pkgs.writers.writeBash "rpcauth" ''
+          exec ${pkgs.python3}/bin/python ${rpcauthSrc} "$@"
+        '';
+      in pkgs.writers.writeBash "generate-secrets" ''
+        set -euo pipefail
+
+        export PATH=${lib.makeBinPath (with pkgs; [ coreutils gnugrep ])}
+
+        makePasswordSecret() {
+          # Passwords have alphabet {a-z, A-Z, 0-9} and ~119 bits of entropy
+          [[ -e $1 ]] || ${pkgs.pwgen}/bin/pwgen -s 20 1 > "$1"
+        }
+        makeBitcoinRPCPassword() {
+          user=$1
+          file=bitcoin-rpcpassword-$user
+          HMACfile=bitcoin-HMAC-$user
+          makePasswordSecret "$file"
+          if [[ $file -nt $HMACfile ]]; then
+            ${rpcauth} $user $(cat "$file") | grep rpcauth | cut -d ':' -f 2 > "$HMACfile"
+          fi
+        }
+        makeCert() {
+          name=$1
+          # Add leading comma if not empty
+          extraAltNames=''${2:+,}''${2:-}
+          if [[ ! -e $name-key ]]; then
+            # Create new key and cert
+            doMakeCert "-newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -keyout $name-key"
+          elif [[ ! -e $name-cert \
+                  || $(cat "$name-cert-alt-names" 2>/dev/null) != $extraAltNames ]]; then
+            # Create cert from existing key
+            doMakeCert "-key $name-key"
+          fi;
+        }
+        doMakeCert() {
+           # This fn uses global variables `name` and `extraAltNames`
+           keyOpts=$1
+           ${pkgs.openssl}/bin/openssl req -x509 \
+             -sha256 -days 3650 $keyOpts -out "$name-cert"  \
+             -subj "/CN=localhost/O=$name" \
+             -addext "subjectAltName=DNS:localhost,IP:127.0.0.1$extraAltNames"
+           echo "$extraAltNames" > "$name-cert-alt-names"
+        }
+
+        umask u=rw,go=
+        ${builtins.concatStringsSep "\n" (builtins.attrValues cfg.generateSecretsCmds)}
+      '';
+    };
   };
 
+  cfg = config.nix-bitcoin;
+in {
+  inherit options;
+
   config = {
     # This target is active when secrets have been setup successfully.
     systemd.targets.nix-bitcoin-secrets = mkIf (cfg.secretsSetupMethod != "manual") {
@@ -107,7 +171,7 @@ in
           cd "${cfg.secretsDir}"
           chown root: .
           chmod 0700 .
-          ${cfg.pkgs.generate-secrets}
+          ${cfg.generateSecretsScript}
         ''}
 
         setupSecret() {
@@ -141,9 +205,11 @@ in
 
         # Make all other files accessible to root only
         unprocessedFiles=$(comm -23 <(printf '%s\n' *) <(printf '%s\n' "''${processedFiles[@]}" | sort))
-        IFS=$'\n'
+        if [[ $unprocessedFiles ]]; then
-        chown root: $unprocessedFiles
+          IFS=$'\n'
-        chmod 0440 $unprocessedFiles
+          chown root: $unprocessedFiles
+          chmod 0440 $unprocessedFiles
+        fi
 
         # Now make the secrets dir accessible to other users
         chmod 0751 "$dir"

modules/spark-wallet.nix

@@ -83,6 +83,13 @@ in {
       } // nbLib.allowedIPAddresses cfg.enforceTor
         // nbLib.nodejs;
     };
+
     nix-bitcoin.secrets.spark-wallet-login.user = cfg.user;
+    nix-bitcoin.generateSecretsCmds.spark-wallet = ''
+      makePasswordSecret spark-wallet-password
+      if [[ spark-wallet-password -nt spark-wallet-login ]]; then
+         echo "login=spark-wallet:$(cat spark-wallet-password)" > spark-wallet-login
+      fi
+    '';
   };
 }

pkgs/default.nix

@@ -9,7 +9,7 @@ let self = {
   spark-wallet = pkgs.callPackage ./spark-wallet { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
   joinmarket = pkgs.callPackage ./joinmarket { inherit (self) nbPython3Packages; };
-  generate-secrets = pkgs.callPackage ./generate-secrets { };
+  generate-secrets = import ./generate-secrets-deprecated.nix;
   nixops19_09 = pkgs.callPackage ./nixops { };
   krops = import ./krops { };
   netns-exec = pkgs.callPackage ./netns-exec { };

pkgs/generate-secrets-deprecated.nix

@@ -0,0 +1,14 @@
+throw ''
+  Please update the `shell.nix` of your node configuration.
+
+  To update, do the following:
+  1. Switch to the directory containing your `configuration.nix` and `shell.nix`.
+  2. Run the following Bash expression (Warning: This overwrites your `shell.nix`):
+
+  # Only update nix-bitcoin-release.nix if it contains a release hash
+  if grep -q sha256 nix-bitcoin-release.nix; then
+    ${toString ../helper/fetch-release} > nix-bitcoin-release.nix && cp ${toString ../examples/shell.nix} .
+  else
+    cp ${toString ../examples/shell.nix} .
+  fi
+''

pkgs/generate-secrets/default.nix

@@ -1,15 +0,0 @@
-{ pkgs }: with pkgs;
-
-let
-  rpcauthSrc = builtins.fetchurl {
-    url = "https://raw.githubusercontent.com/bitcoin/bitcoin/d6cde007db9d3e6ee93bd98a9bbfdce9bfa9b15b/share/rpcauth/rpcauth.py";
-    sha256 = "189mpplam6yzizssrgiyv70c9899ggh8cac76j4n7v0xqzfip07n";
-  };
-  rpcauth = pkgs.writeScriptBin "rpcauth" ''
-    exec ${pkgs.python3}/bin/python ${rpcauthSrc} "$@"
-  '';
-in
-writers.writeBash "generate-secrets" ''
-  export PATH=${lib.makeBinPath [ coreutils pwgen openssl gnugrep rpcauth ]}
-  . ${./generate-secrets.sh} ${./openssl.cnf}
-''

pkgs/generate-secrets/generate-secrets.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-opensslConf=${1:-openssl.cnf}
-
-makePasswordSecret() {
-    # Passwords have alphabet {a-z, A-Z, 0-9} and ~119 bits of entropy
-    [[ -e $1 ]] || pwgen -s 20 1 > "$1"
-}
-makeHMAC() {
-    user=$1
-    rpcauth $user $(cat bitcoin-rpcpassword-$user) | grep rpcauth | cut -d ':' -f 2 > bitcoin-HMAC-$user
-}
-
-makePasswordSecret bitcoin-rpcpassword-privileged
-makePasswordSecret bitcoin-rpcpassword-btcpayserver
-makePasswordSecret bitcoin-rpcpassword-joinmarket-ob-watcher
-makePasswordSecret bitcoin-rpcpassword-public
-makePasswordSecret lnd-wallet-password
-makePasswordSecret liquid-rpcpassword
-makePasswordSecret spark-wallet-password
-makePasswordSecret backup-encryption-password
-makePasswordSecret jm-wallet-password
-
-[[ -e bitcoin-HMAC-privileged ]] || makeHMAC privileged
-[[ -e bitcoin-HMAC-public ]] || makeHMAC public
-[[ -e bitcoin-HMAC-btcpayserver ]] || makeHMAC btcpayserver
-[[ -e bitcoin-HMAC-joinmarket-ob-watcher ]] || makeHMAC joinmarket-ob-watcher
-[[ -e spark-wallet-login   ]] || echo "login=spark-wallet:$(cat spark-wallet-password)" > spark-wallet-login
-[[ -e backup-encryption-env ]] || echo "PASSPHRASE=$(cat backup-encryption-password)" > backup-encryption-env
-
-if [[ ! -e lnd-key || ! -e lnd-cert ]]; then
-    openssl ecparam -genkey -name prime256v1 -out lnd-key
-    openssl req -config $opensslConf -new -sha256 -key lnd-key -out lnd.csr -subj '/CN=localhost/O=lnd'
-    openssl req -config $opensslConf -x509 -sha256 -days 1825 -key lnd-key -in lnd.csr -out lnd-cert
-    rm lnd.csr
-fi
-
-if [[ ! -e loop-key || ! -e loop-cert ]]; then
-    openssl ecparam -genkey -name prime256v1 -out loop-key
-    openssl req -config $opensslConf -new -sha256 -key loop-key -out loop.csr -subj '/CN=localhost/O=loopd'
-    openssl req -config $opensslConf -x509 -sha256 -days 1825 -key loop-key -in loop.csr -out loop-cert
-    rm loop.csr
-fi

pkgs/generate-secrets/openssl.cnf

@@ -1,36 +0,0 @@
-[ req ]
-#default_bits		= 2048
-#default_md		= sha256
-#default_keyfile 	= privkey.pem
-distinguished_name	= req_distinguished_name
-attributes		= req_attributes
-x509_extensions         = v3_ca
-
-[ req_distinguished_name ]
-countryName			= Country Name (2 letter code)
-countryName_min			= 2
-countryName_max			= 2
-stateOrProvinceName		= State or Province Name (full name)
-localityName			= Locality Name (eg, city)
-0.organizationName		= Organization Name (eg, company)
-organizationalUnitName		= Organizational Unit Name (eg, section)
-commonName			= Common Name (eg, fully qualified host name)
-commonName_max			= 64
-emailAddress			= Email Address
-emailAddress_max		= 64
-
-[ req_attributes ]
-challengePassword		= A challenge password
-challengePassword_min		= 4
-challengePassword_max		= 20
-
-[ v3_ca ]
-subjectAltName			= @alt_names
-
-[ alt_names ]
-IP.1 = 127.0.0.1
-DNS.1 = localhost
-# TODO: Remove hardcoded lnd IP
-IP.2 = 169.254.1.14
-# TODO: Remove hardcoded loopd IP
-IP.3 = 169.254.1.22

pkgs/generate-secrets/update-and-generate.nix

@@ -1,10 +0,0 @@
-{ pkgs }: with pkgs;
-
-let
-  generate-secrets = callPackage ./. {};
-in
-writeScript "make-secrets" ''
-  # Update from old secrets format
-  [[ -e secrets.nix ]] && . ${./update-secrets.sh}
-  ${generate-secrets}
-''

pkgs/generate-secrets/update-secrets.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-set -eo pipefail
-
-# Update secrets from the old format to the current one where each secret
-# has a local source file.
-
-reportError() {
-    echo "Updating secrets failed. (Error in line $1)"
-    echo "The secret files have been moved to secrets/old-secrets"
-}
-trap 'reportError $LINENO' ERR
-
-echo "Updating old secrets to the current format."
-
-mkdir old-secrets
-# move all files into old-secrets
-shopt -s extglob dotglob
-mv !(old-secrets) old-secrets
-shopt -u dotglob
-
-secrets=$(cat old-secrets/secrets.nix)
-
-extractPassword() {
-    pwName="$1"
-    destFile="${2:-$pwName}"
-    echo "$secrets" | sed -nE "s/.*?$pwName = \"(.*?)\".*/\1/p" > "$destFile"
-}
-
-rename() {
-    old="old-secrets/$1"
-    if [[ -e $old ]]; then
-        cp "$old" "$2"
-    fi
-}
-
-extractPassword bitcoinrpcpassword bitcoin-rpcpassword
-extractPassword lnd-wallet-password
-extractPassword liquidrpcpassword liquid-rpcpassword
-extractPassword spark-wallet-password
-
-rename lnd.key lnd-key
-rename lnd.cert lnd-cert
-
-rm -r old-secrets

pkgs/krops/get-sha256.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 archive_hash () {
     repo=$1
     rev=$2
-    nix-prefetch-url --unpack "https://github.com/${repo}/archive/${rev}.tar.gz" 2> /dev/null | tail -n 1
+    nix-prefetch-url --unpack "https://github.com/${repo}/archive/${rev}.tar.gz" 2> /dev/null
 }
 
 echo "Fetching latest krops commit"