From cc7149eb785b98014032a816de25da4fc072a7dd Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 16 Mar 2021 12:45:17 +0100 Subject: [PATCH 01/13] examples: improve robustness of deploy scripts - The scripts now work within arbitrary nix-shells. Previously, they failed when run from nix shells other than `examples/shell.nix`. - The scripts now work from arbitrary working dirs. --- examples/deploy-container.sh | 6 ++++-- examples/deploy-krops.sh | 4 +++- examples/deploy-qemu-vm.sh | 4 +++- examples/shell.nix | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/deploy-container.sh b/examples/deploy-container.sh index 8cc424b..ccc3654 100755 --- a/examples/deploy-container.sh +++ b/examples/deploy-container.sh @@ -8,10 +8,12 @@ set -euo pipefail # Run with option `--interactive` or `-i` to start a shell for interacting with # the node. -if [[ ! -v IN_NIX_SHELL ]]; then +if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then echo "Running script in nix shell env..." cd "${BASH_SOURCE[0]%/*}" exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*" +else + cd "$NIX_BITCOIN_EXAMPLES_DIR" fi if [[ $(sysctl -n net.ipv4.ip_forward || sudo sysctl -n net.ipv4.ip_forward) != 1 ]]; then @@ -22,7 +24,7 @@ fi if [[ $EUID != 0 ]]; then # NixOS containers require root permissions - exec sudo "PATH=$PATH" "NIX_PATH=$NIX_PATH" "IN_NIX_SHELL=$IN_NIX_SHELL" "${BASH_SOURCE[0]}" "$@" + exec sudo "PATH=$PATH" "NIX_PATH=$NIX_PATH" "NIX_BITCOIN_EXAMPLES_DIR=$NIX_BITCOIN_EXAMPLES_DIR" "${BASH_SOURCE[0]}" "$@" fi interactive= diff --git a/examples/deploy-krops.sh b/examples/deploy-krops.sh index ca6f42e..cf7de7f 100755 --- a/examples/deploy-krops.sh +++ b/examples/deploy-krops.sh @@ -12,10 +12,12 @@ set -euo pipefail # MAKE SURE TO REPLACE the SSH identity file if you use this script for # anything serious. -if [[ ! -v IN_NIX_SHELL ]]; then +if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then echo "Running script in nix shell env..." cd "${BASH_SOURCE[0]%/*}" exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*" +else + cd "$NIX_BITCOIN_EXAMPLES_DIR" fi source qemu-vm/run-vm.sh diff --git a/examples/deploy-qemu-vm.sh b/examples/deploy-qemu-vm.sh index bff14db..77869b2 100755 --- a/examples/deploy-qemu-vm.sh +++ b/examples/deploy-qemu-vm.sh @@ -11,10 +11,12 @@ set -euo pipefail # MAKE SURE TO REPLACE the SSH identity file if you use this script for # anything serious. -if [[ ! -v IN_NIX_SHELL ]]; then +if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then echo "Running script in nix shell env..." cd "${BASH_SOURCE[0]%/*}" exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*" +else + cd "$NIX_BITCOIN_EXAMPLES_DIR" fi source qemu-vm/run-vm.sh diff --git a/examples/shell.nix b/examples/shell.nix index 7f30899..59e5ac9 100644 --- a/examples/shell.nix +++ b/examples/shell.nix @@ -26,6 +26,8 @@ stdenv.mkDerivation rec { export NIX_PATH="nixpkgs=${nixpkgs-path}:nix-bitcoin=${toString nix-bitcoin-path}:." export PATH="${path}''${PATH:+:}$PATH" + export NIX_BITCOIN_EXAMPLES_DIR="${toString ./.}" + alias fetch-release="${toString nix-bitcoin-path}/helper/fetch-release" krops-deploy() { From 45d0964e27c754f6007347e5ff46c00d15bf9e4a Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 16 Mar 2021 12:45:18 +0100 Subject: [PATCH 02/13] examples/shell.nix: minor improvements - Use idiomatic var name `pkgs` for the imported nixpkgs. - Don't add `figlet` to PATH because it's only used internally. - Only print figlet in interactive shells to avoid interfering with stdout when running `nix-shell --run `. - Define `fetch-release` as a function to enable running it via `nix-shell --run fetch-release` --- examples/shell.nix | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/examples/shell.nix b/examples/shell.nix index 59e5ac9..47b1e47 100644 --- a/examples/shell.nix +++ b/examples/shell.nix @@ -8,19 +8,18 @@ let else nix-bitcoin-release; nixpkgs-path = (import "${toString nix-bitcoin-path}/pkgs/nixpkgs-pinned.nix").nixpkgs; - nixpkgs = import nixpkgs-path {}; - nix-bitcoin = nixpkgs.callPackage nix-bitcoin-path {}; + pkgs = import nixpkgs-path {}; + nix-bitcoin = pkgs.callPackage nix-bitcoin-path {}; nix-bitcoin-unpacked = (import {}).runCommand "nix-bitcoin-src" {} '' mkdir $out; tar xf ${builtins.fetchurl nix-bitcoin-release} -C $out ''; in -with nixpkgs; - +with pkgs; stdenv.mkDerivation rec { name = "nix-bitcoin-environment"; - path = lib.makeBinPath [ nix-bitcoin.extra-container figlet ]; + path = lib.makeBinPath [ nix-bitcoin.extra-container ]; shellHook = '' export NIX_PATH="nixpkgs=${nixpkgs-path}:nix-bitcoin=${toString nix-bitcoin-path}:." @@ -28,7 +27,9 @@ stdenv.mkDerivation rec { export NIX_BITCOIN_EXAMPLES_DIR="${toString ./.}" - alias fetch-release="${toString nix-bitcoin-path}/helper/fetch-release" + fetch-release() { + ${toString nix-bitcoin-path}/helper/fetch-release + } krops-deploy() { # Ensure strict permissions on secrets/ directory before rsyncing it to @@ -37,7 +38,13 @@ stdenv.mkDerivation rec { $(nix-build --no-out-link ${toString ./krops/deploy.nix}) } - figlet "nix-bitcoin" + # Print logo if + # 1. stdout is a TTY, i.e. we're not piping the output + # 2. the shell is interactive + if [[ -t 1 && $- == *i* ]]; then + ${figlet}/bin/figlet "nix-bitcoin" + fi + (mkdir -p secrets; cd secrets; env -i ${nix-bitcoin.generate-secrets}) # Don't run this hook when another nix-shell is run inside this shell From 84b3217c3d8dfbba8264d164733371674d2895d7 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 16 Mar 2021 12:45:19 +0100 Subject: [PATCH 03/13] fetch-release: minor improvements This script is potentially fetched from an untrusted source and should be in good shape to be easily auditable. - Create just one TMPDIR - Improve comments - Use `cut` to extract sha256 - Use camelCase var names like in other scripts --- helper/fetch-release | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/helper/fetch-release b/helper/fetch-release index 6529386..13513d8 100755 --- a/helper/fetch-release +++ b/helper/fetch-release @@ -1,36 +1,40 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p bash coreutils curl jq gnugrep gnupg +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash coreutils curl jq gnupg set -euo pipefail scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) -REPO=fort-nix/nix-bitcoin -if [[ ! -v VERSION ]]; then - VERSION=$(curl --silent "https://api.github.com/repos/$REPO/releases/latest" | jq -r '.tag_name' | tail -c +2) +repo=fort-nix/nix-bitcoin +if [[ ! -v version ]]; then + version=$(curl --silent "https://api.github.com/repos/$repo/releases/latest" | jq -r '.tag_name' | tail -c +2) fi TMPDIR=$(mktemp -d) -GPG_HOME=$(mktemp -d) -trap "rm -rf $TMPDIR $GPG_HOME" EXIT +trap "rm -rf $TMPDIR" EXIT + +GPG_HOME=$TMPDIR/gpg-home +mkdir -p -m 700 "$GPG_HOME" cd $TMPDIR -BASEURL=https://github.com/$REPO/releases/download/v$VERSION -curl --silent -L -O $BASEURL/SHA256SUMS.txt -curl --silent -L -O $BASEURL/SHA256SUMS.txt.asc +baseUrl=https://github.com/$repo/releases/download/v$version +curl --silent -L -O $baseUrl/SHA256SUMS.txt +curl --silent -L -O $baseUrl/SHA256SUMS.txt.asc -# Import key and verify fingerprint +# Import key gpg --homedir $GPG_HOME --import "$scriptDir/key-jonasnick.bin" &> /dev/null +# Verify key fingerprint gpg --homedir $GPG_HOME --list-keys 36C71A37C9D988BDE82508D9B1A70E4F8DCD0366 > /dev/null +# Verify signature for SHA256SUMS.txt gpg --homedir $GPG_HOME --verify SHA256SUMS.txt.asc &> /dev/null || { - echo "ERROR: Signature verification failed. Please open an issue in the project repository." + echo "Error: Signature verification failed. Please open an issue in the project repository." exit 1 } -SHA256=$(cat SHA256SUMS.txt | grep -Eo '^[^ ]+') +sha256=$(cat SHA256SUMS.txt | cut -d\ -f1) cat < Date: Tue, 16 Mar 2021 12:45:20 +0100 Subject: [PATCH 04/13] spark-wallet/generate: remove supplement.json This file is empty and has no effect. --- pkgs/spark-wallet/composition.nix | 6 +----- pkgs/spark-wallet/default.nix | 2 +- pkgs/spark-wallet/generate.sh | 2 +- pkgs/spark-wallet/supplement.json | 2 -- 4 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 pkgs/spark-wallet/supplement.json diff --git a/pkgs/spark-wallet/composition.nix b/pkgs/spark-wallet/composition.nix index 5d13a8b..dd99c0a 100644 --- a/pkgs/spark-wallet/composition.nix +++ b/pkgs/spark-wallet/composition.nix @@ -5,10 +5,6 @@ }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-10_x"}: let - globalBuildInputs = pkgs.lib.attrValues (import ./supplement.nix { - inherit nodeEnv; - inherit (pkgs) fetchurl fetchgit; - }); nodeEnv = import ./node-env.nix { inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile; inherit nodejs; @@ -17,5 +13,5 @@ let in import ./node-packages.nix { inherit (pkgs) fetchurl fetchgit; - inherit nodeEnv globalBuildInputs; + inherit nodeEnv; } \ No newline at end of file diff --git a/pkgs/spark-wallet/default.nix b/pkgs/spark-wallet/default.nix index f4502b9..ea6f456 100644 --- a/pkgs/spark-wallet/default.nix +++ b/pkgs/spark-wallet/default.nix @@ -1,5 +1,5 @@ { stdenv, pkgs, lib }: -lib.head (lib.attrValues (import ./composition.nix { +lib.head (builtins.attrValues (import ./composition.nix { inherit pkgs; inherit (stdenv.hostPlatform) system; })) diff --git a/pkgs/spark-wallet/generate.sh b/pkgs/spark-wallet/generate.sh index 124304c..84434d9 100755 --- a/pkgs/spark-wallet/generate.sh +++ b/pkgs/spark-wallet/generate.sh @@ -20,7 +20,7 @@ jq '.dependencies["qrcode-terminal"] = .optionalDependencies["qrcode-terminal"]' # Run node2nix cp pkg.json $TMPDIR/pkg.json -node2nix --nodejs-10 -i $TMPDIR/pkg.json -c composition.nix --no-copy-node-env --supplement-input supplement.json +node2nix --nodejs-10 -i $TMPDIR/pkg.json -c composition.nix --no-copy-node-env # Use verified source in node-packages.nix url="https://github.com/shesek/spark-wallet/releases/download/v$version/spark-wallet-$version-npm.tgz" diff --git a/pkgs/spark-wallet/supplement.json b/pkgs/spark-wallet/supplement.json deleted file mode 100644 index 0d4f101..0000000 --- a/pkgs/spark-wallet/supplement.json +++ /dev/null @@ -1,2 +0,0 @@ -[ -] From d214605b3239f81a7585fc4979f134530d0ad050 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 16 Mar 2021 12:45:21 +0100 Subject: [PATCH 05/13] spark-wallet: add flakes compatibility Pure flakes can't use NIX_PATH. --- pkgs/spark-wallet/composition.nix | 2 +- pkgs/spark-wallet/generate.sh | 7 +++++++ pkgs/spark-wallet/node-env.nix | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 pkgs/spark-wallet/node-env.nix diff --git a/pkgs/spark-wallet/composition.nix b/pkgs/spark-wallet/composition.nix index dd99c0a..3bc2c21 100644 --- a/pkgs/spark-wallet/composition.nix +++ b/pkgs/spark-wallet/composition.nix @@ -5,7 +5,7 @@ }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-10_x"}: let - nodeEnv = import ./node-env.nix { + nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" { inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile; inherit nodejs; libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; diff --git a/pkgs/spark-wallet/generate.sh b/pkgs/spark-wallet/generate.sh index 84434d9..b1bb841 100755 --- a/pkgs/spark-wallet/generate.sh +++ b/pkgs/spark-wallet/generate.sh @@ -22,6 +22,13 @@ jq '.dependencies["qrcode-terminal"] = .optionalDependencies["qrcode-terminal"]' cp pkg.json $TMPDIR/pkg.json node2nix --nodejs-10 -i $TMPDIR/pkg.json -c composition.nix --no-copy-node-env +# Set node env import. +# The reason for not providing a custom node-env.nix file is the following: +# To be flakes-compatible, we have to locate the nixpgs source via `pkgs.path` instead of ``. +# This requires the `pkgs` variable which is available only in composition.nix, not in node-env.nix. +nodeEnvImport='import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix"' +sed -i "s|import ./node-env.nix|$nodeEnvImport|" composition.nix + # Use verified source in node-packages.nix url="https://github.com/shesek/spark-wallet/releases/download/v$version/spark-wallet-$version-npm.tgz" sed -i '/packageName = "spark-wallet";/!b;n;n;c\ src = fetchurl {\n url = "'$url'";\n sha256 = "'$shasum'";\n };' node-packages.nix diff --git a/pkgs/spark-wallet/node-env.nix b/pkgs/spark-wallet/node-env.nix deleted file mode 100644 index f17b2d6..0000000 --- a/pkgs/spark-wallet/node-env.nix +++ /dev/null @@ -1 +0,0 @@ -import From 09cd3ce5e41648e526b442bf6fb84e0b46b9cf19 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 16 Mar 2021 12:45:22 +0100 Subject: [PATCH 06/13] lnd: show curl error messages --- modules/lnd.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/lnd.nix b/modules/lnd.nix index 00493a5..18bd166 100644 --- a/modules/lnd.nix +++ b/modules/lnd.nix @@ -198,6 +198,7 @@ in { RestartSec = "10s"; ReadWritePaths = cfg.dataDir; ExecStartPost = let + curl = "${pkgs.curl}/bin/curl -s --show-error"; restUrl = "https://${cfg.restAddress}:${toString cfg.restPort}/v1"; in [ (nbLib.script "lnd-create-wallet" '' @@ -212,13 +213,13 @@ in { if [[ ! -f "$mnemonic" ]]; then echo Create lnd seed umask u=r,go= - ${pkgs.curl}/bin/curl -s \ + ${curl} \ --cacert ${secretsDir}/lnd-cert \ -X GET ${restUrl}/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > "$mnemonic" fi echo Create lnd wallet - ${pkgs.curl}/bin/curl -s --output /dev/null --show-error \ + ${curl} --output /dev/null \ --cacert ${secretsDir}/lnd-cert \ -X POST -d "{\"wallet_password\": \"$(cat ${secretsDir}/lnd-wallet-password | tr -d '\n' | base64 -w0)\", \ \"cipher_seed_mnemonic\": $(cat "$mnemonic" | tr -d '\n')}" \ @@ -231,8 +232,7 @@ in { done else echo Unlock lnd wallet - - ${pkgs.curl}/bin/curl -s \ + ${curl} \ -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${networkDir}/admin.macaroon')" \ --cacert ${secretsDir}/lnd-cert \ -X POST \ @@ -251,7 +251,7 @@ in { ${lib.concatMapStrings (macaroon: '' echo "Create custom macaroon ${macaroon}" macaroonPath="$RUNTIME_DIRECTORY/${macaroon}.macaroon" - ${pkgs.curl}/bin/curl -s \ + ${curl} \ -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${networkDir}/admin.macaroon')" \ --cacert ${secretsDir}/lnd-cert \ -X POST \ From cdf27d9d0cf624832a904e312b533164969bd73d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 16 Mar 2021 12:45:23 +0100 Subject: [PATCH 07/13] bitcoind: improve service timeouts - Improve readability by using minutes - set `TimeoutStopSec` like in bitcoin/contrib/init/bitcoind.service. Stopping bitcoind can exceed the default timeout during IBD. --- modules/bitcoind.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix index ad804c3..ed43c31 100644 --- a/modules/bitcoind.nix +++ b/modules/bitcoind.nix @@ -351,7 +351,8 @@ in { NotifyAccess = "all"; User = cfg.user; Group = cfg.group; - TimeoutStartSec = 300; + TimeoutStartSec = "5min"; + TimeoutStopSec = "10min"; ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'"; Restart = "on-failure"; UMask = mkIf cfg.dataDirReadableByGroup "0027"; From 020433cec6c333830439a533c2ed09ea7f01df93 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Mar 2021 13:19:45 +0100 Subject: [PATCH 08/13] services: add helper fn setAllowedIPAddresses Also use 'allowLocalIPAddresses' instead of 'allowTor' in bitcoind-import-banlist which doesn't use Tor. --- modules/bitcoind.nix | 6 ++---- modules/btcpayserver.nix | 10 ++-------- modules/clightning.nix | 5 +---- modules/electrs.nix | 5 +---- modules/lightning-loop.nix | 4 +--- modules/lightning-pool.nix | 4 +--- modules/liquid.nix | 5 +---- modules/lnd.nix | 6 ++---- modules/recurring-donations.nix | 4 +--- modules/spark-wallet.nix | 4 +--- pkgs/lib.nix | 14 +++++++++----- 11 files changed, 22 insertions(+), 45 deletions(-) diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix index ed43c31..6633f40 100644 --- a/modules/bitcoind.nix +++ b/modules/bitcoind.nix @@ -357,9 +357,7 @@ in { Restart = "on-failure"; UMask = mkIf cfg.dataDirReadableByGroup "0027"; ReadWritePaths = cfg.dataDir; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP) + } // nbLib.allowedIPAddresses cfg.enforceTor // optionalAttrs (cfg.zmqpubrawblock != null || cfg.zmqpubrawtx != null) nbLib.allowAnyProtocol; }; @@ -385,7 +383,7 @@ in { User = cfg.user; Group = cfg.group; ReadWritePaths = cfg.dataDir; - } // nbLib.allowTor; + } // nbLib.allowLocalIPAddresses; }; users.users.${cfg.user}.group = cfg.group; diff --git a/modules/btcpayserver.nix b/modules/btcpayserver.nix index d529883..7504144 100644 --- a/modules/btcpayserver.nix +++ b/modules/btcpayserver.nix @@ -155,10 +155,7 @@ in { RestartSec = "10s"; ReadWritePaths = cfg.nbxplorer.dataDir; MemoryDenyWriteExecute = "false"; - } // (if cfg.nbxplorer.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP - ); + } // nbLib.allowedIPAddresses cfg.nbxplorer.enforceTor; }; systemd.services.btcpayserver = let @@ -204,10 +201,7 @@ in { RestartSec = "10s"; ReadWritePaths = cfg.btcpayserver.dataDir; MemoryDenyWriteExecute = "false"; - } // (if cfg.btcpayserver.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP - ); + } // nbLib.allowedIPAddresses cfg.btcpayserver.enforceTor; }; in self; users.users.${cfg.nbxplorer.user} = { diff --git a/modules/clightning.nix b/modules/clightning.nix index 6015b5a..1fbff34 100644 --- a/modules/clightning.nix +++ b/modules/clightning.nix @@ -128,10 +128,7 @@ in { Restart = "on-failure"; RestartSec = "10s"; ReadWritePaths = cfg.dataDir; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP - ); + } // nbLib.allowedIPAddresses cfg.enforceTor; # Wait until the rpc socket appears postStart = '' while [[ ! -e ${cfg.networkDir}/lightning-rpc ]]; do diff --git a/modules/electrs.nix b/modules/electrs.nix index 6e51b59..53a20ad 100644 --- a/modules/electrs.nix +++ b/modules/electrs.nix @@ -102,10 +102,7 @@ in { Restart = "on-failure"; RestartSec = "10s"; ReadWritePaths = "${cfg.dataDir} ${if cfg.high-memory then "${bitcoind.dataDir}" else ""}"; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP - ); + } // nbLib.allowedIPAddresses cfg.enforceTor; }; users.users.${cfg.user} = { diff --git a/modules/lightning-loop.nix b/modules/lightning-loop.nix index 4ba3022..9d40b9c 100644 --- a/modules/lightning-loop.nix +++ b/modules/lightning-loop.nix @@ -102,9 +102,7 @@ in { Restart = "on-failure"; RestartSec = "10s"; ReadWritePaths = cfg.dataDir; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP); + } // nbLib.allowedIPAddresses cfg.enforceTor; }; nix-bitcoin.secrets = { diff --git a/modules/lightning-pool.nix b/modules/lightning-pool.nix index b9d10ca..89dcfec 100644 --- a/modules/lightning-pool.nix +++ b/modules/lightning-pool.nix @@ -100,9 +100,7 @@ in { Restart = "on-failure"; RestartSec = "10s"; ReadWritePaths = cfg.dataDir; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP); + } // (nbLib.allowedIPAddresses cfg.enforceTor); }; }; } diff --git a/modules/liquid.nix b/modules/liquid.nix index 2c6e2f9..c05278a 100644 --- a/modules/liquid.nix +++ b/modules/liquid.nix @@ -239,10 +239,7 @@ in { PIDFile = pidFile; Restart = "on-failure"; ReadWritePaths = cfg.dataDir; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP - ); + } // nbLib.allowedIPAddresses cfg.enforceTor; }; users.users.${cfg.user} = { diff --git a/modules/lnd.nix b/modules/lnd.nix index 18bd166..be59675 100644 --- a/modules/lnd.nix +++ b/modules/lnd.nix @@ -262,10 +262,8 @@ in { '') (attrNames cfg.macaroons)} '') ]; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP - ) // nbLib.allowAnyProtocol; # For ZMQ + } // nbLib.allowedIPAddresses cfg.enforceTor + // nbLib.allowAnyProtocol; # For ZMQ }; users.users.${cfg.user} = { diff --git a/modules/recurring-donations.nix b/modules/recurring-donations.nix index 9386cc2..68d48ab 100644 --- a/modules/recurring-donations.nix +++ b/modules/recurring-donations.nix @@ -83,9 +83,7 @@ in { ExecStart = "${pkgs.bash}/bin/bash ${recurring-donations-script}"; User = "recurring-donations"; Type = "oneshot"; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP); + } // nbLib.allowedIPAddresses cfg.enforceTor; }; systemd.timers.recurring-donations = { requires = [ "clightning.service" ]; diff --git a/modules/spark-wallet.nix b/modules/spark-wallet.nix index 8ffdc21..f9947ec 100644 --- a/modules/spark-wallet.nix +++ b/modules/spark-wallet.nix @@ -79,9 +79,7 @@ in { User = cfg.user; Restart = "on-failure"; RestartSec = "10s"; - } // (if cfg.enforceTor - then nbLib.allowTor - else nbLib.allowAnyIP) + } // nbLib.allowedIPAddresses cfg.enforceTor // nbLib.nodejs; }; nix-bitcoin.secrets.spark-wallet-login.user = cfg.user; diff --git a/pkgs/lib.nix b/pkgs/lib.nix index b6919cf..bf29b68 100644 --- a/pkgs/lib.nix +++ b/pkgs/lib.nix @@ -35,13 +35,17 @@ let self = { # nodejs applications apparently rely on memory write execute nodejs = { MemoryDenyWriteExecute = "false"; }; - # Allow tor traffic. Allow takes precedence over Deny. - allowTor = { + + # Allow takes precedence over Deny. + allowLocalIPAddresses = { IPAddressAllow = "127.0.0.1/32 ::1/128 169.254.0.0/16"; }; - # Allow any traffic - allowAnyIP = { IPAddressAllow = "any"; }; - allowAnyProtocol = { RestrictAddressFamilies = "~"; }; + allowAllIPAddresses = { IPAddressAllow = "any"; }; + allowTor = self.allowLocalIPAddresses; + allowedIPAddresses = onlyLocal: + if onlyLocal + then self.allowLocalIPAddresses + else self.allowAllIPAddresses; enforceTor = mkOption { type = types.bool; From 08fe9ba84ae6b1394679194aa0a98996823cf32a Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Mar 2021 13:19:46 +0100 Subject: [PATCH 09/13] services: add finer-grained address family restrictions Due to a possible NixOS bug, this commit has no effect on NixOS 20.09 where `RestrictAddressFamilies` is a no-op. It's only relevant for NixOS unstable with cgroups v2. bitcoind+zmq: instead of allowing all address families, only add the required AF_NETLINK family. lnd: lnd only runs a zmq client, not a server, therefore it requires no additional address families. lightning-pool, clightning-plugin-zmq: add AF_NETLINK. --- modules/bitcoind.nix | 4 +++- modules/clightning-plugins/zmq.nix | 6 ++++++ modules/lightning-pool.nix | 3 ++- modules/lnd.nix | 3 +-- pkgs/lib.nix | 4 ++++ 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix index 6633f40..9a2a79b 100644 --- a/modules/bitcoind.nix +++ b/modules/bitcoind.nix @@ -55,6 +55,8 @@ let # Extra options ${cfg.extraConfig} ''; + + zmqServerEnabled = (cfg.zmqpubrawblock != null) || (cfg.zmqpubrawtx != null); in { options = { services.bitcoind = { @@ -358,7 +360,7 @@ in { UMask = mkIf cfg.dataDirReadableByGroup "0027"; ReadWritePaths = cfg.dataDir; } // nbLib.allowedIPAddresses cfg.enforceTor - // optionalAttrs (cfg.zmqpubrawblock != null || cfg.zmqpubrawtx != null) nbLib.allowAnyProtocol; + // optionalAttrs zmqServerEnabled nbLib.allowNetlink; }; # Use this to update the banlist: diff --git a/modules/clightning-plugins/zmq.nix b/modules/clightning-plugins/zmq.nix index 19eacf4..5ba2ece 100644 --- a/modules/clightning-plugins/zmq.nix +++ b/modules/clightning-plugins/zmq.nix @@ -4,6 +4,8 @@ with lib; let cfg = config.services.clightning.plugins.zmq; + nbLib = config.nix-bitcoin.lib; + endpoints = [ "channel-opened" "connect" @@ -38,5 +40,9 @@ in plugin=${config.nix-bitcoin.pkgs.clightning-plugins.zmq.path} ${concatStrings (map setEndpoint endpoints)} ''; + + # The zmq server requires AF_NETLINK + systemd.services.clightning.serviceConfig.RestrictAddressFamilies = + mkForce nbLib.allowNetlink.RestrictAddressFamilies; }; } diff --git a/modules/lightning-pool.nix b/modules/lightning-pool.nix index 89dcfec..7736d0a 100644 --- a/modules/lightning-pool.nix +++ b/modules/lightning-pool.nix @@ -100,7 +100,8 @@ in { Restart = "on-failure"; RestartSec = "10s"; ReadWritePaths = cfg.dataDir; - } // (nbLib.allowedIPAddresses cfg.enforceTor); + } // (nbLib.allowedIPAddresses cfg.enforceTor) + // nbLib.allowNetlink; # required by gRPC-Go }; }; } diff --git a/modules/lnd.nix b/modules/lnd.nix index be59675..8093bc9 100644 --- a/modules/lnd.nix +++ b/modules/lnd.nix @@ -262,8 +262,7 @@ in { '') (attrNames cfg.macaroons)} '') ]; - } // nbLib.allowedIPAddresses cfg.enforceTor - // nbLib.allowAnyProtocol; # For ZMQ + } // nbLib.allowedIPAddresses cfg.enforceTor; }; users.users.${cfg.user} = { diff --git a/pkgs/lib.nix b/pkgs/lib.nix index bf29b68..e42b641 100644 --- a/pkgs/lib.nix +++ b/pkgs/lib.nix @@ -33,6 +33,10 @@ let self = { SystemCallArchitectures = "native"; }; + allowNetlink = { + RestrictAddressFamilies = self.defaultHardening.RestrictAddressFamilies + " AF_NETLINK"; + }; + # nodejs applications apparently rely on memory write execute nodejs = { MemoryDenyWriteExecute = "false"; }; From 9ca52af523eba8e58f38bf5ae02b515fe17fae82 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Mar 2021 13:19:47 +0100 Subject: [PATCH 10/13] tests: improve make-test-vm.nix 1. fixedTest: Remove some unneeded layers of function calls. 2. Don't add a modified version of `black` to the global pkgs set. Tests should not affect the pkgs of the tested system modules. Fix the driver build script instead by adding an extra arg to the call to `black`. --- test/lib/make-test-vm.nix | 70 ++++++++++++++++++++++----------------- test/lib/make-test.nix | 13 +++++--- test/run-tests.sh | 2 +- 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/test/lib/make-test-vm.nix b/test/lib/make-test-vm.nix index 8f9928b..7505199 100644 --- a/test/lib/make-test-vm.nix +++ b/test/lib/make-test-vm.nix @@ -1,37 +1,45 @@ -testArgs: - +pkgs: let - pkgs = import { config = {}; overlays = []; }; + pythonTesting = import "${toString pkgs.path}/nixos/lib/testing-python.nix" { + system = builtins.currentSystem; + inherit pkgs; + }; +in - test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs); +args: +let + test = pythonTesting.makeTest args; - fixedTest = { ... }@args: - let - pkgsFixed = pkgs // { - # Fix the black Python code formatter that's used in the test to allow the test - # script to have longer lines. The default width of 88 chars is too restrictive for - # our script. - python3Packages = pkgs.python3Packages // { - black = pkgs.writeScriptBin "black" '' - fileToCheck=''${@:$#} - [[ $fileToCheck = *test-script ]] && extraArgs='--line-length 100' - exec ${pkgs.python3Packages.black}/bin/black $extraArgs "$@" - ''; - }; - }; - test' = test (args // { pkgs = pkgsFixed; }); - in - # See nixpkgs/nixos/lib/testing-python.nix for the original definition - test'.overrideAttrs (_: { - # 1. Save test output - # 2. Add link to driver so that a gcroot to a test prevents the driver from - # being garbage-collected - buildCommand = '' - mkdir $out - LOGFILE=$out/output.xml tests='exec(os.environ["testScript"])' ${test'.driver}/bin/nixos-test-driver - ln -s ${test'.driver} $out/driver - ''; - }) // { inherit (test') nodes driver; } ; + fixedDriver = test.driver.overrideAttrs (old: let + # Allow the test script to have longer lines by fixing the call to the 'black' + # code formatter. + # The default width of 88 chars is too restrictive for our script. + parts = builtins.split ''/nix/store/[^ ]+/black '' old.buildCommand; + preMatch = builtins.elemAt parts 0; + postMatch = builtins.elemAt parts 2; + in { + # See `mkDriver` in nixpkgs/nixos/lib/testing-python.nix for the original definition of `buildCommand` + buildCommand = '' + ${preMatch}${pkgs.python3Packages.black}/bin/black --line-length 100 ${postMatch} + ''; + # Keep reference to the `testDriver` derivation, required by `buildCommand` + testDriverReference = old.buildCommand; + }); + # 1. Use fixed driver + # 2. Save test logging output + # 3. Add link to driver so that a gcroot to a test prevents the driver from + # being garbage-collected + fixedTest = test.overrideAttrs (_: { + # See `runTests` in nixpkgs/nixos/lib/testing-python.nix for the original definition of `buildCommand` + buildCommand = '' + mkdir $out + LOGFILE=$out/output.xml tests='exec(os.environ["testScript"])' ${fixedDriver}/bin/nixos-test-driver + ln -s ${fixedDriver} $out/driver + ''; + }) // { + driver = fixedDriver; + inherit (test) nodes; + }; in fixedTest diff --git a/test/lib/make-test.nix b/test/lib/make-test.nix index c3620fe..23d71d5 100644 --- a/test/lib/make-test.nix +++ b/test/lib/make-test.nix @@ -1,8 +1,11 @@ -scenario: testConfig: - +let + pkgs = import { config = {}; overlays = []; }; + makeVM = import ./make-test-vm.nix pkgs; +in +name: testConfig: { - vm = import ./make-test-vm.nix (pkgs: { - name = "nix-bitcoin-${scenario}"; + vm = makeVM { + name = "nix-bitcoin-${name}"; machine = { imports = [ testConfig ]; @@ -37,7 +40,7 @@ scenario: testConfig: run_tests() '' ]; - }); + }; container = { # The container name has a 11 char length limit diff --git a/test/run-tests.sh b/test/run-tests.sh index 7bfceb7..25a13df 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -212,7 +212,7 @@ vmTestNixExpr() { fi cat < Date: Mon, 22 Mar 2021 13:19:48 +0100 Subject: [PATCH 11/13] tests: optimize building multiple tests at once The result of `import tests.nix {}` is now an attrset of tests. This makes it easier and more efficient to evaluate or build multiple tests in one call to `nix build`. Simplify tests.nix by removing the large module args scope in favor of self-contained scenario module definitions. Add CPU core and memory size defaults to the test configuration to enable building tests without `run-tests.sh`. Add the following top-level args to tests.nix: - `extraScenarios` to provide a nix-level way to define extra scenarios. - `pkgs` to allow building tests with custom pkgs or systems. --- test/lib/make-container.sh | 2 +- test/lib/make-test-vm.nix | 2 +- test/lib/make-test.nix | 15 +++++++-- test/run-tests.sh | 6 ++-- test/tests.nix | 66 ++++++++++++++++++++++++-------------- 5 files changed, 59 insertions(+), 32 deletions(-) diff --git a/test/lib/make-container.sh b/test/lib/make-container.sh index e039dc1..af19185 100755 --- a/test/lib/make-container.sh +++ b/test/lib/make-container.sh @@ -84,6 +84,6 @@ if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.6*) ]] fi read -d '' src < { config = {}; overlays = []; }; makeVM = import ./make-test-vm.nix pkgs; + inherit (pkgs) lib; in + name: testConfig: { vm = makeVM { @@ -9,8 +11,15 @@ name: testConfig: machine = { imports = [ testConfig ]; - # Needed because duplicity requires 270 MB of free temp space, regardless of backup size - virtualisation.diskSize = 1024; + virtualisation = { + # Needed because duplicity requires 270 MB of free temp space, regardless of backup size + diskSize = 1024; + + # Min. 800 MiB needed to avoid 'out of memory' errors + memorySize = lib.mkDefault 2048; + + cores = lib.mkDefault 2; + }; }; testScript = nodes: let diff --git a/test/run-tests.sh b/test/run-tests.sh index 25a13df..f07c835 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -110,7 +110,7 @@ trap 'eval "$runAtExit"' EXIT if [[ $scenario = *' '* ]]; then export scenarioOverridesFile=$(mktemp ${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX) runAtExit+='rm -f "$scenarioOverridesFile";' - echo "{ testEnv, config, pkgs, lib }: with testEnv; with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" + echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" scenario=tmp fi @@ -120,7 +120,7 @@ run() { export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) runAtExit+="rm -rf $TMPDIR;" - nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver + nix-build --out-link $TMPDIR/driver -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver # Variable 'tests' contains the Python code that is executed by the driver on startup if [[ $1 == --interactive ]]; then @@ -212,7 +212,7 @@ vmTestNixExpr() { fi cat < Date: Mon, 22 Mar 2021 14:38:31 +0100 Subject: [PATCH 12/13] test.py: improve composability of test 'banlist-and-restart' The test now works if not all services previously used in the `systemctl restart` command are available. --- test/tests.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/test/tests.py b/test/tests.py index 48cc4ca..c009f1d 100644 --- a/test/tests.py +++ b/test/tests.py @@ -352,10 +352,9 @@ def _(): # Current time in µs pre_restart = succeed("date +%s.%6N").rstrip() - # Sanity-check system by restarting all services - succeed( - "systemctl restart bitcoind clightning lnd lightning-loop lightning-pool spark-wallet liquidd" - ) + # Sanity-check system by restarting bitcoind. + # This also restarts all services depending on bitcoind. + succeed("systemctl restart bitcoind") # Now that the bitcoind restart triggered a banlist import restart, check that # re-importing already banned addresses works @@ -367,27 +366,37 @@ def _(): @test("regtest") def _(): - if "electrs" in enabled_tests: + def enabled(unit): + if unit in enabled_tests: + # Wait because the unit might have been restarted in the preceding + # 'banlist-and-restart' test + machine.wait_for_unit(unit) + return True + else: + return False + + if enabled("electrs"): + machine.wait_for_unit("onion-addresses") machine.wait_until_succeeds(log_has_string("electrs", "BlockchainInfo")) get_block_height_cmd = ( """echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'""" f" | nc -N {ip('electrs')} 50001 | jq -M .result.height" ) assert_full_match(get_block_height_cmd, "10\n") - if "clightning" in enabled_tests: + if enabled("clightning"): machine.wait_until_succeeds( "[[ $(runuser -u operator -- lightning-cli getinfo | jq -M .blockheight) == 10 ]]" ) - if "lnd" in enabled_tests: + if enabled("lnd"): machine.wait_until_succeeds( "[[ $(runuser -u operator -- lncli getinfo | jq -M .block_height) == 10 ]]" ) - if "lightning-loop" in enabled_tests: + if enabled("lightning-loop"): machine.wait_until_succeeds( log_has_string("lightning-loop", "Starting event loop at height 10") ) succeed("runuser -u operator -- loop getparams") - if "lightning-pool" in enabled_tests: + if enabled("lightning-pool"): machine.wait_until_succeeds( log_has_string("lightning-pool", "lnd is now fully synced to its chain backend") ) From 4cddf284e96adf8e2e70e228df55ff1efd9d9649 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Mar 2021 14:38:32 +0100 Subject: [PATCH 13/13] treewide: remove use of deprecated `stdenv.lib` --- pkgs/clboss/default.nix | 4 ++-- pkgs/elementsd/default.nix | 4 ++-- pkgs/hwi/default.nix | 4 +--- pkgs/hwi/ecdsa/default.nix | 5 +++-- pkgs/hwi/hidapi/default.nix | 10 +++++----- pkgs/secp256k1/default.nix | 6 ++---- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pkgs/clboss/default.nix b/pkgs/clboss/default.nix index f7f5725..cb3eb97 100644 --- a/pkgs/clboss/default.nix +++ b/pkgs/clboss/default.nix @@ -1,4 +1,4 @@ -{ stdenv, fetchurl, pkgconfig, curl, libev, sqlite }: +{ lib, stdenv, fetchurl, pkgconfig, curl, libev, sqlite }: let curlWithGnuTLS = curl.override { gnutlsSupport = true; sslSupport = false; }; @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; - meta = with stdenv.lib; { + meta = with lib; { description = "Automated C-Lightning Node Manager"; homepage = "https://github.com/ZmnSCPxj/clboss"; maintainers = with maintainers; [ nixbitcoin ]; diff --git a/pkgs/elementsd/default.nix b/pkgs/elementsd/default.nix index a46a1a3..1699b1e 100644 --- a/pkgs/elementsd/default.nix +++ b/pkgs/elementsd/default.nix @@ -1,8 +1,8 @@ -{ stdenv, fetchurl, pkgconfig, autoreconfHook, openssl, db48, boost, zeromq, rapidcheck +{ lib, stdenv, fetchurl, pkgconfig, autoreconfHook, openssl, db48, boost, zeromq, rapidcheck , zlib, miniupnpc, qtbase ? null, qttools ? null, wrapQtAppsHook ? null, utillinux, protobuf, python3, qrencode, libevent , withGui }: -with stdenv.lib; +with lib; stdenv.mkDerivation rec { pname = "elements${optionalString (!withGui) "d"}"; version = "0.18.1.9"; diff --git a/pkgs/hwi/default.nix b/pkgs/hwi/default.nix index 86d074c..2ab6ce0 100644 --- a/pkgs/hwi/default.nix +++ b/pkgs/hwi/default.nix @@ -1,6 +1,4 @@ -{ pkgs, stdenv, fetchurl, fetchFromGitHub, python3 }: - -with stdenv.lib; +{ lib, pkgs, stdenv, fetchurl, fetchFromGitHub, python3 }: let python = python3.override { diff --git a/pkgs/hwi/ecdsa/default.nix b/pkgs/hwi/ecdsa/default.nix index d9c43f5..9ab0397 100644 --- a/pkgs/hwi/ecdsa/default.nix +++ b/pkgs/hwi/ecdsa/default.nix @@ -1,4 +1,5 @@ -{ stdenv +{ lib +, stdenv , buildPythonPackage , fetchPypi , pkgs @@ -18,7 +19,7 @@ buildPythonPackage rec { # Only needed for tests checkInputs = [ pkgs.openssl ]; - meta = with stdenv.lib; { + meta = with lib; { description = "ECDSA cryptographic signature library"; homepage = "https://github.com/warner/python-ecdsa"; license = licenses.mit; diff --git a/pkgs/hwi/hidapi/default.nix b/pkgs/hwi/hidapi/default.nix index cd70a7b..0d2692e 100644 --- a/pkgs/hwi/hidapi/default.nix +++ b/pkgs/hwi/hidapi/default.nix @@ -1,4 +1,4 @@ -{ stdenv, libusb1, udev, darwin, fetchPypi, buildPythonPackage, cython }: +{ lib, stdenv, libusb1, udev, darwin, fetchPypi, buildPythonPackage, cython }: buildPythonPackage rec { pname = "hidapi"; @@ -10,18 +10,18 @@ buildPythonPackage rec { }; propagatedBuildInputs = - stdenv.lib.optionals stdenv.isLinux [ libusb1 udev ] ++ - stdenv.lib.optionals stdenv.isDarwin [ darwin.IOKit darwin.apple_sdk.frameworks.CoreFoundation ] ++ + lib.optionals stdenv.isLinux [ libusb1 udev ] ++ + lib.optionals stdenv.isDarwin [ darwin.IOKit darwin.apple_sdk.frameworks.CoreFoundation ] ++ [ cython ]; # Fix the USB backend library lookup - postPatch = stdenv.lib.optionalString stdenv.isLinux '' + postPatch = lib.optionalString stdenv.isLinux '' libusb=${libusb1.dev}/include/libusb-1.0 test -d $libusb || { echo "ERROR: $libusb doesn't exist, please update/fix this build expression."; exit 1; } sed -i -e "s|/usr/include/libusb-1.0|$libusb|" setup.py ''; - meta = with stdenv.lib; { + meta = with lib; { description = "A Cython interface to the hidapi from https://github.com/signal11/hidapi"; homepage = "https://github.com/trezor/cython-hidapi"; # license can actually be either bsd3 or gpl3 diff --git a/pkgs/secp256k1/default.nix b/pkgs/secp256k1/default.nix index a52adb3..2f9f28f 100644 --- a/pkgs/secp256k1/default.nix +++ b/pkgs/secp256k1/default.nix @@ -1,6 +1,4 @@ -{ stdenv, fetchFromGitHub, autoreconfHook }: - -let inherit (stdenv.lib) optionals; in +{ lib, stdenv, fetchFromGitHub, autoreconfHook }: stdenv.mkDerivation { pname = "secp256k1"; @@ -18,7 +16,7 @@ stdenv.mkDerivation { configureFlags = ["--enable-module-recovery" "--disable-jni" "--enable-experimental" "--enable-module-ecdh" "--enable-benchmark=no" ]; - meta = with stdenv.lib; { + meta = with lib; { description = "Optimized C library for EC operations on curve secp256k1"; homepage = "https://github.com/bitcoin-core/secp256k1"; license = with licenses; [ mit ];