Merge #308: Update nixpkgs-unstable, bitcoind

b114d0c3b1 bitcoind: use systemd startup notification (Erik Arvstedt)
332d0e70c8 bitcoind: support onion address announcing (Erik Arvstedt)
9662c19ab1 onionServices: use actual user name of services (Erik Arvstedt)
5c09845e6f bitcoind: tag incoming connections as onion on enforceTor (Erik Arvstedt)
8f9ea61d6e update nixpkgs-unstable (Erik Arvstedt)
05e5ec99ec modules packages: build electrs, lightning-loop with nixpkgs stable (Erik Arvstedt)
44546561fc run-tests: allow defining scenarios via cmdline args (Erik Arvstedt)
fc40776689 improve backup test (Erik Arvstedt)
9a67a32779 fix build-to-cachix (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK b114d0c3b1
  nixbitcoin:
    ACK b114d0c3b1

Tree-SHA512: fbf4810fb0e1aeb46618c53cea3d6bde582eb92837afabe77e5bbf5c4add82277af4eba6bfeae6331f8992902e5dbe5081f2a016121a02dab3e797d53f232dd2
This commit is contained in:
Jonas Nick 2021-02-01 10:07:02 +00:00
commit cba1188db8
No known key found for this signature in database
GPG Key ID: 4861DBF262123605
13 changed files with 107 additions and 32 deletions

View File

@ -39,4 +39,4 @@ task:
build_script: build_script:
- echo "sandbox = true" >> /etc/nix/nix.conf - echo "sandbox = true" >> /etc/nix/nix.conf
- export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)" - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)"
- nix run -f '<nixpkgs>' bash cachix -c ./ci/build.sh - nix run -f '<nixpkgs>' bash coreutils cachix -c ./ci/build.sh

View File

@ -13,11 +13,17 @@ trap 'echo Error at line $LINENO' ERR
atExit() { atExit() {
rm -rf $tmpDir rm -rf $tmpDir
if [[ -v cachixPid ]]; then kill $cachixPid; fi if [[ -v cachixPid ]]; then stopCachix; fi
} }
tmpDir=$(mktemp -d -p /tmp) tmpDir=$(mktemp -d -p /tmp)
trap atExit EXIT trap atExit EXIT
stopCachix() {
kill $cachixPid 2>/dev/null || true
# Wait for cachix to finish
tail --pid=$cachixPid -f /dev/null
}
## Instantiate ## Instantiate
time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null
@ -44,6 +50,7 @@ fi
nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
if [[ $CACHIX_SIGNING_KEY ]]; then if [[ $CACHIX_SIGNING_KEY ]]; then
stopCachix
cachix push $cachixCache $outPath cachix push $cachixCache $outPath
fi fi

View File

@ -55,5 +55,11 @@ c systemctl status bitcoind
# Explore a single feature # Explore a single feature
./run-tests.sh --scenario electrs container ./run-tests.sh --scenario electrs container
# Run a command in a container
./run-tests.sh --scenario '{
services.clightning.enable = true;
nix-bitcoin.nodeinfo.enable = true;
}' container --run c nodeinfo
``` ```
See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation. See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.

View File

@ -28,6 +28,10 @@
# LND and electrs are not compatible with pruning. # LND and electrs are not compatible with pruning.
# services.bitcoind.prune = 100000; # services.bitcoind.prune = 100000;
# #
# Set this to accounce the onion service address to peers.
# The onion service allows accepting incoming connections via Tor.
# nix-bitcoin.onionServices.bitcoind.public = true;
#
# You can add options that are not defined in modules/bitcoind.nix as follows # You can add options that are not defined in modules/bitcoind.nix as follows
# services.bitcoind.extraConfig = '' # services.bitcoind.extraConfig = ''
# maxorphantx=110 # maxorphantx=110

View File

@ -11,6 +11,8 @@ let
# We're already logging via journald # We're already logging via journald
nodebuglogfile=1 nodebuglogfile=1
startupnotify=/run/current-system/systemd/bin/systemd-notify --ready
${optionalString cfg.regtest '' ${optionalString cfg.regtest ''
regtest=1 regtest=1
[regtest] [regtest]
@ -22,7 +24,7 @@ let
${optionalString (cfg.assumevalid != null) "assumevalid=${cfg.assumevalid}"} ${optionalString (cfg.assumevalid != null) "assumevalid=${cfg.assumevalid}"}
# Connection options # Connection options
${optionalString cfg.listen "bind=${cfg.address}"} ${optionalString cfg.listen "bind=${cfg.address}${optionalString cfg.enforceTor "=onion"}"}
port=${toString cfg.port} port=${toString cfg.port}
${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"} ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
listen=${if cfg.listen then "1" else "0"} listen=${if cfg.listen then "1" else "0"}
@ -67,6 +69,14 @@ in {
default = 8333; default = 8333;
description = "Port to listen for peer connections."; description = "Port to listen for peer connections.";
}; };
getPublicAddressCmd = mkOption {
type = types.str;
default = "";
description = ''
Bash expression which outputs the public service address to announce to peers.
If left empty, no address is announced.
'';
};
package = mkOption { package = mkOption {
type = types.package; type = types.package;
default = config.nix-bitcoin.pkgs.bitcoind; default = config.nix-bitcoin.pkgs.bitcoind;
@ -328,21 +338,22 @@ in {
${extraRpcauth} ${extraRpcauth}
${/* Enable bitcoin-cli for group 'bitcoin' */ ""} ${/* Enable bitcoin-cli for group 'bitcoin' */ ""}
printf "rpcuser=${cfg.rpc.users.privileged.name}\nrpcpassword="; cat "${secretsDir}/bitcoin-rpcpassword-privileged"; printf "rpcuser=${cfg.rpc.users.privileged.name}\nrpcpassword="; cat "${secretsDir}/bitcoin-rpcpassword-privileged";
echo
${optionalString (cfg.getPublicAddressCmd != "") ''
echo "externalip=$(${cfg.getPublicAddressCmd})"
''}
) )
confFile='${cfg.dataDir}/bitcoin.conf' confFile='${cfg.dataDir}/bitcoin.conf'
if [[ ! -e $confFile || $cfg != $(cat $confFile) ]]; then if [[ ! -e $confFile || $cfg != $(cat $confFile) ]]; then
install -o '${cfg.user}' -g '${cfg.group}' -m 640 <(echo "$cfg") $confFile install -o '${cfg.user}' -g '${cfg.group}' -m 640 <(echo "$cfg") $confFile
fi fi
''; '';
postStart = ''
# Poll until bitcoind accepts commands. This can take a long time.
while ! ${cfg.cli}/bin/bitcoin-cli getnetworkinfo &> /dev/null; do
sleep 1
done
'';
serviceConfig = nix-bitcoin-services.defaultHardening // { serviceConfig = nix-bitcoin-services.defaultHardening // {
Type = "notify";
NotifyAccess = "all";
User = "${cfg.user}"; User = "${cfg.user}";
Group = "${cfg.group}"; Group = "${cfg.group}";
TimeoutStartSec = 300;
ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'"; ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'";
Restart = "on-failure"; Restart = "on-failure";
UMask = mkIf cfg.dataDirReadableByGroup "0027"; UMask = mkIf cfg.dataDirReadableByGroup "0027";

View File

@ -27,13 +27,22 @@ in {
/var/lib/onion-addresses/myuser. /var/lib/onion-addresses/myuser.
''; '';
}; };
services = mkOption {
type = with types; listOf str;
default = [];
description = ''
Services that can access their onion address via file
/var/lib/onion-addresses/<service>
The file is readable only by the service user.
'';
};
dataDir = mkOption { dataDir = mkOption {
readOnly = true; readOnly = true;
default = "/var/lib/onion-addresses"; default = "/var/lib/onion-addresses";
}; };
}; };
config = mkIf (cfg.access != {}) { config = mkIf (cfg.access != {} || cfg.services != []) {
systemd.services.onion-addresses = { systemd.services.onion-addresses = {
wantedBy = [ "tor.service" ]; wantedBy = [ "tor.service" ];
bindsTo = [ "tor.service" ]; bindsTo = [ "tor.service" ];
@ -42,6 +51,7 @@ in {
Type = "oneshot"; Type = "oneshot";
RemainAfterExit = true; RemainAfterExit = true;
StateDirectory = "onion-addresses"; StateDirectory = "onion-addresses";
StateDirectoryMode = "771";
PrivateNetwork = "true"; # This service needs no network access PrivateNetwork = "true"; # This service needs no network access
PrivateUsers = "false"; PrivateUsers = "false";
CapabilityBoundingSet = "CAP_CHOWN CAP_FSETID CAP_SETFCAP CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_IPC_OWNER"; CapabilityBoundingSet = "CAP_CHOWN CAP_FSETID CAP_SETFCAP CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_IPC_OWNER";
@ -70,6 +80,13 @@ in {
'') '')
(builtins.attrNames cfg.access) (builtins.attrNames cfg.access)
} }
${concatMapStrings (service: ''
onionFile=/var/lib/tor/onion/${service}/hostname
if [[ -e $onionFile ]]; then
install -o ${config.systemd.services.${service}.serviceConfig.User} -m 400 $onionFile ${service}
fi
'') cfg.services}
''; '';
}; };
}; };

View File

@ -71,12 +71,12 @@ in {
); );
}; };
# Enable public services to access their own onion addresses nix-bitcoin.onionAddresses = {
nix-bitcoin.onionAddresses.access = ( # Enable public services to access their own onion addresses
genAttrs publicServices singleton services = publicServices;
) // {
# Allow the operator user to access onion addresses for all active services # Allow the operator user to access onion addresses for all active services
${config.nix-bitcoin.operator.name} = mkIf config.nix-bitcoin.operator.enable activeServices; access.${config.nix-bitcoin.operator.name} = mkIf config.nix-bitcoin.operator.enable activeServices;
}; };
systemd.services = let systemd.services = let
onionAddresses = [ "onion-addresses.service" ]; onionAddresses = [ "onion-addresses.service" ];
@ -96,7 +96,7 @@ in {
in srv.public && srv.enable in srv.public && srv.enable
) services; ) services;
in genAttrs publicServices' (service: { in genAttrs publicServices' (service: {
getPublicAddressCmd = "cat ${config.nix-bitcoin.onionAddresses.dataDir}/${service}/${service}"; getPublicAddressCmd = "cat ${config.nix-bitcoin.onionAddresses.dataDir}/${service}";
}); });
} }

View File

@ -12,7 +12,7 @@ in
sha256 = "1vjh0np1rlirbhhj9b2d0zhrqdmiji5svxh9baqq7r3680af1iif"; sha256 = "1vjh0np1rlirbhhj9b2d0zhrqdmiji5svxh9baqq7r3680af1iif";
}; };
nixpkgs-unstable = fetch { nixpkgs-unstable = fetch {
rev = "296793637b22bdb4d23b479879eba0a71c132a66"; rev = "891f607d5301d6730cb1f9dcf3618bcb1ab7f10e";
sha256 = "0j09yih9693w5vjx64ikfxyja1ha7pisygrwrpg3wfz3sssglg69"; sha256 = "1cr39f0sbr0h5d83dv1q34mcpwnkwwbdk5fqlyqp2mnxghzwssng";
}; };
} }

View File

@ -13,9 +13,6 @@ in
lnd lnd
nbxplorer nbxplorer
btcpayserver; btcpayserver;
inherit (nixBitcoinPkgsUnstable)
electrs
lightning-loop;
stable = nixBitcoinPkgsStable; stable = nixBitcoinPkgsStable;
unstable = nixBitcoinPkgsUnstable; unstable = nixBitcoinPkgsUnstable;

View File

@ -4,10 +4,15 @@ let
nbPkgs = import ../pkgs { inherit pkgs; }; nbPkgs = import ../pkgs { inherit pkgs; };
pkgsUnstable = with nbPkgs; [ pkgsUnstable = with nbPkgs; [
electrs electrs
elementsd
hwi hwi
joinmarket joinmarket
lightning-loop lightning-loop
## elementsd fails with error
# test/key_properties.cpp:16:10: fatal error: rapidcheck/boost_test.h: No such file or directory
# 16 | #include <rapidcheck/boost_test.h>
# | ^~~~~~~~~~~~~~~~~~~~~~~~~
# elementsd
]; ];
in in
pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable) pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable)

View File

@ -10,7 +10,13 @@
# Test specific scenario # Test specific scenario
# ./run-tests.sh --scenario|-s <scenario> # ./run-tests.sh --scenario|-s <scenario>
# #
# When <scenario> is undefined, the test is run with an adhoc scenario # - When <scenario> contains a space, <scenario> is treated as nix code defining
# a scenario. It is evaluated in the same context as other scenarios in ./tests.nix
#
# Example:
# ./run-tests.sh -s "{ nix-bitcoin.nodeinfo.enable = true; }" container --run c nodeinfo
#
# - When <scenario> does not name a scenario, the test is run with an adhoc scenario
# where services.<scenario> is enabled. # where services.<scenario> is enabled.
# #
# Example: # Example:
@ -80,13 +86,24 @@ numCPUs=${numCPUs:-$(nproc)}
# Min. 800 MiB needed to avoid 'out of memory' errors # Min. 800 MiB needed to avoid 'out of memory' errors
memoryMiB=${memoryMiB:-2048} memoryMiB=${memoryMiB:-2048}
export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs) export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..")
runAtExit=
trap 'eval "$runAtExit"' EXIT
# Support explicit scenario definitions
if [[ $scenario = *' '* ]]; then
export scenarioOverridesFile=$(mktemp ${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX)
runAtExit+='rm -f "$scenarioOverridesFile";'
echo "{ testEnv, config, pkgs, lib }: with testEnv; with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
scenario=tmp
fi
# Run the test. No temporary files are left on the host system. # Run the test. No temporary files are left on the host system.
run() { run() {
# TMPDIR is also used by the test driver for VM tmp files # TMPDIR is also used by the test driver for VM tmp files
export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
trap "rm -rf $TMPDIR" EXIT runAtExit+="rm -rf $TMPDIR;"
nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
@ -134,7 +151,8 @@ instantiate() {
} }
container() { container() {
. "$scriptDir/lib/make-container.sh" "$@" export scriptDir scenario
"$scriptDir/lib/make-container.sh" "$@"
} }
doBuild() { doBuild() {

View File

@ -187,6 +187,7 @@ let testEnv = rec {
services.bitcoind.regtest = true; services.bitcoind.regtest = true;
systemd.services.bitcoind.postStart = mkAfter '' systemd.services.bitcoind.postStart = mkAfter ''
cli=${config.services.bitcoind.cli}/bin/bitcoin-cli cli=${config.services.bitcoind.cli}/bin/bitcoin-cli
$cli createwallet "test"
address=$($cli getnewaddress) address=$($cli getnewaddress)
$cli generatetoaddress 10 $address $cli generatetoaddress 10 $address
''; '';

View File

@ -286,6 +286,9 @@ def _():
# Impure: stops bitcoind (and dependent services) # Impure: stops bitcoind (and dependent services)
@test("backups") @test("backups")
def _(): def _():
# For testing that bitcoind wallets are backed up
succeed("bitcoin-cli -named createwallet wallet_name=test blank=true >/dev/null")
succeed("systemctl stop bitcoind") succeed("systemctl stop bitcoind")
succeed("systemctl start duplicity") succeed("systemctl start duplicity")
machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded.")) machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded."))
@ -296,12 +299,18 @@ def _():
"0 differences found", "0 differences found",
) )
# Backup should include important files # Backup should include important files
files = succeed(f"{run_duplicity} list-current-files file:///var/lib/localBackups") files = {
assert "var/lib/clightning/bitcoin/hsm_secret" in files "bitcoind": "var/lib/bitcoind/test/wallet.dat",
assert "secrets/lnd-seed-mnemonic" in files "clightning": "var/lib/clightning/bitcoin/hsm_secret",
assert "secrets/jm-wallet-seed" in files "lnd": "secrets/lnd-seed-mnemonic",
assert "var/lib/bitcoind/wallet.dat" in files "joinmarket": "secrets/jm-wallet-seed",
assert "var/backup/postgresql/btcpaydb.sql.gz" in files "btcpayserver": "var/backup/postgresql/btcpaydb.sql.gz",
}
actual_files = succeed(f"{run_duplicity} list-current-files file:///var/lib/localBackups")
for test, file in files.items():
if test in enabled_tests and file not in actual_files:
raise Exception(f"Backup file '{file}' is missing.")
# Impure: restarts services # Impure: restarts services