From 852c112603fb93d552904f8664dba3c798ebaea9 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Tue, 2 Nov 2021 13:44:58 +0000 Subject: [PATCH 01/10] Use HTTPS URL for spark-wallet GitHub node packages Manually patch spark-wallet until https://github.com/svanderburg/node2nix/pull/269 is merged --- pkgs/spark-wallet/node-packages.nix | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/spark-wallet/node-packages.nix b/pkgs/spark-wallet/node-packages.nix index 0d5ed5a..072853c 100644 --- a/pkgs/spark-wallet/node-packages.nix +++ b/pkgs/spark-wallet/node-packages.nix @@ -1399,12 +1399,12 @@ let sha512 = "dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw=="; }; }; - "proxy-agent-git://github.com/shesek/node-proxy-agent#0d916427f586e41a1a4485db88509218017f9696" = { + "proxy-agent-https://github.com/shesek/node-proxy-agent#0d916427f586e41a1a4485db88509218017f9696" = { name = "proxy-agent"; packageName = "proxy-agent"; version = "3.1.1"; src = fetchgit { - url = "git://github.com/shesek/node-proxy-agent"; + url = "https://github.com/shesek/node-proxy-agent"; rev = "0d916427f586e41a1a4485db88509218017f9696"; sha256 = "9582aafb90887f3a600806240c0f71fd09ef2b8805a37281894b56e717a35d98"; }; @@ -1823,12 +1823,12 @@ let sha512 = "OUDHEssirmplo3F+1HWKUrUjvnQuA+nZI6i/JJBdXb5eq9IyEQwPyPpqND+SSsxf6TygpBEkUjISVRN4/VOpeg=="; }; }; - "superagent-proxy-git://github.com/shesek/superagent-proxy#b512f8bbcf1baa196058922ee3b6cfe45375ebc9" = { + "superagent-proxy-https://github.com/shesek/superagent-proxy#b512f8bbcf1baa196058922ee3b6cfe45375ebc9" = { name = "superagent-proxy"; packageName = "superagent-proxy"; version = "2.0.0"; src = fetchgit { - url = "git://github.com/shesek/superagent-proxy"; + url = "https://github.com/shesek/superagent-proxy"; rev = "b512f8bbcf1baa196058922ee3b6cfe45375ebc9"; sha256 = "ca984f1f2dc7cfed2aa301de1daf4305c359e89276d897fffa92e7c80850d03c"; }; @@ -2208,7 +2208,7 @@ in sources."prelude-ls-1.1.2" sources."process-nextick-args-2.0.1" sources."proxy-addr-2.0.6" - (sources."proxy-agent-git://github.com/shesek/node-proxy-agent#0d916427f586e41a1a4485db88509218017f9696" // { + (sources."proxy-agent-https://github.com/shesek/node-proxy-agent#0d916427f586e41a1a4485db88509218017f9696" // { dependencies = [ sources."debug-4.3.0" sources."ms-2.1.2" @@ -2273,7 +2273,7 @@ in sources."semver-7.3.2" ]; }) - (sources."superagent-proxy-git://github.com/shesek/superagent-proxy#b512f8bbcf1baa196058922ee3b6cfe45375ebc9" // { + (sources."superagent-proxy-https://github.com/shesek/superagent-proxy#b512f8bbcf1baa196058922ee3b6cfe45375ebc9" // { dependencies = [ sources."debug-3.2.6" sources."ms-2.1.2" @@ -2318,4 +2318,4 @@ in bypassCache = true; reconstructLock = true; }; -} \ No newline at end of file +} From b83fd845c29787504f2e3378bb7ca1198f25ac3d Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Mon, 1 Nov 2021 12:49:36 +0000 Subject: [PATCH 02/10] update nixpkgs Includes - update stable - btcpayserver: 1.2.4 -> 1.3.1 - electrs: 0.9.1 -> 0.9.2 - nbxplorer: 2.2.11 -> 2.2.16 --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 25f8436..f6fb8e2 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1631561581, - "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "lastModified": 1634851050, + "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=", "owner": "numtide", "repo": "flake-utils", - "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1633422542, - "narHash": "sha256-JYz2PmVogNRO8DhcvXzL/QhZzboTspJz2YSRlnAj8aM=", + "lastModified": 1635719588, + "narHash": "sha256-pWjdy0NheM97NsPE6+jUnr5LYyeA0sBGTdw4mfXMGZQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aff647e2704fa1223994604887bb78276dc57083", + "rev": "f0869b1a2c0b150aac26e10bb5c2364ffb2e804f", "type": "github" }, "original": { @@ -33,11 +33,11 @@ }, "nixpkgsUnstable": { "locked": { - "lastModified": 1635295995, - "narHash": "sha256-sGYiXjFlxTTMNb4NSkgvX+knOOTipE6gqwPUQpxNF+c=", + "lastModified": 1635781173, + "narHash": "sha256-nTA2y2jIJiVj5RawHUNhlZUIy5J/Q2CA6YP4T1qBkLo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "22a500a3f87bbce73bd8d777ef920b43a636f018", + "rev": "7053541084bf5ce2921ef307e5585d39d7ba8b3f", "type": "github" }, "original": { From 49086abcc53f80134e5fa62cea5e5edbe9f5e8da Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:38 +0100 Subject: [PATCH 03/10] liquidd: use systemd startup notification --- modules/btcpayserver.nix | 6 +++--- modules/liquid.nix | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/btcpayserver.nix b/modules/btcpayserver.nix index daa9f16..5bb0f7f 100644 --- a/modules/btcpayserver.nix +++ b/modules/btcpayserver.nix @@ -159,10 +159,10 @@ in { lbtcnodeendpoint=${nbLib.addressWithPort liquidd.address bitcoind.whitelistedPort} ''} ''; - in { + in rec { wantedBy = [ "multi-user.target" ]; - requires = [ "bitcoind.service" ]; - after = [ "bitcoind.service" ]; + requires = [ "bitcoind.service" ] ++ optional cfg.btcpayserver.lbtc "liquidd.service"; + after = requires; preStart = '' install -m 600 ${configFile} '${cfg.nbxplorer.dataDir}/settings.config' { diff --git a/modules/liquid.nix b/modules/liquid.nix index 36abe7e..2c072f8 100644 --- a/modules/liquid.nix +++ b/modules/liquid.nix @@ -168,6 +168,11 @@ let bitcoind = config.services.bitcoind; configFile = pkgs.writeText "elements.conf" '' + # We're already logging via journald + nodebuglogfile=1 + + startupnotify=/run/current-system/systemd/bin/systemd-notify --ready + chain=${bitcoind.makeNetworkName "liquidv1" '' regtest [regtest]'' # Add [regtest] config section @@ -253,7 +258,8 @@ in { } >> '${cfg.dataDir}/elements.conf' ''; serviceConfig = nbLib.defaultHardening // { - Type = "simple"; + Type = "notify"; + NotifyAccess = "all"; User = cfg.user; Group = cfg.group; ExecStart = "${nbPkgs.elementsd}/bin/elementsd -datadir='${cfg.dataDir}'"; From 82c92df162b6635708f7b281b749ae8eb25defa4 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:39 +0100 Subject: [PATCH 04/10] tests/regtest: fix restarting bitcoind Previously, the test wallet was not loaded after restarting bitcoind and generating blocks failed. --- test/tests.nix | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/tests.nix b/test/tests.nix index 1e13ad3..2766fd4 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -229,10 +229,11 @@ let services.bitcoind.regtest = true; systemd.services.bitcoind.postStart = mkAfter '' cli=${config.services.bitcoind.cli}/bin/bitcoin-cli - # Don't fail when wallet already exists - $cli createwallet "test" || true - address=$($cli getnewaddress) - $cli generatetoaddress 10 $address + if ! $cli listwallets | ${pkgs.jq}/bin/jq -e 'index("test")'; then + $cli -named createwallet wallet_name=test load_on_startup=true + address=$($cli -rpcwallet=test getnewaddress) + $cli generatetoaddress 10 $address + fi ''; # lightning-loop contains no builtin swap server for regtest. From 083e141e3e6dabeceb83221fbd2a8296f98f42db Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:40 +0100 Subject: [PATCH 05/10] tests/btcpayserver: test bitcoind P2P connection in regtest nbxplorer requires at least 100 blocks (coinbase maturity) in regtest. --- test/tests.nix | 3 ++- test/tests.py | 21 ++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/test/tests.nix b/test/tests.nix index 2766fd4..1f50398 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -225,6 +225,7 @@ let regtestBase = { config, ... }: { tests.regtest = true; + test.data.num_blocks = 100; services.bitcoind.regtest = true; systemd.services.bitcoind.postStart = mkAfter '' @@ -232,7 +233,7 @@ let if ! $cli listwallets | ${pkgs.jq}/bin/jq -e 'index("test")'; then $cli -named createwallet wallet_name=test load_on_startup=true address=$($cli -rpcwallet=test getnewaddress) - $cli generatetoaddress 10 $address + $cli generatetoaddress ${toString config.test.data.num_blocks} $address fi ''; diff --git a/test/tests.py b/test/tests.py index 99d0682..2e0a09a 100644 --- a/test/tests.py +++ b/test/tests.py @@ -189,8 +189,10 @@ def _(): def _(): assert_running("nbxplorer") machine.wait_until_succeeds(log_has_string("nbxplorer", "BTC: RPC connection successful")) - machine.wait_until_succeeds(log_has_string("nbxplorer", "LBTC: RPC connection successful")) + if "liquidd" in enabled_tests: + machine.wait_until_succeeds(log_has_string("nbxplorer", "LBTC: RPC connection successful")) wait_for_open_port(ip("nbxplorer"), 24444) + assert_running("btcpayserver") machine.wait_until_succeeds(log_has_string("btcpayserver", "Listening on")) wait_for_open_port(ip("btcpayserver"), 23000) @@ -359,6 +361,8 @@ def _(): else: return False + num_blocks = test_data["num_blocks"] + if enabled("electrs"): machine.wait_for_unit("onion-addresses") machine.wait_until_succeeds(log_has_string("electrs", "serving Electrum RPC")) @@ -366,18 +370,18 @@ def _(): """echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'""" f" | nc {ip('electrs')} 50001 | head -1 | jq -M .result.height" ) - assert_full_match(get_block_height_cmd, "10\n") + assert_full_match(get_block_height_cmd, f"{num_blocks}\n") if enabled("clightning"): machine.wait_until_succeeds( - "[[ $(runuser -u operator -- lightning-cli getinfo | jq -M .blockheight) == 10 ]]" + f"[[ $(runuser -u operator -- lightning-cli getinfo | jq -M .blockheight) == {num_blocks} ]]" ) if enabled("lnd"): machine.wait_until_succeeds( - "[[ $(runuser -u operator -- lncli getinfo | jq -M .block_height) == 10 ]]" + f"[[ $(runuser -u operator -- lncli getinfo | jq -M .block_height) == {num_blocks} ]]" ) if enabled("lightning-loop"): machine.wait_until_succeeds( - log_has_string("lightning-loop", "Starting event loop at height 10") + log_has_string("lightning-loop", f"Starting event loop at height {num_blocks}") ) succeed("runuser -u operator -- loop getparams") if enabled("lightning-pool"): @@ -385,6 +389,13 @@ def _(): log_has_string("lightning-pool", "lnd is now fully synced to its chain backend") ) succeed("runuser -u operator -- pool orders list") + if enabled("btcpayserver"): + machine.wait_until_succeeds(log_has_string("nbxplorer", f"BTC: Starting scan at block {num_blocks}")) + # nbxplorer 2.2.16 currently fails with with lbtc (liquidd) on regtest + # LBTC: Full node version detected: 210000 + # LBTC: RPC connection successful + # LBTC: Failed to connect to RPC + # System.IO.EndOfStreamException: No more byte to read if "netns-isolation" in enabled_tests: def ip(name): From 8616254d63a391a1f7948cb6b4c3273b134d7496 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:41 +0100 Subject: [PATCH 06/10] bitcoind-rpc-public-whitelist: remove waitfornewblock This is an internal testing function and it's no longer used by electrs. --- modules/bitcoind-rpc-public-whitelist.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/bitcoind-rpc-public-whitelist.nix b/modules/bitcoind-rpc-public-whitelist.nix index ccbbe62..70ce406 100644 --- a/modules/bitcoind-rpc-public-whitelist.nix +++ b/modules/bitcoind-rpc-public-whitelist.nix @@ -25,7 +25,6 @@ "gettxoutsetinfo" "scantxoutset" "verifytxoutproof" - "waitfornewblock" # Mining "getblocktemplate" "getmininginfo" From 0478354477c67de6df65ca1c070deb2b200d30ea Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:42 +0100 Subject: [PATCH 07/10] versioning: move variable The variable is an internal implementation detail. Move it close to where it's used. --- modules/versioning.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/versioning.nix b/modules/versioning.nix index fd61035..5fd03d6 100644 --- a/modules/versioning.nix +++ b/modules/versioning.nix @@ -22,8 +22,6 @@ let }; }; - version = config.nix-bitcoin.configVersion; - # Sorted by increasing version numbers changes = [ { @@ -197,6 +195,8 @@ let ''; }; + version = config.nix-bitcoin.configVersion; + incompatibleChanges = optionals (version != null && versionOlder lastChange) (builtins.filter (change: versionOlder change && (change.condition or true)) changes); From 6584540828242d15ab9f50831b47581d9ff793a2 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:43 +0100 Subject: [PATCH 08/10] makeShell: make help message extensible Users can now override help and print more help messages alongside `nixBitcoinHelp`. --- helper/makeShell.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helper/makeShell.nix b/helper/makeShell.nix index 19386be..3e10418 100644 --- a/helper/makeShell.nix +++ b/helper/makeShell.nix @@ -22,7 +22,7 @@ pkgs.stdenv.mkDerivation { # 2. the shell is interactive if [[ -t 1 && $- == *i* ]]; then isInteractive=1; else isInteractive=; fi - help() { + nixBitcoinHelp() { echo "nix-bitcoin path: ${toString ../.}" echo echo "Available commands" @@ -46,6 +46,7 @@ pkgs.stdenv.mkDerivation { echo "update-nix-bitcoin" echo " Fetch and use the latest version of nix-bitcoin" } + help() { nixBitcoinHelp; } h() { help; } fetch-release() { From c30fe1919b81d89e0d6e22a58028c7b7e112c16d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 13:07:44 +0100 Subject: [PATCH 09/10] netns-isolation: don't auto-assign IPv6 addrs to peer links This simplifies the host's address configuration. This also removes unused addresses that are returned when resolving container hostnames via nss-mymachines: `getent ahosts nb-test` --- modules/netns-isolation.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/netns-isolation.nix b/modules/netns-isolation.nix index 49484e8..a2dc63f 100644 --- a/modules/netns-isolation.nix +++ b/modules/netns-isolation.nix @@ -173,6 +173,8 @@ in { ${ip} link add ${veth} type veth peer name ${peer} ${ip} link set ${veth} netns ${netnsName} ${ipNetns} addr add ${v.address}/24 dev ${veth} + # The peer link is never used directly, so don't auto-assign an IPv6 address + echo 1 > /proc/sys/net/ipv6/conf/${peer}/disable_ipv6 ${ip} link set ${peer} up ${ipNetns} link set ${veth} up ${ip} link set ${peer} master nb-br From b3e868d0af9a1b724d8466affbaa4c7d5454084d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Tue, 2 Nov 2021 15:15:29 +0100 Subject: [PATCH 10/10] tests/regtest: disable incompatible `validatepegin` for liquidd Otherwise liquidd startup fails. This bug was not covered by our tests, because we're not combining `regtest` with `secure-node`. But nixbitcoin.org does, which should suffice for now. --- modules/liquid.nix | 6 ++++++ test/tests.nix | 3 +++ 2 files changed, 9 insertions(+) diff --git a/modules/liquid.nix b/modules/liquid.nix index 2c072f8..dd8d884 100644 --- a/modules/liquid.nix +++ b/modules/liquid.nix @@ -234,6 +234,12 @@ in { inherit options; config = mkIf cfg.enable { + assertions = [ + { assertion = bitcoind.regtest -> cfg.validatepegin != true; + message = "liquidd: `validatepegin` is incompatible with regtest."; + } + ]; + services.bitcoind.enable = true; environment.systemPackages = [ diff --git a/test/tests.nix b/test/tests.nix index 1f50398..ca924d8 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -248,6 +248,9 @@ let services.lightning-pool.extraConfig = '' auctionserver=localhost ''; + + # `validatepegin` is incompatible with regtest + services.liquidd.validatepegin = mkForce false; }; ## Examples / debug helper