diff --git a/examples/configuration.nix b/examples/configuration.nix index 26f3cfc..7f47bc7 100644 --- a/examples/configuration.nix +++ b/examples/configuration.nix @@ -93,10 +93,6 @@ ### ELECTRS # Set this to enable electrs, an efficient Electrum server implemented in Rust. # services.electrs.enable = true; - # - # If you have more than 8GB memory, enable this option so electrs will - # sync faster. Only available if hardware wallets are disabled. - # services.electrs.high-memory = true; ### BTCPayServer # Set this to enable BTCPayServer, a self-hosted, open-source @@ -150,7 +146,6 @@ ### Hardware wallets # Enable the following to allow using hardware wallets. # See https://github.com/bitcoin-core/HWI for more information. - # Only available if electrs.high-memory is disabled. # # Ledger must be initialized through the official ledger live app and the Bitcoin app must # be installed and running on the device. @@ -264,5 +259,5 @@ # The nix-bitcoin release version that your config is compatible with. # When upgrading to a backwards-incompatible release, nix-bitcoin will display an # an error and provide hints for migrating your config to the new release. - nix-bitcoin.configVersion = "0.0.51"; + nix-bitcoin.configVersion = "0.0.53"; } diff --git a/flake.lock b/flake.lock index e2abc25..945a198 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1631993469, - "narHash": "sha256-McX1mqzbXtfyf8w9cA5gJHD2mDjen+4ze6xpHHKXuRI=", + "lastModified": 1633422542, + "narHash": "sha256-JYz2PmVogNRO8DhcvXzL/QhZzboTspJz2YSRlnAj8aM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6120ac5cd201f6cb593d1b80e861be0342495be9", + "rev": "aff647e2704fa1223994604887bb78276dc57083", "type": "github" }, "original": { @@ -33,16 +33,16 @@ }, "nixpkgsUnstable": { "locked": { - "lastModified": 1632025165, - "narHash": "sha256-vFHkvmdwbJAFg2d60CQ4fbqpEAUFXR0SO3HRWI4+bXM=", + "lastModified": 1633514490, + "narHash": "sha256-wQrUBgyF4EXlz9HgEHrQEj9vbgh6+nO8iXc3XCTQkLA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a54d2e72e282f2bc68c49f82c735cf664244ec75", + "rev": "1c1b567985bd1be77601657562ed20299d169529", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "master", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 302b612..65ef974 100644 --- a/flake.nix +++ b/flake.nix @@ -6,7 +6,7 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05"; - nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + nixpkgsUnstable.url = "github:NixOS/nixpkgs/master"; flake-utils.url = "github:numtide/flake-utils"; }; diff --git a/helper/makeShell.nix b/helper/makeShell.nix index d11008d..19386be 100644 --- a/helper/makeShell.nix +++ b/helper/makeShell.nix @@ -6,7 +6,7 @@ let nbPkgs = import ../pkgs { inherit pkgs; }; cfgDir = toString configDir; path = lib.optionalString pkgs.stdenv.isLinux '' - export PATH="${lib.makeBinPath [ nbPkgs.extra-container ]}''${PATH:+:}$PATH" + export PATH="${lib.makeBinPath [ nbPkgs.pinned.extra-container ]}''${PATH:+:}$PATH" ''; in pkgs.stdenv.mkDerivation { diff --git a/modules/bitcoind-rpc-public-whitelist.nix b/modules/bitcoind-rpc-public-whitelist.nix index 70ce406..ccbbe62 100644 --- a/modules/bitcoind-rpc-public-whitelist.nix +++ b/modules/bitcoind-rpc-public-whitelist.nix @@ -25,6 +25,7 @@ "gettxoutsetinfo" "scantxoutset" "verifytxoutproof" + "waitfornewblock" # Mining "getblocktemplate" "getmininginfo" diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix index 07da64d..d2f3769 100644 --- a/modules/bitcoind.nix +++ b/modules/bitcoind.nix @@ -253,7 +253,7 @@ let ${optionalString (cfg.assumevalid != null) "assumevalid=${cfg.assumevalid}"} # Connection options - ${optionalString cfg.listen "bind=${cfg.address}${optionalString cfg.enforceTor "=onion"}"} + ${optionalString cfg.listen "bind=${cfg.address}"} port=${toString cfg.port} ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"} listen=${if cfg.listen then "1" else "0"} diff --git a/modules/electrs.nix b/modules/electrs.nix index 15cad4c..8072ef9 100644 --- a/modules/electrs.nix +++ b/modules/electrs.nix @@ -19,13 +19,6 @@ let default = "/var/lib/electrs"; description = "The data directory for electrs."; }; - high-memory = mkOption { - type = types.bool; - default = false; - description = '' - If enabled, the electrs service will sync faster on high-memory systems (≥ 8GB). - ''; - }; monitoringPort = mkOption { type = types.port; default = 4224; @@ -63,7 +56,12 @@ in { } ]; - services.bitcoind.enable = true; + services.bitcoind = { + enable = true; + # Enable p2p connections + listen = true; + extraConfig = "whitelist=download@${nbLib.address cfg.address}"; + }; systemd.tmpfiles.rules = [ "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -" @@ -80,37 +78,31 @@ in { serviceConfig = nbLib.defaultHardening // { RuntimeDirectory = "electrs"; RuntimeDirectoryMode = "700"; + # electrs only uses the working directory for reading electrs.toml WorkingDirectory = "/run/electrs"; ExecStart = '' - ${config.nix-bitcoin.pkgs.electrs}/bin/electrs -vvv \ - ${if cfg.high-memory then - traceIf (!bitcoind.dataDirReadableByGroup) '' - Warning: For optimal electrs syncing performance, enable services.bitcoind.dataDirReadableByGroup. - Note that this disables wallet support in bitcoind. - '' "" - else - "--jsonrpc-import --index-batch-size=10" - } \ + ${config.nix-bitcoin.pkgs.electrs}/bin/electrs -vv \ --network=${bitcoind.makeNetworkName "bitcoin" "regtest"} \ --db-dir='${cfg.dataDir}' \ --daemon-dir='${bitcoind.dataDir}' \ --electrum-rpc-addr=${cfg.address}:${toString cfg.port} \ --monitoring-addr=${cfg.address}:${toString cfg.monitoringPort} \ --daemon-rpc-addr=${nbLib.addressWithPort bitcoind.rpc.address bitcoind.rpc.port} \ + --daemon-p2p-addr=${nbLib.addressWithPort bitcoind.address bitcoind.port} \ ${cfg.extraArgs} ''; User = cfg.user; Group = cfg.group; Restart = "on-failure"; RestartSec = "10s"; - ReadWritePaths = "${cfg.dataDir} ${if cfg.high-memory then "${bitcoind.dataDir}" else ""}"; + ReadWritePaths = cfg.dataDir; } // nbLib.allowedIPAddresses cfg.enforceTor; }; users.users.${cfg.user} = { isSystemUser = true; group = cfg.group; - extraGroups = [ "bitcoinrpc-public" ] ++ optionals cfg.high-memory [ bitcoind.user ]; + extraGroups = [ "bitcoinrpc-public" ]; }; users.groups.${cfg.group} = {}; }; diff --git a/modules/hardware-wallets.nix b/modules/hardware-wallets.nix index cf95dc4..807fb66 100644 --- a/modules/hardware-wallets.nix +++ b/modules/hardware-wallets.nix @@ -38,7 +38,6 @@ in { { assertion = (config.services.bitcoind.disablewallet == null || !config.services.bitcoind.disablewallet); message = '' Hardware-Wallets are not compatible with bitcoind.disablewallet. - Note that this option is active when enabling electrs.high-memory. ''; } ]; diff --git a/modules/obsolete-options.nix b/modules/obsolete-options.nix index fc10181..137165f 100644 --- a/modules/obsolete-options.nix +++ b/modules/obsolete-options.nix @@ -26,5 +26,12 @@ in { (mkRenamedAnnounceTorOption "clightning") (mkRenamedAnnounceTorOption "lnd") + + # 0.0.53 + (mkRemovedOptionModule [ "services" "electrs" "high-memory" ] '' + This option is no longer supported by electrs 0.9.0. Electrs now always uses + bitcoin peer connections for syncing blocks. This performs well on low and high + memory systems. + '') ]; } diff --git a/modules/presets/secure-node.nix b/modules/presets/secure-node.nix index 0318851..7b72aa6 100644 --- a/modules/presets/secure-node.nix +++ b/modules/presets/secure-node.nix @@ -35,7 +35,6 @@ in { services.bitcoind = { enable = true; listen = true; - dataDirReadableByGroup = mkIf cfg.electrs.high-memory true; dbCache = 1000; }; diff --git a/modules/versioning.nix b/modules/versioning.nix index c57add6..6fd8fa0 100644 --- a/modules/versioning.nix +++ b/modules/versioning.nix @@ -138,6 +138,37 @@ let [1] https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/fidelity-bonds.md ''; } + { + version = "0.0.53"; + condition = config.services.electrs.enable; + message = let + dbPath = "${config.services.electrs.dataDir}/mainnet"; + in '' + Electrs 0.9.0 has switched to a new, more space efficient database format, + reducing storage demands by ~60% [1]. + When started, electrs will automatically reindex the bitcoin blockchain. + This can take a few hours, depending on your hardware. The electrs server is + inactive during reindexing. + + To upgrade, do the following: + + - If you have less than 40 GB of free space [2] on the electrs data dir volume: + 1. Delete the database: + systemctl stop electrs + rm -r '${dbPath}' + 2. Deploy the new system config to your node + + - Otherwise: + 1. Deploy the new system config to your node + 2. Check that electrs works as expected and delete the old database: + rm -r '${dbPath}' + + [1] https://github.com/romanz/electrs/blob/557911e3baf9a000f883a6f619f0518945a7678d/doc/usage.md#upgrading + [2] This is based on the bitcoin blockchain size as of 2021-09. + The general formula is, approximately, size_of(${dbPath}) * 0.6 + This includes the final database size (0.4) plus some extra storage (0.2). + ''; + } ]; mkOnionServiceChange = service: { diff --git a/pkgs/default.nix b/pkgs/default.nix index 39df452..2ab1e1f 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -13,7 +13,6 @@ let self = { nixops19_09 = pkgs.callPackage ./nixops { }; krops = import ./krops { }; netns-exec = pkgs.callPackage ./netns-exec { }; - extra-container = pkgs.callPackage ./extra-container { }; clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages); clboss = pkgs.callPackage ./clboss { }; secp256k1 = pkgs.callPackage ./secp256k1 { }; diff --git a/pkgs/extra-container/default.nix b/pkgs/extra-container/default.nix deleted file mode 100644 index 40f8308..0000000 --- a/pkgs/extra-container/default.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ stdenv, lib, nixos-container, openssh -, glibcLocales -}: - -stdenv.mkDerivation rec { - pname = "extra-container"; - # Update this file when changing the version: ../../test/lib/make-container.sh - version = "0.7"; - - src = builtins.fetchTarball { - url = "https://github.com/erikarvstedt/extra-container/archive/${version}.tar.gz"; - sha256 = "1hcbi611vm0kn8rl7q974wcjkihpddan6m3p7hx8l8jnv18ydng8"; - }; - - buildCommand = '' - install -D $src/extra-container $out/bin/extra-container - patchShebangs $out/bin - share=$out/share/extra-container - install $src/eval-config.nix -Dt $share - - # Use existing PATH for systemctl and machinectl - scriptPath="export PATH=${lib.makeBinPath [ nixos-container openssh ]}:\$PATH" - - sed -i \ - -e "s|evalConfig=.*|evalConfig=$share/eval-config.nix|" \ - -e "s|LOCALE_ARCHIVE=.*|LOCALE_ARCHIVE=${glibcLocales}/lib/locale/locale-archive|" \ - -e "2i$scriptPath" \ - $out/bin/extra-container - ''; - - meta = with lib; { - description = "Run declarative containers without full system rebuilds"; - homepage = https://github.com/erikarvstedt/extra-container; - license = licenses.mit; - maintainers = [ maintainers.earvstedt ]; - }; -} diff --git a/pkgs/pinned.nix b/pkgs/pinned.nix index 10c6042..90954f8 100644 --- a/pkgs/pinned.nix +++ b/pkgs/pinned.nix @@ -12,6 +12,7 @@ pkgs: pkgsUnstable: clightning electrs elementsd + extra-container hwi lightning-loop lightning-pool diff --git a/test/lib/make-container.sh b/test/lib/make-container.sh index 70803fb..845cbc4 100755 --- a/test/lib/make-container.sh +++ b/test/lib/make-container.sh @@ -77,9 +77,10 @@ while [[ $# > 0 ]]; do done containerBin=$(type -P extra-container) || true -if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.7*) ]]; then - echo "Building extra-container. Skip this step by adding extra-container 0.7 to PATH." - nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs -A extra-container >/dev/null +if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.8*) ]]; then + echo "Building extra-container. Skip this step by adding extra-container 0.8 to PATH." + nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \ + -A pinned.extra-container >/dev/null export PATH="/tmp/extra-container/bin${PATH:+:}$PATH" fi diff --git a/test/tests.nix b/test/tests.nix index 4e93ae4..6f5d741 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -74,6 +74,8 @@ let tests.charge-lnd = cfg.charge-lnd.enable; tests.electrs = cfg.electrs.enable; + # Sigterm is broken during IBD in version 0.9.0 https://github.com/romanz/electrs/issues/532 + systemd.services.electrs.serviceConfig.KillSignal = "SIGKILL"; tests.liquidd = cfg.liquidd.enable; services.liquidd.extraConfig = mkIf config.test.noConnections "connect=0"; diff --git a/test/tests.py b/test/tests.py index a8ab0bb..e8a9534 100644 --- a/test/tests.py +++ b/test/tests.py @@ -106,10 +106,13 @@ def _(): assert_running("electrs") wait_for_open_port(ip("electrs"), 4224) # prometeus metrics provider # Check RPC connection to bitcoind - machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo")) + if not "regtest" in enabled_tests: + machine.wait_until_succeeds( + log_has_string("electrs", "waiting for 0 blocks to download") + ) # Impure: Stops electrs -# Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages +# Stop electrs from spamming the test log with 'waiting for 0 blocks to download' messages @test("stop-electrs") def _(): succeed("systemctl stop electrs") @@ -353,10 +356,10 @@ def _(): if enabled("electrs"): machine.wait_for_unit("onion-addresses") - machine.wait_until_succeeds(log_has_string("electrs", "BlockchainInfo")) + machine.wait_until_succeeds(log_has_string("electrs", "serving Electrum RPC")) get_block_height_cmd = ( """echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'""" - f" | nc -N {ip('electrs')} 50001 | jq -M .result.height" + f" | nc {ip('electrs')} 50001 | head -1 | jq -M .result.height" ) assert_full_match(get_block_height_cmd, "10\n") if enabled("clightning"):