From 6f42fa81814ace4e327d16f1ce67638a15fcda22 Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Tue, 5 Oct 2021 16:52:00 +0200
Subject: [PATCH 1/5] update nixpkgs (electrs: 0.8.11 -> 0.9.0)

---
 flake.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/flake.lock b/flake.lock
index e2abc25..3efab6e 100644
--- a/flake.lock
+++ b/flake.lock
@@ -17,11 +17,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1631993469,
-        "narHash": "sha256-McX1mqzbXtfyf8w9cA5gJHD2mDjen+4ze6xpHHKXuRI=",
+        "lastModified": 1633267966,
+        "narHash": "sha256-gFKvZ5AmV/dDTKXVxacPbXe4R0BsFpwtVaQxuIm2nnk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6120ac5cd201f6cb593d1b80e861be0342495be9",
+        "rev": "7daf35532d2d8bf5e6f7f962e6cd13a66d01a71d",
         "type": "github"
       },
       "original": {
@@ -33,11 +33,11 @@
     },
     "nixpkgsUnstable": {
       "locked": {
-        "lastModified": 1632025165,
-        "narHash": "sha256-vFHkvmdwbJAFg2d60CQ4fbqpEAUFXR0SO3HRWI4+bXM=",
+        "lastModified": 1633329294,
+        "narHash": "sha256-0LpQLS4KMgxslMgmDHmxG/5twFlXDBW9z4Or1iOrCvU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a54d2e72e282f2bc68c49f82c735cf664244ec75",
+        "rev": "ee084c02040e864eeeb4cf4f8538d92f7c675671",
         "type": "github"
       },
       "original": {

From 75b89f3957b824c1feb3436f8c3c27ab33573526 Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Tue, 5 Oct 2021 16:52:02 +0200
Subject: [PATCH 2/5] electrs: adapt to version 0.9.0

- `waitfornewblock` was previously not included in the public RPC
  whitelist because it's reserved for testing and marked as hidden
  in bitcoind.

- electrs changed its verbosity settings. `-vv` is now the best choice
  for normal usage.

- bitcoind option `dataDirReadableByGroup` is now unused.
  Because it can be valuable for other use cases and implementing
  it is intricate, we're keeping it for now.

- test: keep `nc` connection open because otherwise the electrs
  RPC server would now close the connection before sending a response.
---
 examples/configuration.nix                |  7 +----
 modules/bitcoind-rpc-public-whitelist.nix |  1 +
 modules/electrs.nix                       | 30 ++++++++--------------
 modules/hardware-wallets.nix              |  1 -
 modules/obsolete-options.nix              |  7 +++++
 modules/presets/secure-node.nix           |  1 -
 modules/versioning.nix                    | 31 +++++++++++++++++++++++
 test/tests.nix                            |  2 ++
 test/tests.py                             | 11 +++++---
 9 files changed, 60 insertions(+), 31 deletions(-)

diff --git a/examples/configuration.nix b/examples/configuration.nix
index 26f3cfc..7f47bc7 100644
--- a/examples/configuration.nix
+++ b/examples/configuration.nix
@@ -93,10 +93,6 @@
   ### ELECTRS
   # Set this to enable electrs, an efficient Electrum server implemented in Rust.
   # services.electrs.enable = true;
-  #
-  # If you have more than 8GB memory, enable this option so electrs will
-  # sync faster. Only available if hardware wallets are disabled.
-  # services.electrs.high-memory = true;
 
   ### BTCPayServer
   # Set this to enable BTCPayServer, a self-hosted, open-source
@@ -150,7 +146,6 @@
   ### Hardware wallets
   # Enable the following to allow using hardware wallets.
   # See https://github.com/bitcoin-core/HWI for more information.
-  # Only available if electrs.high-memory is disabled.
   #
   # Ledger must be initialized through the official ledger live app and the Bitcoin app must
   # be installed and running on the device.
@@ -264,5 +259,5 @@
   # The nix-bitcoin release version that your config is compatible with.
   # When upgrading to a backwards-incompatible release, nix-bitcoin will display an
   # an error and provide hints for migrating your config to the new release.
-  nix-bitcoin.configVersion = "0.0.51";
+  nix-bitcoin.configVersion = "0.0.53";
 }
diff --git a/modules/bitcoind-rpc-public-whitelist.nix b/modules/bitcoind-rpc-public-whitelist.nix
index 70ce406..ccbbe62 100644
--- a/modules/bitcoind-rpc-public-whitelist.nix
+++ b/modules/bitcoind-rpc-public-whitelist.nix
@@ -25,6 +25,7 @@
   "gettxoutsetinfo"
   "scantxoutset"
   "verifytxoutproof"
+  "waitfornewblock"
   # Mining
   "getblocktemplate"
   "getmininginfo"
diff --git a/modules/electrs.nix b/modules/electrs.nix
index 15cad4c..8072ef9 100644
--- a/modules/electrs.nix
+++ b/modules/electrs.nix
@@ -19,13 +19,6 @@ let
       default = "/var/lib/electrs";
       description = "The data directory for electrs.";
     };
-    high-memory = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        If enabled, the electrs service will sync faster on high-memory systems (≥ 8GB).
-      '';
-    };
     monitoringPort = mkOption {
       type = types.port;
       default = 4224;
@@ -63,7 +56,12 @@ in {
       }
     ];
 
-    services.bitcoind.enable = true;
+    services.bitcoind = {
+      enable = true;
+      # Enable p2p connections
+      listen = true;
+      extraConfig = "whitelist=download@${nbLib.address cfg.address}";
+    };
 
     systemd.tmpfiles.rules = [
       "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
@@ -80,37 +78,31 @@ in {
       serviceConfig = nbLib.defaultHardening // {
         RuntimeDirectory = "electrs";
         RuntimeDirectoryMode = "700";
+        # electrs only uses the working directory for reading electrs.toml
         WorkingDirectory = "/run/electrs";
         ExecStart = ''
-          ${config.nix-bitcoin.pkgs.electrs}/bin/electrs -vvv \
-          ${if cfg.high-memory then
-              traceIf (!bitcoind.dataDirReadableByGroup) ''
-                Warning: For optimal electrs syncing performance, enable services.bitcoind.dataDirReadableByGroup.
-                Note that this disables wallet support in bitcoind.
-              '' ""
-            else
-              "--jsonrpc-import --index-batch-size=10"
-          } \
+          ${config.nix-bitcoin.pkgs.electrs}/bin/electrs -vv \
           --network=${bitcoind.makeNetworkName "bitcoin" "regtest"} \
           --db-dir='${cfg.dataDir}' \
           --daemon-dir='${bitcoind.dataDir}' \
           --electrum-rpc-addr=${cfg.address}:${toString cfg.port} \
           --monitoring-addr=${cfg.address}:${toString cfg.monitoringPort} \
           --daemon-rpc-addr=${nbLib.addressWithPort bitcoind.rpc.address bitcoind.rpc.port} \
+          --daemon-p2p-addr=${nbLib.addressWithPort bitcoind.address bitcoind.port} \
           ${cfg.extraArgs}
         '';
         User = cfg.user;
         Group = cfg.group;
         Restart = "on-failure";
         RestartSec = "10s";
-        ReadWritePaths = "${cfg.dataDir} ${if cfg.high-memory then "${bitcoind.dataDir}" else ""}";
+        ReadWritePaths = cfg.dataDir;
       } // nbLib.allowedIPAddresses cfg.enforceTor;
     };
 
     users.users.${cfg.user} = {
       isSystemUser = true;
       group = cfg.group;
-      extraGroups = [ "bitcoinrpc-public" ] ++ optionals cfg.high-memory [ bitcoind.user ];
+      extraGroups = [ "bitcoinrpc-public" ];
     };
     users.groups.${cfg.group} = {};
   };
diff --git a/modules/hardware-wallets.nix b/modules/hardware-wallets.nix
index cf95dc4..807fb66 100644
--- a/modules/hardware-wallets.nix
+++ b/modules/hardware-wallets.nix
@@ -38,7 +38,6 @@ in {
         { assertion = (config.services.bitcoind.disablewallet == null || !config.services.bitcoind.disablewallet);
           message = ''
             Hardware-Wallets are not compatible with bitcoind.disablewallet.
-            Note that this option is active when enabling electrs.high-memory.
           '';
         }
       ];
diff --git a/modules/obsolete-options.nix b/modules/obsolete-options.nix
index fc10181..137165f 100644
--- a/modules/obsolete-options.nix
+++ b/modules/obsolete-options.nix
@@ -26,5 +26,12 @@ in {
 
     (mkRenamedAnnounceTorOption "clightning")
     (mkRenamedAnnounceTorOption "lnd")
+
+    # 0.0.53
+    (mkRemovedOptionModule [ "services" "electrs" "high-memory" ] ''
+      This option is no longer supported by electrs 0.9.0. Electrs now always uses
+      bitcoin peer connections for syncing blocks. This performs well on low and high
+      memory systems.
+    '')
   ];
 }
diff --git a/modules/presets/secure-node.nix b/modules/presets/secure-node.nix
index 0318851..7b72aa6 100644
--- a/modules/presets/secure-node.nix
+++ b/modules/presets/secure-node.nix
@@ -35,7 +35,6 @@ in {
     services.bitcoind = {
       enable = true;
       listen = true;
-      dataDirReadableByGroup = mkIf cfg.electrs.high-memory true;
       dbCache = 1000;
     };
 
diff --git a/modules/versioning.nix b/modules/versioning.nix
index c57add6..6fd8fa0 100644
--- a/modules/versioning.nix
+++ b/modules/versioning.nix
@@ -138,6 +138,37 @@ let
         [1] https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/fidelity-bonds.md
       '';
     }
+    {
+      version = "0.0.53";
+      condition = config.services.electrs.enable;
+      message = let
+        dbPath = "${config.services.electrs.dataDir}/mainnet";
+      in ''
+        Electrs 0.9.0 has switched to a new, more space efficient database format,
+        reducing storage demands by ~60% [1].
+        When started, electrs will automatically reindex the bitcoin blockchain.
+        This can take a few hours, depending on your hardware. The electrs server is
+        inactive during reindexing.
+
+        To upgrade, do the following:
+
+        - If you have less than 40 GB of free space [2] on the electrs data dir volume:
+          1. Delete the database:
+             systemctl stop electrs
+             rm -r '${dbPath}'
+          2. Deploy the new system config to your node
+
+        - Otherwise:
+          1. Deploy the new system config to your node
+          2. Check that electrs works as expected and delete the old database:
+             rm -r '${dbPath}'
+
+        [1] https://github.com/romanz/electrs/blob/557911e3baf9a000f883a6f619f0518945a7678d/doc/usage.md#upgrading
+        [2] This is based on the bitcoin blockchain size as of 2021-09.
+            The general formula is, approximately, size_of(${dbPath}) * 0.6
+            This includes the final database size (0.4) plus some extra storage (0.2).
+      '';
+    }
   ];
 
   mkOnionServiceChange = service: {
diff --git a/test/tests.nix b/test/tests.nix
index 4e93ae4..6f5d741 100644
--- a/test/tests.nix
+++ b/test/tests.nix
@@ -74,6 +74,8 @@ let
       tests.charge-lnd = cfg.charge-lnd.enable;
 
       tests.electrs = cfg.electrs.enable;
+      # Sigterm is broken during IBD in version 0.9.0 https://github.com/romanz/electrs/issues/532
+      systemd.services.electrs.serviceConfig.KillSignal = "SIGKILL";
 
       tests.liquidd = cfg.liquidd.enable;
       services.liquidd.extraConfig = mkIf config.test.noConnections "connect=0";
diff --git a/test/tests.py b/test/tests.py
index a8ab0bb..e8a9534 100644
--- a/test/tests.py
+++ b/test/tests.py
@@ -106,10 +106,13 @@ def _():
     assert_running("electrs")
     wait_for_open_port(ip("electrs"), 4224)  # prometeus metrics provider
     # Check RPC connection to bitcoind
-    machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
+    if not "regtest" in enabled_tests:
+        machine.wait_until_succeeds(
+            log_has_string("electrs", "waiting for 0 blocks to download")
+        )
 
 # Impure: Stops electrs
-# Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages
+# Stop electrs from spamming the test log with 'waiting for 0 blocks to download' messages
 @test("stop-electrs")
 def _():
     succeed("systemctl stop electrs")
@@ -353,10 +356,10 @@ def _():
 
     if enabled("electrs"):
         machine.wait_for_unit("onion-addresses")
-        machine.wait_until_succeeds(log_has_string("electrs", "BlockchainInfo"))
+        machine.wait_until_succeeds(log_has_string("electrs", "serving Electrum RPC"))
         get_block_height_cmd = (
             """echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'"""
-            f" | nc -N {ip('electrs')} 50001 | jq -M .result.height"
+            f" | nc {ip('electrs')} 50001 | head -1 | jq -M .result.height"
         )
         assert_full_match(get_block_height_cmd, "10\n")
     if enabled("clightning"):

From 265fc1911d9eb9cf124e1e99ad4de20445cbd544 Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Tue, 5 Oct 2021 16:52:03 +0200
Subject: [PATCH 3/5] extra-container: pin to nixpkgs-unstable

extra-container is now part of nixpkgs.
---
 helper/makeShell.nix             |  2 +-
 pkgs/default.nix                 |  1 -
 pkgs/extra-container/default.nix | 37 --------------------------------
 pkgs/pinned.nix                  |  1 +
 test/lib/make-container.sh       |  7 +++---
 5 files changed, 6 insertions(+), 42 deletions(-)
 delete mode 100644 pkgs/extra-container/default.nix

diff --git a/helper/makeShell.nix b/helper/makeShell.nix
index d11008d..19386be 100644
--- a/helper/makeShell.nix
+++ b/helper/makeShell.nix
@@ -6,7 +6,7 @@ let
   nbPkgs = import ../pkgs { inherit pkgs; };
   cfgDir = toString configDir;
   path = lib.optionalString pkgs.stdenv.isLinux ''
-    export PATH="${lib.makeBinPath [ nbPkgs.extra-container ]}''${PATH:+:}$PATH"
+    export PATH="${lib.makeBinPath [ nbPkgs.pinned.extra-container ]}''${PATH:+:}$PATH"
   '';
 in
 pkgs.stdenv.mkDerivation {
diff --git a/pkgs/default.nix b/pkgs/default.nix
index 39df452..2ab1e1f 100644
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -13,7 +13,6 @@ let self = {
   nixops19_09 = pkgs.callPackage ./nixops { };
   krops = import ./krops { };
   netns-exec = pkgs.callPackage ./netns-exec { };
-  extra-container = pkgs.callPackage ./extra-container { };
   clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages);
   clboss = pkgs.callPackage ./clboss { };
   secp256k1 = pkgs.callPackage ./secp256k1 { };
diff --git a/pkgs/extra-container/default.nix b/pkgs/extra-container/default.nix
deleted file mode 100644
index 40f8308..0000000
--- a/pkgs/extra-container/default.nix
+++ /dev/null
@@ -1,37 +0,0 @@
-{ stdenv, lib, nixos-container, openssh
-, glibcLocales
-}:
-
-stdenv.mkDerivation rec {
-  pname = "extra-container";
-  # Update this file when changing the version: ../../test/lib/make-container.sh
-  version = "0.7";
-
-  src = builtins.fetchTarball {
-    url = "https://github.com/erikarvstedt/extra-container/archive/${version}.tar.gz";
-    sha256 = "1hcbi611vm0kn8rl7q974wcjkihpddan6m3p7hx8l8jnv18ydng8";
-  };
-
-  buildCommand = ''
-    install -D $src/extra-container $out/bin/extra-container
-    patchShebangs $out/bin
-    share=$out/share/extra-container
-    install $src/eval-config.nix -Dt $share
-
-    # Use existing PATH for systemctl and machinectl
-    scriptPath="export PATH=${lib.makeBinPath [ nixos-container openssh ]}:\$PATH"
-
-    sed -i \
-      -e "s|evalConfig=.*|evalConfig=$share/eval-config.nix|" \
-      -e "s|LOCALE_ARCHIVE=.*|LOCALE_ARCHIVE=${glibcLocales}/lib/locale/locale-archive|" \
-      -e "2i$scriptPath" \
-      $out/bin/extra-container
-  '';
-
-  meta = with lib; {
-    description = "Run declarative containers without full system rebuilds";
-    homepage = https://github.com/erikarvstedt/extra-container;
-    license = licenses.mit;
-    maintainers = [ maintainers.earvstedt ];
-  };
-}
diff --git a/pkgs/pinned.nix b/pkgs/pinned.nix
index 10c6042..90954f8 100644
--- a/pkgs/pinned.nix
+++ b/pkgs/pinned.nix
@@ -12,6 +12,7 @@ pkgs: pkgsUnstable:
     clightning
     electrs
     elementsd
+    extra-container
     hwi
     lightning-loop
     lightning-pool
diff --git a/test/lib/make-container.sh b/test/lib/make-container.sh
index 70803fb..845cbc4 100755
--- a/test/lib/make-container.sh
+++ b/test/lib/make-container.sh
@@ -77,9 +77,10 @@ while [[ $# > 0 ]]; do
 done
 
 containerBin=$(type -P extra-container) || true
-if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.7*) ]]; then
-    echo "Building extra-container. Skip this step by adding extra-container 0.7 to PATH."
-    nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs -A extra-container >/dev/null
+if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.8*) ]]; then
+    echo "Building extra-container. Skip this step by adding extra-container 0.8 to PATH."
+    nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \
+      -A pinned.extra-container >/dev/null
     export PATH="/tmp/extra-container/bin${PATH:+:}$PATH"
 fi
 

From b9301ce0d95cf210815127041461c1305761bfd2 Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Wed, 6 Oct 2021 12:26:35 +0200
Subject: [PATCH 4/5] emergency fix: lnd: 0.13.1-beta -> 0.13.3-beta

Fixes CVE-2021-41593.
Temporarily switch to nixpkgs/master.
---
 flake.lock | 14 +++++++-------
 flake.nix  |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/flake.lock b/flake.lock
index 3efab6e..945a198 100644
--- a/flake.lock
+++ b/flake.lock
@@ -17,11 +17,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1633267966,
-        "narHash": "sha256-gFKvZ5AmV/dDTKXVxacPbXe4R0BsFpwtVaQxuIm2nnk=",
+        "lastModified": 1633422542,
+        "narHash": "sha256-JYz2PmVogNRO8DhcvXzL/QhZzboTspJz2YSRlnAj8aM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7daf35532d2d8bf5e6f7f962e6cd13a66d01a71d",
+        "rev": "aff647e2704fa1223994604887bb78276dc57083",
         "type": "github"
       },
       "original": {
@@ -33,16 +33,16 @@
     },
     "nixpkgsUnstable": {
       "locked": {
-        "lastModified": 1633329294,
-        "narHash": "sha256-0LpQLS4KMgxslMgmDHmxG/5twFlXDBW9z4Or1iOrCvU=",
+        "lastModified": 1633514490,
+        "narHash": "sha256-wQrUBgyF4EXlz9HgEHrQEj9vbgh6+nO8iXc3XCTQkLA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ee084c02040e864eeeb4cf4f8538d92f7c675671",
+        "rev": "1c1b567985bd1be77601657562ed20299d169529",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "master",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 302b612..65ef974 100644
--- a/flake.nix
+++ b/flake.nix
@@ -6,7 +6,7 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05";
-    nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgsUnstable.url = "github:NixOS/nixpkgs/master";
     flake-utils.url = "github:numtide/flake-utils";
   };
 

From 8938eadf0c4d4995381e7eb79188a7fea8c87471 Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Wed, 6 Oct 2021 16:55:41 +0200
Subject: [PATCH 5/5] bitcoind: don't tag all incoming connections as 'Tor'

We're also accepting local, non-Tor connections.
---
 modules/bitcoind.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix
index 07da64d..d2f3769 100644
--- a/modules/bitcoind.nix
+++ b/modules/bitcoind.nix
@@ -253,7 +253,7 @@ let
     ${optionalString (cfg.assumevalid != null) "assumevalid=${cfg.assumevalid}"}
 
     # Connection options
-    ${optionalString cfg.listen "bind=${cfg.address}${optionalString cfg.enforceTor "=onion"}"}
+    ${optionalString cfg.listen "bind=${cfg.address}"}
     port=${toString cfg.port}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     listen=${if cfg.listen then "1" else "0"}