Merge #144: Electrs fixes

5596bcf4fb bitcoind: set default rpcuser (Erik Arvstedt)
c4cf323873 electrs: add option 'extraArgs' (Erik Arvstedt)
e731d71232 electrs: add option 'address' (Erik Arvstedt)
1e62456ed1 electrs: test RPC connection to bitcoind (Erik Arvstedt)
0be67c325e electrs: use cfg.user, cfg.group (Erik Arvstedt)
48be5a79fa electrs.enable: use mkEnableOption (Erik Arvstedt)
b75b2a1626 electrs: improve description (Erik Arvstedt)
fa3455d01f electrs: don't leak bitcoinrpc secret through process ARGV (Erik Arvstedt)
f30aadbef2 electrs: enable unstable build, pin pkg to unstable (Erik Arvstedt)
5c6571654e electrs: 0.7.1 -> 0.8.3 (Erik Arvstedt)
47481b2642 electrs: quote dataDir in shell cmd (Erik Arvstedt)
8fb33d1099 electrs: use bitcoind.dataDir option (Erik Arvstedt)
45ba1f1fb3 electrs: don't print timestamps to log (Erik Arvstedt)
88080a58bf electrs: wrap long lines in preStart (Erik Arvstedt)
301bb91ae5 simplify setting high-memory options (Erik Arvstedt)
93fd2329b8 electrs: make nginx TLS proxy optional (Erik Arvstedt)
acde24ce43 electrs: move user/group definitions to bottom (Erik Arvstedt)
148327326b electrs: formatting (Erik Arvstedt)
cce9932b62 make pinned pkgs accessible through pkgs/default.nix (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 5596bcf4fb

Tree-SHA512: 2064b352839a1787ccb05930ac2cf1f0d3596aaea81135086e8a91b9eebf319868087a27cdf9f2fd0152ab652d338680cdf9e866185e86777fcdd87931651b39

modules/bitcoind.nix

@@ -135,7 +135,7 @@ in {
 
       rpcuser = mkOption {
           type = types.nullOr types.str;
-          default = null;
+          default = "bitcoinrpc";
           description = "Username for JSON-RPC connections";
       };
       rpcpassword = mkOption {

modules/electrs.nix

@@ -1,22 +1,17 @@
 { config, lib, pkgs, ... }:
 
 with lib;
-
 let
   cfg = config.services.electrs;
   inherit (config) nix-bitcoin-services;
   secretsDir = config.nix-bitcoin.secretsDir;
-  index-batch-size = "${if cfg.high-memory then "" else "--index-batch-size=10"}";
-  jsonrpc-import = "${if cfg.high-memory then "" else "--jsonrpc-import"}";
 in {
+  imports = [
+    (mkRenamedOptionModule [ "services" "electrs" "nginxport" ] [ "services" "electrs" "TLSProxy" "port" ])
+  ];
+
   options.services.electrs = {
-    enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        If enabled, the electrs service will be installed.
-      '';
-    };
+    enable = mkEnableOption "electrs";
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/electrs";
@@ -36,53 +31,65 @@ in {
       type = types.bool;
       default = false;
       description = ''
-      If enabled, the electrs service will sync faster on high-memory systems (≥ 8GB).
+        If enabled, the electrs service will sync faster on high-memory systems (≥ 8GB).
       '';
     };
+    address = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = "RPC listening address.";
+    };
     port = mkOption {
-        type = types.ints.u16;
-        default = 50001;
-        description = "RPC port.";
+      type = types.ints.u16;
+      default = 50001;
+      description = "RPC port.";
     };
     onionport = mkOption {
-        type = types.ints.u16;
-        default = 50002;
-        description = "Port on which to listen for tor client connections.";
+      type = types.ints.u16;
+      default = 50002;
+      description = "Port on which to listen for tor client connections.";
     };
-    nginxport = mkOption {
+    extraArgs = mkOption {
+      type = types.separatedString " ";
+      default = "";
+      description = "Extra command line arguments passed to electrs.";
+    };
+    TLSProxy = {
+      enable = mkEnableOption "Nginx TLS proxy";
+      port = mkOption {
         type = types.ints.u16;
         default = 50003;
         description = "Port on which to listen for TLS client connections.";
+      };
     };
     enforceTor = nix-bitcoin-services.enforceTor;
   };
 
-  config = mkIf cfg.enable {
-    users.users.${cfg.user} = {
-        description = "electrs User";
-        group = cfg.group;
-        extraGroups = [ "bitcoinrpc" "bitcoin"];
-        home = cfg.dataDir;
-    };
-    users.groups.${cfg.group} = {};
-
+  config = mkIf cfg.enable (mkMerge [{
     systemd.services.electrs = {
-      description = "Run electrs";
+      description = "Electrs Electrum Server";
       wantedBy = [ "multi-user.target" ];
-      requires = [ "bitcoind.service" "nginx.service"];
+      requires = [ "bitcoind.service" ];
       after = [ "bitcoind.service" ];
-      # create shell script to start up electrs safely with password parameter
       preStart = ''
         mkdir -m 0770 -p ${cfg.dataDir}
         chown -R '${cfg.user}:${cfg.group}' ${cfg.dataDir}
-        echo "${pkgs.nix-bitcoin.electrs}/bin/electrs -vvv ${index-batch-size} ${jsonrpc-import} --timestamp --db-dir ${cfg.dataDir} --daemon-dir /var/lib/bitcoind --cookie=${config.services.bitcoind.rpcuser}:$(cat ${secretsDir}/bitcoin-rpcpassword) --electrum-rpc-addr=127.0.0.1:${toString cfg.port}" > /run/electrs/startscript.sh
-        '';	
-      serviceConfig = rec {
+        echo "cookie = \"${config.services.bitcoind.rpcuser}:$(cat ${secretsDir}/bitcoin-rpcpassword)\"" \
+          > electrs.toml
+        '';
+      serviceConfig = {
         RuntimeDirectory = "electrs";
         RuntimeDirectoryMode = "700";
+        WorkingDirectory = "/run/electrs";
         PermissionsStartOnly = "true";
-        ExecStart = "${pkgs.bash}/bin/bash /run/${RuntimeDirectory}/startscript.sh";
-        User = "electrs";
+        ExecStart = ''
+          ${pkgs.nix-bitcoin.electrs}/bin/electrs -vvv \
+          ${optionalString (!cfg.high-memory) "--jsonrpc-import --index-batch-size=10"} \
+          --db-dir '${cfg.dataDir}' --daemon-dir '${config.services.bitcoind.dataDir}' \
+          --electrum-rpc-addr=${toString cfg.address}:${toString cfg.port} ${cfg.extraArgs}
+        '';
+        User = cfg.user;
+        Group = cfg.group;
         Restart = "on-failure";
         RestartSec = "10s";
       } // nix-bitcoin-services.defaultHardening
@@ -92,16 +99,34 @@ in {
         );
     };
 
+    users.users.${cfg.user} = {
+      description = "electrs User";
+      group = cfg.group;
+      extraGroups = [ "bitcoinrpc" "bitcoin"];
+      home = cfg.dataDir;
+    };
+    users.groups.${cfg.group} = {};
+  }
+
+  (mkIf cfg.TLSProxy.enable {
     services.nginx = {
       enable = true;
-      appendConfig = ''
+      appendConfig = let
+        address =
+          if cfg.address == "0.0.0.0" then
+            "127.0.0.1"
+          else if cfg.address == "::" then
+            "::1"
+          else
+            cfg.address;
+      in ''
         stream {
           upstream electrs {
-            server 127.0.0.1:${toString config.services.electrs.port};
+            server ${address}:${toString cfg.port};
           }
 
           server {
-            listen ${toString config.services.electrs.nginxport} ssl;
+            listen ${toString cfg.TLSProxy.port} ssl;
             proxy_pass electrs;
 
             ssl_certificate ${secretsDir}/nginx-cert;
@@ -114,9 +139,12 @@ in {
         }
       '';
     };
-    systemd.services.nginx = {
-      requires = [ "nix-bitcoin-secrets.target" ];
-      after = [ "nix-bitcoin-secrets.target" ];
+    systemd.services = {
+      electrs.wants = [ "nginx.service" ];
+      nginx = {
+        requires = [ "nix-bitcoin-secrets.target" ];
+        after = [ "nix-bitcoin-secrets.target" ];
+      };
     };
     nix-bitcoin.secrets = rec {
       nginx-key = {
@@ -125,5 +153,6 @@ in {
       };
       nginx-cert = nginx-key;
     };
-  };
+  })
+  ]);
 }

modules/modules.nix

@@ -1,15 +1,6 @@
 { config, pkgs, lib, ... }:
-let
-  nixpkgs-pinned = import ../pkgs/nixpkgs-pinned.nix;
-  unstable = import nixpkgs-pinned.nixpkgs-unstable {};
 
-  allPackages = pkgs: (import ../pkgs { inherit pkgs; }) // {
-    bitcoin = unstable.bitcoin.override { miniupnpc = null; };
-    bitcoind = unstable.bitcoind.override { miniupnpc = null; };
-    clightning = unstable.clightning;
-    lnd = unstable.lnd;
-  };
-in {
+{
   imports = [
     ./bitcoind.nix
     ./clightning.nix
@@ -37,7 +28,10 @@ in {
 
   config = {
     nixpkgs.overlays = [ (self: super: {
-      nix-bitcoin = allPackages super;
+      nix-bitcoin = let
+        pkgs = import ../pkgs { pkgs = super; };
+      in
+        pkgs // pkgs.pinned;
     }) ];
   };
 }

modules/nix-bitcoin.nix

@@ -54,7 +54,6 @@ in {
     services.bitcoind.proxy = config.services.tor.client.socksListenAddress;
     services.bitcoind.enforceTor = true;
     services.bitcoind.port = 8333;
-    services.bitcoind.rpcuser = "bitcoinrpc";
     services.bitcoind.zmqpubrawblock = "tcp://127.0.0.1:28332";
     services.bitcoind.zmqpubrawtx = "tcp://127.0.0.1:28333";
     services.bitcoind.assumevalid = "00000000000000000000e5abc3a74fe27dc0ead9c70ea1deb456f11c15fd7bc6";
@@ -145,10 +144,11 @@ in {
     services.electrs.port = 50001;
     services.electrs.enforceTor = true;
     services.electrs.onionport = 50002;
-    services.electrs.nginxport = 50003;
+    services.electrs.TLSProxy.enable = true;
+    services.electrs.TLSProxy.port = 50003;
     services.tor.hiddenServices.electrs = {
       map = [{
-        port = config.services.electrs.onionport; toPort = config.services.electrs.nginxport;
+        port = config.services.electrs.onionport; toPort = config.services.electrs.TLSProxy.port;
       }];
       version = 3;
     };

pkgs/default.nix

@@ -4,10 +4,12 @@
   lightning-charge = pkgs.callPackage ./lightning-charge { };
   nanopos = pkgs.callPackage ./nanopos { };
   spark-wallet = pkgs.callPackage ./spark-wallet { };
-  electrs = (pkgs.callPackage ./electrs { }).rootCrate.build;
+  electrs = pkgs.callPackage ./electrs { };
   elementsd = pkgs.callPackage ./elementsd { withGui = false; };
   hwi = pkgs.callPackage ./hwi { };
   pylightning = pkgs.python3Packages.callPackage ./pylightning { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
   generate-secrets = pkgs.callPackage ./generate-secrets { };
+
+  pinned = import ./pinned.nix;
 }

pkgs/electrs/generate.sh

@@ -1,45 +0,0 @@
-#! /usr/bin/env nix-shell
-#! nix-shell -i bash -p git gnupg dirmngr
-set -e
-
-# Creating temporary directory
-echo "Creating temporary directory"
-DIR="$(mktemp -d)"
-cd $DIR
-git clone https://github.com/romanz/electrs
-
-# Checking out latest release
-echo "Checking out latest release"
-cd electrs
-latesttagelectrs=$(git describe --tags `git rev-list --tags --max-count=1`)
-git checkout ${latesttagelectrs}
-echo "Latest release is ${latesttagelectrs}"
-
-# Optional GPG Verification
-read -p "Do you want to import Roman Zeyde's PGP Key (46917CBB)? [yN]" -r
-echo
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-echo "Getting Roman Zeyde's PGP key"
-gpg --recv-keys 15C8C3574AE4F1E25F3F35C587CAE5FA46917CBB
-echo "Verifying latest release"
-git verify-tag ${latesttagelectrs}
-fi
-
-echo "Generating crate2nix expression"
-git clone https://github.com/kolloch/crate2nix ../crate2nix
-cd ../crate2nix
-git checkout e311fc6f88b61e1eda85e8c588e7c23dea03b532 # latest commit that works
-cd ../electrs
-nix-shell -v ../crate2nix/shell.nix --run "crate2nix generate"
-
-echo "Fixing nix expression"
-sed -i 's/(path+file.*)/(registry+https:\/\/github.com\/romanz\/electrs)/g' default.nix
-sed -i '/crateName = "electrs";/i\        name = "electrs-${version}";' default.nix
-sed -i 's/src = (builtins.filterSource sourceFilter .\/.);/sha256 = "<insert correct hash here>";/g' default.nix
-# @jb55's fixes from https://github.com/jb55/electrs/commit/e3bed69c17dac1af1be34d18e5be2c815c20838c
-sed -i '/lib? pkgs.lib/a\  llvmPackages ? pkgs.llvmPackages,' default.nix
-sed -i 's/resolvedDefaultFeatures = \[ "bzip2" "default" "lz4" "snappy" "static" "zlib" "zstd" \]/resolvedDefaultFeatures = \[ "bzip2" "default" "lz4" "snappy" "static" "zlib" \]/g' default.nix
-sed -i '/crateName = "librocksdb-sys";/a\\n        enableParallelBuilding = true;\n        buildInputs = [ pkgs.clang ];\n        LIBCLANG_PATH = "${llvmPackages.libclang}/lib";\n' default.nix
-sed -i 's/"default" = \[ "snappy" "lz4" "zstd" "zlib" "bzip2" \];/"default" = \[ "snappy" "zstd" "zlib" "bzip2" \];/g' default.nix
-echo "Done. You now have your pkgs/electrs/default.nix expression in $DIR/electrs/default.nix. Just replace the electrs sha256 and you'll be good to go."

pkgs/pinned.nix

@@ -0,0 +1,13 @@
+let
+  nixpkgsPinned = import ./nixpkgs-pinned.nix;
+  unstable = import nixpkgsPinned.nixpkgs-unstable { config = {}; overlays = []; };
+  nixBitcoinPkgsUnstable = import ./. { pkgs = unstable; };
+in
+{
+  bitcoin = unstable.bitcoin.override { miniupnpc = null; };
+  bitcoind = unstable.bitcoind.override { miniupnpc = null; };
+  inherit (unstable)
+    clightning
+    lnd;
+  inherit (nixBitcoinPkgsUnstable) electrs;
+}

test/test-script.py

@@ -39,6 +39,8 @@ assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
 
 assert_running("electrs")
 machine.wait_for_open_port(4224)  # prometeus metrics provider
+# Check RPC connection to bitcoind
+machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
 assert_running("nginx")
 # SSL stratum server via nginx. Only check for open port, no content is served here
 # as electrs isn't ready.