Merge #144: Electrs fixes

5596bcf4fb bitcoind: set default rpcuser (Erik Arvstedt) c4cf323873 electrs: add option 'extraArgs' (Erik Arvstedt) e731d71232 electrs: add option 'address' (Erik Arvstedt) 1e62456ed1 electrs: test RPC connection to bitcoind (Erik Arvstedt) 0be67c325e electrs: use cfg.user, cfg.group (Erik Arvstedt) 48be5a79fa electrs.enable: use mkEnableOption (Erik Arvstedt) b75b2a1626 electrs: improve description (Erik Arvstedt) fa3455d01f electrs: don't leak bitcoinrpc secret through process ARGV (Erik Arvstedt) f30aadbef2 electrs: enable unstable build, pin pkg to unstable (Erik Arvstedt) 5c6571654e electrs: 0.7.1 -> 0.8.3 (Erik Arvstedt) 47481b2642 electrs: quote dataDir in shell cmd (Erik Arvstedt) 8fb33d1099 electrs: use bitcoind.dataDir option (Erik Arvstedt) 45ba1f1fb3 electrs: don't print timestamps to log (Erik Arvstedt) 88080a58bf electrs: wrap long lines in preStart (Erik Arvstedt) 301bb91ae5 simplify setting high-memory options (Erik Arvstedt) 93fd2329b8 electrs: make nginx TLS proxy optional (Erik Arvstedt) acde24ce43 electrs: move user/group definitions to bottom (Erik Arvstedt) 148327326b electrs: formatting (Erik Arvstedt) cce9932b62 make pinned pkgs accessible through pkgs/default.nix (Erik Arvstedt) Pull request description: ACKs for top commit: jonasnick: ACK 5596bcf4fb Tree-SHA512: 2064b352839a1787ccb05930ac2cf1f0d3596aaea81135086e8a91b9eebf319868087a27cdf9f2fd0152ab652d338680cdf9e866185e86777fcdd87931651b39
2020-03-04 21:03:38 +00:00 · 2020-03-04 21:03:38 +00:00 · d62dac450a
commit d62dac450a
parent aca23197d3 5596bcf4fb
9 changed files with 121 additions and 3029 deletions
--- a/modules/bitcoind.nix
+++ b/modules/bitcoind.nix
@ -135,7 +135,7 @@ in {

      rpcuser = mkOption {
          type = types.nullOr types.str;
-          default = null;
+          default = "bitcoinrpc";
          description = "Username for JSON-RPC connections";
      };
      rpcpassword = mkOption {
--- a/modules/electrs.nix
+++ b/modules/electrs.nix
@ -1,22 +1,17 @@
 { config, lib, pkgs, ... }:

 with lib;
-
 let
  cfg = config.services.electrs;
  inherit (config) nix-bitcoin-services;
  secretsDir = config.nix-bitcoin.secretsDir;
-  index-batch-size = "${if cfg.high-memory then "" else "--index-batch-size=10"}";
-  jsonrpc-import = "${if cfg.high-memory then "" else "--jsonrpc-import"}";
 in {
+  imports = [
+    (mkRenamedOptionModule [ "services" "electrs" "nginxport" ] [ "services" "electrs" "TLSProxy" "port" ])
+  ];
+
  options.services.electrs = {
-    enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        If enabled, the electrs service will be installed.
-      '';
-    };
+    enable = mkEnableOption "electrs";
    dataDir = mkOption {
      type = types.path;
      default = "/var/lib/electrs";
@ -36,53 +31,65 @@ in {
      type = types.bool;
      default = false;
      description = ''
-      If enabled, the electrs service will sync faster on high-memory systems (≥ 8GB).
+        If enabled, the electrs service will sync faster on high-memory systems (≥ 8GB).
      '';
    };
+    address = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = "RPC listening address.";
+    };
    port = mkOption {
-        type = types.ints.u16;
-        default = 50001;
-        description = "RPC port.";
+      type = types.ints.u16;
+      default = 50001;
+      description = "RPC port.";
    };
    onionport = mkOption {
-        type = types.ints.u16;
-        default = 50002;
-        description = "Port on which to listen for tor client connections.";
+      type = types.ints.u16;
+      default = 50002;
+      description = "Port on which to listen for tor client connections.";
    };
-    nginxport = mkOption {
+    extraArgs = mkOption {
+      type = types.separatedString " ";
+      default = "";
+      description = "Extra command line arguments passed to electrs.";
+    };
+    TLSProxy = {
+      enable = mkEnableOption "Nginx TLS proxy";
+      port = mkOption {
        type = types.ints.u16;
        default = 50003;
        description = "Port on which to listen for TLS client connections.";
+      };
    };
    enforceTor = nix-bitcoin-services.enforceTor;
  };

-  config = mkIf cfg.enable {
-    users.users.${cfg.user} = {
-        description = "electrs User";
-        group = cfg.group;
-        extraGroups = [ "bitcoinrpc" "bitcoin"];
-        home = cfg.dataDir;
-    };
-    users.groups.${cfg.group} = {};
-
+  config = mkIf cfg.enable (mkMerge [{
    systemd.services.electrs = {
-      description = "Run electrs";
+      description = "Electrs Electrum Server";
      wantedBy = [ "multi-user.target" ];
-      requires = [ "bitcoind.service" "nginx.service"];
+      requires = [ "bitcoind.service" ];
      after = [ "bitcoind.service" ];
-      # create shell script to start up electrs safely with password parameter
      preStart = ''
        mkdir -m 0770 -p ${cfg.dataDir}
        chown -R '${cfg.user}:${cfg.group}' ${cfg.dataDir}
-        echo "${pkgs.nix-bitcoin.electrs}/bin/electrs -vvv ${index-batch-size} ${jsonrpc-import} --timestamp --db-dir ${cfg.dataDir} --daemon-dir /var/lib/bitcoind --cookie=${config.services.bitcoind.rpcuser}:$(cat ${secretsDir}/bitcoin-rpcpassword) --electrum-rpc-addr=127.0.0.1:${toString cfg.port}" > /run/electrs/startscript.sh
+        echo "cookie = \"${config.services.bitcoind.rpcuser}:$(cat ${secretsDir}/bitcoin-rpcpassword)\"" \
+          > electrs.toml
        '';
-      serviceConfig = rec {
+      serviceConfig = {
        RuntimeDirectory = "electrs";
        RuntimeDirectoryMode = "700";
+        WorkingDirectory = "/run/electrs";
        PermissionsStartOnly = "true";
-        ExecStart = "${pkgs.bash}/bin/bash /run/${RuntimeDirectory}/startscript.sh";
-        User = "electrs";
+        ExecStart = ''
+          ${pkgs.nix-bitcoin.electrs}/bin/electrs -vvv \
+          ${optionalString (!cfg.high-memory) "--jsonrpc-import --index-batch-size=10"} \
+          --db-dir '${cfg.dataDir}' --daemon-dir '${config.services.bitcoind.dataDir}' \
+          --electrum-rpc-addr=${toString cfg.address}:${toString cfg.port} ${cfg.extraArgs}
+        '';
+        User = cfg.user;
+        Group = cfg.group;
        Restart = "on-failure";
        RestartSec = "10s";
      } // nix-bitcoin-services.defaultHardening
@ -92,16 +99,34 @@ in {
        );
    };

+    users.users.${cfg.user} = {
+      description = "electrs User";
+      group = cfg.group;
+      extraGroups = [ "bitcoinrpc" "bitcoin"];
+      home = cfg.dataDir;
+    };
+    users.groups.${cfg.group} = {};
+  }
+
+  (mkIf cfg.TLSProxy.enable {
    services.nginx = {
      enable = true;
-      appendConfig = ''
+      appendConfig = let
+        address =
+          if cfg.address == "0.0.0.0" then
+            "127.0.0.1"
+          else if cfg.address == "::" then
+            "::1"
+          else
+            cfg.address;
+      in ''
        stream {
          upstream electrs {
-            server 127.0.0.1:${toString config.services.electrs.port};
+            server ${address}:${toString cfg.port};
          }

          server {
-            listen ${toString config.services.electrs.nginxport} ssl;
+            listen ${toString cfg.TLSProxy.port} ssl;
            proxy_pass electrs;

            ssl_certificate ${secretsDir}/nginx-cert;
@ -114,9 +139,12 @@ in {
        }
      '';
    };
-    systemd.services.nginx = {
-      requires = [ "nix-bitcoin-secrets.target" ];
-      after = [ "nix-bitcoin-secrets.target" ];
+    systemd.services = {
+      electrs.wants = [ "nginx.service" ];
+      nginx = {
+        requires = [ "nix-bitcoin-secrets.target" ];
+        after = [ "nix-bitcoin-secrets.target" ];
+      };
    };
    nix-bitcoin.secrets = rec {
      nginx-key = {
@ -125,5 +153,6 @@ in {
      };
      nginx-cert = nginx-key;
    };
-  };
+  })
+  ]);
 }
--- a/modules/modules.nix
+++ b/modules/modules.nix
@ -1,15 +1,6 @@
 { config, pkgs, lib, ... }:
-let
-  nixpkgs-pinned = import ../pkgs/nixpkgs-pinned.nix;
-  unstable = import nixpkgs-pinned.nixpkgs-unstable {};

-  allPackages = pkgs: (import ../pkgs { inherit pkgs; }) // {
-    bitcoin = unstable.bitcoin.override { miniupnpc = null; };
-    bitcoind = unstable.bitcoind.override { miniupnpc = null; };
-    clightning = unstable.clightning;
-    lnd = unstable.lnd;
-  };
-in {
+{
  imports = [
    ./bitcoind.nix
    ./clightning.nix
@ -37,7 +28,10 @@ in {

  config = {
    nixpkgs.overlays = [ (self: super: {
-      nix-bitcoin = allPackages super;
+      nix-bitcoin = let
+        pkgs = import ../pkgs { pkgs = super; };
+      in
+        pkgs // pkgs.pinned;
    }) ];
  };
 }
--- a/modules/nix-bitcoin.nix
+++ b/modules/nix-bitcoin.nix
@ -54,7 +54,6 @@ in {
    services.bitcoind.proxy = config.services.tor.client.socksListenAddress;
    services.bitcoind.enforceTor = true;
    services.bitcoind.port = 8333;
-    services.bitcoind.rpcuser = "bitcoinrpc";
    services.bitcoind.zmqpubrawblock = "tcp://127.0.0.1:28332";
    services.bitcoind.zmqpubrawtx = "tcp://127.0.0.1:28333";
    services.bitcoind.assumevalid = "00000000000000000000e5abc3a74fe27dc0ead9c70ea1deb456f11c15fd7bc6";
@ -145,10 +144,11 @@ in {
    services.electrs.port = 50001;
    services.electrs.enforceTor = true;
    services.electrs.onionport = 50002;
-    services.electrs.nginxport = 50003;
+    services.electrs.TLSProxy.enable = true;
+    services.electrs.TLSProxy.port = 50003;
    services.tor.hiddenServices.electrs = {
      map = [{
-        port = config.services.electrs.onionport; toPort = config.services.electrs.nginxport;
+        port = config.services.electrs.onionport; toPort = config.services.electrs.TLSProxy.port;
      }];
      version = 3;
    };
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@ -4,10 +4,12 @@
  lightning-charge = pkgs.callPackage ./lightning-charge { };
  nanopos = pkgs.callPackage ./nanopos { };
  spark-wallet = pkgs.callPackage ./spark-wallet { };
-  electrs = (pkgs.callPackage ./electrs { }).rootCrate.build;
+  electrs = pkgs.callPackage ./electrs { };
  elementsd = pkgs.callPackage ./elementsd { withGui = false; };
  hwi = pkgs.callPackage ./hwi { };
  pylightning = pkgs.python3Packages.callPackage ./pylightning { };
  liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
  generate-secrets = pkgs.callPackage ./generate-secrets { };
+
+  pinned = import ./pinned.nix;
 }
--- a/pkgs/electrs/default.nix
+++ b/pkgs/electrs/default.nix
--- a/pkgs/electrs/generate.sh
+++ b/pkgs/electrs/generate.sh
@ -1,45 +0,0 @@
-#! /usr/bin/env nix-shell
-#! nix-shell -i bash -p git gnupg dirmngr
-set -e
-
-# Creating temporary directory
-echo "Creating temporary directory"
-DIR="$(mktemp -d)"
-cd $DIR
-git clone https://github.com/romanz/electrs
-
-# Checking out latest release
-echo "Checking out latest release"
-cd electrs
-latesttagelectrs=$(git describe --tags `git rev-list --tags --max-count=1`)
-git checkout ${latesttagelectrs}
-echo "Latest release is ${latesttagelectrs}"
-
-# Optional GPG Verification
-read -p "Do you want to import Roman Zeyde's PGP Key (46917CBB)? [yN]" -r
-echo
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-echo "Getting Roman Zeyde's PGP key"
-gpg --recv-keys 15C8C3574AE4F1E25F3F35C587CAE5FA46917CBB
-echo "Verifying latest release"
-git verify-tag ${latesttagelectrs}
-fi
-
-echo "Generating crate2nix expression"
-git clone https://github.com/kolloch/crate2nix ../crate2nix
-cd ../crate2nix
-git checkout e311fc6f88b61e1eda85e8c588e7c23dea03b532 # latest commit that works
-cd ../electrs
-nix-shell -v ../crate2nix/shell.nix --run "crate2nix generate"
-
-echo "Fixing nix expression"
-sed -i 's/(path+file.*)/(registry+https:\/\/github.com\/romanz\/electrs)/g' default.nix
-sed -i '/crateName = "electrs";/i\        name = "electrs-${version}";' default.nix
-sed -i 's/src = (builtins.filterSource sourceFilter .\/.);/sha256 = "<insert correct hash here>";/g' default.nix
-# @jb55's fixes from https://github.com/jb55/electrs/commit/e3bed69c17dac1af1be34d18e5be2c815c20838c
-sed -i '/lib? pkgs.lib/a\  llvmPackages ? pkgs.llvmPackages,' default.nix
-sed -i 's/resolvedDefaultFeatures = \[ "bzip2" "default" "lz4" "snappy" "static" "zlib" "zstd" \]/resolvedDefaultFeatures = \[ "bzip2" "default" "lz4" "snappy" "static" "zlib" \]/g' default.nix
-sed -i '/crateName = "librocksdb-sys";/a\\n        enableParallelBuilding = true;\n        buildInputs = [ pkgs.clang ];\n        LIBCLANG_PATH = "${llvmPackages.libclang}/lib";\n' default.nix
-sed -i 's/"default" = \[ "snappy" "lz4" "zstd" "zlib" "bzip2" \];/"default" = \[ "snappy" "zstd" "zlib" "bzip2" \];/g' default.nix
-echo "Done. You now have your pkgs/electrs/default.nix expression in $DIR/electrs/default.nix. Just replace the electrs sha256 and you'll be good to go."
--- a/pkgs/pinned.nix
+++ b/pkgs/pinned.nix
@ -0,0 +1,13 @@
+let
+  nixpkgsPinned = import ./nixpkgs-pinned.nix;
+  unstable = import nixpkgsPinned.nixpkgs-unstable { config = {}; overlays = []; };
+  nixBitcoinPkgsUnstable = import ./. { pkgs = unstable; };
+in
+{
+  bitcoin = unstable.bitcoin.override { miniupnpc = null; };
+  bitcoind = unstable.bitcoind.override { miniupnpc = null; };
+  inherit (unstable)
+    clightning
+    lnd;
+  inherit (nixBitcoinPkgsUnstable) electrs;
+}
--- a/test/test-script.py
+++ b/test/test-script.py
@ -39,6 +39,8 @@ assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')

 assert_running("electrs")
 machine.wait_for_open_port(4224)  # prometeus metrics provider
+# Check RPC connection to bitcoind
+machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
 assert_running("nginx")
 # SSL stratum server via nginx. Only check for open port, no content is served here
 # as electrs isn't ready.