Merge fort-nix/nix-bitcoin#380: joinmarket: 0.8.3 -> 0.9.1

9730be9282 joinmarket-yieldgenerator: simplify start script (Erik Arvstedt)
179b86d19c joinmarket: allow recreating wallet from seed (Erik Arvstedt)
7c5ef32b50 versioning: move list of changes to the top (Erik Arvstedt)
b15d71605e joinmarket: fix leaking passwords (Erik Arvstedt)
5c14453389 joinmarket-ob-watcher: don't assert running, assert rpc failure (nixbitcoin)
00a0759884 joinmarket-ob-watcher: extra permissions & functionality for fidelity bonds (nixbitcoin)
d7f9e33e1c joinmarket-ob-watcher: move resource files to extra dir (Erik Arvstedt)
32d0f08d77 docs: fix usage steps numbering (nixbitcoin)
e95abf6c7e joinmarket: 0.8.3 -> 0.9.1 (nixbitcoin)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK 9730be9282

Tree-SHA512: b6e693d3e293ad3d590479eefdb5d1e144a5d7b16c4160fc7cf4ba890a78b6e94b170c43f61a541363a17dddc3cf4441917270e23ece643b7cff4c0cb4581337

docs/usage.md

@@ -252,10 +252,15 @@ For clarity reasons, nix-bitcoin renames all scripts to `jm-*` without `.py`, fo
 example `wallet-tool.py` becomes `jm-wallet-tool`. The rest of this section
 details nix-bitcoin specific workflows for JoinMarket.
 
-## Initialize JoinMarket Wallet
+## Wallets
 
-By default, nix-bitcoin's JoinMarket module automatically generates a wallet for
+By default, a wallet is automatically generated at service startup.
-you. If however, you want to manually initialize your wallet, follow these steps.
+It's stored at `/var/lib/joinmarket/wallets/wallet.jmdat`, and its mnmenoic recovery
+seed phrase is stored at `/var/lib/joinmarket/jm-wallet-seed`.
+
+A missing wallet file is automatically recreated if the seed file is still present.
+
+If you want to manually initialize your wallet instead, follow these steps:
 
 1. Enable JoinMarket in your node configuration
 
@@ -301,7 +306,7 @@ to run it accross SSH sessions. You can also use tmux in the same fashion.
     screen -S "tumbler"
     ```
 
-2. Start the tumbler
+3. Start the tumbler
 
     Example: Tumbling into your wallet after buying from an exchange to improve privacy:
 
@@ -314,19 +319,19 @@ to run it accross SSH sessions. You can also use tmux in the same fashion.
 
     Get more information [here](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/tumblerguide.md)
 
-3. Detach the screen session to leave the tumbler running in the background
+4. Detach the screen session to leave the tumbler running in the background
 
     ```
     Ctrl-a d or Ctrl-a Ctrl-d
     ```
 
-4. Re-attach to the screen session
+5. Re-attach to the screen session
 
     ```console
     screen -r tumbler
     ```
 
-5. End screen session
+6. End screen session
 
     Type exit when tumbler is done
 

examples/configuration.nix

@@ -260,6 +260,6 @@
   # The nix-bitcoin release version that your config is compatible with.
   # When upgrading to a backwards-incompatible release, nix-bitcoin will display an
   # an error and provide hints for migrating your config to the new release.
-  nix-bitcoin.configVersion = "0.0.49";
+  nix-bitcoin.configVersion = "0.0.51";
 
 }

modules/joinmarket-ob-watcher.nix

@@ -5,7 +5,11 @@ let
   cfg = config.services.joinmarket-ob-watcher;
   nbLib = config.nix-bitcoin.lib;
   nbPkgs = config.nix-bitcoin.pkgs;
+  secretsDir = config.nix-bitcoin.secretsDir;
 
+  inherit (config.services) bitcoind;
+
+  torAddress = config.services.tor.client.socksListenAddress;
   socks5Settings = with config.services.tor.client.socksListenAddress; ''
     socks5 = true
     socks5_host = ${addr}
@@ -14,7 +18,11 @@ let
 
   configFile = builtins.toFile "config" ''
     [BLOCKCHAIN]
-    blockchain_source = no-blockchain
+    blockchain_source = bitcoin-rpc
+    network = ${bitcoind.network}
+    rpc_host = ${bitcoind.rpc.address}
+    rpc_port = ${toString bitcoind.rpc.port}
+    rpc_user = ${bitcoind.rpc.users.joinmarket-ob-watcher.name}
 
     [MESSAGING:server1]
     host = darkirc6tqgpnwd3blln3yfv5ckl47eg7llfxkmtovrv7c7iwohhb6ad.onion
@@ -48,6 +56,16 @@ in {
       default = "/var/lib/joinmarket-ob-watcher";
       description = "The data directory for JoinMarket orderbook watcher.";
     };
+    user = mkOption {
+      type = types.str;
+      default = "joinmarket-ob-watcher";
+      description = "The user as which to run JoinMarket.";
+    };
+    group = mkOption {
+      type = types.str;
+      default = cfg.user;
+      description = "The group as which to run JoinMarket.";
+    };
     # This option is only used by netns-isolation
     enforceTor = mkOption {
       readOnly = true;
@@ -56,12 +74,23 @@ in {
   };
 
   config = mkIf cfg.enable {
+    services.bitcoind.rpc.users.joinmarket-ob-watcher = {
+      passwordHMACFromFile = true;
+      rpcwhitelist = bitcoind.rpc.users.public.rpcwhitelist ++ [
+        "listwallets"
+      ];
+    };
+
     # Joinmarket is Tor-only
     services.tor = {
       enable = true;
       client.enable = true;
     };
 
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
+    ];
+
     systemd.services.joinmarket-ob-watcher = {
       wantedBy = [ "multi-user.target" ];
       requires = [ "tor.service" ];
@@ -69,15 +98,20 @@ in {
       # The service writes to HOME/.config/matplotlib
       environment.HOME = cfg.dataDir;
       preStart = ''
-        ln -snf ${configFile} ${cfg.dataDir}/joinmarket.cfg
+        {
+          cat ${configFile}
+          echo
+          echo '[BLOCKCHAIN]'
+          echo "rpc_password = $(cat ${secretsDir}/bitcoin-rpcpassword-joinmarket-ob-watcher)"
+        } > '${cfg.dataDir}/joinmarket.cfg'
       '';
       serviceConfig = nbLib.defaultHardening // rec {
-        DynamicUser = true;
         StateDirectory = "joinmarket-ob-watcher";
         StateDirectoryMode = "770";
         WorkingDirectory = cfg.dataDir; # The service creates dir 'logs' in the working dir
+        User = cfg.user;
         ExecStart = ''
-          ${nbPkgs.joinmarket}/bin/ob-watcher --datadir=${cfg.dataDir} \
+          ${nbPkgs.joinmarket}/bin/jm-ob-watcher --datadir=${cfg.dataDir} \
             --host=${cfg.address} --port=${toString cfg.port}
         '';
         SystemCallFilter = nbLib.defaultHardening.SystemCallFilter ++ [ "mbind" ] ;
@@ -85,5 +119,17 @@ in {
         RestartSec = "10s";
       } // nbLib.allowTor;
     };
+
+    users.users.${cfg.user} = {
+      isSystemUser = true;
+      group = cfg.group;
+      home = cfg.dataDir;
+    };
+    users.groups.${cfg.group} = {};
+
+    nix-bitcoin.secrets = {
+      bitcoin-rpcpassword-joinmarket-ob-watcher.user = cfg.user;
+      bitcoin-HMAC-joinmarket-ob-watcher.user = bitcoind.user;
+    };
   };
 }

modules/joinmarket.nix

@@ -33,7 +33,6 @@ let
     rpc_host = ${bitcoind.rpc.address}
     rpc_port = ${toString bitcoind.rpc.port}
     rpc_user = ${bitcoind.rpc.users.privileged.name}
-    @@RPC_PASSWORD@@
     ${optionalString (cfg.rpcWalletFile != null) "rpc_wallet_file = ${cfg.rpcWalletFile}"}
 
     [MESSAGING:server1]
@@ -64,6 +63,8 @@ let
     tx_broadcast = self
     minimum_makers = 4
     max_sats_freeze_reuse = -1
+    interest_rate = """ + _DEFAULT_INTEREST_RATE + """
+    bondless_makers_allowance = """ + _DEFAULT_BONDLESS_MAKERS_ALLOWANCE + """
     taker_utxo_retries = 3
     taker_utxo_age = 5
     taker_utxo_amtpercent = 20
@@ -235,10 +236,12 @@ in {
       requires = [ "bitcoind.service" ];
       after = [ "bitcoind.service" ];
       preStart = ''
-        install -o '${cfg.user}' -g '${cfg.group}' -m 640 ${configFile} ${cfg.dataDir}/joinmarket.cfg
+        {
-          sed -i \
+          cat ${configFile}
-            "s|@@RPC_PASSWORD@@|rpc_password = $(cat ${secretsDir}/bitcoin-rpcpassword-privileged)|" \
+          echo
-            '${cfg.dataDir}/joinmarket.cfg'
+          echo '[BLOCKCHAIN]'
+          echo "rpc_password = $(cat ${secretsDir}/bitcoin-rpcpassword-privileged)"
+        } > '${cfg.dataDir}/joinmarket.cfg'
       '';
       # Generating wallets (jmclient/wallet.py) is only supported for mainnet or testnet
       postStart = mkIf (bitcoind.network == "mainnet") ''
@@ -247,15 +250,31 @@ in {
         if [[ ! -f $wallet ]]; then
           ${optionalString (cfg.rpcWalletFile != null) ''
             echo "Create watch-only wallet ${cfg.rpcWalletFile}"
-            ${bitcoind.cli}/bin/bitcoin-cli -named createwallet \
+            if ! output=$(${bitcoind.cli}/bin/bitcoin-cli -named createwallet \
-              wallet_name="${cfg.rpcWalletFile}" disable_private_keys=true
+                          wallet_name="${cfg.rpcWalletFile}" disable_private_keys=true 2>&1); then
+              # Ignore error if bitcoind wallet already exists
+              if [[ $output != *"already exists"* ]]; then
+                echo "$output"
+                exit 1
+              fi
+            fi
           ''}
-          pw=$(cat "${secretsDir}"/jm-wallet-password)
+          # Restore wallet from seed if available
+          seed=
+          if [[ -e jm-wallet-seed ]]; then
+            seed="--recovery-seed-file jm-wallet-seed"
+          fi
           cd ${cfg.dataDir}
-          if ! ${nbPkgs.joinmarket}/bin/jm-genwallet --datadir=${cfg.dataDir} $walletname $pw \
+          # Strip trailing newline from password file
-                 | grep 'recovery_seed' \
+          if ! tr -d "\n" <"${secretsDir}/jm-wallet-password" \
-                 | cut -d ':' -f2 \
+               | ${nbPkgs.joinmarket}/bin/jm-genwallet \
-                 | (umask u=r,go=; cat > jm-wallet-seed); then
+                   --datadir=${cfg.dataDir} --wallet-password-stdin $seed $walletname \
+               | (if [[ ! $seed ]]; then
+                    umask u=r,go=
+                    grep -ohP '(?<=recovery_seed:).*' > jm-wallet-seed
+                  else
+                    cat > /dev/null
+                  fi); then
             echo "wallet creation failed"
             rm -f "$wallet" jm-wallet-seed
             exit 1
@@ -293,21 +312,15 @@ in {
       wantedBy = [ "joinmarket.service" ];
       requires = [ "joinmarket.service" ];
       after = [ "joinmarket.service" ];
-      preStart = let
+      script = ''
-        start = ''
+        tr -d "\n" <"${secretsDir}/jm-wallet-password" \
-          exec ${nbPkgs.joinmarket}/bin/jm-yg-privacyenhanced --datadir='${cfg.dataDir}' --wallet-password-stdin wallet.jmdat
+        | ${nbPkgs.joinmarket}/bin/jm-yg-privacyenhanced --datadir='${cfg.dataDir}' \
-        '';
+          --wallet-password-stdin wallet.jmdat
-      in ''
-        pw=$(cat "${secretsDir}"/jm-wallet-password)
-        echo "echo -n $pw | ${start}" > $RUNTIME_DIRECTORY/start
       '';
       serviceConfig = nbLib.defaultHardening // rec {
-        RuntimeDirectory = "joinmarket-yieldgenerator"; # Only used to create start script
-        RuntimeDirectoryMode = "700";
         WorkingDirectory = cfg.dataDir; # The service creates dir 'logs' in the working dir
-        ExecStart = "${pkgs.bash}/bin/bash /run/${RuntimeDirectory}/start";
         # Show "joinmarket-yieldgenerator" instead of "bash" in the journal.
-        # The parent bash start process has to run alongside the main process
+        # The start script has to run alongside the main process
         # because it provides the wallet password via stdin to the main process
         SyslogIdentifier = "joinmarket-yieldgenerator";
         User = cfg.user;

modules/netns-isolation.nix

@@ -251,6 +251,7 @@ in {
       };
       joinmarket-ob-watcher = {
         id = 26;
+        connections = [ "bitcoind" ];
       };
       lightning-pool = {
         id = 27;

modules/versioning.nix

@@ -10,19 +10,7 @@ let
   version = config.nix-bitcoin.configVersion;
 
   # Sorted by increasing version numbers
-  changes = let
+  changes = [
-    mkOnionServiceChange = service: {
-      version = "0.0.30";
-      condition = config.services.${service}.enable;
-      message = ''
-        The onion service for ${service} has been disabled in the default
-        configuration (`secure-node.nix`).
-
-        To enable the onion service, add the following to your configuration:
-        nix-bitcon.onionServices.${service}.enable = true;
-      '';
-    };
-  in [
     {
       version = "0.0.26";
       condition = config.services.joinmarket.enable;
@@ -112,8 +100,43 @@ let
         [1] https://github.com/bitcoin/bitcoin/pull/15454
       '';
     }
+    {
+      version = "0.0.51";
+      condition = config.services.joinmarket.enable;
+      message = let
+        jmDataDir = config.services.joinmarket.dataDir;
+      in ''
+        Joinmarket 0.9.1 has added support for Fidelity Bonds [1].
+
+        If you've used joinmarket before, do the following to enable Fidelity Bonds in your existing wallet.
+        Enabling Fidelity Bonds has no effect if you don't use them.
+
+        1. Deploy the new system config to your node
+        2. Run the following on your node:
+           # Ensure that the wallet seed exists and rename the wallet
+           ls ${jmDataDir}/jm-wallet-seed && mv ${jmDataDir}/wallets/wallet.jmdat{,.bak}
+           # This automatically recreates the wallet with Fidelity Bonds support
+           systemctl restart joinmarket
+           # Remove wallet backup if update was successful
+           rm ${jmDataDir}/wallets/wallet.jmdat.bak
+
+        [1] https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/fidelity-bonds.md
+      '';
+    }
   ];
 
+  mkOnionServiceChange = service: {
+    version = "0.0.30";
+    condition = config.services.${service}.enable;
+    message = ''
+        The onion service for ${service} has been disabled in the default
+        configuration (`secure-node.nix`).
+
+        To enable the onion service, add the following to your configuration:
+        nix-bitcon.onionServices.${service}.enable = true;
+      '';
+  };
+
   incompatibleChanges = optionals
     (version != null && versionOlder lastChange)
     (builtins.filter (change: versionOlder change && (change.condition or true)) changes);

pkgs/generate-secrets/generate-secrets.sh

@@ -15,6 +15,7 @@ makeHMAC() {
 
 makePasswordSecret bitcoin-rpcpassword-privileged
 makePasswordSecret bitcoin-rpcpassword-btcpayserver
+makePasswordSecret bitcoin-rpcpassword-joinmarket-ob-watcher
 makePasswordSecret bitcoin-rpcpassword-public
 makePasswordSecret lnd-wallet-password
 makePasswordSecret liquid-rpcpassword
@@ -25,6 +26,7 @@ makePasswordSecret jm-wallet-password
 [[ -e bitcoin-HMAC-privileged ]] || makeHMAC privileged
 [[ -e bitcoin-HMAC-public ]] || makeHMAC public
 [[ -e bitcoin-HMAC-btcpayserver ]] || makeHMAC btcpayserver
+[[ -e bitcoin-HMAC-joinmarket-ob-watcher ]] || makeHMAC joinmarket-ob-watcher
 [[ -e spark-wallet-login   ]] || echo "login=spark-wallet:$(cat spark-wallet-password)" > spark-wallet-login
 [[ -e backup-encryption-env ]] || echo "PASSPHRASE=$(cat backup-encryption-password)" > backup-encryption-env
 

pkgs/joinmarket/default.nix

@@ -1,10 +1,20 @@
-{ stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }:
+{ stdenv, lib, fetchurl, applyPatches, fetchpatch, python3, nbPython3Packages, pkgs }:
 
 let
-  version = "0.8.3";
+  version = "0.9.1";
+  src = applyPatches {
     src = fetchurl {
       url = "https://github.com/JoinMarket-Org/joinmarket-clientserver/archive/v${version}.tar.gz";
-    sha256 = "0kcgp8lsgnbaxfv13lrg6x7vcbdi5yj526lq9vmvbbidyw4km3r2";
+      sha256 = "0a8jlzi3ll1dw60fwnqs5awmcfxdjynh6i1gfmcc29qhwjpx5djl";
+    };
+    patches = [
+      (fetchpatch {
+        # https://github.com/JoinMarket-Org/joinmarket-clientserver/pull/999
+        name = "improve-genwallet";
+        url = "https://patch-diff.githubusercontent.com/raw/JoinMarket-Org/joinmarket-clientserver/pull/999.patch";
+        sha256 = "08x2i1q8qsn5rxmfmmj4i8s1d2yc862i152riw3d8zwz7x2cq40h";
+      })
+    ];
   };
 
   runtimePackages = with nbPython3Packages; [
@@ -32,7 +42,6 @@ stdenv.mkDerivation {
     }
 
     cp scripts/joinmarketd.py $out/bin/joinmarketd
-    cp scripts/obwatch/ob-watcher.py $out/bin/ob-watcher
     cpBin add-utxo.py
     cpBin convert_old_wallet.py
     cpBin receive-payjoin.py
@@ -46,8 +55,13 @@ stdenv.mkDerivation {
     chmod +x -R $out/bin
     patchShebangs $out/bin
 
+    ## ob-watcher
+    obw=$out/libexec/joinmarket-ob-watcher
+    install -D scripts/obwatch/ob-watcher.py $obw/ob-watcher
+    patchShebangs $obw/ob-watcher
+    ln -s $obw/ob-watcher $out/bin/jm-ob-watcher
+
     # These files must be placed in the same dir as ob-watcher
-    cp scripts/obwatch/orderbook.html $out/bin/orderbook.html
+    cp -r scripts/obwatch/{orderbook.html,sybil_attack_calculations.py,vendor} $obw
-    cp -r scripts/obwatch/vendor $out/bin/vendor
   '';
 }

pkgs/python-packages/jmclient/default.nix

@@ -1,4 +1,4 @@
-{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon }:
+{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon }:
 
 buildPythonPackage rec {
   pname = "joinmarketclient";
@@ -6,7 +6,7 @@ buildPythonPackage rec {
 
   postUnpack = "sourceRoot=$sourceRoot/jmclient";
 
-  checkInputs = [ joinmarketbitcoin txtorcon ];
+  checkInputs = [ joinmarketbitcoin joinmarketdaemon txtorcon ];
 
   # configparser may need to be compiled with python_version<"3.2"
   propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes ];

test/tests.py

@@ -221,8 +221,9 @@ def _():
 
 @test("joinmarket-ob-watcher")
 def _():
-    assert_running("joinmarket-ob-watcher")
+    # joinmarket-ob-watcher fails on non-synced mainnet nodes.
-    machine.wait_until_succeeds(log_has_string("joinmarket-ob-watcher", "Starting ob-watcher"))
+    # Also, it doesn't support any of the test networks.
+    machine.wait_until_succeeds(log_has_string("joinmarket-ob-watcher", "unknown error in JSON-RPC"))
 
 @test("nodeinfo")
 def _():