Merge fort-nix/nix-bitcoin#410: joinmarket: 0.9.2 -> 0.9.3

d5ce1c43a8 test: make joinmarket work with regtest (nixbitcoin)
a10aa21c69 joinmarket: 0.9.2 -> 0.9.3 (nixbitcoin)
721ba1aeba python-packages: separate `specific-versions` pkgs (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK d5ce1c43a8
  jonasnick:
    light utACK d5ce1c43a8

Tree-SHA512: 5ddccbf9a88640086d14051283b59e704364d4d5f3f6aa6c698d88d8a6634ac9a7b525b11cf1670c9aaa6a797635bc23e135972d9bc8c909ec51b58fe57e8f5c

.cirrus.yml

@@ -31,9 +31,10 @@ task:
         - environment:
             scenario: netnsRegtest
 
-    - name: pkgs_unstable
-      environment:
-        nixpkgs: nixpkgs-unstable
+    # Disabled because this test is currently empty
+    # - name: pkgs_unstable
+    #   environment:
+    #     nixpkgs: nixpkgs-unstable
 
   # This script is run as root
   build_script:

modules/joinmarket.nix

@@ -4,6 +4,22 @@ with lib;
 let
   options.services.joinmarket = {
     enable = mkEnableOption "JoinMarket";
+    payjoinAddress = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = ''
+        The address where payjoin onion connections are forwarded to.
+        This address is never used directly, it only serves as the internal endpoint
+        for the payjoin onion service.
+        The onion service is automatically setup by joinmarket and accepts
+        connections at port 80.
+      '';
+    };
+    payjoinPort = mkOption {
+      type = types.port;
+      default = 64180; # A random private port
+      description = "The port corresponding to option `payjoinAddress`.";
+    };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/joinmarket";
@@ -78,7 +94,7 @@ let
           The average transaction fee you're adding to coinjoin transactions
         '';
       };
-      txfee_factor = mkOption {
+      txfee_contribution_factor = mkOption {
         type = types.float;
         default = 0.3;
         description = ''
@@ -153,8 +169,8 @@ let
     use_ssl = false
 
     [BLOCKCHAIN]
-    blockchain_source = bitcoin-rpc
-    network = ${bitcoind.network}
+    blockchain_source = ${bitcoind.makeNetworkName "bitcoin-rpc" "regtest"}
+    network = ${bitcoind.makeNetworkName "mainnet" "testnet"}
     rpc_host = ${nbLib.address bitcoind.rpc.address}
     rpc_port = ${toString bitcoind.rpc.port}
     rpc_user = ${bitcoind.rpc.users.privileged.name}
@@ -171,6 +187,7 @@ let
     native = true
     merge_algorithm = default
     tx_fees = 3
+    tx_fees_factor = 0.2
     absurd_fee_per_kb = 350000
     max_sweep_fee_change = 0.8
     tx_broadcast = self
@@ -192,6 +209,8 @@ let
     onion_socks5_host = ${torAddress.addr}
     onion_socks5_port = ${toString torAddress.port}
     tor_control_host = unix:/run/tor/control
+    onion_serving_host = ${cfg.payjoinAddress}
+    onion_serving_port = ${toString cfg.payjoinPort}
     hidden_service_ssl = false
 
     [YIELDGENERATOR]
@@ -199,8 +218,8 @@ let
     cjfee_a = ${toString yg.cjfee_a}
     cjfee_r = ${toString yg.cjfee_r}
     cjfee_factor = ${toString yg.cjfee_factor}
-    txfee = ${toString yg.txfee}
-    txfee_factor = ${toString yg.txfee_factor}
+    txfee_contribution = 0
+    txfee_contribution_factor = ${toString yg.txfee_contribution_factor}
     minsize = ${toString yg.minsize}
     size_factor = ${toString yg.size_factor}
     gaplimit = 6
@@ -263,15 +282,16 @@ in {
           echo "rpc_password = $(cat ${secretsDir}/bitcoin-rpcpassword-privileged)"
         } > '${cfg.dataDir}/joinmarket.cfg'
       '';
-      # Generating wallets (jmclient/wallet.py) is only supported for mainnet or testnet
-      postStart = mkIf (bitcoind.network == "mainnet") ''
+      postStart = ''
         walletname=wallet.jmdat
         wallet=${cfg.dataDir}/wallets/$walletname
         if [[ ! -f $wallet ]]; then
           ${optionalString (cfg.rpcWalletFile != null) ''
             echo "Create watch-only wallet ${cfg.rpcWalletFile}"
             if ! output=$(${bitcoind.cli}/bin/bitcoin-cli -named createwallet \
-                          wallet_name="${cfg.rpcWalletFile}" disable_private_keys=true 2>&1); then
+                            wallet_name="${cfg.rpcWalletFile}" \
+                            ${optionalString (!bitcoind.regtest) "disable_private_keys=true"} 2>&1
+                         ); then
               # Ignore error if bitcoind wallet already exists
               if [[ $output != *"already exists"* ]]; then
                 echo "$output"

modules/netns-isolation.nix

@@ -313,7 +313,10 @@ in {
     services.nbxplorer.address = netns.nbxplorer.address;
     services.btcpayserver.address = netns.btcpayserver.address;
 
-    services.joinmarket.cliExec = mkCliExec "joinmarket";
+    services.joinmarket = {
+      payjoinAddress = netns.joinmarket.address;
+      cliExec = mkCliExec "joinmarket";
+    };
     systemd.services.joinmarket-yieldgenerator.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-joinmarket";
 
     services.joinmarket-ob-watcher.address = netns.joinmarket-ob-watcher.address;

pkgs/joinmarket/default.nix

@@ -1,10 +1,10 @@
 { stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }:
 
 let
-  version = "0.9.2";
+  version = "0.9.3";
   src = fetchurl {
     url = "https://github.com/JoinMarket-Org/joinmarket-clientserver/archive/v${version}.tar.gz";
-    sha256 = "0nzhnf2324d0qc9pljcjwpzpvhcclqg0ijvzb1skmn73s4f25akg";
+    sha256 = "0j00jjqbppvcj52dpyjfqzwsm86xf9h2yf15j35ah5gsdr317dgq";
   };
 
   runtimePackages = with nbPython3Packages; [

pkgs/python-packages/default.nix

@@ -11,14 +11,6 @@ in {
   urldecode = callPackage ./urldecode {};
   chromalog = callPackage ./chromalog {};
   txzmq = callPackage ./txzmq {};
-  recommonmark = callPackage ./recommonmark { inherit (super) recommonmark; };
-
-  # cryptography 3.3.2, required by joinmarketdaemon
-  cryptography = callPackage ./cryptography {};
-  cryptography_vectors = callPackage ./cryptography/vectors.nix {};
-
-  # twisted 20.3.0, required by joinmarketbase
-  twisted = callPackage ./twisted {};
 
   joinmarketbase = joinmarketPkg ./jmbase;
   joinmarketclient = joinmarketPkg ./jmclient;
@@ -28,4 +20,25 @@ in {
   pyln-client = clightningPkg ./pyln-client;
   pyln-proto = clightningPkg ./pyln-proto;
   pylightning = clightningPkg ./pylightning;
+
+  ## Specific versions of packages that already exist in nixpkgs
+
+  # cryptography 3.3.2, required by joinmarketdaemon
+  cryptography = callPackage ./specific-versions/cryptography {};
+  cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {};
+
+  # twisted 20.3.0, required by joinmarketbase
+  twisted = callPackage ./specific-versions/twisted.nix {};
+
+  # autobahn 20.12.3, required by joinmarketclient
+  autobahn = callPackage ./specific-versions/autobahn.nix {};
+
+  # klein 20.6.0, required by joinmarketclient
+  klein = callPackage ./specific-versions/klein.nix {};
+
+  # tubes 0.2.0, required by klein
+  tubes = callPackage ./specific-versions/tubes.nix {};
+
+  # recommonmark 0.7.1, required by pyln-client
+  recommonmark = callPackage ./specific-versions/recommonmark.nix { inherit (super) recommonmark; };
 }

pkgs/python-packages/jmclient/default.nix

@@ -1,4 +1,4 @@
-{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon }:
+{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon, klein, pyjwt, autobahn, cryptography }:
 
 buildPythonPackage rec {
   pname = "joinmarketclient";
@@ -9,7 +9,7 @@ buildPythonPackage rec {
   checkInputs = [ joinmarketbitcoin joinmarketdaemon txtorcon ];
 
   # configparser may need to be compiled with python_version<"3.2"
-  propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes ];
+  propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes klein pyjwt autobahn cryptography ];
 
   meta = with lib; {
     description = "Client library for Bitcoin coinjoins";

pkgs/python-packages/specific-versions/autobahn.nix

@@ -0,0 +1,36 @@
+# Like nixpkgs revision 8d668463b0883b2e21ba2e2635cd5f9bbc409b18
+# but without Python 2 support
+
+{ lib, buildPythonPackage, fetchPypi,
+  six, txaio, twisted, zope_interface, cffi,
+  mock, pytest, cryptography, pynacl
+}:
+buildPythonPackage rec {
+  pname = "autobahn";
+  version = "20.12.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "15b8zm7jalwisfwc08szxy3bh2bnn0hd41dbsnswi0lqwbh962j1";
+  };
+
+  propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ];
+
+  checkInputs = [ mock pytest ];
+  checkPhase = ''
+    runHook preCheck
+    USE_TWISTED=true py.test $out
+    runHook postCheck
+  '';
+
+  # Tests do no seem to be compatible yet with pytest 5.1
+  # https://github.com/crossbario/autobahn-python/issues/1235
+  doCheck = false;
+
+  meta = with lib; {
+    description = "WebSocket and WAMP in Python for Twisted and asyncio.";
+    homepage    = "https://crossbar.io/autobahn";
+    license     = licenses.mit;
+    maintainers = with maintainers; [ nand0p ];
+  };
+}

pkgs/python-packages/specific-versions/klein.nix

@@ -0,0 +1,29 @@
+{ lib, buildPythonPackage, fetchPypi, python
+, attrs, enum34, hyperlink, incremental, six, twisted, typing, tubes, werkzeug, zope_interface
+, hypothesis, treq
+}:
+
+buildPythonPackage rec {
+  pname = "klein";
+  version = "20.6.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-ZYS5zf9JWbnc7pWhwcIAEPUhoqEsT/PN2LkDqbDpk/Y=";
+  };
+
+  propagatedBuildInputs = [ attrs enum34 hyperlink incremental six twisted typing tubes werkzeug zope_interface ];
+
+  checkInputs = [ hypothesis treq ];
+
+  checkPhase = ''
+    ${python.interpreter} -m twisted.trial -j $NIX_BUILD_CORES klein
+  '';
+
+  meta = with lib; {
+    description = "Klein Web Micro-Framework";
+    homepage    = "https://github.com/twisted/klein";
+    license     = licenses.mit;
+    maintainers = with maintainers; [ exarkun ];
+  };
+}

pkgs/python-packages/specific-versions/tubes.nix

@@ -0,0 +1,29 @@
+{ lib, buildPythonPackage, fetchPypi, python
+, characteristic, six, twisted
+}:
+
+buildPythonPackage rec {
+  pname = "tubes";
+  version = "0.2.0";
+
+  src = fetchPypi {
+    pname = "Tubes";
+    inherit version;
+    sha256 = "0sg1gg2002h1xsgxigznr1zk1skwmhss72dzk6iysb9k9kdgymcd";
+  };
+
+  propagatedBuildInputs = [ characteristic six twisted ];
+
+  checkPhase = ''
+    ${python.interpreter} -m twisted.trial -j $NIX_BUILD_CORES tubes
+  '';
+
+  pythonImportsCheck = [ "tubes" ];
+
+  meta = with lib; {
+    description = "a data-processing and flow-control engine for event-driven programs";
+    homepage    = "https://github.com/twisted/tubes";
+    license     = licenses.mit;
+    maintainers = with maintainers; [ exarkun ];
+  };
+}

test/pkgs-unstable.nix

@@ -3,7 +3,8 @@ let
   pkgs = import pinned.nixpkgs-unstable {};
   nbPkgs = import ../pkgs { inherit pkgs; };
   pkgsUnstable = with nbPkgs; [
-    joinmarket
+    # Disabled because joinmarket dependencies currently don't build on on unstable.
+    # joinmarket
   ];
 in
 pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable)

test/tests.nix

@@ -248,9 +248,6 @@ let
       services.lightning-pool.extraConfig = ''
         auctionserver=localhost
       '';
-
-      # Needs wallet support which is unavailable for regtest
-      services.joinmarket.yieldgenerator.enable = mkForce false;
     };
 
     ## Examples / debug helper

test/tests.py

@@ -218,9 +218,12 @@ def _():
 
 @test("joinmarket-yieldgenerator")
 def _():
-    machine.wait_until_succeeds(
-        log_has_string("joinmarket-yieldgenerator", "Critical error updating blockheight.")
-    )
+    if "regtest" in enabled_tests:
+        expected_log_msg = "You do not have the minimum required amount of coins to be a maker"
+    else:
+        expected_log_msg = "Critical error updating blockheight."
+
+    machine.wait_until_succeeds(log_has_string("joinmarket-yieldgenerator", expected_log_msg))
 
 @test("joinmarket-ob-watcher")
 def _():