Merge #237: BTCPayServer from upstream & postgresqlBackup

73f4275d2a backups: add btcpayserver database (nixbitcoin)
0784e2d479 Revert "temp: mirror erikarvstedt btcpayserver" (nixbitcoin)
7764f36405 pkgs: update pinned nixpkgs (nixbitcoin)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK 73f4275d2a

Tree-SHA512: 6cd9ed61139918c762cfd66cfdc6f92d3d4d173fa29f8a7244b5f613a7982c5d48e10f0f0bce374cb40a51a84b4765ef8afb1b74fde5166967709d973107aa52

.travis.yml

@@ -27,10 +27,6 @@ env:
   - PKG=elementsd STABLE=0
   - PKG=electrs STABLE=1
   - PKG=electrs STABLE=0
-  - PKG=nbxplorer STABLE=1
-  - PKG=nbxplorer STABLE=0
-  - PKG=btcpayserver STABLE=1
-  - PKG=btcpayserver STABLE=0
   - PKG=liquid-swap STABLE=1
   - PKG=lightning-loop STABLE=0
   - PKG=nixops19_09 STABLE=1

modules/backups.nix

@@ -20,6 +20,7 @@ let
     ${config.services.btcpayserver.dataDir}
     ${config.services.joinmarket.dataDir}
     /secrets/jm-wallet-seed
+    ${config.services.postgresqlBackup.location}/btcpaydb.sql.gz
     /var/lib/tor
     # Extra files
     ${cfg.extraFiles}
@@ -69,8 +70,8 @@ in {
     };
   };
 
-  config = mkMerge [
+  config = mkIf (cfg.enable && cfg.program == "duplicity") (mkMerge [
-    (mkIf (cfg.enable && cfg.program == "duplicity") {
+    {
       environment.systemPackages = [ pkgs.duplicity ];
 
       services.duplicity = {
@@ -85,7 +86,17 @@ in {
       };
 
       nix-bitcoin.secrets.backup-encryption-env.user = "root";
-
+    }
+    (mkIf config.services.btcpayserver.enable {
+      services.postgresqlBackup = {
+        enable = true;
+        databases = [ "btcpaydb" ];
+        startAt = [];
+      };
+      systemd.services.duplicity = rec {
+        wants = [ "postgresqlBackup-btcpaydb.service" ];
+        after = wants;
+      };
     })
-  ];
+  ]);
 }

pkgs/btcpayserver/default.nix

@@ -1,66 +0,0 @@
-{ lib, stdenv, fetchFromGitHub, fetchurl, linkFarmFromDrvs, makeWrapper,
-  dotnetPackages, dotnetCorePackages, writeScript, bash
-}:
-
-let
-  deps = import ./deps.nix {
-    fetchNuGet = { name, version, sha256 }: fetchurl {
-      name = "nuget-${name}-${version}.nupkg";
-      url = "https://www.nuget.org/api/v2/package/${name}/${version}";
-      inherit sha256;
-    };
-  };
-  dotnetSdk = dotnetCorePackages.sdk_3_1;
-in
-
-stdenv.mkDerivation rec {
-  pname = "btcpayserver";
-  version = "1.0.5.5";
-
-  src = fetchFromGitHub {
-    owner = pname;
-    repo = pname;
-    rev = "v${version}";
-    sha256 = "11h1nrmb7f44msbhhiz9ddqh5ss2kz6d8ysnvd070x3xj5krgnxz";
-  };
-
-  nativeBuildInputs = [ dotnetSdk dotnetPackages.Nuget ];
-
-  # Due to a bug in btcpayserver, we can't just `dotnet publish` to create a binary.
-  # Build with `dotnet build` instead and add a custom `dotnet run` script.
-  buildPhase = ''
-    export HOME=$TMP/home
-    export DOTNET_CLI_TELEMETRY_OPTOUT=1
-    export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
-
-    nuget sources Add -Name tmpsrc -Source $TMP/nuget
-    nuget init ${linkFarmFromDrvs "deps" deps} $TMP/nuget
-
-    dotnet restore --source $TMP/nuget BTCPayServer/BTCPayServer.csproj
-    dotnet build -c Release BTCPayServer/BTCPayServer.csproj
-  '';
-
-  runScript =  ''
-    #!${bash}/bin/bash
-    DOTNET_CLI_TELEMETRY_OPTOUT=1 exec ${dotnetSdk}/bin/dotnet run --no-launch-profile --no-build \
-      -c Release -p @@SHARE@@/BTCPayServer/BTCPayServer.csproj -- "$@"
-  '';
-
-  installPhase = ''
-    cd ..
-    share=$out/share/$pname
-    mkdir -p $share
-    mv -T source $share
-    install -D -m500 <(echo "$runScript" | sed "s|@@SHARE@@|$share|") $out/bin/$pname
-  '';
-
-  dontStrip = true;
-
-  meta = with lib; {
-    description = "Self-hosted, open-source cryptocurrency payment processor";
-    homepage = "https://btcpayserver.org";
-    maintainers = with maintainers; [ kcalvinalvin earvstedt ];
-    license = lib.licenses.mit;
-    platforms = lib.platforms.linux;
-  };
-}

pkgs/btcpayserver/update.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
-
-"$scriptDir"/../nbxplorer/util/update-common.sh btcpayserver "$scriptDir"/deps.nix

pkgs/default.nix

@@ -1,5 +1,5 @@
 { pkgs ? import <nixpkgs> {} }:
-let self = {
+{
   lightning-charge = pkgs.callPackage ./lightning-charge { };
   nanopos = pkgs.callPackage ./nanopos { };
   spark-wallet = pkgs.callPackage ./spark-wallet { };
@@ -13,13 +13,8 @@ let self = {
   nixops19_09 = pkgs.callPackage ./nixops { };
   netns-exec = pkgs.callPackage ./netns-exec { };
   lightning-loop = pkgs.callPackage ./lightning-loop { };
-  btcpayserver = pkgs.callPackage ./btcpayserver { inherit (self) linkFarmFromDrvs; };
-  nbxplorer = pkgs.callPackage ./nbxplorer { inherit (self) linkFarmFromDrvs; };
-
-  # Temporary backport for btcpayserver
-  linkFarmFromDrvs = pkgs.linkFarmFromDrvs or self.pinned.nixpkgsUnstable.linkFarmFromDrvs;
 
   pinned = import ./pinned.nix;
 
   lib = import ./lib.nix { inherit (pkgs) lib; };
-}; in self
+}

pkgs/nbxplorer/default.nix

@@ -1,54 +0,0 @@
-{ lib, stdenv, fetchFromGitHub, fetchurl, linkFarmFromDrvs, makeWrapper,
-  dotnetPackages, dotnetCorePackages
-}:
-
-let
-  deps = import ./deps.nix {
-    fetchNuGet = { name, version, sha256 }: fetchurl {
-      name = "nuget-${name}-${version}.nupkg";
-      url = "https://www.nuget.org/api/v2/package/${name}/${version}";
-      inherit sha256;
-    };
-  };
-  dotnetSdk = dotnetCorePackages.sdk_3_1;
-in
-
-stdenv.mkDerivation rec {
-  pname = "nbxplorer";
-  version = "2.1.42";
-
-  src = fetchFromGitHub {
-    owner = "dgarage";
-    repo = "NBXplorer";
-    rev = "v${version}";
-    sha256 = "01q6n7095rrha00xs3l7igzfb9rd743z8crxa2dcz4q5srapfzpi";
-  };
-
-  nativeBuildInputs = [ dotnetSdk dotnetPackages.Nuget makeWrapper ];
-
-  buildPhase = ''
-    export HOME=$TMP/home
-    export DOTNET_CLI_TELEMETRY_OPTOUT=1
-    export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
-
-    nuget sources Add -Name tmpsrc -Source $TMP/nuget
-    nuget init ${linkFarmFromDrvs "deps" deps} $TMP/nuget
-
-    dotnet restore --source $TMP/nuget NBXplorer/NBXplorer.csproj
-    dotnet publish --no-restore --output $out/share/$pname -c Release NBXplorer/NBXplorer.csproj
-  '';
-
-  installPhase = ''
-    makeWrapper $out/share/$pname/NBXplorer $out/bin/$pname \
-      --set DOTNET_ROOT "${dotnetSdk}"
-  '';
-
-  dontStrip = true;
-
-  meta = with lib; {
-    description = "Minimalist UTXO tracker for HD Cryptocurrency Wallets";
-    maintainers = with maintainers; [ kcalvinalvin earvstedt ];
-    license = lib.licenses.mit;
-    platforms = lib.platforms.linux;
-  };
-}

pkgs/nbxplorer/update.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
-
-getVersionFromTags=1 "$scriptDir"/util/update-common.sh nbxplorer "$scriptDir"/deps.nix

pkgs/nbxplorer/util/create-deps.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p dotnet-sdk_3
-set -euo pipefail
-
-# Writes deps for dotnet package in $pkgSrc to $depsFile.
-# Expects $pkgSrc to contain a single .sln file.
-
-pkgSrc=$1
-depsFile=$2
-
-sln=$(cd "$pkgSrc"; find * -maxdepth 0 -name '*.sln' | head -1)
-[[ $sln ]] || { echo "No .sln file in $pkgSrc" ; exit 1; }
-
-tmpdir=$(mktemp -d /tmp/$pkgName-src.XXX)
-trap "rm -rf $tmpdir" EXIT
-echo "Using tmp dir: $tmpdir"
-cp -rT "$pkgSrc" "$tmpdir"
-chmod -R +w "$tmpdir"
-
-pushd "$tmpdir" > /dev/null
-mkdir home
-echo "Running dotnet restore for $sln"
-HOME=home DOTNET_CLI_TELEMETRY_OPTOUT=1 \
-  dotnet restore -v normal --no-cache "$sln" > restore_log
-
-echo "{ fetchNuGet }: [" > "$depsFile"
-while read pkgSpec; do
-  { read name; read version; } < <(
-    # Ignore build version part: 1.0.0-beta2+77df2220 -> 1.0.0-beta2
-    sed -nE 's/.*<id>([^<]*).*/\1/p; s/.*<version>([^<+]*).*/\1/p' "$pkgSpec"
-  )
-  sha256=$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkgSpec")"/*.nupkg)
-  cat >> "$depsFile" <<EOF
-  (fetchNuGet {
-    name = "$name";
-    version = "$version";
-    sha256 = "$sha256";
-  })
-EOF
-done < <(find home/.nuget/packages -name '*.nuspec' | LC_ALL=C sort)
-echo "]" >> "$depsFile"
-
-echo "Created $depsFile"
-
-popd > /dev/null

pkgs/nbxplorer/util/update-common.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p coreutils curl jq common-updater-scripts dotnet-sdk_3
-set -euo pipefail
-
-# This script uses the following env vars:
-# getVersionFromTags
-# onlyCreateDeps
-
-pkgName=$1
-depsFile=$2
-
-: ${getVersionFromTags:=}
-: ${onlyCreateDeps:=}
-
-scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
-nbPkgs=$(realpath "$scriptDir"/../..)
-
-evalNbPkgs() {
-  nix eval --raw "(with import \"$nbPkgs\" {}; $1)"
-}
-
-getRepo() {
-  url=$(evalNbPkgs $pkgName.src.meta.homepage)
-  echo $(basename $(dirname $url))/$(basename $url)
-}
-
-getLatestVersionTag() {
-  unstable=$(nix eval --raw "(import \"$nbPkgs/nixpkgs-pinned.nix\").nixpkgs-unstable")
-  $unstable/pkgs/common-updater/scripts/list-git-tags https://github.com/$(getRepo) 2>/dev/null \
-    | sort -V | tail -1 | sed 's|^v||'
-}
-
-if [[ ! $onlyCreateDeps ]]; then
-  oldVersion=$(evalNbPkgs "$pkgName.version")
-  if [[ $getVersionFromTags ]]; then
-    newVersion=$(getLatestVersionTag)
-  else
-    newVersion=$(curl -s "https://api.github.com/repos/$(getRepo)/releases" | jq -r '.[0].name')
-  fi
-
-  if [[ $newVersion == $oldVersion ]]; then
-    echo "$pkgName is up to date: $newVersion"
-  else
-    echo "Please manually update $pkgName: $oldVersion -> $newVersion"
-  fi
-fi
-
-echo "Creating deps.nix"
-storeSrc="$(nix-build "$nbPkgs" -A $pkgName.src --no-out-link)"
-. "$scriptDir"/create-deps.sh "$storeSrc" "$depsFile"

pkgs/nixpkgs-pinned.nix

@@ -8,11 +8,11 @@ in
 {
   # To update, run ../helper/fetch-channel REV
   nixpkgs = fetch {
-    rev = "42674051d12540d4a996504990c6ea3619505953";
+    rev = "21d8e70a69f704a6ab971b2d8265d40cc7bb69b1";
-    sha256 = "1hz1n1hghilgzk4zlya498xm5lvhsf0r5b49yii7q86h3616fhwy";
+    sha256 = "0d6ym23bzx8c4ani7lp3k9qmbv7j9bf15vfmiff0f5lbz326bdgi";
   };
   nixpkgs-unstable = fetch {
-    rev = "a31736120c5de6e632f5a0ba1ed34e53fc1c1b00";
+    rev = "1179840f9a88b8a548f4b11d1a03aa25a790c379";
-    sha256 = "0xfjizw6w84w1fj47hxzw2vwgjlszzmsjb8k8cgqhb379vmkxjfl";
+    sha256 = "00jy37wj04bvh299xgal2iik2my9l0nq6cw50r1b2kdfrji8d563";
   };
 }

pkgs/pinned.nix

@@ -10,12 +10,13 @@ in
     bitcoin
     bitcoind
     clightning
-    lnd;
+    lnd
+    nbxplorer
+    btcpayserver;
   inherit (nixBitcoinPkgsUnstable)
     electrs
     lightning-loop;
 
   stable = nixBitcoinPkgsStable;
   unstable = nixBitcoinPkgsUnstable;
-  inherit nixpkgsUnstable;
 }

test/base.py

@@ -166,6 +166,10 @@ def run_tests(extra_tests):
         "export $(cat /secrets/backup-encryption-env); duplicity list-current-files 'file:///var/lib/localBackups'",
         "var/lib/bitcoind/wallet.dat",
     )
+    assert_matches(
+        "export $(cat /secrets/backup-encryption-env); duplicity list-current-files 'file:///var/lib/localBackups'",
+        "var/backup/postgresql/btcpaydb.sql.gz",
+    )
 
     ### Check that all extra_tests have been run
     assert len(extra_tests) == 0

test/test.nix

@@ -19,6 +19,9 @@ import ./make-test.nix rec {
       # hardened
     ];
 
+    # needed because duplicity requires 270 MB of free temp space, regardless of backup size.
+    virtualisation.diskSize = 1024;
+
     nix-bitcoin.netns-isolation.enable = (scenario == "withnetns");
 
     services.bitcoind.extraConfig = mkForce "connect=0";