Merge #276: Use Cirrus CI

1c0233c0a8 use Cirrus CI (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 1c0233c0a8

Tree-SHA512: 504c3a7808b6138e97ad35d1c3fa0f8666eeff1b09d35d914745cc51f45af183fc9baa08ff4702335e2a67e0bb09164902d9f7357beb12c009ed2e6f9629dbf6

.cirrus.yml

@@ -0,0 +1,42 @@
+container:
+  image: nixos/nix
+  # Besides virtualization, this also enables privileged containers which are required for
+  # sandboxed builds
+  kvm: true
+  # Needed for package builds
+  memory: 8G
+
+environment:
+  CACHIX_SIGNING_KEY: ENCRYPTED[!cec502ed813cbcd0237697d2031f750186ff20eed5815b1ad950ad2f2d701702ae6ba2f0cb4cb1985687a696c8ee492c!]
+  # Save some traffic by excluding the full git history
+  CIRRUS_CLONE_DEPTH: 1
+
+task:
+  # Use the maximum timeout. Needed when rebuilding packages on a channel update.
+  timeout_in: 120m
+
+  matrix:
+    - name: modules_test
+      environment:
+        nixpkgs: nixpkgs
+      container:
+        # A maximum of 16 CPUs is shared among all concurrent tasks.
+        # https://cirrus-ci.org/faq/#are-there-any-limits
+        cpu: 4
+      matrix:
+        - environment:
+            scenario: default
+        - environment:
+            scenario: netns
+        - environment:
+            scenario: netnsRegtest
+
+    - name: pkgs_unstable
+      environment:
+        nixpkgs: nixpkgs-unstable
+
+  # This script is run as root
+  build_script:
+    - echo "sandbox = true" >> /etc/nix/nix.conf
+    - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)"
+    - nix run -f '<nixpkgs>' bash cachix -c ./ci/build.sh

.travis.yml

@@ -1,74 +0,0 @@
-dist: bionic # needed for KVM
-language: shell
-
-install:
-  - curl -L https://nixos.org/nix/install | sh
-  - . /home/travis/.nix-profile/etc/profile.d/nix.sh
-  - if [[ $STABLE == 1 ]]; then export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)"; fi
-  - if [[ $STABLE == 0 ]]; then export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs-unstable)"; fi
-  - nix-env -iA cachix -f https://cachix.org/api/v1/install
-  - cachix use nix-bitcoin
-  - VER="$(nix eval nixpkgs.lib.version)"
-env:
-  global:
-    # CACHIX_SIGNING_KEY
-  - secure: "xXCFZ7g+k5YmCGm8R8l3bZElVmt+RD1KscG3kGr5w4HyyDPTzFetPo+sT8bUpysDU0u3HWhfVhHtpog2mhNhwVl3tQwKXea3dHKC1i6ypBg3gjDngmJRR5wo++ocYDpK8qPaU7m/jHQTNFnTA4CbmMcc05GcYx/1Ai/ZGkNwWFjdIcVeOUoiol33gykMOXIGDg2qlXudt33wP53FHbX8L4fxzodWfAuxKK4AoGprxy5eSnU7LCaXxxJmu4HwuV+Ux2U1NfE/E33cvhlUvTQCswVSZFG06mg8rwhMG1ozsDvlL2itZlu/BeUQH5y3XMMlnJIUXUazkRBibf1w/ebVjpOF+anqkqmq8tcbFEa7T+RJeVTIsvP+L8rE8fcmuZtdg9hNmgRnLmaeT0vVwD1L2UqW9HdRyujdoS0jPYuoc1W7f1JQWfAPhBPQ1SrtKyNNqcbVJ34aN7b+4vCzRpQL1JTbmjzQIWhkiKN1qMo1v/wbIydW8yka4hc4JOfdQLaAJEPI1eAC1MLotSAegMnwKWE1dzm66MuPSipksYjZrvsB28cV4aCVUffIuRhrSr1i2afRHwTpNbK9U4/576hah15ftUdR79Sfkcoi1ekSQTFGRvkRIPYtkKLYwFa3jVA41qz7+IIZCf4TsApy3XDdFx91cRub7yPq9BeZ83A+qYQ="
-  jobs:
-  - TestModules=1 STABLE=1 SCENARIO=default
-  - TestModules=1 STABLE=1 SCENARIO=netns
-  - TestModules=1 STABLE=1 SCENARIO=netnsRegtest
-  - PKG=hwi STABLE=1
-  - PKG=hwi STABLE=0
-  - PKG=lightning-charge STABLE=1
-  - PKG=lightning-charge STABLE=0
-  - PKG=nanopos STABLE=1
-  - PKG=nanopos STABLE=0
-  - PKG=spark-wallet STABLE=1
-  - PKG=elementsd STABLE=1
-  - PKG=elementsd STABLE=0
-  - PKG=electrs STABLE=1
-  - PKG=electrs STABLE=0
-  - PKG=liquid-swap STABLE=1
-  - PKG=lightning-loop STABLE=0
-  - PKG=nixops19_09 STABLE=1
-  - PKG=joinmarket STABLE=1
-  - PKG=joinmarket STABLE=0
-  - PKG=clightning-plugins-all STABLE=1
-script:
-  - printf '%s (%s)\n' "$NIX_PATH" "$VER"
-  - |
-    getBuildExpr() {
-        if [[ $TestModules ]]; then
-            if [[ ! -e /dev/kvm ]]; then
-                >&2 echo "No KVM available on VM Host."
-                exit 1
-            fi
-            sudo chmod go+rw /dev/kvm
-            test/run-tests.sh --scenario $SCENARIO exprForCI
-        else
-            echo "(import ./. {}).$PKG"
-        fi
-    }
-  - buildExpr=$(getBuildExpr)
-  - time nix-instantiate -E "$buildExpr" --add-root ./drv --indirect
-  - outPath=$(nix-store --query ./drv)
-  - |
-    if nix path-info --store https://nix-bitcoin.cachix.org $outPath &>/dev/null; then
-      echo "$outPath" has already been built successfully.
-      travis_terminate 0
-    fi
-    # Travis doesn't expose secrets to pull-request builds,
-    # so skip cache uploading in this case
-  - |
-    if [[ $CACHIX_SIGNING_KEY ]]; then
-      cachix push nix-bitcoin --watch-store &
-      cachixPid=$!
-    fi
-  - nix-build ./drv
-  - |
-    if [[ $CACHIX_SIGNING_KEY ]]; then
-      # Wait until cachix has finished uploading
-      # Run as root because yama/ptrace_scope != 0
-      ruby=$(nix-build '<nixpkgs>' -A ruby)/bin/ruby
-      time sudo $ruby helper/wait-for-network-idle.rb $cachixPid
-    fi

README.md

@@ -1,7 +1,7 @@
 nix-bitcoin
 ===
 
-[![Build Status](https://travis-ci.org/fort-nix/nix-bitcoin.svg?branch=master)](https://travis-ci.org/fort-nix/nix-bitcoin)
+[![Build Status](https://api.cirrus-ci.com/github/fort-nix/nix-bitcoin.svg?branch=master)](https://cirrus-ci.com/github/fort-nix/nix-bitcoin)
 
 Nix packages and nixos modules for easily installing Bitcoin nodes and higher layer protocols with an emphasis on security.
 This is a work in progress - don't expect it to be bug-free, secure or stable.

ci/build.nix

@@ -0,0 +1,14 @@
+let
+  pkgs = import <nixpkgs> {};
+  nbPkgs = import ../pkgs { inherit pkgs; };
+  ciPkgs = with nbPkgs; [
+    electrs
+    elementsd
+    hwi
+    joinmarket
+    lightning-charge
+    lightning-loop
+    nanopos
+  ];
+in
+pkgs.writeText "ci-pkgs" (pkgs.lib.concatMapStringsSep "\n" toString ciPkgs)

ci/build.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# This script can also be run locally for testing:
+#   scenario=default ./build.sh
+#
+# WARNING: This script fetches contents from an untrusted $cachixCache to your local nix-store.
+#
+# When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system.
+
+set -euo pipefail
+
+scenario=${scenario:-}
+CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-}
+cachixCache=nix-bitcoin
+
+trap 'echo Error at line $LINENO' ERR
+
+if [[ -v CIRRUS_CI ]]; then
+    tmpDir=/tmp
+    if [[ $scenario ]]; then
+        if [[ ! -e /dev/kvm ]]; then
+            >&2 echo "No KVM available on VM host."
+            exit 1
+        fi
+        # Enable KVM access for nixbld users
+        chmod o+rw /dev/kvm
+    fi
+else
+    atExit() {
+        rm -rf $tmpDir
+        if [[ -v cachixPid ]]; then kill $cachixPid; fi
+    }
+    tmpDir=$(mktemp -d -p /tmp)
+    trap atExit EXIT
+    # Prevent cachix from writing to HOME
+    export HOME=$tmpDir
+fi
+
+cachix use $cachixCache
+cd "${BASH_SOURCE[0]%/*}"
+
+## Build
+
+echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))"
+
+if [[ $scenario ]]; then
+    buildExpr=$(../test/run-tests.sh --scenario $scenario exprForCI)
+else
+    buildExpr="import ./build.nix"
+fi
+
+time nix-instantiate -E "$buildExpr" --add-root $tmpDir/drv --indirect > /dev/null
+printf "instantiated "; realpath $tmpDir/drv
+
+outPath=$(nix-store --query $tmpDir/drv)
+if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then
+    echo "$outPath" has already been built successfully.
+    exit 0
+fi
+
+# Cirrus doesn't expose secrets to pull-request builds,
+# so skip cache uploading in this case
+if [[ $CACHIX_SIGNING_KEY ]]; then
+    # Speed up task by uploading store paths as soon as they are created
+    cachix push $cachixCache --watch-store &
+    cachixPid=$!
+fi
+
+nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
+
+if [[ $CACHIX_SIGNING_KEY ]]; then
+    cachix push $cachixCache $outPath
+fi
+
+echo $outPath

helper/wait-for-network-idle.rb

@@ -1,29 +0,0 @@
-#!/usr/bin/env ruby
-require 'open3'
-
-# Wait until the given PID had no network activity for `Timeout` seconds, then exit.
-
-pid = ARGV.first
-Timeout = 2
-
-stdin, out, err, wait_thread = Open3.popen3("strace -f -e trace=network -s 1 -q -p #{pid}")
-while IO.select([err], nil, nil, Timeout)
-  begin
-    out = err.read_nonblock(1 << 10)
-  rescue EOFError
-    status = wait_thread.value
-    if status.success?
-      puts "Monitored process #{pid} exited"
-      exit 0
-    else
-      puts "Strace failed with exit code #{status.to_i}. Last output:\n#{out}"
-      # strace often fails with code 256 which looks like success to shells. fail with 1 instead.
-      exit 1
-    end
-  end
-end
-
-# If we exit without an explicit kill,
-# ptrace can fail on reattachment:  ptrace(PTRACE_SEIZE, $PID): Operation not permitted
-# Only relevant for testing.
-Process.kill("TERM", wait_thread.pid)

pkgs/default.nix

@@ -24,8 +24,4 @@ let self = {
   lib = import ./lib.nix { inherit (pkgs) lib; };
 
   modulesPkgs = self // self.pinned;
-
-  # Used in ../.travis.yml
-  clightning-plugins-all = pkgs.writeText "clightning-plugins"
-    (pkgs.lib.concatMapStringsSep "\n" toString (builtins.attrValues self.clightning-plugins));
 }; in self

test/run-tests.sh

@@ -141,7 +141,7 @@ buildTest() {
 # On continuous integration nodes there are few other processes running alongside the
 # test, so use more memory here for maximum performance.
 exprForCI() {
-    memoryMiB=3072
+    memoryMiB=4096
     memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
     memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
     # Round down to nearest multiple of 50 MiB for improved test build caching