Merge #282: Improve tests

9977fa69af ci: use run-tests.sh (Erik Arvstedt)
a82f0f5f48 add test 'pkgsUnstable' (Erik Arvstedt)
95bc1237e2 run-tests: rename testDir -> scriptDir (Erik Arvstedt)
466d23deaa ci: extract build-to-cachix.sh (Erik Arvstedt)
a70c3bf210 make-test-vm: remove unneeded leftover arg attrs (Erik Arvstedt)
ed65e78a2b make-test: expose test config (Erik Arvstedt)
7265742655 run-tests: add 'instantiate' command (Erik Arvstedt)
8cbdef8bf6 run-tests: fix CLI (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    Very nice! ACK 9977fa69af

Tree-SHA512: bb7f97096cc6e21f053c7db72a584a25ad62bca28af99e51fa83c15d2f75a198ada801428657821fc35f2cf01831176af8a9cd471e21dd0a7f5185f9d58efea1

ci/build-to-cachix.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Build a single-output derivation and store it in 'cachixCache'.
+# Skip the build if it is already cached.
+# Accepts the same arguments as nix-instantiate.
+
+set -euo pipefail
+
+CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-}
+cachixCache=nix-bitcoin
+
+trap 'echo Error at line $LINENO' ERR
+
+atExit() {
+    rm -rf $tmpDir
+    if [[ -v cachixPid ]]; then kill $cachixPid; fi
+}
+tmpDir=$(mktemp -d -p /tmp)
+trap atExit EXIT
+
+## Instantiate
+
+time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null
+printf "instantiated "; realpath $tmpDir/drv
+
+outPath=$(nix-store --query $tmpDir/drv)
+if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then
+    echo "$outPath has already been built successfully."
+    exit 0
+fi
+
+## Build
+
+if [[ -v CIRRUS_CI ]]; then
+    cachix use $cachixCache
+fi
+
+if [[ $CACHIX_SIGNING_KEY ]]; then
+    # Speed up task by uploading store paths as soon as they are created
+    cachix push $cachixCache --watch-store &
+    cachixPid=$!
+fi
+
+nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
+
+if [[ $CACHIX_SIGNING_KEY ]]; then
+    cachix push $cachixCache $outPath
+fi
+
+echo $outPath

ci/build.nix

@@ -1,12 +0,0 @@
-let
-  pkgs = import <nixpkgs> {};
-  nbPkgs = import ../pkgs { inherit pkgs; };
-  ciPkgs = with nbPkgs; [
-    electrs
-    elementsd
-    hwi
-    joinmarket
-    lightning-loop
-  ];
-in
-pkgs.writeText "ci-pkgs" (pkgs.lib.concatMapStringsSep "\n" toString ciPkgs)

ci/build.sh

@@ -3,73 +3,27 @@
 # This script can also be run locally for testing:
 #   scenario=default ./build.sh
 #
-# WARNING: This script fetches contents from an untrusted $cachixCache to your local nix-store.
-#
 # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system.
 
 set -euo pipefail
 
 scenario=${scenario:-}
-CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-}
-cachixCache=nix-bitcoin
 
-trap 'echo Error at line $LINENO' ERR
+if [[ -v CIRRUS_CI && $scenario ]]; then
-
+    if [[ ! -e /dev/kvm ]]; then
-if [[ -v CIRRUS_CI ]]; then
+        >&2 echo "No KVM available on VM host."
-    tmpDir=/tmp
+        exit 1
-    if [[ $scenario ]]; then
-        if [[ ! -e /dev/kvm ]]; then
-            >&2 echo "No KVM available on VM host."
-            exit 1
-        fi
-        # Enable KVM access for nixbld users
-        chmod o+rw /dev/kvm
     fi
-else
+    # Enable KVM access for nixbld users
-    atExit() {
+    chmod o+rw /dev/kvm
-        rm -rf $tmpDir
-        if [[ -v cachixPid ]]; then kill $cachixPid; fi
-    }
-    tmpDir=$(mktemp -d -p /tmp)
-    trap atExit EXIT
-    # Prevent cachix from writing to HOME
-    export HOME=$tmpDir
 fi
 
-cachix use $cachixCache
-cd "${BASH_SOURCE[0]%/*}"
-
-## Build
-
 echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))"
 
 if [[ $scenario ]]; then
-    buildExpr=$(../test/run-tests.sh --scenario $scenario exprForCI)
+    testArgs="--scenario $scenario"
 else
-    buildExpr="import ./build.nix"
+    testArgs=pkgsUnstable
 fi
 
-time nix-instantiate -E "$buildExpr" --add-root $tmpDir/drv --indirect > /dev/null
+"${BASH_SOURCE[0]%/*}/../test/run-tests.sh" --ci $testArgs
-printf "instantiated "; realpath $tmpDir/drv
-
-outPath=$(nix-store --query $tmpDir/drv)
-if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then
-    echo "$outPath" has already been built successfully.
-    exit 0
-fi
-
-# Cirrus doesn't expose secrets to pull-request builds,
-# so skip cache uploading in this case
-if [[ $CACHIX_SIGNING_KEY ]]; then
-    # Speed up task by uploading store paths as soon as they are created
-    cachix push $cachixCache --watch-store &
-    cachixPid=$!
-fi
-
-nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
-
-if [[ $CACHIX_SIGNING_KEY ]]; then
-    cachix push $cachixCache $outPath
-fi
-
-echo $outPath

test/lib/make-test-vm.nix

@@ -5,7 +5,7 @@ let
 
   test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs);
 
-  fixedTest = { system ? builtins.currentSystem, ... }@args:
+  fixedTest = { ... }@args:
     let
        pkgsFixed = pkgs // {
          # Fix the black Python code formatter that's used in the test to allow the test

test/lib/make-test.nix

@@ -41,11 +41,13 @@ scenario: testConfig:
 
   container = {
     # The container name has a 11 char length limit
-    containers.nb-test = { config, ...}: {
+    containers.nb-test = { config, ... }: {
       config = {
         extra = config.config.test.container;
         config = testConfig;
       };
     };
   };
+
+  config = testConfig;
 }

test/pkgs-unstable.nix

@@ -0,0 +1,13 @@
+let
+  pinned = import ../pkgs/nixpkgs-pinned.nix;
+  pkgs = import pinned.nixpkgs-unstable {};
+  nbPkgs = import ../pkgs { inherit pkgs; };
+  pkgsUnstable = with nbPkgs; [
+    electrs
+    elementsd
+    hwi
+    joinmarket
+    lightning-loop
+  ];
+in
+pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable)

test/run-tests.sh

@@ -40,8 +40,11 @@
 
 set -eo pipefail
 
+scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+
 scenario=
 outLinkPrefix=
+ciBuild=
 while :; do
     case $1 in
         --scenario|-s)
@@ -64,6 +67,10 @@ while :; do
                 exit 1
             fi
             ;;
+        --ci)
+            shift
+            ciBuild=1
+            ;;
         *)
             break
     esac
@@ -73,9 +80,7 @@ numCPUs=${numCPUs:-$(nproc)}
 # Min. 800 MiB needed to avoid 'out of memory' errors
 memoryMiB=${memoryMiB:-2048}
 
-testDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs)
-
-export NIX_PATH=nixpkgs=$(nix eval --raw -f "$testDir/../pkgs/nixpkgs-pinned.nix" nixpkgs)
 
 # Run the test. No temporary files are left on the host system.
 run() {
@@ -83,7 +88,7 @@ run() {
     export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
     trap "rm -rf $TMPDIR" EXIT
 
-    nix-build --out-link $TMPDIR/driver -E "(import \"$testDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
+    nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
 
     # Variable 'tests' contains the Python code that is executed by the driver on startup
     if [[ $1 == --interactive ]]; then
@@ -124,42 +129,57 @@ evalTest() {
     echo # nix eval doesn't print a newline
 }
 
+instantiate() {
+    nix-instantiate -E "$(vmTestNixExpr)" "$@"
+}
+
 container() {
-  . "$testDir/lib/make-container.sh" "$@"
+  . "$scriptDir/lib/make-container.sh" "$@"
+}
+
+doBuild() {
+    name=$1
+    shift
+    if [[ $ciBuild ]]; then
+        "$scriptDir/../ci/build-to-cachix.sh" "$@"
+    else
+        if [[ $outLinkPrefix ]]; then
+            outLink="--out-link $outLinkPrefix-$name"
+        else
+            outLink=--no-out-link
+        fi
+        nix-build $outLink "$@"
+    fi
 }
 
 # Run the test by building the test derivation
 buildTest() {
-    if [[ $outLinkPrefix ]]; then
+    vmTestNixExpr | doBuild $scenario $outLinkArg "$@" -
-        buildArgs="--out-link $outLinkPrefix-$scenario"
-    else
-        buildArgs=--no-out-link
-    fi
-    vmTestNixExpr | nix-build $buildArgs "$@" -
-}
-
-# On continuous integration nodes there are few other processes running alongside the
-# test, so use more memory here for maximum performance.
-exprForCI() {
-    memoryMiB=4096
-    memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
-    memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
-    # Round down to nearest multiple of 50 MiB for improved test build caching
-    ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50))
-    ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB
-    >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-    >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB"
-
-    # VMX is usually not available on CI nodes due to recursive virtualisation.
-    # Explicitly disable VMX, otherwise QEMU 4.20 fails with message
-    # "error: failed to set MSR 0x48b to 0x159ff00000000"
-    vmTestNixExpr "-cpu host,-vmx"
 }
 
 vmTestNixExpr() {
-  extraQEMUOpts="$1"
+    extraQEMUOpts=
-  cat <<EOF
+
-    ((import "$testDir/tests.nix" { scenario = "$scenario"; }).vm {}).overrideAttrs (old: rec {
+    if [[ $ciBuild ]]; then
+        # On continuous integration nodes there are few other processes running alongside the
+        # test, so use more memory here for maximum performance.
+        memoryMiB=4096
+        memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
+        memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
+        # Round down to nearest multiple of 50 MiB for improved test build caching
+        ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50))
+        ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB
+        >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
+        >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB"
+
+        # VMX is usually not available on CI nodes due to recursive virtualisation.
+        # Explicitly disable VMX, otherwise QEMU 4.20 fails with message
+        # "error: failed to set MSR 0x48b to 0x159ff00000000"
+        extraQEMUOpts="-cpu host,-vmx"
+    fi
+
+    cat <<EOF
+    ((import "$scriptDir/tests.nix" { scenario = "$scenario"; }).vm {}).overrideAttrs (old: rec {
       buildCommand = ''
         export QEMU_OPTS="-smp $numCPUs -m $memoryMiB $extraQEMUOpts"
         echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
@@ -168,6 +188,10 @@ vmTestNixExpr() {
 EOF
 }
 
+pkgsUnstable() {
+    doBuild pkgs-unstable "$scriptDir/pkgs-unstable.nix"
+}
+
 # A basic subset of tests to keep the total runtime within
 # manageable bounds (<4 min on desktop systems).
 # These are also run on the CI server.
@@ -175,30 +199,31 @@ basic() {
     scenario=default buildTest "$@"
     scenario=netns buildTest "$@"
     scenario=netnsRegtest buildTest "$@"
+    pkgsUnstable
 }
 
 all() {
-    scenario=default buildTest "$@"
+    basic
-    scenario=netns buildTest "$@"
     scenario=full buildTest "$@"
     scenario=regtest buildTest "$@"
-    scenario=netnsRegtest buildTest "$@"
 }
 
+# An alias for buildTest
 build() {
-    if [[ $scenario ]]; then
+    buildTest "$@"
-        buildTest "$@"
-    else
-        basic "$@"
-    fi
 }
 
-command="${1:-build}"
+if [[ $# > 0 && $1 != -* ]]; then
-shift || true
+    # An explicit command was provided
-if [[ $command != build  ]]; then
+    command=$1
+    shift
+    if [[ $command == eval ]]; then
+        command=evalTest
+    fi
     : ${scenario:=default}
-fi
+elif [[ $scenario ]]; then
-if [[ $command == eval ]]; then
+    command=buildTest
-    command=evalTest
+else
+    command=basic
 fi
 $command "$@"