diff --git a/ci/build-to-cachix.sh b/ci/build-to-cachix.sh new file mode 100755 index 0000000..ab99535 --- /dev/null +++ b/ci/build-to-cachix.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Build a single-output derivation and store it in 'cachixCache'. +# Skip the build if it is already cached. +# Accepts the same arguments as nix-instantiate. + +set -euo pipefail + +CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-} +cachixCache=nix-bitcoin + +trap 'echo Error at line $LINENO' ERR + +atExit() { + rm -rf $tmpDir + if [[ -v cachixPid ]]; then kill $cachixPid; fi +} +tmpDir=$(mktemp -d -p /tmp) +trap atExit EXIT + +## Instantiate + +time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null +printf "instantiated "; realpath $tmpDir/drv + +outPath=$(nix-store --query $tmpDir/drv) +if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then + echo "$outPath has already been built successfully." + exit 0 +fi + +## Build + +if [[ -v CIRRUS_CI ]]; then + cachix use $cachixCache +fi + +if [[ $CACHIX_SIGNING_KEY ]]; then + # Speed up task by uploading store paths as soon as they are created + cachix push $cachixCache --watch-store & + cachixPid=$! +fi + +nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null + +if [[ $CACHIX_SIGNING_KEY ]]; then + cachix push $cachixCache $outPath +fi + +echo $outPath diff --git a/ci/build.nix b/ci/build.nix deleted file mode 100644 index 16cb6a2..0000000 --- a/ci/build.nix +++ /dev/null @@ -1,12 +0,0 @@ -let - pkgs = import {}; - nbPkgs = import ../pkgs { inherit pkgs; }; - ciPkgs = with nbPkgs; [ - electrs - elementsd - hwi - joinmarket - lightning-loop - ]; -in -pkgs.writeText "ci-pkgs" (pkgs.lib.concatMapStringsSep "\n" toString ciPkgs) diff --git a/ci/build.sh b/ci/build.sh index 8a69c5a..3c3f2e1 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -3,73 +3,27 @@ # This script can also be run locally for testing: # scenario=default ./build.sh # -# WARNING: This script fetches contents from an untrusted $cachixCache to your local nix-store. -# # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system. set -euo pipefail scenario=${scenario:-} -CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-} -cachixCache=nix-bitcoin -trap 'echo Error at line $LINENO' ERR - -if [[ -v CIRRUS_CI ]]; then - tmpDir=/tmp - if [[ $scenario ]]; then - if [[ ! -e /dev/kvm ]]; then - >&2 echo "No KVM available on VM host." - exit 1 - fi - # Enable KVM access for nixbld users - chmod o+rw /dev/kvm +if [[ -v CIRRUS_CI && $scenario ]]; then + if [[ ! -e /dev/kvm ]]; then + >&2 echo "No KVM available on VM host." + exit 1 fi -else - atExit() { - rm -rf $tmpDir - if [[ -v cachixPid ]]; then kill $cachixPid; fi - } - tmpDir=$(mktemp -d -p /tmp) - trap atExit EXIT - # Prevent cachix from writing to HOME - export HOME=$tmpDir + # Enable KVM access for nixbld users + chmod o+rw /dev/kvm fi -cachix use $cachixCache -cd "${BASH_SOURCE[0]%/*}" - -## Build - echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))" if [[ $scenario ]]; then - buildExpr=$(../test/run-tests.sh --scenario $scenario exprForCI) + testArgs="--scenario $scenario" else - buildExpr="import ./build.nix" + testArgs=pkgsUnstable fi -time nix-instantiate -E "$buildExpr" --add-root $tmpDir/drv --indirect > /dev/null -printf "instantiated "; realpath $tmpDir/drv - -outPath=$(nix-store --query $tmpDir/drv) -if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then - echo "$outPath" has already been built successfully. - exit 0 -fi - -# Cirrus doesn't expose secrets to pull-request builds, -# so skip cache uploading in this case -if [[ $CACHIX_SIGNING_KEY ]]; then - # Speed up task by uploading store paths as soon as they are created - cachix push $cachixCache --watch-store & - cachixPid=$! -fi - -nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null - -if [[ $CACHIX_SIGNING_KEY ]]; then - cachix push $cachixCache $outPath -fi - -echo $outPath +"${BASH_SOURCE[0]%/*}/../test/run-tests.sh" --ci $testArgs diff --git a/test/lib/make-test-vm.nix b/test/lib/make-test-vm.nix index 90d3e64..8f9928b 100644 --- a/test/lib/make-test-vm.nix +++ b/test/lib/make-test-vm.nix @@ -5,7 +5,7 @@ let test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs); - fixedTest = { system ? builtins.currentSystem, ... }@args: + fixedTest = { ... }@args: let pkgsFixed = pkgs // { # Fix the black Python code formatter that's used in the test to allow the test diff --git a/test/lib/make-test.nix b/test/lib/make-test.nix index a0baa17..c3620fe 100644 --- a/test/lib/make-test.nix +++ b/test/lib/make-test.nix @@ -41,11 +41,13 @@ scenario: testConfig: container = { # The container name has a 11 char length limit - containers.nb-test = { config, ...}: { + containers.nb-test = { config, ... }: { config = { extra = config.config.test.container; config = testConfig; }; }; }; + + config = testConfig; } diff --git a/test/pkgs-unstable.nix b/test/pkgs-unstable.nix new file mode 100644 index 0000000..3ad8583 --- /dev/null +++ b/test/pkgs-unstable.nix @@ -0,0 +1,13 @@ +let + pinned = import ../pkgs/nixpkgs-pinned.nix; + pkgs = import pinned.nixpkgs-unstable {}; + nbPkgs = import ../pkgs { inherit pkgs; }; + pkgsUnstable = with nbPkgs; [ + electrs + elementsd + hwi + joinmarket + lightning-loop + ]; +in +pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable) diff --git a/test/run-tests.sh b/test/run-tests.sh index c48064a..c3fd248 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -40,8 +40,11 @@ set -eo pipefail +scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) + scenario= outLinkPrefix= +ciBuild= while :; do case $1 in --scenario|-s) @@ -64,6 +67,10 @@ while :; do exit 1 fi ;; + --ci) + shift + ciBuild=1 + ;; *) break esac @@ -73,9 +80,7 @@ numCPUs=${numCPUs:-$(nproc)} # Min. 800 MiB needed to avoid 'out of memory' errors memoryMiB=${memoryMiB:-2048} -testDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) - -export NIX_PATH=nixpkgs=$(nix eval --raw -f "$testDir/../pkgs/nixpkgs-pinned.nix" nixpkgs) +export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs) # Run the test. No temporary files are left on the host system. run() { @@ -83,7 +88,7 @@ run() { export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) trap "rm -rf $TMPDIR" EXIT - nix-build --out-link $TMPDIR/driver -E "(import \"$testDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver + nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver # Variable 'tests' contains the Python code that is executed by the driver on startup if [[ $1 == --interactive ]]; then @@ -124,42 +129,57 @@ evalTest() { echo # nix eval doesn't print a newline } +instantiate() { + nix-instantiate -E "$(vmTestNixExpr)" "$@" +} + container() { - . "$testDir/lib/make-container.sh" "$@" + . "$scriptDir/lib/make-container.sh" "$@" +} + +doBuild() { + name=$1 + shift + if [[ $ciBuild ]]; then + "$scriptDir/../ci/build-to-cachix.sh" "$@" + else + if [[ $outLinkPrefix ]]; then + outLink="--out-link $outLinkPrefix-$name" + else + outLink=--no-out-link + fi + nix-build $outLink "$@" + fi } # Run the test by building the test derivation buildTest() { - if [[ $outLinkPrefix ]]; then - buildArgs="--out-link $outLinkPrefix-$scenario" - else - buildArgs=--no-out-link - fi - vmTestNixExpr | nix-build $buildArgs "$@" - -} - -# On continuous integration nodes there are few other processes running alongside the -# test, so use more memory here for maximum performance. -exprForCI() { - memoryMiB=4096 - memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo) - memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo) - # Round down to nearest multiple of 50 MiB for improved test build caching - ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50)) - ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB - >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB" - - # VMX is usually not available on CI nodes due to recursive virtualisation. - # Explicitly disable VMX, otherwise QEMU 4.20 fails with message - # "error: failed to set MSR 0x48b to 0x159ff00000000" - vmTestNixExpr "-cpu host,-vmx" + vmTestNixExpr | doBuild $scenario $outLinkArg "$@" - } vmTestNixExpr() { - extraQEMUOpts="$1" - cat <&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" + >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB" + + # VMX is usually not available on CI nodes due to recursive virtualisation. + # Explicitly disable VMX, otherwise QEMU 4.20 fails with message + # "error: failed to set MSR 0x48b to 0x159ff00000000" + extraQEMUOpts="-cpu host,-vmx" + fi + + cat < 0 && $1 != -* ]]; then + # An explicit command was provided + command=$1 + shift + if [[ $command == eval ]]; then + command=evalTest + fi : ${scenario:=default} -fi -if [[ $command == eval ]]; then - command=evalTest +elif [[ $scenario ]]; then + command=buildTest +else + command=basic fi $command "$@"