Merge #282: Improve tests

9977fa69af ci: use run-tests.sh (Erik Arvstedt)
a82f0f5f48 add test 'pkgsUnstable' (Erik Arvstedt)
95bc1237e2 run-tests: rename testDir -> scriptDir (Erik Arvstedt)
466d23deaa ci: extract build-to-cachix.sh (Erik Arvstedt)
a70c3bf210 make-test-vm: remove unneeded leftover arg attrs (Erik Arvstedt)
ed65e78a2b make-test: expose test config (Erik Arvstedt)
7265742655 run-tests: add 'instantiate' command (Erik Arvstedt)
8cbdef8bf6 run-tests: fix CLI (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    Very nice! ACK 9977fa69af

Tree-SHA512: bb7f97096cc6e21f053c7db72a584a25ad62bca28af99e51fa83c15d2f75a198ada801428657821fc35f2cf01831176af8a9cd471e21dd0a7f5185f9d58efea1
This commit is contained in:
Jonas Nick 2020-12-11 21:25:02 +00:00
commit 7928495e45
No known key found for this signature in database
GPG Key ID: 4861DBF262123605
7 changed files with 147 additions and 115 deletions

50
ci/build-to-cachix.sh Executable file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Build a single-output derivation and store it in 'cachixCache'.
# Skip the build if it is already cached.
# Accepts the same arguments as nix-instantiate.
set -euo pipefail
CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-}
cachixCache=nix-bitcoin
trap 'echo Error at line $LINENO' ERR
atExit() {
rm -rf $tmpDir
if [[ -v cachixPid ]]; then kill $cachixPid; fi
}
tmpDir=$(mktemp -d -p /tmp)
trap atExit EXIT
## Instantiate
time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null
printf "instantiated "; realpath $tmpDir/drv
outPath=$(nix-store --query $tmpDir/drv)
if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then
echo "$outPath has already been built successfully."
exit 0
fi
## Build
if [[ -v CIRRUS_CI ]]; then
cachix use $cachixCache
fi
if [[ $CACHIX_SIGNING_KEY ]]; then
# Speed up task by uploading store paths as soon as they are created
cachix push $cachixCache --watch-store &
cachixPid=$!
fi
nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
if [[ $CACHIX_SIGNING_KEY ]]; then
cachix push $cachixCache $outPath
fi
echo $outPath

View File

@ -1,12 +0,0 @@
let
pkgs = import <nixpkgs> {};
nbPkgs = import ../pkgs { inherit pkgs; };
ciPkgs = with nbPkgs; [
electrs
elementsd
hwi
joinmarket
lightning-loop
];
in
pkgs.writeText "ci-pkgs" (pkgs.lib.concatMapStringsSep "\n" toString ciPkgs)

View File

@ -3,21 +3,13 @@
# This script can also be run locally for testing:
# scenario=default ./build.sh
#
# WARNING: This script fetches contents from an untrusted $cachixCache to your local nix-store.
#
# When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system.
set -euo pipefail
scenario=${scenario:-}
CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-}
cachixCache=nix-bitcoin
trap 'echo Error at line $LINENO' ERR
if [[ -v CIRRUS_CI ]]; then
tmpDir=/tmp
if [[ $scenario ]]; then
if [[ -v CIRRUS_CI && $scenario ]]; then
if [[ ! -e /dev/kvm ]]; then
>&2 echo "No KVM available on VM host."
exit 1
@ -25,51 +17,13 @@ if [[ -v CIRRUS_CI ]]; then
# Enable KVM access for nixbld users
chmod o+rw /dev/kvm
fi
else
atExit() {
rm -rf $tmpDir
if [[ -v cachixPid ]]; then kill $cachixPid; fi
}
tmpDir=$(mktemp -d -p /tmp)
trap atExit EXIT
# Prevent cachix from writing to HOME
export HOME=$tmpDir
fi
cachix use $cachixCache
cd "${BASH_SOURCE[0]%/*}"
## Build
echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))"
if [[ $scenario ]]; then
buildExpr=$(../test/run-tests.sh --scenario $scenario exprForCI)
testArgs="--scenario $scenario"
else
buildExpr="import ./build.nix"
testArgs=pkgsUnstable
fi
time nix-instantiate -E "$buildExpr" --add-root $tmpDir/drv --indirect > /dev/null
printf "instantiated "; realpath $tmpDir/drv
outPath=$(nix-store --query $tmpDir/drv)
if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then
echo "$outPath" has already been built successfully.
exit 0
fi
# Cirrus doesn't expose secrets to pull-request builds,
# so skip cache uploading in this case
if [[ $CACHIX_SIGNING_KEY ]]; then
# Speed up task by uploading store paths as soon as they are created
cachix push $cachixCache --watch-store &
cachixPid=$!
fi
nix-build --out-link $tmpDir/result $tmpDir/drv >/dev/null
if [[ $CACHIX_SIGNING_KEY ]]; then
cachix push $cachixCache $outPath
fi
echo $outPath
"${BASH_SOURCE[0]%/*}/../test/run-tests.sh" --ci $testArgs

View File

@ -5,7 +5,7 @@ let
test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs);
fixedTest = { system ? builtins.currentSystem, ... }@args:
fixedTest = { ... }@args:
let
pkgsFixed = pkgs // {
# Fix the black Python code formatter that's used in the test to allow the test

View File

@ -48,4 +48,6 @@ scenario: testConfig:
};
};
};
config = testConfig;
}

13
test/pkgs-unstable.nix Normal file
View File

@ -0,0 +1,13 @@
let
pinned = import ../pkgs/nixpkgs-pinned.nix;
pkgs = import pinned.nixpkgs-unstable {};
nbPkgs = import ../pkgs { inherit pkgs; };
pkgsUnstable = with nbPkgs; [
electrs
elementsd
hwi
joinmarket
lightning-loop
];
in
pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable)

View File

@ -40,8 +40,11 @@
set -eo pipefail
scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
scenario=
outLinkPrefix=
ciBuild=
while :; do
case $1 in
--scenario|-s)
@ -64,6 +67,10 @@ while :; do
exit 1
fi
;;
--ci)
shift
ciBuild=1
;;
*)
break
esac
@ -73,9 +80,7 @@ numCPUs=${numCPUs:-$(nproc)}
# Min. 800 MiB needed to avoid 'out of memory' errors
memoryMiB=${memoryMiB:-2048}
testDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
export NIX_PATH=nixpkgs=$(nix eval --raw -f "$testDir/../pkgs/nixpkgs-pinned.nix" nixpkgs)
export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs)
# Run the test. No temporary files are left on the host system.
run() {
@ -83,7 +88,7 @@ run() {
export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
trap "rm -rf $TMPDIR" EXIT
nix-build --out-link $TMPDIR/driver -E "(import \"$testDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver
# Variable 'tests' contains the Python code that is executed by the driver on startup
if [[ $1 == --interactive ]]; then
@ -124,23 +129,40 @@ evalTest() {
echo # nix eval doesn't print a newline
}
instantiate() {
nix-instantiate -E "$(vmTestNixExpr)" "$@"
}
container() {
. "$testDir/lib/make-container.sh" "$@"
. "$scriptDir/lib/make-container.sh" "$@"
}
doBuild() {
name=$1
shift
if [[ $ciBuild ]]; then
"$scriptDir/../ci/build-to-cachix.sh" "$@"
else
if [[ $outLinkPrefix ]]; then
outLink="--out-link $outLinkPrefix-$name"
else
outLink=--no-out-link
fi
nix-build $outLink "$@"
fi
}
# Run the test by building the test derivation
buildTest() {
if [[ $outLinkPrefix ]]; then
buildArgs="--out-link $outLinkPrefix-$scenario"
else
buildArgs=--no-out-link
fi
vmTestNixExpr | nix-build $buildArgs "$@" -
vmTestNixExpr | doBuild $scenario $outLinkArg "$@" -
}
vmTestNixExpr() {
extraQEMUOpts=
if [[ $ciBuild ]]; then
# On continuous integration nodes there are few other processes running alongside the
# test, so use more memory here for maximum performance.
exprForCI() {
memoryMiB=4096
memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
@ -153,13 +175,11 @@ exprForCI() {
# VMX is usually not available on CI nodes due to recursive virtualisation.
# Explicitly disable VMX, otherwise QEMU 4.20 fails with message
# "error: failed to set MSR 0x48b to 0x159ff00000000"
vmTestNixExpr "-cpu host,-vmx"
}
extraQEMUOpts="-cpu host,-vmx"
fi
vmTestNixExpr() {
extraQEMUOpts="$1"
cat <<EOF
((import "$testDir/tests.nix" { scenario = "$scenario"; }).vm {}).overrideAttrs (old: rec {
((import "$scriptDir/tests.nix" { scenario = "$scenario"; }).vm {}).overrideAttrs (old: rec {
buildCommand = ''
export QEMU_OPTS="-smp $numCPUs -m $memoryMiB $extraQEMUOpts"
echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
@ -168,6 +188,10 @@ vmTestNixExpr() {
EOF
}
pkgsUnstable() {
doBuild pkgs-unstable "$scriptDir/pkgs-unstable.nix"
}
# A basic subset of tests to keep the total runtime within
# manageable bounds (<4 min on desktop systems).
# These are also run on the CI server.
@ -175,30 +199,31 @@ basic() {
scenario=default buildTest "$@"
scenario=netns buildTest "$@"
scenario=netnsRegtest buildTest "$@"
pkgsUnstable
}
all() {
scenario=default buildTest "$@"
scenario=netns buildTest "$@"
basic
scenario=full buildTest "$@"
scenario=regtest buildTest "$@"
scenario=netnsRegtest buildTest "$@"
}
# An alias for buildTest
build() {
if [[ $scenario ]]; then
buildTest "$@"
else
basic "$@"
fi
}
command="${1:-build}"
shift || true
if [[ $command != build ]]; then
: ${scenario:=default}
fi
if [[ $# > 0 && $1 != -* ]]; then
# An explicit command was provided
command=$1
shift
if [[ $command == eval ]]; then
command=evalTest
fi
: ${scenario:=default}
elif [[ $scenario ]]; then
command=buildTest
else
command=basic
fi
$command "$@"