From b625325487afa0f060b0c68f44e0b860ebe6adac Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 27 Aug 2022 13:34:39 +0200 Subject: [PATCH 1/5] ci/modules_test: use flake-enabled nix --- .cirrus.yml | 11 +++++------ test/ci/build.sh | 4 +--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 297a74f..93a699b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -7,11 +7,13 @@ task: # Use the maximum timeout. Needed when rebuilding packages on a channel update. timeout_in: 120m + container: + # Defined in https://github.com/nix-community/docker-nixpkgs + image: nixpkgs/nix-flakes:nixos-22.05 + matrix: - name: modules_test container: - # Use Nix release 2.3.x, the default on NixOS stable - image: nixos/nix:2.3.12 # Besides virtualization, this also enables privileged containers which are required for # sandboxed builds kvm: true @@ -28,12 +30,9 @@ task: # This script is run as root build_script: - echo "sandbox = true" >> /etc/nix/nix.conf - - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)" - - nix run -f '' bash coreutils cachix -c ./test/ci/build.sh + - nix shell --inputs-from . nixpkgs#{bash,coreutils,gawk,cachix} -c ./test/ci/build.sh - name: flake - container: - image: nixpkgs/nix-flakes build_script: - nix flake check - ./test/nixos-search/ci-test.sh diff --git a/test/ci/build.sh b/test/ci/build.sh index da711b8..3d48523 100755 --- a/test/ci/build.sh +++ b/test/ci/build.sh @@ -16,6 +16,4 @@ if [[ -v CIRRUS_CI ]]; then chmod o+rw /dev/kvm fi -echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))" - -"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario $scenario +"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario" From 9a92d29111af74365a7694467847df2afd086f9f Mon Sep 17 00:00:00 2001 From: Otto Sabart Date: Tue, 16 Aug 2022 21:00:00 +0200 Subject: [PATCH 2/5] tests: add shellcheck --- .cirrus.yml | 4 ++++ test/run-tests.sh | 8 +++++++- test/shellcheck.sh | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100755 test/shellcheck.sh diff --git a/.cirrus.yml b/.cirrus.yml index 93a699b..3b56692 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -36,3 +36,7 @@ task: build_script: - nix flake check - ./test/nixos-search/ci-test.sh + + - name: shellcheck + build_script: + - nix shell --inputs-from . nixpkgs#{shellcheck,findutils,gnugrep} -c ./test/shellcheck.sh diff --git a/test/run-tests.sh b/test/run-tests.sh index 9824a16..0d8ceae 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -312,8 +312,14 @@ examples() { (cd "$scriptDir/../examples" && nix-shell --run "$script") } +shellcheck() { + if ! checkFlakeSupport "shellcheck"; then return; fi + nix shell --inputs-from "$scriptDir/.." nixpkgs#shellcheck -c "$scriptDir/shellcheck.sh" +} + all() { - buildable + buildable "$@" + shellcheck examples flake nixosSearch diff --git a/test/shellcheck.sh b/test/shellcheck.sh new file mode 100755 index 0000000..822f5d9 --- /dev/null +++ b/test/shellcheck.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "${BASH_SOURCE[0]%/*}/.." +{ + # Skip .git dir in all find commands + find . -type f ! -path './.git/*' -name '*.sh' + # Find files without extensions that have a shell shebang + find . -type f ! -path './.git/*' ! -name "*.*" -exec grep -lP '\A^#! */usr/bin/env (?:nix-shell|bash)' {} \; +} | while IFS= read -r path; do + echo "$path" + file=${path##*/} + dir=${path%/*} + # Switch working directory so that shellcheck can access external sources + # (via arg `--external-sources`) + pushd "$dir" > /dev/null + shellcheck --external-sources --shell bash "$file" + popd > /dev/null +done From a59c3b4b8a99ee2fe04d233826fbf1e8f4457b8c Mon Sep 17 00:00:00 2001 From: Otto Sabart Date: Tue, 16 Aug 2022 21:00:00 +0200 Subject: [PATCH 3/5] shellcheck: fix lint of package helper bash scripts --- pkgs/clightning-plugins/get-sha256.sh | 2 +- pkgs/clightning-rest/generate.sh | 16 +++++----- pkgs/joinmarket/get-sha256.sh | 14 ++++----- pkgs/krops/fetch-release.sh | 2 +- pkgs/lndinit/get-sha256.sh | 14 ++++----- .../python-bitcointx/get-sha256.sh | 12 ++++---- pkgs/rtl/generate.sh | 19 ++++++------ pkgs/spark-wallet/generate.sh | 29 ++++++++++--------- 8 files changed, 55 insertions(+), 53 deletions(-) diff --git a/pkgs/clightning-plugins/get-sha256.sh b/pkgs/clightning-plugins/get-sha256.sh index 3def505..886d5f8 100755 --- a/pkgs/clightning-plugins/get-sha256.sh +++ b/pkgs/clightning-plugins/get-sha256.sh @@ -11,4 +11,4 @@ archive_hash () { echo "Fetching latest lightningd/plugins release" latest=$(git ls-remote https://github.com/lightningd/plugins master | cut -f 1) echo "rev = \"${latest}\";" -echo "sha256 = \"$(archive_hash lightningd/plugins $latest)\";" +echo "sha256 = \"$(archive_hash lightningd/plugins "$latest")\";" diff --git a/pkgs/clightning-rest/generate.sh b/pkgs/clightning-rest/generate.sh index 75f75fd..8970391 100755 --- a/pkgs/clightning-rest/generate.sh +++ b/pkgs/clightning-rest/generate.sh @@ -8,18 +8,18 @@ repo=https://github.com/Ride-The-Lightning/c-lightning-REST scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) updateSrc() { - TMPDIR="$(mktemp -d /tmp/clightning-rest.XXX)" - trap "rm -rf $TMPDIR" EXIT + TMPDIR=$(mktemp -d /tmp/clightning-rest.XXX) + trap 'rm -rf $TMPDIR' EXIT # Fetch and verify source tarball export GNUPGHOME=$TMPDIR # Fetch saubyk's key gpg --keyserver hkps://keyserver.ubuntu.com --recv-key 3E9BD4436C288039CA827A9200C9E2BC2E45666F file=v${version}.tar.gz - wget -P $TMPDIR $repo/archive/refs/tags/$file - wget -P $TMPDIR $repo/releases/download/v${version}/$file.asc - gpg --verify $TMPDIR/$file.asc $TMPDIR/$file - hash=$(nix hash file $TMPDIR/$file) + wget -P "$TMPDIR" "${repo}/archive/refs/tags/${file}" + wget -P "$TMPDIR" "${repo}/releases/download/v${version}/${file}.asc" + gpg --verify "${TMPDIR}/${file}.asc" "${TMPDIR}/${file}" + hash=$(nix hash file "${TMPDIR}/${file}") sed -i " s|\bversion = .*;|version = \"$version\";| @@ -28,7 +28,7 @@ updateSrc() { } updateNodeModulesHash() { - $scriptDir/../../helper/update-fixed-output-derivation.sh ./default.nix clightning-rest.nodeModules nodeModules + "$scriptDir/../../helper/update-fixed-output-derivation.sh" ./default.nix clightning-rest.nodeModules nodeModules } if [[ $# == 0 ]]; then @@ -36,5 +36,5 @@ if [[ $# == 0 ]]; then updateSrc updateNodeModulesHash else - eval "$@" + "$@" fi diff --git a/pkgs/joinmarket/get-sha256.sh b/pkgs/joinmarket/get-sha256.sh index 6bdb149..111aa68 100755 --- a/pkgs/joinmarket/get-sha256.sh +++ b/pkgs/joinmarket/get-sha256.sh @@ -3,23 +3,23 @@ set -euo pipefail TMPDIR="$(mktemp -d -p /tmp)" -trap "rm -rf $TMPDIR" EXIT -cd $TMPDIR +trap 'rm -rf $TMPDIR' EXIT +cd "$TMPDIR" echo "Fetching latest release" git clone https://github.com/joinmarket-org/joinmarket-clientserver 2> /dev/null cd joinmarket-clientserver -latest=$(git describe --tags `git rev-list --tags --max-count=1`) -echo "Latest release is ${latest}" +latest=$(git describe --tags "$(git rev-list --tags --max-count=1)") +echo "Latest release is $latest" # GPG verification export GNUPGHOME=$TMPDIR echo "Fetching Adam Gibson's key" gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 2B6FC204D9BF332D062B461A141001A1AF77F20B 2> /dev/null echo "Verifying latest release" -git verify-tag ${latest} +git verify-tag "$latest" -echo "tag: ${latest}" +echo "tag: $latest" # The prefix option is necessary because GitHub prefixes the archive contents in this format echo "sha256: $(nix-hash --type sha256 --flat --base32 \ - <(git archive --format tar.gz --prefix=joinmarket-clientserver-"${latest//v}"/ ${latest}))" + <(git archive --format tar.gz --prefix=joinmarket-clientserver-"${latest//v}"/ "$latest"))" diff --git a/pkgs/krops/fetch-release.sh b/pkgs/krops/fetch-release.sh index cf414c6..8fa5c5a 100755 --- a/pkgs/krops/fetch-release.sh +++ b/pkgs/krops/fetch-release.sh @@ -14,4 +14,4 @@ version=$( | sed -E 's|refs/tags/||g; s|((v)?(.*))|\1 \3|g' | sort -k 2 -V | tail -1 | cut -f 1 -d' ' ) echo "rev: ${version}" -echo "sha256: $(archive_hash krebs/krops $version)" +echo "sha256: $(archive_hash krebs/krops "$version")" diff --git a/pkgs/lndinit/get-sha256.sh b/pkgs/lndinit/get-sha256.sh index ba65d22..cca5bca 100755 --- a/pkgs/lndinit/get-sha256.sh +++ b/pkgs/lndinit/get-sha256.sh @@ -2,15 +2,15 @@ #! nix-shell -i bash -p git gnupg curl jq set -euo pipefail -TMPDIR="$(mktemp -d -p /tmp)" -trap "rm -rf $TMPDIR" EXIT -cd $TMPDIR +TMPDIR=$(mktemp -d -p /tmp) +trap 'rm -rf $TMPDIR' EXIT +cd "$TMPDIR" echo "Fetching latest release" repo=lightninglabs/lndinit -latest=$(curl -fsS https://api.github.com/repos/$repo/releases/latest | jq -r .tag_name) +latest=$(curl -fsS "https://api.github.com/repos/$repo/releases/latest" | jq -r .tag_name) echo "Latest release is $latest" -git clone --depth 1 --branch $latest https://github.com/lightninglabs/lndinit 2>/dev/null +git clone --depth 1 --branch "$latest" https://github.com/lightninglabs/lndinit 2>/dev/null cd lndinit # GPG verification @@ -18,9 +18,9 @@ export GNUPGHOME=$TMPDIR echo "Fetching Oliver Gugger's key" gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys F4FC70F07310028424EFC20A8E4256593F177720 2> /dev/null echo "Verifying latest release" -git verify-tag $latest +git verify-tag "$latest" echo "tag: $latest" -git checkout -q tags/$latest +git checkout -q "tags/$latest" rm -rf .git nix hash path . diff --git a/pkgs/python-packages/python-bitcointx/get-sha256.sh b/pkgs/python-packages/python-bitcointx/get-sha256.sh index ad38735..745ffb1 100755 --- a/pkgs/python-packages/python-bitcointx/get-sha256.sh +++ b/pkgs/python-packages/python-bitcointx/get-sha256.sh @@ -2,9 +2,9 @@ #! nix-shell -i bash -p git gnupg set -euo pipefail -TMPDIR="$(mktemp -d -p /tmp)" -trap "rm -rf $TMPDIR" EXIT -cd $TMPDIR +TMPDIR=$(mktemp -d -p /tmp) +trap 'rm -rf $TMPDIR' EXIT +cd "$TMPDIR" echo "Fetching latest release" git clone https://github.com/simplexum/python-bitcointx 2> /dev/null @@ -17,8 +17,8 @@ export GNUPGHOME=$TMPDIR echo "Fetching Dimitry Pethukov's Key" gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys B17A35BBA187395784E2A6B32301D26BDC15160D 2> /dev/null echo "Verifying latest release" -git verify-commit ${latest} +git verify-commit "$latest" -echo "tag: ${latest}" +echo "tag: $latest" # The prefix option is necessary because GitHub prefixes the archive contents in this format -echo "sha256: $(git archive --format tar.gz --prefix=python-bitcointx-"${latest}"/ ${latest} | sha256sum | cut -d\ -f1)" +echo "sha256: $(git archive --format tar.gz --prefix=python-bitcointx-"$latest"/ "$latest" | sha256sum | cut -d\ -f1)" diff --git a/pkgs/rtl/generate.sh b/pkgs/rtl/generate.sh index baae4ee..fabe9ab 100755 --- a/pkgs/rtl/generate.sh +++ b/pkgs/rtl/generate.sh @@ -8,18 +8,19 @@ repo=https://github.com/Ride-The-Lightning/RTL scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) updateSrc() { - TMPDIR="$(mktemp -d /tmp/rtl.XXX)" - trap "rm -rf $TMPDIR" EXIT + TMPDIR=$(mktemp -d /tmp/rtl.XXX) + trap 'rm -rf $TMPDIR' EXIT # Fetch and verify source tarball export GNUPGHOME=$TMPDIR + # Fetch saubyk's key gpg --keyserver hkps://keyserver.ubuntu.com --recv-key 3E9BD4436C288039CA827A9200C9E2BC2E45666F - file=v${version}.tar.gz - wget -P $TMPDIR $repo/archive/refs/tags/$file - wget -P $TMPDIR $repo/releases/download/v${version}/$file.asc - gpg --verify $TMPDIR/$file.asc $TMPDIR/$file - hash=$(nix hash file $TMPDIR/$file) + file=v$version.tar.gz + wget -P "$TMPDIR" "$repo/archive/refs/tags/$file" + wget -P "$TMPDIR" "$repo/releases/download/v$version/$file.asc" + gpg --verify "$TMPDIR/$file.asc" "$TMPDIR/$file" + hash=$(nix hash file "$TMPDIR/$file") sed -i " s|\bversion = .*;|version = \"$version\";| @@ -28,7 +29,7 @@ updateSrc() { } updateNodeModulesHash() { - $scriptDir/../../helper/update-fixed-output-derivation.sh ./default.nix rtl.nodeModules nodeModules + "$scriptDir/../../helper/update-fixed-output-derivation.sh" ./default.nix rtl.nodeModules nodeModules } if [[ $# == 0 ]]; then @@ -36,5 +37,5 @@ if [[ $# == 0 ]]; then updateSrc updateNodeModulesHash else - eval "$@" + "$@" fi diff --git a/pkgs/spark-wallet/generate.sh b/pkgs/spark-wallet/generate.sh index 0fa672e..50c3a83 100755 --- a/pkgs/spark-wallet/generate.sh +++ b/pkgs/spark-wallet/generate.sh @@ -2,44 +2,45 @@ #! nix-shell -i bash -p nodePackages.node2nix gnupg wget jq moreutils gnused set -euo pipefail -TMPDIR="$(mktemp -d -p /tmp)" -trap "rm -rf $TMPDIR" EXIT +TMPDIR=$(mktemp -d -p /tmp) +trap 'rm -rf $TMPDIR' EXIT version="0.3.1" repo=https://github.com/shesek/spark-wallet # Fetch and verify source tarball file=spark-wallet-${version}-npm.tgz -url=$repo/releases/download/v$version/$file +url=$repo/releases/download/v${version}/$file export GNUPGHOME=$TMPDIR gpg --keyserver hkps://keyserver.ubuntu.com --recv-key FCF19B67866562F08A43AAD681F6104CD0F150FC -wget -P $TMPDIR $url -wget -P $TMPDIR $repo/releases/download/v$version/SHA256SUMS.asc -gpg --verify $TMPDIR/SHA256SUMS.asc -(cd $TMPDIR; sha256sum --check --ignore-missing SHA256SUMS.asc) -hash=$(nix hash file $TMPDIR/$file) +wget -P "$TMPDIR" "$url" +wget -P "$TMPDIR" "$repo/releases/download/v${version}/SHA256SUMS.asc" +gpg --verify "$TMPDIR/SHA256SUMS.asc" +(cd "$TMPDIR"; sha256sum --check --ignore-missing SHA256SUMS.asc) +hash=$(nix hash file "$TMPDIR/$file") # Extract source src=$TMPDIR/src -mkdir $src -tar xvf $TMPDIR/$file -C $src --strip-components 1 >/dev/null +mkdir "$src" +tar xvf "$TMPDIR/$file" -C "$src" --strip-components 1 >/dev/null # Make qrcode-terminal a strict dependency so that node2nix includes it in the package derivation. -jq '.dependencies["qrcode-terminal"] = .optionalDependencies["qrcode-terminal"]' $src/package.json | sponge $src/package.json +jq '.dependencies["qrcode-terminal"] = .optionalDependencies["qrcode-terminal"]' "$src/package.json" | sponge "$src/package.json" node2nix \ --nodejs-14 \ - --input $src/package.json \ - --lock $src/npm-shrinkwrap.json \ + --input "$src/package.json" \ + --lock "$src/npm-shrinkwrap.json" \ --composition composition.nix \ --no-copy-node-env # Use node-env.nix from nixpkgs +# shellcheck disable=SC2016 nodeEnvImport='import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix"' sed -i "s|import ./node-env.nix|$nodeEnvImport|" composition.nix # Use the verified package src -read -d '' fetchurl < Date: Tue, 16 Aug 2022 21:00:00 +0200 Subject: [PATCH 4/5] shellcheck: fix lint of scripts in tests --- examples/deploy-container-minimal.sh | 9 ++++---- examples/deploy-container.sh | 2 +- examples/deploy-krops.sh | 12 +++++----- examples/deploy-qemu-vm.sh | 4 ++-- examples/qemu-vm/run-vm.sh | 18 ++++++++------- examples/start-bash-session.sh | 2 ++ test/ci/build-to-cachix.sh | 20 ++++++++-------- test/ci/build.sh | 1 + test/lib/copy-src.sh | 13 +++++++---- test/lib/create-git-repo.sh | 4 +++- test/lib/make-container.sh | 10 ++++---- test/lib/make-test-vm.nix | 4 ++-- test/nixos-search/ci-test.sh | 8 ++++--- test/run-tests.sh | 34 +++++++++++++++++----------- 14 files changed, 82 insertions(+), 59 deletions(-) diff --git a/examples/deploy-container-minimal.sh b/examples/deploy-container-minimal.sh index 99b67e8..d2414fb 100755 --- a/examples/deploy-container-minimal.sh +++ b/examples/deploy-container-minimal.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -euo pipefail if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then echo "Running script in nix shell env..." @@ -9,16 +10,16 @@ else fi tmpDir=$(mktemp -d /tmp/nix-bitcoin-minimal-container.XXX) -trap "rm -rf $tmpDir" EXIT +trap 'rm -rf $tmpDir' EXIT # Modify importable-configuration.nix to use the local # source instead of fetchTarball ;|; s|system.extraDependencies = .*|| -' > $tmpDir/importable-configuration.nix +' > "$tmpDir/importable-configuration.nix" -cat > $tmpDir/configuration.nix < "$tmpDir/configuration.nix" < $tmpDir/configuration.nix < { configuration = { config, lib, ... }: { imports = [ ]; @@ -43,11 +43,11 @@ vmNumCPUs=4 vmMemoryMiB=2048 sshPort=60734 # Start the VM in the background -runVM $tmpDir/vm $vmNumCPUs $vmMemoryMiB $sshPort +runVM "$tmpDir/vm" "$vmNumCPUs" "$vmMemoryMiB" "$sshPort" # Build the krops deploy script export sshPort -nix-build --out-link $tmpDir/krops-deploy - <<'EOF' +nix-build --out-link "$tmpDir/krops-deploy" - <<'EOF' let krops = (import {}).krops; @@ -85,7 +85,7 @@ EOF echo "Building the nix-bitcoin node" # Pre-build the nix-bitcoin node outside of the VM to save some time -nix-build --out-link $tmpDir/store-paths -E ' +nix-build --out-link "$tmpDir/store-paths" -E ' let system = (import { configuration = ; }).system; pkgsUnstable = (import ).nixpkgs-unstable; @@ -98,7 +98,7 @@ vmWaitForSSH # Add the store paths that include the nix-bitcoin node # to the nix store db in the VM -c "nix-store --load-db < $(realpath $tmpDir/store-paths)/registration" +c "nix-store --load-db < $(realpath "$tmpDir/store-paths")/registration" echo echo "Generate secrets" @@ -106,7 +106,7 @@ nix-shell --run generate-secrets echo echo "Deploy with krops" -$tmpDir/krops-deploy +"$tmpDir/krops-deploy" echo echo "Bitcoind service:" diff --git a/examples/deploy-qemu-vm.sh b/examples/deploy-qemu-vm.sh index 53c971c..955a8e7 100755 --- a/examples/deploy-qemu-vm.sh +++ b/examples/deploy-qemu-vm.sh @@ -22,7 +22,7 @@ fi source qemu-vm/run-vm.sh echo "Building VM" -nix-build --out-link $tmpDir/vm - <<'EOF' +nix-build --out-link "$tmpDir/vm" - <<'EOF' (import { configuration = { imports = [ @@ -37,7 +37,7 @@ EOF vmNumCPUs=4 vmMemoryMiB=2048 sshPort=60734 -runVM $tmpDir/vm $vmNumCPUs $vmMemoryMiB $sshPort +runVM "$tmpDir/vm" "$vmNumCPUs" "$vmMemoryMiB" "$sshPort" vmWaitForSSH printf "Waiting until services are ready" diff --git a/examples/qemu-vm/run-vm.sh b/examples/qemu-vm/run-vm.sh index 0181796..51b7133 100644 --- a/examples/qemu-vm/run-vm.sh +++ b/examples/qemu-vm/run-vm.sh @@ -1,22 +1,23 @@ qemuDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) +# shellcheck disable=SC1091 source "$qemuDir/wait-until.sh" tmpDir=/tmp/nix-bitcoin-qemu-vm -mkdir -p $tmpDir +mkdir -p "$tmpDir" # Cleanup on exit cleanup() { set +eu if [[ $qemuPID ]]; then - kill -9 $qemuPID + kill -9 "$qemuPID" fi - rm -rf $tmpDir + rm -rf "$tmpDir" } trap "cleanup" EXIT identityFile=$qemuDir/id-vm -chmod 0600 $identityFile +chmod 0600 "$identityFile" runVM() { vm=$1 @@ -24,9 +25,10 @@ runVM() { vmMemoryMiB=$3 sshPort=$4 - export NIX_DISK_IMAGE=$tmpDir/img - export QEMU_NET_OPTS=hostfwd=tcp::$sshPort-:22 - /dev/null & + export NIX_DISK_IMAGE="$tmpDir/img" + export QEMU_NET_OPTS="hostfwd=tcp::${sshPort}-:22" + # shellcheck disable=SC2211 + /dev/null & qemuPID=$! } @@ -39,7 +41,7 @@ vmWaitForSSH() { # Run command in VM c() { - ssh -p $sshPort -i $identityFile -o ConnectTimeout=1 \ + ssh -p "$sshPort" -i "$identityFile" -o ConnectTimeout=1 \ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ -o ControlMaster=auto -o ControlPath=$tmpDir/ssh-connection -o ControlPersist=60 \ root@127.0.0.1 "$@" diff --git a/examples/start-bash-session.sh b/examples/start-bash-session.sh index a91d91f..599e635 100644 --- a/examples/start-bash-session.sh +++ b/examples/start-bash-session.sh @@ -11,6 +11,8 @@ c systemctl status bitcoind # BASH_ENVIRONMENT contains definitions of read-only variables like 'BASHOPTS' that # cause warnings on evaluation. Suppress these warnings while sourcing. +# +# shellcheck disable=SC2016 BASH_ENVIRONMENT=<(declare -p; declare -pf) \ USAGE_INFO="$USAGE_INFO" \ bash --rcfile <(echo ' diff --git a/test/ci/build-to-cachix.sh b/test/ci/build-to-cachix.sh index 0fbbd5a..88bfe6e 100755 --- a/test/ci/build-to-cachix.sh +++ b/test/ci/build-to-cachix.sh @@ -6,21 +6,21 @@ set -euo pipefail -CACHIX_SIGNING_KEY=${CACHIX_SIGNING_KEY:-} +CACHIX_SIGNING_KEY="${CACHIX_SIGNING_KEY:-}" cachixCache=nix-bitcoin trap 'echo Error at line $LINENO' ERR tmpDir=$(mktemp -d -p /tmp) -trap "rm -rf $tmpDir" EXIT +trap 'rm -rf $tmpDir' EXIT ## Instantiate -time nix-instantiate "$@" --add-root $tmpDir/drv --indirect > /dev/null -printf "instantiated "; realpath $tmpDir/drv +time nix-instantiate "$@" --add-root "$tmpDir/drv" --indirect > /dev/null +printf "instantiated "; realpath "$tmpDir/drv" -outPath=$(nix-store --query $tmpDir/drv) -if nix path-info --store https://$cachixCache.cachix.org $outPath &>/dev/null; then +outPath=$(nix-store --query "$tmpDir/drv") +if nix path-info --store "https://${cachixCache}.cachix.org" "$outPath" &>/dev/null; then echo "$outPath has already been built successfully." exit 0 fi @@ -28,7 +28,7 @@ fi ## Build if [[ -v CIRRUS_CI ]]; then - cachix use $cachixCache + cachix use "$cachixCache" fi if [[ $CACHIX_SIGNING_KEY ]]; then @@ -38,10 +38,10 @@ else buildCmd=nix-build fi -$buildCmd --out-link $tmpDir/result $tmpDir/drv >/dev/null +$buildCmd --out-link "$tmpDir/result" "$tmpDir/drv" >/dev/null if [[ $CACHIX_SIGNING_KEY ]]; then - cachix push $cachixCache $outPath + cachix push "$cachixCache" "$outPath" fi -echo $outPath +echo "$outPath" diff --git a/test/ci/build.sh b/test/ci/build.sh index 3d48523..af80ffe 100755 --- a/test/ci/build.sh +++ b/test/ci/build.sh @@ -16,4 +16,5 @@ if [[ -v CIRRUS_CI ]]; then chmod o+rw /dev/kvm fi +# shellcheck disable=SC2154 "${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario" diff --git a/test/lib/copy-src.sh b/test/lib/copy-src.sh index 39e6995..e19134c 100644 --- a/test/lib/copy-src.sh +++ b/test/lib/copy-src.sh @@ -4,15 +4,18 @@ tmp=$(mktemp -d '/tmp/nix-bitcoin-src.XXXXX') # Move source cache if it exists (atomic) -mv /tmp/nix-bitcoin-src $tmp/src 2>/dev/null || true +mv /tmp/nix-bitcoin-src "$tmp/src" 2>/dev/null || true atExit() { # Set the current src as the source cache (atomic) - mv -T $tmp/src /tmp/nix-bitcoin-src 2>/dev/null || true - rm -rf $tmp + mv -T "$tmp/src" /tmp/nix-bitcoin-src 2>/dev/null || true + rm -rf "$tmp" } trap "atExit" EXIT -rsync -a --delete --exclude='.git*' "$scriptDir/../" $tmp/src +# shellcheck disable=SC2154 +rsync -a --delete --exclude='.git*' "$scriptDir/../" "$tmp/src" echo "Copied src" -_nixBitcoinInCopiedSrc=1 $tmp/src/test/run-tests.sh "${args[@]}" + +# shellcheck disable=SC2154 +_nixBitcoinInCopiedSrc=1 "$tmp/src/test/run-tests.sh" "${args[@]}" diff --git a/test/lib/create-git-repo.sh b/test/lib/create-git-repo.sh index 4f635c5..110dfdf 100644 --- a/test/lib/create-git-repo.sh +++ b/test/lib/create-git-repo.sh @@ -1,13 +1,15 @@ # Create and maintain a minimal git repo at the root of the copied src ( + # shellcheck disable=SC2154,SC2164 cd "$scriptDir/.." amend=--amend + if [[ ! -e .git ]]; then git init amend= fi git add . if ! git diff --quiet --cached; then - git commit -a $amend -m - + git commit -a "$amend" -m - fi ) >/dev/null diff --git a/test/lib/make-container.sh b/test/lib/make-container.sh index 89944b5..19d009f 100755 --- a/test/lib/make-container.sh +++ b/test/lib/make-container.sh @@ -57,6 +57,8 @@ if [[ $EUID != 0 ]]; then # NixOS containers require root permissions. # By using sudo here and not at the user's call-site extra-container can detect if it is running # inside an existing shell session (by checking an internal environment variable). + # + # shellcheck disable=SC2154 exec sudo scenario="$scenario" scriptDir="$scriptDir" NIX_PATH="$NIX_PATH" PATH="$PATH" \ scenarioOverridesFile="${scenarioOverridesFile:-}" "$scriptDir/lib/make-container.sh" "$@" fi @@ -64,7 +66,7 @@ fi export containerName=nb-test containerCommand=shell -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do case $1 in --command|-c) shift @@ -77,14 +79,14 @@ while [[ $# > 0 ]]; do done containerBin=$(type -P extra-container) || true -if [[ ! ($containerBin && $(realpath $containerBin) == *extra-container-0.10*) ]]; then +if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) ]]; then echo "Building extra-container. Skip this step by adding extra-container 0.10 to PATH." nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \ -A pinned.extra-container >/dev/null export PATH="/tmp/extra-container/bin${PATH:+:}$PATH" fi -read -d '' src <> /etc/nix/nix.conf -export PATH=$(nix shell -L .#flake-info .#cachix -c sh -c 'echo $PATH') + +# shellcheck disable=SC2016 +PATH=$(nix shell -L .#flake-info .#cachix -c sh -c 'echo $PATH') if [[ ${CACHIX_SIGNING_KEY:-} ]]; then - cachix push $cachixCache $(type -P flake-info); + cachix push "$cachixCache" "$(type -P flake-info)"; fi echo "Running flake-info (nixos-search)" diff --git a/test/run-tests.sh b/test/run-tests.sh index 0d8ceae..54f25a9 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -109,14 +109,18 @@ numCPUs=${numCPUs:-$(nproc)} # Min. 800 MiB needed to avoid 'out of memory' errors memoryMiB=${memoryMiB:-2048} -export NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..") +NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..") +export NIX_PATH runAtExit= trap 'eval "$runAtExit"' EXIT # Support explicit scenario definitions if [[ $scenario = *' '* ]]; then - export scenarioOverridesFile=$(mktemp ${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX) + scenarioOverridesFile=$(mktemp "${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX") + export scenarioOverridesFile + + # shellcheck disable=SC2016 runAtExit+='rm -f "$scenarioOverridesFile";' echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" scenario=tmp @@ -125,10 +129,11 @@ fi # Run the test. No temporary files are left on the host system. run() { # TMPDIR is also used by the test driver for VM tmp files - export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) - runAtExit+="rm -rf $TMPDIR;" + TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) + export TMPDIR + runAtExit+="rm -rf ${TMPDIR};" - nix-build --out-link $TMPDIR/driver -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver + nix-build --out-link "$TMPDIR/driver" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver # Variable 'tests' contains the Python code that is executed by the driver on startup if [[ $1 == --interactive ]]; then @@ -150,14 +155,14 @@ run() { echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on' - cd $TMPDIR # The VM creates a VDE control socket in $PWD + cd "$TMPDIR" # The VM creates a VDE control socket in $PWD env -i \ NIX_PATH="$NIX_PATH" \ TMPDIR="$TMPDIR" \ USE_TMPDIR=1 \ QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ QEMU_NET_OPTS="$QEMU_NET_OPTS" \ - $TMPDIR/driver/bin/nixos-test-driver <(echo "$tests") + "$TMPDIR/driver/bin/nixos-test-driver" <(echo "$tests") } debug() { @@ -179,18 +184,20 @@ container() { # Run a regular NixOS VM vm() { - export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) + TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) + export TMPDIR runAtExit+="rm -rf $TMPDIR;" - nix-build --out-link $TMPDIR/vm -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vmWithoutTests" + nix-build --out-link "$TMPDIR/vm" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vmWithoutTests" echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" [[ $NB_TEST_ENABLE_NETWORK ]] || export QEMU_NET_OPTS="restrict=on,$QEMU_NET_OPTS" + # shellcheck disable=SC2211 USE_TMPDIR=1 \ NIX_DISK_IMAGE=$TMPDIR/img.qcow2 \ QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ - $TMPDIR/vm/bin/run-*-vm + "$TMPDIR"/vm/bin/run-*-vm } doBuild() { @@ -223,6 +230,7 @@ vmTestNixExpr() { memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo) memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo) # Round down to nearest multiple of 50 MiB for improved test build caching + # shellcheck disable=SC2017 ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50)) ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" @@ -276,10 +284,10 @@ nixosSearch() { if [[ $outLinkPrefix ]]; then # Add gcroots for flake-info - nix build $scriptDir/nixos-search#flake-info -o "$outLinkPrefix-flake-info" + nix build "$scriptDir/nixos-search#flake-info" -o "$outLinkPrefix-flake-info" fi echo "Running flake-info (nixos-search)" - nix run $scriptDir/nixos-search#flake-info -- flake "$scriptDir/.." + nix run "$scriptDir/nixos-search#flake-info" -- flake "$scriptDir/.." } # A basic subset of tests to keep the total runtime within @@ -330,7 +338,7 @@ build() { buildTest "$@" } -if [[ $# > 0 && $1 != -* ]]; then +if [[ $# -gt 0 && $1 != -* ]]; then # An explicit command was provided command=$1 shift From 91a03ce7d2a4d92edcb875938454da91b4178a21 Mon Sep 17 00:00:00 2001 From: Otto Sabart Date: Tue, 16 Aug 2022 21:00:00 +0200 Subject: [PATCH 5/5] shellcheck: fix lint of scripts in /helper --- helper/fetch-release | 10 ++++---- helper/push-release.sh | 30 ++++++++++++------------ helper/update-fixed-output-derivation.sh | 6 ++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/helper/fetch-release b/helper/fetch-release index ee14de8..2066b3b 100755 --- a/helper/fetch-release +++ b/helper/fetch-release @@ -12,10 +12,10 @@ if [[ ! -v version ]]; then fi TMPDIR=$(mktemp -d) -trap "rm -rf $TMPDIR" EXIT +trap 'rm -rf $TMPDIR' EXIT export GNUPGHOME=$TMPDIR/gpg-home -mkdir -p -m 700 "$GNUPGHOME" +mkdir -m 700 "$GNUPGHOME" # Import key gpg --import "$scriptDir/key-jonasnick.bin" &> /dev/null @@ -25,10 +25,10 @@ gpg --import "$scriptDir/key-jonasnick.bin" &> /dev/null gpg --list-keys "36C7 1A37 C9D9 88BD E825 08D9 B1A7 0E4F 8DCD 0366" > /dev/null # Fetch nar-hash of release -cd $TMPDIR +cd "$TMPDIR" baseUrl=https://github.com/$repo/releases/download/v$version -curl -fsS -L -O $baseUrl/nar-hash.txt -curl -fsS -L -O $baseUrl/nar-hash.txt.asc +curl -fsS -L -O "$baseUrl/nar-hash.txt" +curl -fsS -L -O "$baseUrl/nar-hash.txt.asc" # Verify signature for nar-hash gpg --verify nar-hash.txt.asc &> /dev/null || { diff --git a/helper/push-release.sh b/helper/push-release.sh index 9e833a3..91407f5 100755 --- a/helper/push-release.sh +++ b/helper/push-release.sh @@ -37,11 +37,11 @@ fi cd "${BASH_SOURCE[0]%/*}" RESPONSE=$(curl https://api.github.com/repos/$REPO/releases/latest 2> /dev/null) -echo "Latest release" $(echo $RESPONSE | jq -r '.tag_name' | tail -c +2) +echo "Latest release" "$(echo "$RESPONSE" | jq -r '.tag_name' | tail -c +2)" if [[ ! $DRY_RUN ]]; then while true; do - read -p "Create release $TAG_NAME? [yn] " yn + read -rp "Create release ${TAG_NAME}? [yn] " yn case $yn in [Yy]* ) break;; [Nn]* ) exit;; @@ -51,22 +51,22 @@ if [[ ! $DRY_RUN ]]; then fi TMPDIR=$(mktemp -d) -if [[ ! $DRY_RUN ]]; then trap "rm -rf $TMPDIR" EXIT; fi +if [[ ! $DRY_RUN ]]; then trap 'rm -rf $TMPDIR' EXIT; fi ARCHIVE_NAME=nix-bitcoin-$TAG_NAME.tar.gz ARCHIVE=$TMPDIR/$ARCHIVE_NAME # Need to be in the repo root directory for archiving -(cd $(git rev-parse --show-toplevel); git archive --format=tar.gz -o $ARCHIVE $BRANCH) +(cd "$(git rev-parse --show-toplevel)"; git archive --format=tar.gz -o "$ARCHIVE" "$BRANCH") SHA256SUMS=$TMPDIR/SHA256SUMS.txt # Use relative path with sha256sums because it'll output the first # argument -(cd $TMPDIR; sha256sum $ARCHIVE_NAME > $SHA256SUMS) -gpg -o $SHA256SUMS.asc -a --detach-sig $SHA256SUMS +(cd "$TMPDIR"; sha256sum "$ARCHIVE_NAME" > "$SHA256SUMS") +gpg -o "$SHA256SUMS.asc" -a --detach-sig "$SHA256SUMS" -pushd $TMPDIR >/dev/null +pushd "$TMPDIR" >/dev/null -nix hash to-sri --type sha256 $(nix-prefetch-url --unpack file://$ARCHIVE 2> /dev/null) > nar-hash.txt +nix hash to-sri --type sha256 "$(nix-prefetch-url --unpack "file://$ARCHIVE" 2> /dev/null)" > nar-hash.txt gpg -o nar-hash.txt.asc -a --detach-sig nar-hash.txt if [[ $DRY_RUN ]]; then @@ -76,7 +76,7 @@ fi POST_DATA="{ \"tag_name\": \"v$TAG_NAME\", \"name\": \"nix-bitcoin-$TAG_NAME\", \"body\": \"nix-bitcoin-$TAG_NAME\", \"target_comitish\": \"$BRANCH\" }" RESPONSE=$(curl -H "Authorization: token $OAUTH_TOKEN" -d "$POST_DATA" https://api.github.com/repos/$REPO/releases 2> /dev/null) -ID=$(echo $RESPONSE | jq -r '.id') +ID=$(echo "$RESPONSE" | jq -r '.id') if [[ $ID == null ]]; then echo "Failed to create release with $POST_DATA" exit 1 @@ -85,20 +85,20 @@ fi post_asset() { GH_ASSET="https://uploads.github.com/repos/$REPO/releases/$ID/assets?name=" curl -H "Authorization: token $OAUTH_TOKEN" --data-binary "@$1" -H "Content-Type: application/octet-stream" \ - $GH_ASSET/$(basename $1) &> /dev/null + "$GH_ASSET/$(basename "$1")" &> /dev/null } post_asset nar-hash.txt post_asset nar-hash.txt.asc # Post additional assets for backwards compatibility. # This allows older nix-bitcoin installations to upgrade via `fetch-release`. -post_asset $ARCHIVE -post_asset $SHA256SUMS -post_asset $SHA256SUMS.asc +post_asset "$ARCHIVE" +post_asset "$SHA256SUMS" +post_asset "$SHA256SUMS.asc" popd >/dev/null if [[ ! $DRY_RUN ]]; then - git push $GIT_REMOTE $BRANCH:release + git push "$GIT_REMOTE" "${BRANCH}:release" fi -echo "Successfully created" $(echo $POST_DATA | jq -r .tag_name) +echo "Successfully created" "$(echo "$POST_DATA" | jq -r .tag_name)" diff --git a/helper/update-fixed-output-derivation.sh b/helper/update-fixed-output-derivation.sh index 65e99d0..dafceb4 100755 --- a/helper/update-fixed-output-derivation.sh +++ b/helper/update-fixed-output-derivation.sh @@ -7,10 +7,10 @@ flakeOutput=$2 # A pattern in a line preceding the hash that should be updated patternPrecedingHash=$3 -sed -i "/$patternPrecedingHash/,/hash/ s|hash = .*|hash = \"\";|" $file +sed -i "/$patternPrecedingHash/,/hash/ s|hash = .*|hash = \"\";|" "$file" # Display stderr and capture it. stdbuf is required to disable output buffering. stderr=$( - nix build --no-link -L .#$flakeOutput |& + nix build --no-link -L ".#$flakeOutput" |& stdbuf -oL grep -v '\berror:.*failed to build$' | tee /dev/stderr || : ) @@ -20,5 +20,5 @@ if [[ ! $hash ]]; then echo "Error: No hash in build output." exit 1 fi -sed -i "/$patternPrecedingHash/,/hash/ s|hash = .*|hash = \"$hash\";|" $file +sed -i "/$patternPrecedingHash/,/hash/ s|hash = .*|hash = \"$hash\";|" "$file" echo "(Note: The above hash mismatch message is not an error. It is part of the fetching process.)"