From 349f904e79544c9d80a40f7552beaa3bebb29434 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:25 +0100 Subject: [PATCH 01/13] secrets: use assertion instead of error in default value This enables generating module option documentation. --- modules/secrets/secrets.nix | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/secrets/secrets.nix b/modules/secrets/secrets.nix index 4776bca..4ad7752 100644 --- a/modules/secrets/secrets.nix +++ b/modules/secrets/secrets.nix @@ -67,16 +67,7 @@ let secretsSetupMethod = mkOption { type = types.str; - default = throw '' - Error: No secrets setup method has been defined. - To fix this, choose one of the following: - - - Use one of the deployment methods in ${toString ./../deployment} - - - Set `nix-bitcoin.generateSecrets = true` to automatically generate secrets - - - Set `nix-bitcoin.secretsSetupMethod = "manual"` if you want to manually setup secrets - ''; + default = null; }; generateSecretsScript = mkOption { @@ -141,6 +132,21 @@ in { inherit options; config = { + assertions = [ + { assertion = cfg.secretsSetupMethod != null; + message = '' + No secrets setup method has been defined. + To fix this, choose one of the following: + + - Use one of the deployment methods in ${toString ./../deployment} + + - Set `nix-bitcoin.generateSecrets = true` to automatically generate secrets + + - Set `nix-bitcoin.secretsSetupMethod = "manual"` if you want to manually setup secrets + ''; + } + ]; + # This target is active when secrets have been setup successfully. systemd.targets.nix-bitcoin-secrets = mkIf (cfg.secretsSetupMethod != "manual") { # This ensures that the secrets target is always activated when switching From f0096371bf2f287b29346ecd8a8d2913c1a11037 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:26 +0100 Subject: [PATCH 02/13] btcpayserver: expand `nbPkgs` in option defaults Required by commit `nix-bitcoin.nix: add `defaultText` (automatic)`. --- modules/btcpayserver.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/btcpayserver.nix b/modules/btcpayserver.nix index 94f6074..b1926e1 100644 --- a/modules/btcpayserver.nix +++ b/modules/btcpayserver.nix @@ -18,9 +18,9 @@ let package = mkOption { type = types.package; default = if cfg.btcpayserver.lbtc then - nbPkgs.btcpayserver.override { altcoinSupport = true; } + config.nix-bitcoin.pkgs.btcpayserver.override { altcoinSupport = true; } else - nbPkgs.btcpayserver; + config.nix-bitcoin.pkgs.btcpayserver; description = "The package providing btcpayserver binaries."; }; dataDir = mkOption { @@ -68,7 +68,7 @@ let }; package = mkOption { type = types.package; - default = nbPkgs.nbxplorer; + default = config.nix-bitcoin.pkgs.nbxplorer; description = "The package providing nbxplorer binaries."; }; address = mkOption { @@ -102,7 +102,6 @@ let cfg = config.services; nbLib = config.nix-bitcoin.lib; - nbPkgs = config.nix-bitcoin.pkgs; inherit (config.services) bitcoind liquidd; in { From d24c02943585150c8b79b879dd6262f86134c5a1 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:27 +0100 Subject: [PATCH 03/13] treewide: add `defaultText` (manual) This enables generating module option documentation. Add `defaultText` to options where the default value can't be evaluated during metadata generation. --- modules/joinmarket.nix | 2 ++ modules/nix-bitcoin.nix | 5 +++++ modules/nodeinfo.nix | 1 + 3 files changed, 8 insertions(+) diff --git a/modules/joinmarket.nix b/modules/joinmarket.nix index 79565ad..053a376 100644 --- a/modules/joinmarket.nix +++ b/modules/joinmarket.nix @@ -44,11 +44,13 @@ let }; cli = mkOption { default = cli; + defaultText = "(See source)"; }; # Used by ./joinmarket-ob-watcher.nix ircServers = mkOption { readOnly = true; default = ircServers; + defaultText = "(See source)"; }; # This option is only used by netns-isolation. # Tor is always enabled. diff --git a/modules/nix-bitcoin.nix b/modules/nix-bitcoin.nix index 605c115..0ad9eb8 100644 --- a/modules/nix-bitcoin.nix +++ b/modules/nix-bitcoin.nix @@ -7,17 +7,20 @@ with lib; pkgs = mkOption { type = types.attrs; default = (import ../pkgs { inherit pkgs; }).modulesPkgs; + defaultText = "nix-bitcoin/pkgs.modulesPkgs"; }; lib = mkOption { readOnly = true; default = import ../pkgs/lib.nix lib pkgs; + defaultText = "nix-bitcoin/pkgs/lib.nix"; }; torClientAddressWithPort = mkOption { readOnly = true; default = with config.services.tor.client.socksListenAddress; "${addr}:${toString port}"; + defaultText = "(See source)"; }; # Torify binary that works with custom Tor SOCKS addresses @@ -29,6 +32,7 @@ with lib; --address ${config.services.tor.client.socksListenAddress.addr} \ "$@" ''; + defaultText = "(See source)"; }; # A helper for using doas instead of sudo when doas is enabled @@ -38,6 +42,7 @@ with lib; # TODO-EXTERNAL: Use absolute path until https://github.com/NixOS/nixpkgs/pull/133622 is available. then "/run/wrappers/bin/doas -u" else "sudo -u"; + defaultText = "(See source)"; }; }; }; diff --git a/modules/nodeinfo.nix b/modules/nodeinfo.nix index bafd38d..1c3d544 100644 --- a/modules/nodeinfo.nix +++ b/modules/nodeinfo.nix @@ -8,6 +8,7 @@ let program = mkOption { readOnly = true; default = script; + defaultText = "(See source)"; }; }; }; From 3681f118f72e53e215a9bdcd453fa06dab81eafe Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:28 +0100 Subject: [PATCH 04/13] nix-bitcoin.nix: add `defaultText` (automatic) This enables generating module option documentation. This commit was genereated by running the following script inside the repo root dir: def add_default_text(file) src = File.read(file) src2 = src.gsub(/( = mkOption\s+\{[^{]*?)(\n\s+default = )(.*?);$(.*?\})/m) do |str| pre, defaultVar, default, post = Regexp.last_match.captures replacement = if !post.include?('defaultText =') if default =~ /\bpkgs\b/ defaultText = default.lines.length == 1 ? default : "(See source)" "#{pre}#{defaultVar}#{default};#{defaultVar.sub('default', 'defaultText')}#{defaultText.inspect};#{post}" end end replacement or str end File.write(file, src2) if src2 != src end Dir["modules/**/*.nix"].each do |f| next if File.basename(f) == "nix-bitcoin.nix" add_default_text f end --- modules/bitcoind.nix | 2 ++ modules/btcpayserver.nix | 2 ++ modules/clightning-plugins/clboss.nix | 1 + modules/clightning.nix | 1 + modules/lightning-loop.nix | 2 ++ modules/lightning-pool.nix | 2 ++ modules/liquid.nix | 2 ++ modules/lnd-rest-onion-service.nix | 1 + modules/lnd.nix | 2 ++ 9 files changed, 15 insertions(+) diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix index 2d1ea64..f158cce 100644 --- a/modules/bitcoind.nix +++ b/modules/bitcoind.nix @@ -57,6 +57,7 @@ let package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.bitcoind; + defaultText = "config.nix-bitcoin.pkgs.bitcoind"; description = "The package providing bitcoin binaries."; }; extraConfig = mkOption { @@ -265,6 +266,7 @@ let default = pkgs.writeScriptBin "bitcoin-cli" '' exec ${cfg.package}/bin/bitcoin-cli -datadir='${cfg.dataDir}' "$@" ''; + defaultText = "(See source)"; description = "Binary to connect with the bitcoind instance."; }; tor = nbLib.tor; diff --git a/modules/btcpayserver.nix b/modules/btcpayserver.nix index b1926e1..47b152e 100644 --- a/modules/btcpayserver.nix +++ b/modules/btcpayserver.nix @@ -21,6 +21,7 @@ let config.nix-bitcoin.pkgs.btcpayserver.override { altcoinSupport = true; } else config.nix-bitcoin.pkgs.btcpayserver; + defaultText = "(See source)"; description = "The package providing btcpayserver binaries."; }; dataDir = mkOption { @@ -69,6 +70,7 @@ let package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.nbxplorer; + defaultText = "config.nix-bitcoin.pkgs.nbxplorer"; description = "The package providing nbxplorer binaries."; }; address = mkOption { diff --git a/modules/clightning-plugins/clboss.nix b/modules/clightning-plugins/clboss.nix index fe48358..970a9cc 100644 --- a/modules/clightning-plugins/clboss.nix +++ b/modules/clightning-plugins/clboss.nix @@ -17,6 +17,7 @@ let cfg = config.services.clightning.plugins.clboss; in package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.clboss; + defaultText = "config.nix-bitcoin.pkgs.clboss"; description = "The package providing clboss binaries."; }; }; diff --git a/modules/clightning.nix b/modules/clightning.nix index 4e1f3dc..965ae81 100644 --- a/modules/clightning.nix +++ b/modules/clightning.nix @@ -69,6 +69,7 @@ let default = pkgs.writeScriptBin "lightning-cli" '' ${nbPkgs.clightning}/bin/lightning-cli --lightning-dir='${cfg.dataDir}' "$@" ''; + defaultText = "(See source)"; description = "Binary to connect with the clightning instance."; }; getPublicAddressCmd = mkOption { diff --git a/modules/lightning-loop.nix b/modules/lightning-loop.nix index 2d2441a..8d9039b 100644 --- a/modules/lightning-loop.nix +++ b/modules/lightning-loop.nix @@ -27,6 +27,7 @@ let package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.lightning-loop; + defaultText = "config.nix-bitcoin.pkgs.lightning-loop"; description = "The package providing lightning-loop binaries."; }; dataDir = mkOption { @@ -54,6 +55,7 @@ let --macaroonpath '${cfg.dataDir}/${network}/loop.macaroon' \ --tlscertpath '${secretsDir}/loop-cert' "$@" ''; + defaultText = "(See source)"; description = "Binary to connect with the lightning-loop instance."; }; tor = nbLib.tor; diff --git a/modules/lightning-pool.nix b/modules/lightning-pool.nix index c93aa6a..092a83f 100644 --- a/modules/lightning-pool.nix +++ b/modules/lightning-pool.nix @@ -27,6 +27,7 @@ let package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.lightning-pool; + defaultText = "config.nix-bitcoin.pkgs.lightning-pool"; description = "The package providing lightning-pool binaries."; }; dataDir = mkOption { @@ -54,6 +55,7 @@ let --network ${network} \ --basedir '${cfg.dataDir}' "$@" ''; + defaultText = "(See source)"; description = "Binary to connect with the lightning-pool instance."; }; tor = nbLib.tor; diff --git a/modules/liquid.nix b/modules/liquid.nix index 6e7e6ad..3e45992 100644 --- a/modules/liquid.nix +++ b/modules/liquid.nix @@ -148,12 +148,14 @@ let default = pkgs.writeScriptBin "elements-cli" '' ${nbPkgs.elementsd}/bin/elements-cli -datadir='${cfg.dataDir}' "$@" ''; + defaultText = "(See source)"; description = "Binary to connect with the liquidd instance."; }; swapCli = mkOption { default = pkgs.writeScriptBin "liquidswap-cli" '' ${nbPkgs.liquid-swap}/bin/liquidswap-cli -c '${cfg.dataDir}/elements.conf' "$@" ''; + defaultText = "(See source)"; description = "Binary for managing liquid swaps."; }; tor = nbLib.tor; diff --git a/modules/lnd-rest-onion-service.nix b/modules/lnd-rest-onion-service.nix index 3d845c1..f2c94d5 100644 --- a/modules/lnd-rest-onion-service.nix +++ b/modules/lnd-rest-onion-service.nix @@ -15,6 +15,7 @@ let package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.lndconnect; + defaultText = "config.nix-bitcoin.pkgs.lndconnect"; description = "The package providing lndconnect binaries."; }; }; diff --git a/modules/lnd.nix b/modules/lnd.nix index 5d14846..409f1a7 100644 --- a/modules/lnd.nix +++ b/modules/lnd.nix @@ -81,6 +81,7 @@ let package = mkOption { type = types.package; default = config.nix-bitcoin.pkgs.lnd; + defaultText = "config.nix-bitcoin.pkgs.lnd"; description = "The package providing lnd binaries."; }; cli = mkOption { @@ -92,6 +93,7 @@ let --tlscertpath '${cfg.certPath}' \ --macaroonpath '${networkDir}/admin.macaroon' "$@" ''; + defaultText = "(See source)"; description = "Binary to connect with the lnd instance."; }; getPublicAddressCmd = mkOption { From bbebd0b3832fd584da0aa0eb71144cfba6a0f6a6 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:29 +0100 Subject: [PATCH 05/13] treewide: remove invalid docbook XML from option descriptions NixOS will soon use CommonMark Markdown for option descriptions. We can then switch back the old, slightly clearer bracket syntax. --- modules/bitcoind.nix | 2 +- modules/liquid.nix | 2 +- modules/onion-addresses.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/bitcoind.nix b/modules/bitcoind.nix index f158cce..817a80f 100644 --- a/modules/bitcoind.nix +++ b/modules/bitcoind.nix @@ -120,7 +120,7 @@ let example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae"; description = '' Password HMAC-SHA-256 for JSON-RPC connections. Must be a string of the - format $. + format `salt-hex$hmac-hex`. ''; }; passwordHMACFromFile = mkOption { diff --git a/modules/liquid.nix b/modules/liquid.nix index 3e45992..352bf84 100644 --- a/modules/liquid.nix +++ b/modules/liquid.nix @@ -224,7 +224,7 @@ let example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae"; description = '' Password HMAC-SHA-256 for JSON-RPC connections. Must be a string of the - format $. + format `salt-hex$hmac-hex`. ''; }; }; diff --git a/modules/onion-addresses.nix b/modules/onion-addresses.nix index 07466c5..9ddd1c0 100644 --- a/modules/onion-addresses.nix +++ b/modules/onion-addresses.nix @@ -28,7 +28,7 @@ let default = []; description = '' Services that can access their onion address via file - /var/lib/onion-addresses/ + `/var/lib/onion-addresses/$service` The file is readable only by the service user. ''; }; From 96df81a4d0898e94635302f655afb903a3d4f6b4 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:30 +0100 Subject: [PATCH 06/13] run-tests: move `flake` next to other tests --- test/run-tests.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/run-tests.sh b/test/run-tests.sh index a4cddfe..22d8803 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -246,6 +246,14 @@ vmTestNixExpr() { EOF } +flake() { + if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then + echo "Skipping flake test. Nix flake support is not enabled." + else + nix flake check "$scriptDir/.." + fi +} + # A basic subset of tests to keep the total runtime within # manageable bounds (<4 min on desktop systems). # These are also run on the CI server. @@ -275,14 +283,6 @@ examples() { (cd "$scriptDir/../examples" && nix-shell --run "$script") } -flake() { - if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then - echo "Skipping flake test. Nix flake support is not enabled." - else - nix flake check "$scriptDir/.." - fi -} - all() { buildable examples From 77d8c00c535ddb9e76c16374327a39aaf9666d8e Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:31 +0100 Subject: [PATCH 07/13] run-tests.sh: extract `checkFlakeSupport` Required by the following commit. --- test/run-tests.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/test/run-tests.sh b/test/run-tests.sh index 22d8803..c052079 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -246,12 +246,25 @@ vmTestNixExpr() { EOF } -flake() { - if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then - echo "Skipping flake test. Nix flake support is not enabled." - else - nix flake check "$scriptDir/.." +checkFlakeSupport() { + testName=$1 + if [[ ! -v hasFlakes ]]; then + if [[ $(nix flake 2>&1) == *"requires a sub-command"* ]]; then + hasFlakes=1 + else + hasFlakes= + fi fi + if [[ ! $hasFlakes ]]; then + echo "Skipping test '$testName'. Nix flake support is not enabled." + return 1 + fi +} + +flake() { + if ! checkFlakeSupport "flake"; then return; fi + + nix flake check "$scriptDir/.." } # A basic subset of tests to keep the total runtime within From eaf98f92dbeaef780285fab1fabc7be3ba500a1c Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:32 +0100 Subject: [PATCH 08/13] test: add test `nixosSearch` This checks that creating module docs for search.nixos.org succceeds. Errors can happen when complex `default` values can't be evaluated or on malformed Docbook XML in descriptions. --- helper/update-flake.sh | 5 ++++ test/nixos-search/flake.lock | 44 ++++++++++++++++++++++++++++++++++++ test/nixos-search/flake.nix | 9 ++++++++ test/run-tests.sh | 13 +++++++++++ 4 files changed, 71 insertions(+) create mode 100644 test/nixos-search/flake.lock create mode 100644 test/nixos-search/flake.nix diff --git a/helper/update-flake.sh b/helper/update-flake.sh index 939786f..ca48f72 100755 --- a/helper/update-flake.sh +++ b/helper/update-flake.sh @@ -26,11 +26,16 @@ if [[ ${1:-} != -f ]] && ! git diff --quiet ../flake.{nix,lock}; then exit 1 fi +echo "Updating flake 'nixos-search'" +nix flake update ../test/nixos-search +echo + versions=$(nix eval --json -f update-flake.nix versions) ## Uncomment the following to generate a version change message for testing # versions=$(echo "$versions" | sed 's|1|0|g') +echo "Updating main flake" nix flake update .. echo diff --git a/test/nixos-search/flake.lock b/test/nixos-search/flake.lock new file mode 100644 index 0000000..87ec42f --- /dev/null +++ b/test/nixos-search/flake.lock @@ -0,0 +1,44 @@ +{ + "nodes": { + "nixos-search": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1638831015, + "narHash": "sha256-WmCsa6gSY01Yt0wminCbNV8Mn+4iU1OEF5r3YUh6SVs=", + "owner": "nixos", + "repo": "nixos-search", + "rev": "fa5fe2a61c36ed90506af75c19bfaf2f9d537d87", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixos-search", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1631118067, + "narHash": "sha256-tEcFvm3a6ToeBGwHdjfB2mVQwa4LZCZTQYE2LnY3ycA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "09cd65b33c5653d7d2954fef4b9f0e718c899743", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-unstable", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixos-search": "nixos-search" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/test/nixos-search/flake.nix b/test/nixos-search/flake.nix new file mode 100644 index 0000000..f92bf29 --- /dev/null +++ b/test/nixos-search/flake.nix @@ -0,0 +1,9 @@ +# This flake just mirrors input `nixos-search`. +# Because `nixos-search` is a dev-only dependency, we don't add +# it to the main flake. +{ + inputs.nixos-search.url = "github:nixos/nixos-search"; + outputs = { self, nixos-search }: { + inherit (nixos-search) packages; + }; +} diff --git a/test/run-tests.sh b/test/run-tests.sh index c052079..649fd81 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -267,6 +267,18 @@ flake() { nix flake check "$scriptDir/.." } +# Test generating module documentation for search.nixos.org +nixosSearch() { + if ! checkFlakeSupport "nixosSearch"; then return; fi + + if [[ $outLinkPrefix ]]; then + # Add gcroots for flake-info + nix build $scriptDir/nixos-search#flake-info -o "$outLinkPrefix-flake-info" + fi + echo "Running flake-info (nixos-search)" + nix run $scriptDir/nixos-search#flake-info -- flake ../. +} + # A basic subset of tests to keep the total runtime within # manageable bounds (<4 min on desktop systems). # These are also run on the CI server. @@ -300,6 +312,7 @@ all() { buildable examples flake + nixosSearch } # An alias for buildTest From 0fd3be534315582fd08b86f2e7996ded3976d845 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Wed, 8 Dec 2021 04:07:33 +0100 Subject: [PATCH 09/13] move ci/ to test/ --- .cirrus.yml | 2 +- {ci => test/ci}/build-to-cachix.sh | 0 {ci => test/ci}/build.sh | 2 +- test/run-tests.sh | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename {ci => test/ci}/build-to-cachix.sh (100%) rename {ci => test/ci}/build.sh (90%) diff --git a/.cirrus.yml b/.cirrus.yml index 56f8a49..131cc47 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -34,4 +34,4 @@ task: build_script: - echo "sandbox = true" >> /etc/nix/nix.conf - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)" - - nix run -f '' bash coreutils cachix -c ./ci/build.sh + - nix run -f '' bash coreutils cachix -c ./test/ci/build.sh diff --git a/ci/build-to-cachix.sh b/test/ci/build-to-cachix.sh similarity index 100% rename from ci/build-to-cachix.sh rename to test/ci/build-to-cachix.sh diff --git a/ci/build.sh b/test/ci/build.sh similarity index 90% rename from ci/build.sh rename to test/ci/build.sh index 3c3f2e1..dd04779 100755 --- a/ci/build.sh +++ b/test/ci/build.sh @@ -26,4 +26,4 @@ else testArgs=pkgsUnstable fi -"${BASH_SOURCE[0]%/*}/../test/run-tests.sh" --ci $testArgs +"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario $scenario diff --git a/test/run-tests.sh b/test/run-tests.sh index 649fd81..06ac514 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -199,7 +199,7 @@ doBuild() { name=$1 shift if [[ $ciBuild ]]; then - "$scriptDir/../ci/build-to-cachix.sh" "$@" + "$scriptDir/ci/build-to-cachix.sh" "$@" else if [[ $outLinkPrefix ]]; then outLink="--out-link $outLinkPrefix-$name" From 26a53f0d59a0865e56e2c4c3e7ae6bc602ff46cd Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Fri, 10 Dec 2021 14:07:56 +0100 Subject: [PATCH 10/13] ci: simplify test Remove obsolete features that were required for pkgsUnstable. --- .cirrus.yml | 16 ++++++---------- test/ci/build.sh | 10 +--------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 131cc47..03ff10f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,22 +16,18 @@ task: timeout_in: 120m name: modules_test - environment: - nixpkgs: nixpkgs container: # A maximum of 16 CPUs is shared among all concurrent tasks. # https://cirrus-ci.org/faq/#are-there-any-limits cpu: 4 - matrix: - - environment: - scenario: default - - environment: - scenario: netns - - environment: - scenario: netnsRegtest + environment: + matrix: + - scenario: default + - scenario: netns + - scenario: netnsRegtest # This script is run as root build_script: - echo "sandbox = true" >> /etc/nix/nix.conf - - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix $nixpkgs)" + - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)" - nix run -f '' bash coreutils cachix -c ./test/ci/build.sh diff --git a/test/ci/build.sh b/test/ci/build.sh index dd04779..da711b8 100755 --- a/test/ci/build.sh +++ b/test/ci/build.sh @@ -7,9 +7,7 @@ set -euo pipefail -scenario=${scenario:-} - -if [[ -v CIRRUS_CI && $scenario ]]; then +if [[ -v CIRRUS_CI ]]; then if [[ ! -e /dev/kvm ]]; then >&2 echo "No KVM available on VM host." exit 1 @@ -20,10 +18,4 @@ fi echo "$NIX_PATH ($(nix eval --raw nixpkgs.lib.version))" -if [[ $scenario ]]; then - testArgs="--scenario $scenario" -else - testArgs=pkgsUnstable -fi - "${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario $scenario From d3c8d7bd6f0bc2b4d1bfc73717e640655654836f Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Fri, 10 Dec 2021 14:07:57 +0100 Subject: [PATCH 11/13] .cirrus.yml: refactor Required by the following commit. --- .cirrus.yml | 56 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 03ff10f..294b910 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,33 +1,31 @@ -container: - image: nixos/nix - # Besides virtualization, this also enables privileged containers which are required for - # sandboxed builds - kvm: true - # Needed for package builds - memory: 8G - -environment: - CACHIX_SIGNING_KEY: ENCRYPTED[!cec502ed813cbcd0237697d2031f750186ff20eed5815b1ad950ad2f2d701702ae6ba2f0cb4cb1985687a696c8ee492c!] - # Save some traffic by excluding the full git history - CIRRUS_CLONE_DEPTH: 1 - task: + environment: + CACHIX_SIGNING_KEY: ENCRYPTED[!cec502ed813cbcd0237697d2031f750186ff20eed5815b1ad950ad2f2d701702ae6ba2f0cb4cb1985687a696c8ee492c!] + # Save some traffic by excluding the full git history + CIRRUS_CLONE_DEPTH: 1 + # Use the maximum timeout. Needed when rebuilding packages on a channel update. timeout_in: 120m - name: modules_test - container: - # A maximum of 16 CPUs is shared among all concurrent tasks. - # https://cirrus-ci.org/faq/#are-there-any-limits - cpu: 4 - environment: - matrix: - - scenario: default - - scenario: netns - - scenario: netnsRegtest - - # This script is run as root - build_script: - - echo "sandbox = true" >> /etc/nix/nix.conf - - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)" - - nix run -f '' bash coreutils cachix -c ./test/ci/build.sh + matrix: + - name: modules_test + container: + image: nixos/nix + # Besides virtualization, this also enables privileged containers which are required for + # sandboxed builds + kvm: true + # Needed for package builds + memory: 8G + # A maximum of 16 CPUs is shared among all concurrent tasks. + # https://cirrus-ci.org/faq/#are-there-any-limits + cpu: 4 + environment: + matrix: + - scenario: default + - scenario: netns + - scenario: netnsRegtest + # This script is run as root + build_script: + - echo "sandbox = true" >> /etc/nix/nix.conf + - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)" + - nix run -f '' bash coreutils cachix -c ./test/ci/build.sh From 1ef40ac230fd4acdf3183357b19f393dc297e788 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Fri, 10 Dec 2021 14:07:58 +0100 Subject: [PATCH 12/13] ci: add flake test - Check flake - Check nixos-search doc generation --- .cirrus.yml | 7 +++++++ test/nixos-search/ci-test.sh | 23 +++++++++++++++++++++++ test/nixos-search/flake.nix | 4 ++++ 3 files changed, 34 insertions(+) create mode 100755 test/nixos-search/ci-test.sh diff --git a/.cirrus.yml b/.cirrus.yml index 294b910..2dff4ac 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -29,3 +29,10 @@ task: - echo "sandbox = true" >> /etc/nix/nix.conf - export NIX_PATH="nixpkgs=$(nix eval --raw -f pkgs/nixpkgs-pinned.nix nixpkgs)" - nix run -f '' bash coreutils cachix -c ./test/ci/build.sh + + - name: flake + container: + image: nixpkgs/nix-flakes + build_script: + - nix flake check + - ./test/nixos-search/ci-test.sh diff --git a/test/nixos-search/ci-test.sh b/test/nixos-search/ci-test.sh new file mode 100755 index 0000000..e719fb4 --- /dev/null +++ b/test/nixos-search/ci-test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "${BASH_SOURCE[0]%/*}" + +# Use cachix to cache the `flake-info` build +cachixCache=nix-bitcoin + +nix run .#cachix -- use $cachixCache + +# We're running in a basic, unprivileged container that doesn't support sandboxing. +# Sandboxing is unnneeded because we're only building the 3rd-party `flake-info` tool. +echo "sandbox = false" >> /etc/nix/nix.conf +export PATH=$(nix shell -L .#flake-info .#cachix -c sh -c 'echo $PATH') + +if [[ ${CACHIX_SIGNING_KEY:-} ]]; then + cachix push $cachixCache $(type -P flake-info); +fi + +# flake-info requires '' +export NIX_PATH=nixpkgs=$(nix eval --raw .#nixpkgsPath) +echo "Running flake-info (nixos-search)" +flake-info flake ./. diff --git a/test/nixos-search/flake.nix b/test/nixos-search/flake.nix index f92bf29..76eb49e 100644 --- a/test/nixos-search/flake.nix +++ b/test/nixos-search/flake.nix @@ -5,5 +5,9 @@ inputs.nixos-search.url = "github:nixos/nixos-search"; outputs = { self, nixos-search }: { inherit (nixos-search) packages; + + # Used by ./ci-test.sh + inherit (nixos-search.inputs.nixpkgs) legacyPackages; + nixpkgsPath = toString nixos-search.inputs.nixpkgs; }; } From 5e59c784ed2c7092d7ab8df365a56e7a4bd05dbe Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Fri, 10 Dec 2021 14:07:59 +0100 Subject: [PATCH 13/13] flake: remove internal pkg `netns-exec` from packages --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 34e635e..b994e5f 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ nbPkgs = self.mkNbPkgs { inherit system pkgs; }; packages = flake-utils.lib.flattenTree (removeAttrs nbPkgs [ - "pinned" "modulesPkgs" "nixops19_09" "krops" "generate-secrets" + "pinned" "modulesPkgs" "nixops19_09" "krops" "generate-secrets" "netns-exec" ]) // { runVM = mkVMScript packages.vm;