Merge #215: Improve abstraction in test scenarios

4ece2da8db tests: move nginx check in scenarios lib (Jonas Nick)
5c0170c6b8 tests: add post-clightning extra tests to scenarios (Jonas Nick)
5fa0602a18 tests: add extra_tests argument to scenario lib run_tests() (Jonas Nick)
6f9349b0a4 tests: create run_tests() function in the scenario lib (Jonas Nick)
3d2b366156 tests: move common code in tests to lib (Jonas Nick)
e13c532dbf tests: move common code of scenarios python scripts to the top (Jonas Nick)

Pull request description:

ACKs for top commit:
  nixbitcoin:
    ACK 4ece2da8db

Tree-SHA512: 72893ff186915dd1b1439f2a81bc75d3618f89ee21f1229448cabaced6ca10c244705a74e9f4e6f8825691df9cd351a6dc9e142ec87ecab8ee17a17e78c72292

test/scenarios/default.py

@@ -1,100 +1,41 @@
-### Tests
+def electrs():
+    machine.wait_for_open_port(4224)  # prometeus metrics provider
 
-assert_running("setup-secrets")
-# Unused secrets should be inaccessible
-succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
 
-assert_running("bitcoind")
+def spark_wallet():
-machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
+    machine.wait_for_open_port(9737)
-assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
+    spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
-# Test RPC Whitelist
+    assert_matches(f"curl -s {spark_auth}@localhost:9737", "Spark")
-machine.wait_until_succeeds("su operator -c 'bitcoin-cli help'")
-# Restating rpcuser & rpcpassword overrides privileged credentials
-machine.fail(
-    "bitcoin-cli -rpcuser=publicrpc -rpcpassword=$(cat /secrets/bitcoin-rpcpassword-public) help"
-)
-machine.wait_until_succeeds(
-    log_has_string("bitcoind", "RPC User publicrpc not allowed to call method help")
-)
 
-assert_running("electrs")
-machine.wait_for_open_port(4224)  # prometeus metrics provider
-# Check RPC connection to bitcoind
-machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
-assert_running("nginx")
-# Stop electrs from spamming the test log with 'wait for bitcoind sync' messages
-succeed("systemctl stop electrs")
 
-assert_running("liquidd")
+def lightning_charge():
-machine.wait_until_succeeds("elements-cli getnetworkinfo")
+    machine.wait_for_open_port(9112)
-assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
+    charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
-succeed("su operator -c 'liquidswap-cli --help'")
+    assert_matches(f"curl -s api-token:{charge_auth}@localhost:9112/info | jq", '"id"')
 
-assert_running("clightning")
-assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
 
-assert_running("spark-wallet")
+def nanopos():
-spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
+    machine.wait_for_open_port(9116)
-machine.wait_for_open_port(9737)
+    assert_matches("curl localhost:9116", "tshirt")
-assert_matches(f"curl -s {spark_auth}@localhost:9737", "Spark")
 
-assert_running("lightning-charge")
-charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
-machine.wait_for_open_port(9112)
-assert_matches(f"curl -s api-token:{charge_auth}@localhost:9112/info | jq", '"id"')
 
-assert_running("nanopos")
+def web_index():
-machine.wait_for_open_port(9116)
+    machine.wait_for_open_port(80)
-assert_matches("curl localhost:9116", "tshirt")
+    assert_matches("curl localhost", "nix-bitcoin")
+    assert_matches("curl -L localhost/store", "tshirt")
 
-assert_running("onion-chef")
 
-# FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
+def post_clightning():
-# to incomplete unit dependencies.
+    pass
-# 'create-web-index' implicitly tests 'nodeinfo'.
-machine.wait_for_unit("create-web-index")
-machine.wait_for_open_port(80)
-assert_matches("curl localhost", "nix-bitcoin")
-assert_matches("curl -L localhost/store", "tshirt")
 
-machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
-assert_no_failure("bitcoind-import-banlist")
 
-# test that `systemctl status` can't leak credentials
+extra_tests = {
-assert_matches(
+    "electrs": electrs,
-    "sudo -u electrs systemctl status clightning 2>&1 >/dev/null",
+    "spark-wallet": spark_wallet,
-    "Failed to dump process list for 'clightning.service', ignoring: Access denied",
+    "lightning-charge": lightning_charge,
-)
+    "nanopos": nanopos,
-machine.succeed("grep -Fq hidepid=2 /proc/mounts")
+    "web-index": web_index,
+    "post-clightning": post_clightning,
+}
 
-### Additional tests
+run_tests(extra_tests)
-
-# Current time in µs
-pre_restart = succeed("date +%s.%6N").rstrip()
-
-# Sanity-check system by restarting all services
-succeed("systemctl restart bitcoind clightning spark-wallet lightning-charge nanopos liquidd")
-
-# Now that the bitcoind restart triggered a banlist import restart, check that
-# re-importing already banned addresses works
-machine.wait_until_succeeds(
-    log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
-)
-assert_no_failure("bitcoind-import-banlist")
-
-### Test lnd
-
-succeed("systemctl stop nanopos lightning-charge spark-wallet clightning")
-succeed("systemctl start lnd")
-assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
-assert_no_failure("lnd")
-
-### Test loopd
-
-succeed("systemctl start lightning-loop")
-assert_matches("su operator -c 'loop --version'", "version")
-# Check that lightning-loop fails with the right error, making sure
-# lightning-loop can connect to lnd
-machine.wait_until_succeeds(
-    log_has_string("lightning-loop", "chain notifier RPC isstill in the process of starting")
-)

test/scenarios/lib.py

@@ -32,3 +32,106 @@ def assert_running(unit):
 # Don't execute the following test suite when this script is running in interactive mode
 if "is_interactive" in vars():
     raise Exception()
+
+### Tests
+
+# The argument extra_tests is a dictionary from strings to functions. The string
+# determines at which point of run_tests the corresponding function is executed.
+def run_tests(extra_tests):
+    assert_running("setup-secrets")
+    # Unused secrets should be inaccessible
+    succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
+
+    assert_running("bitcoind")
+    machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
+    assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
+    # Test RPC Whitelist
+    machine.wait_until_succeeds("su operator -c 'bitcoin-cli help'")
+    # Restating rpcuser & rpcpassword overrides privileged credentials
+    machine.fail(
+        "bitcoin-cli -rpcuser=publicrpc -rpcpassword=$(cat /secrets/bitcoin-rpcpassword-public) help"
+    )
+    machine.wait_until_succeeds(
+        log_has_string("bitcoind", "RPC User publicrpc not allowed to call method help")
+    )
+
+    assert_running("electrs")
+    extra_tests.pop("electrs")()
+    # Check RPC connection to bitcoind
+    machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
+    # Stop electrs from spamming the test log with 'wait for bitcoind sync' messages
+    succeed("systemctl stop electrs")
+
+    assert_running("liquidd")
+    machine.wait_until_succeeds("elements-cli getnetworkinfo")
+    assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
+    succeed("su operator -c 'liquidswap-cli --help'")
+
+    assert_running("clightning")
+    assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+
+    assert_running("spark-wallet")
+    extra_tests.pop("spark-wallet")()
+
+    assert_running("lightning-charge")
+    extra_tests.pop("lightning-charge")()
+
+    assert_running("nanopos")
+    extra_tests.pop("nanopos")()
+
+    assert_running("onion-chef")
+
+    # FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
+    # to incomplete unit dependencies.
+    # 'create-web-index' implicitly tests 'nodeinfo'.
+    machine.wait_for_unit("create-web-index")
+    assert_running("nginx")
+    extra_tests.pop("web-index")()
+
+    machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
+    assert_no_failure("bitcoind-import-banlist")
+
+    # test that `systemctl status` can't leak credentials
+    assert_matches(
+        "sudo -u electrs systemctl status clightning 2>&1 >/dev/null",
+        "Failed to dump process list for 'clightning.service', ignoring: Access denied",
+    )
+    machine.succeed("grep -Fq hidepid=2 /proc/mounts")
+
+    ### Additional tests
+
+    # Current time in µs
+    pre_restart = succeed("date +%s.%6N").rstrip()
+
+    # Sanity-check system by restarting all services
+    succeed("systemctl restart bitcoind clightning spark-wallet lightning-charge nanopos liquidd")
+
+    # Now that the bitcoind restart triggered a banlist import restart, check that
+    # re-importing already banned addresses works
+    machine.wait_until_succeeds(
+        log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
+    )
+    assert_no_failure("bitcoind-import-banlist")
+
+    extra_tests.pop("post-clightning")()
+
+    ### Test lnd
+
+    stopped_services = "nanopos lightning-charge spark-wallet clightning"
+    succeed("systemctl stop " + stopped_services)
+    succeed("systemctl start lnd")
+    assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
+    assert_no_failure("lnd")
+
+    ### Test loopd
+
+    succeed("systemctl start lightning-loop")
+    assert_matches("su operator -c 'loop --version'", "version")
+    # Check that lightning-loop fails with the right error, making sure
+    # lightning-loop can connect to lnd
+    machine.wait_until_succeeds(
+        log_has_string("lightning-loop", "chain notifier RPC isstill in the process of starting")
+    )
+
+    ### Check that all extra_tests have been run
+    assert len(extra_tests) == 0

test/scenarios/withnetns.py

@@ -10,153 +10,91 @@ nanopos_ip = "169.254.1.19"
 recurringdonations_ip = "169.254.1.20"
 nginx_ip = "169.254.1.21"
 
-### Tests
 
-assert_running("setup-secrets")
+def electrs():
-# Unused secrets should be inaccessible
+    machine.wait_until_succeeds(
-succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
+        "ip netns exec nb-electrs nc -z localhost 4224"
+    )  # prometeus metrics provider
 
-assert_running("bitcoind")
-machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
-assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
-# Test RPC Whitelist
-machine.wait_until_succeeds("su operator -c 'bitcoin-cli help'")
-# Restating rpcuser & rpcpassword overrides privileged credentials
-machine.fail(
-    "bitcoin-cli -rpcuser=publicrpc -rpcpassword=$(cat /secrets/bitcoin-rpcpassword-public) help"
-)
-machine.wait_until_succeeds(
-    log_has_string("bitcoind", "RPC User publicrpc not allowed to call method help")
-)
 
-assert_running("electrs")
+def spark_wallet():
-machine.wait_until_succeeds(
+    machine.wait_until_succeeds("ip netns exec nb-spark-wallet nc -z %s 9737" % sparkwallet_ip)
-    "ip netns exec nb-electrs nc -z localhost 4224"
+    spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
-)  # prometeus metrics provider
+    assert_matches(
-# Check RPC connection to bitcoind
+        f"ip netns exec nb-spark-wallet curl -s {spark_auth}@%s:9737" % sparkwallet_ip, "Spark"
-machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
+    )
-assert_running("nginx")
-# Stop electrs from spamming the test log with 'wait for bitcoind sync' messages
-succeed("systemctl stop electrs")
 
-assert_running("liquidd")
-machine.wait_until_succeeds("elements-cli getnetworkinfo")
-assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
-succeed("su operator -c 'liquidswap-cli --help'")
 
-assert_running("clightning")
+def lightning_charge():
-assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+    machine.wait_until_succeeds("ip netns exec nb-nanopos nc -z %s 9112" % lightningcharge_ip)
+    charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
+    assert_matches(
+        f"ip netns exec nb-nanopos curl -s api-token:{charge_auth}@%s:9112/info | jq"
+        % lightningcharge_ip,
+        '"id"',
+    )
 
-assert_running("spark-wallet")
-spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
-machine.wait_until_succeeds("ip netns exec nb-spark-wallet nc -z %s 9737" % sparkwallet_ip)
-assert_matches(
-    f"ip netns exec nb-spark-wallet curl -s {spark_auth}@%s:9737" % sparkwallet_ip, "Spark"
-)
 
-assert_running("lightning-charge")
+def nanopos():
-charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
+    machine.wait_until_succeeds("ip netns exec nb-lightning-charge nc -z %s 9116" % nanopos_ip)
-machine.wait_until_succeeds("ip netns exec nb-nanopos nc -z %s 9112" % lightningcharge_ip)
+    assert_matches("ip netns exec nb-lightning-charge curl %s:9116" % nanopos_ip, "tshirt")
-assert_matches(
-    f"ip netns exec nb-nanopos curl -s api-token:{charge_auth}@%s:9112/info | jq"
-    % lightningcharge_ip,
-    '"id"',
-)
 
-assert_running("nanopos")
-machine.wait_until_succeeds("ip netns exec nb-lightning-charge nc -z %s 9116" % nanopos_ip)
-assert_matches("ip netns exec nb-lightning-charge curl %s:9116" % nanopos_ip, "tshirt")
 
-assert_running("onion-chef")
+def web_index():
+    machine.wait_until_succeeds("ip netns exec nb-nginx nc -z localhost 80")
+    assert_matches("ip netns exec nb-nginx curl localhost", "nix-bitcoin")
+    assert_matches("ip netns exec nb-nginx curl -L localhost/store", "tshirt")
 
-# FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
-# to incomplete unit dependencies.
-# 'create-web-index' implicitly tests 'nodeinfo'.
-machine.wait_for_unit("create-web-index")
-machine.wait_until_succeeds("ip netns exec nb-nginx nc -z localhost 80")
-assert_matches("ip netns exec nb-nginx curl localhost", "nix-bitcoin")
-assert_matches("ip netns exec nb-nginx curl -L localhost/store", "tshirt")
 
-machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
+def post_clightning():
-assert_no_failure("bitcoind-import-banlist")
+    ping_bitcoind = "ip netns exec nb-bitcoind ping -c 1 -w 1"
+    ping_nanopos = "ip netns exec nb-nanopos ping -c 1 -w 1"
 
-### Security tests
+    # Positive ping tests (non-exhaustive)
+    machine.succeed(
+        "%s %s &&" % (ping_bitcoind, bitcoind_ip)
+        + "%s %s &&" % (ping_bitcoind, clightning_ip)
+        + "%s %s &&" % (ping_bitcoind, liquidd_ip)
+        + "%s %s &&" % (ping_nanopos, lightningcharge_ip)
+        + "%s %s &&" % (ping_nanopos, nanopos_ip)
+        + "%s %s" % (ping_nanopos, nginx_ip)
+    )
 
-ping_bitcoind = "ip netns exec nb-bitcoind ping -c 1 -w 1"
+    # Negative ping tests (non-exhaustive)
-ping_nanopos = "ip netns exec nb-nanopos ping -c 1 -w 1"
+    machine.fail(
+        "%s %s ||" % (ping_bitcoind, sparkwallet_ip)
+        + "%s %s ||" % (ping_bitcoind, lightningcharge_ip)
+        + "%s %s ||" % (ping_bitcoind, nanopos_ip)
+        + "%s %s ||" % (ping_bitcoind, recurringdonations_ip)
+        + "%s %s ||" % (ping_bitcoind, nginx_ip)
+        + "%s %s ||" % (ping_nanopos, bitcoind_ip)
+        + "%s %s ||" % (ping_nanopos, clightning_ip)
+        + "%s %s ||" % (ping_nanopos, lnd_ip)
+        + "%s %s ||" % (ping_nanopos, liquidd_ip)
+        + "%s %s ||" % (ping_nanopos, electrs_ip)
+        + "%s %s ||" % (ping_nanopos, sparkwallet_ip)
+        + "%s %s" % (ping_nanopos, recurringdonations_ip)
+    )
 
-# Positive ping tests (non-exhaustive)
+    # test that netns-exec can't be run for unauthorized namespace
-machine.succeed(
+    machine.fail("netns-exec nb-electrs ip a")
-    "%s %s &&" % (ping_bitcoind, bitcoind_ip)
-    + "%s %s &&" % (ping_bitcoind, clightning_ip)
-    + "%s %s &&" % (ping_bitcoind, liquidd_ip)
-    + "%s %s &&" % (ping_nanopos, lightningcharge_ip)
-    + "%s %s &&" % (ping_nanopos, nanopos_ip)
-    + "%s %s" % (ping_nanopos, nginx_ip)
-)
 
-# Negative ping tests (non-exhaustive)
+    # test that netns-exec drops capabilities
-machine.fail(
+    assert_matches_exactly(
-    "%s %s ||" % (ping_bitcoind, sparkwallet_ip)
+        "su operator -c 'netns-exec nb-bitcoind capsh --print | grep Current '", "Current: =\n"
-    + "%s %s ||" % (ping_bitcoind, lightningcharge_ip)
+    )
-    + "%s %s ||" % (ping_bitcoind, nanopos_ip)
-    + "%s %s ||" % (ping_bitcoind, recurringdonations_ip)
-    + "%s %s ||" % (ping_bitcoind, nginx_ip)
-    + "%s %s ||" % (ping_nanopos, bitcoind_ip)
-    + "%s %s ||" % (ping_nanopos, clightning_ip)
-    + "%s %s ||" % (ping_nanopos, lnd_ip)
-    + "%s %s ||" % (ping_nanopos, liquidd_ip)
-    + "%s %s ||" % (ping_nanopos, electrs_ip)
-    + "%s %s ||" % (ping_nanopos, sparkwallet_ip)
-    + "%s %s" % (ping_nanopos, recurringdonations_ip)
-)
 
-# test that netns-exec can't be run for unauthorized namespace
+    # test that netns-exec can not be executed by users that are not operator
-machine.fail("netns-exec nb-electrs ip a")
+    machine.fail("sudo -u clightning netns-exec nb-bitcoind ip a")
 
-# test that netns-exec drops capabilities
-assert_matches_exactly(
-    "su operator -c 'netns-exec nb-bitcoind capsh --print | grep Current '", "Current: =\n"
-)
 
-# test that netns-exec can not be executed by users that are not operator
+extra_tests = {
-machine.fail("sudo -u clightning netns-exec nb-bitcoind ip a")
+    "electrs": electrs,
+    "spark-wallet": spark_wallet,
+    "lightning-charge": lightning_charge,
+    "nanopos": nanopos,
+    "web-index": web_index,
+    "post-clightning": post_clightning,
+}
 
-# test that `systemctl status` can't leak credentials
+run_tests(extra_tests)
-assert_matches(
-    "sudo -u electrs systemctl status clightning 2>&1 >/dev/null",
-    "Failed to dump process list for 'clightning.service', ignoring: Access denied",
-)
-machine.succeed("grep -Fq hidepid=2 /proc/mounts")
-
-### Additional tests
-
-# Current time in µs
-pre_restart = succeed("date +%s.%6N").rstrip()
-
-# Sanity-check system by restarting all services
-succeed("systemctl restart bitcoind clightning spark-wallet lightning-charge nanopos liquidd")
-
-# Now that the bitcoind restart triggered a banlist import restart, check that
-# re-importing already banned addresses works
-machine.wait_until_succeeds(
-    log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
-)
-assert_no_failure("bitcoind-import-banlist")
-
-### Test lnd
-
-succeed("systemctl stop nanopos lightning-charge spark-wallet clightning")
-succeed("systemctl start lnd")
-assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
-assert_no_failure("lnd")
-
-### Test loopd
-
-succeed("systemctl start lightning-loop")
-assert_matches("su operator -c 'loop --version'", "version")
-# Check that lightning-loop fails with the right error, making sure
-# lightning-loop can connect to lnd
-machine.wait_until_succeeds(
-    log_has_string("lightning-loop", "chain notifier RPC isstill in the process of starting")
-)