Merge fort-nix/nix-bitcoin#423: Misc. improvements

4a74b7de08 clightning: work around unsupported seccomp syscall (Erik Arvstedt)
38a843d005 clightning: update python pkgs to new version (Erik Arvstedt)
6ad7107ddb update nixpkgs (Erik Arvstedt)
f58d67677e netns-isolation: separate host and netns setup (Erik Arvstedt)
cb6e5ef702 netns-isolation: fix routing issues due to netns restarting (Erik Arvstedt)
7f77147b60 makeShell: minor improvements (Erik Arvstedt)
a5730eb736 makeShell: make the help msg a shell derivation variable (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 4a74b7de08

Tree-SHA512: 75454b51db6d7ab41590d8579e0a5136e5ac1be78d5c2f547c6ef1982c0de679968879bb9bac57dd66413f59a4659236601ab75414486b0137c7c43d73d22759

flake.lock

@@ -17,11 +17,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1635719588,
+        "lastModified": 1636333654,
-        "narHash": "sha256-pWjdy0NheM97NsPE6+jUnr5LYyeA0sBGTdw4mfXMGZQ=",
+        "narHash": "sha256-3wh9PtCzcaJQuZrgZ+ygKfhltkDNNqT6zOzGsRbjZEo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f0869b1a2c0b150aac26e10bb5c2364ffb2e804f",
+        "rev": "e74894146a42ba552ebafa19ab2d1df7ccbc1738",
         "type": "github"
       },
       "original": {
@@ -33,11 +33,11 @@
     },
     "nixpkgsUnstable": {
       "locked": {
-        "lastModified": 1635781173,
+        "lastModified": 1636470166,
-        "narHash": "sha256-nTA2y2jIJiVj5RawHUNhlZUIy5J/Q2CA6YP4T1qBkLo=",
+        "narHash": "sha256-0tyWSS5rgMtML5p41ZdE5ocuAnRdtOGvdsqQyMUBYAI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7053541084bf5ce2921ef307e5585d39d7ba8b3f",
+        "rev": "f096b7122ab08e93c8b052c92461ca71b80c0cc8",
         "type": "github"
       },
       "original": {

helper/makeShell.nix

@@ -1,52 +1,58 @@
 { configDir, shellVersion ? null, extraShellInitCmds ? (pkgs: "") }:
 let
+  pinned = import ../pkgs/nixpkgs-pinned.nix;
+  pkgs = import nixpkgs { config = {}; overlays = []; };
   inherit (pkgs) lib;
-  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+  inherit (pinned) nixpkgs;
-  pkgs = import nixpkgs {};
   nbPkgs = import ../pkgs { inherit pkgs; };
   cfgDir = toString configDir;
-  path = lib.optionalString pkgs.stdenv.isLinux ''
+  setPath = lib.optionalString pkgs.stdenv.isLinux ''
     export PATH="${lib.makeBinPath [ nbPkgs.pinned.extra-container ]}''${PATH:+:}$PATH"
   '';
 in
 pkgs.stdenv.mkDerivation {
   name = "nix-bitcoin-environment";
 
+  helpMessage = ''
+    nix-bitcoin path: ${toString ../.}
+
+    Available commands
+    ==================
+    deploy
+      Run krops-deploy and eval-config in parallel.
+      This ensures that eval failures appear quickly when deploying.
+      In this case, deployment is stopped.
+
+    krops-deploy
+      Deploy your node via krops
+
+    eval-config
+      Evaluate your node system configuration
+
+    generate-secrets
+      Create secrets required by your node configuration.
+      Secrets are written to ./secrets/
+      This function is automatically called by krops-deploy.
+
+    update-nix-bitcoin
+      Fetch and use the latest version of nix-bitcoin
+  '';
+
   shellHook = ''
     export NIX_PATH="nixpkgs=${nixpkgs}:nix-bitcoin=${toString ../.}:."
-    ${path}
+    ${setPath}
     export NIX_BITCOIN_EXAMPLES_DIR="${cfgDir}"
+    export nixpkgsUnstable="${pinned.nixpkgs-unstable}"
 
     # Set isInteractive=1 if
     # 1. stdout is a TTY, i.e. we're not piping the output
     # 2. the shell is interactive
     if [[ -t 1 && $- == *i* ]]; then isInteractive=1; else isInteractive=; fi
 
-    nixBitcoinHelp() {
+    # Make this a non-environment var
-        echo "nix-bitcoin path: ${toString ../.}"
+    export -n helpMessage
-        echo
+
-        echo "Available commands"
+    help() { echo "$helpMessage"; }
-        echo "=================="
-        echo "deploy"
-        echo "  Run krops-deploy and eval-config in parallel."
-        echo "  This ensures that eval failures appear quickly when deploying."
-        echo "  In this case, deployment is stopped."
-        echo
-        echo "krops-deploy"
-        echo "  Deploy your node via krops"
-        echo
-        echo "eval-config"
-        echo "  Evaluate your node system configuration"
-        echo
-        echo "generate-secrets"
-        echo "  Create secrets required by your node configuration."
-        echo "  Secrets are written to ./secrets/"
-        echo "  This function is automatically called by krops-deploy."
-        echo
-        echo "update-nix-bitcoin"
-        echo "  Fetch and use the latest version of nix-bitcoin"
-    }
-    help() { nixBitcoinHelp; }
     h() { help; }
 
     fetch-release() {

modules/clightning.nix

@@ -137,6 +137,14 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
+
+        # TODO-EXTERNAL:
+        # The seccomp version used by systemd in NixOS 21.05 doesn't support
+        # handling syscall 436 (close_range), which has only recently been added:
+        # https://github.com/seccomp/libseccomp/commit/ac849e7960547d418009a783da654d5917dbfe2d
+        #
+        # Disable seccomp filtering because clightning depends on this syscall.
+        SystemCallFilter = [];
       } // nbLib.allowedIPAddresses cfg.enforceTor;
       # Wait until the rpc socket appears
       postStart = ''

modules/netns-isolation.nix

@@ -155,41 +155,55 @@ in {
         veth = "nb-veth-${toString v.id}";
         peer = "nb-veth-br-${toString v.id}";
         inherit (v) netnsName;
-        ipNetns = "${ip} -n ${netnsName}";
+        nsenter = "${pkgs.utillinux}/bin/nsenter";
-        netnsIptables = "${ip} netns exec ${netnsName} ${config.networking.firewall.package}/bin/iptables";
         allowedAddresses = concatMapStringsSep "," (available: netns.${available}.address) v.availableNetns;
+
+        setup = ''
+          ${ip} netns add ${netnsName}
+          ${ip} link add ${veth} type veth peer name ${peer}
+          ${ip} link set ${veth} netns ${netnsName}
+          # The peer link is never used directly, so don't auto-assign an IPv6 address
+          echo 1 > /proc/sys/net/ipv6/conf/${peer}/disable_ipv6
+          ${ip} link set ${peer} up
+          ${ip} link set ${peer} master nb-br
+          exec ${nsenter} --net=/run/netns/${netnsName} ${script "in-netns" setupInNetns}
+        '';
+
+        setupInNetns = ''
+          ${ip} link set lo up
+          ${ip} addr add ${v.address}/24 dev ${veth}
+          ${ip} link set ${veth} up
+          ${ip} route add default via ${bridgeIp}
+
+          ${iptables} -w -P INPUT DROP
+          ${iptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
+          # allow return traffic to outgoing connections initiated by the service itself
+          ${iptables} -w -A INPUT -m conntrack --ctstate ESTABLISHED -j ACCEPT
+        '' + optionalString (config.services.${n}.enforceTor or false) ''
+          ${iptables} -w -P OUTPUT DROP
+          ${iptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
+        '' + optionalString (v.availableNetns != []) ''
+          ${iptables} -w -A INPUT -s ${allowedAddresses} -j ACCEPT
+          ${iptables} -w -A OUTPUT -d ${allowedAddresses} -j ACCEPT
+        '';
+
+        script = name: src: pkgs.writers.writeDash name ''
+          set -e
+          ${src}
+        '';
       in {
         "${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}";
 
         "netns-${n}" = rec {
           requires = [ "nb-netns-bridge.service" ];
           after = [ "nb-netns-bridge.service" ];
-          bindsTo = [ "${n}.service" ];
+          requiredBy = [ "${n}.service" ];
-          requiredBy = bindsTo;
+          before = requiredBy;
-          before = bindsTo;
+          serviceConfig = {
-          script = ''
+            Type = "oneshot";
-            ${ip} netns add ${netnsName}
+            RemainAfterExit = true;
-            ${ipNetns} link set lo up
+            ExecStart = script "setup" setup;
-            ${ip} link add ${veth} type veth peer name ${peer}
+          };
-            ${ip} link set ${veth} netns ${netnsName}
-            ${ipNetns} addr add ${v.address}/24 dev ${veth}
-            # The peer link is never used directly, so don't auto-assign an IPv6 address
-            echo 1 > /proc/sys/net/ipv6/conf/${peer}/disable_ipv6
-            ${ip} link set ${peer} up
-            ${ipNetns} link set ${veth} up
-            ${ip} link set ${peer} master nb-br
-            ${ipNetns} route add default via ${bridgeIp}
-            ${netnsIptables} -w -P INPUT DROP
-            ${netnsIptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
-            # allow return traffic to outgoing connections initiated by the service itself
-            ${netnsIptables} -w -A INPUT -m conntrack --ctstate ESTABLISHED -j ACCEPT
-          '' + optionalString (config.services.${n}.enforceTor or false) ''
-            ${netnsIptables} -w -P OUTPUT DROP
-            ${netnsIptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
-          '' + optionalString (v.availableNetns != []) ''
-            ${netnsIptables} -w -A INPUT -s ${allowedAddresses} -j ACCEPT
-            ${netnsIptables} -w -A OUTPUT -d ${allowedAddresses} -j ACCEPT
-          '';
           # Link deletion is implicit in netns deletion, but it sometimes only happens
           # after `netns delete` finishes. Add an extra `link del` to ensure that
           # the link is deleted before the service stops, which is needed for service
@@ -198,10 +212,7 @@ in {
             ${ip} netns delete ${netnsName}
             ${ip} link del ${peer} 2> /dev/null || true
           '';
-          serviceConfig = {
+
-            Type = "oneshot";
-            RemainAfterExit = true;
-          };
         };
       };
     in foldl (services: n:

pkgs/python-packages/default.nix

@@ -19,6 +19,7 @@ in {
 
   pyln-client = clightningPkg ./pyln-client;
   pyln-proto = clightningPkg ./pyln-proto;
+  pyln-bolt7 = clightningPkg ./pyln-bolt7;
   pylightning = clightningPkg ./pylightning;
 
   ## Specific versions of packages that already exist in nixpkgs

pkgs/python-packages/pyln-bolt7/default.nix

@@ -0,0 +1,22 @@
+{ buildPythonPackage, clightning, pyln-proto }:
+
+buildPythonPackage rec {
+  pname = "pyln-bolt7";
+
+  # See fn `bolt_meta` in
+  # https://github.com/ElementsProject/lightning/blob/master/contrib/pyln-spec/bolt7/setup.py
+  version = "1.0.2.186";
+
+  inherit (clightning) src;
+
+  propagatedBuildInputs = [ pyln-proto ];
+
+  postUnpack = "sourceRoot=$sourceRoot/contrib/pyln-spec/bolt7";
+
+  # TODO-EXTERNAL
+  # Remove when this fix is released
+  # https://github.com/ElementsProject/lightning/pull/4910
+  postPatch = ''
+    sed -i 's|pyln.proto|pyln-proto|' requirements.txt
+  '';
+}

pkgs/python-packages/pyln-client/default.nix

@@ -1,12 +1,14 @@
-{ buildPythonPackage, clightning, recommonmark }:
+{ buildPythonPackage, clightning, pyln-bolt7, recommonmark, setuptools-scm }:
 
 buildPythonPackage rec {
   pname = "pyln-client";
-  version = "0.10.1"; # defined in ${src}/contrib/pyln-client/pyln/client/__init__.py
+  version = clightning.version;
 
   inherit (clightning) src;
 
-  propagatedBuildInputs = [ recommonmark ];
+  propagatedBuildInputs = [ pyln-bolt7 recommonmark setuptools-scm ];
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
 }

pkgs/python-packages/pyln-proto/default.nix

@@ -4,11 +4,12 @@
 , coincurve
 , base58
 , mypy
+, setuptools-scm
 }:
 
 buildPythonPackage rec {
   pname = "pyln-proto";
-  version = "0.10.1"; # defined in ${src}/contrib/pyln-proto/setup.py
+  version = clightning.version;
 
   inherit (clightning) src;
 
@@ -18,8 +19,17 @@ buildPythonPackage rec {
     coincurve
     base58
     mypy
+    setuptools-scm
   ];
 
-  postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
+  postUnpack = "sourceRoot=$sourceRoot/contrib/pyln-proto";
+  postPatch = ''
+    sed -i '
+      s|coincurve ~= 13.0|coincurve == 15.0.0|
+      s|base58 ~= 2.0.1|base58 == 2.1.0|
+      s|mypy==0.790|mypy == 0.812|
+    ' requirements.txt
+  '';
 }