From 17507835fc63ceb9592796f21a37f2fe4e844dc7 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Thu, 6 Jan 2022 12:40:52 +0000 Subject: [PATCH] clightning: native database replication Don't put `clightning.replication` options in `examples/configuration.nix` until it is more "battle-tested." --- README.md | 3 +- docs/services.md | 99 ++++++++++++- modules/clightning-replication.nix | 227 +++++++++++++++++++++++++++++ modules/clightning.nix | 10 ++ modules/modules.nix | 1 + pkgs/default.nix | 2 +- test/clightning-replication.nix | 153 +++++++++++++++++++ test/lib/make-test.nix | 27 +++- test/run-tests.sh | 3 +- test/tests.nix | 14 +- test/tests.py | 8 + 11 files changed, 538 insertions(+), 9 deletions(-) create mode 100644 modules/clightning-replication.nix create mode 100644 test/clightning-replication.nix diff --git a/README.md b/README.md index 41c4aa8..1b51aa0 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ A [configuration preset](modules/presets/secure-node.nix) for setting up a secur NixOS modules ([src](modules/modules.nix)) * Application services * [bitcoind](https://github.com/bitcoin/bitcoin) - * [clightning](https://github.com/ElementsProject/lightning) with support for announcing an onion service\ + * [clightning](https://github.com/ElementsProject/lightning) with support for announcing an onion service + and [database replication](docs/services.md#setup-clightning-database-replication).\ Available plugins: * [clboss](https://github.com/ZmnSCPxj/clboss): automated C-Lightning Node Manager * [commando](https://github.com/lightningd/plugins/tree/master/commando): control your node over lightning diff --git a/docs/services.md b/docs/services.md index 90a71dc..27c9698 100644 --- a/docs/services.md +++ b/docs/services.md @@ -26,6 +26,104 @@ systemctl cat bitcoind systemctl show bitcoind ``` +# clightning database replication + +The clightning database can be replicated to a local path +or to a remote SSH target.\ +When remote replication is enabled, nix-bitcoin mounts a SSHFS to a local path.\ +Optionally, backups can be encrypted via `gocryptfs`. + +Note: You should also backup the static file `hsm_secret` (located at +`/var/lib/clightning/bitcoin/hsm_secret` by default), either manually +or via the `services.backups` module. + +## Remote target via SSHFS + +1. Add this to your `configuration.nix`: + ```nix + services.clightning.replication = { + enable = true; + sshfs.destination = "user@hostname:directory"; + # This is optional + encrypt = true; + }; + programs.ssh.knownHosts."hostname".publicKey = ""; + ``` + + Leave out the `encrypt` line if you want to store data on your destination + in plaintext.\ + Adjust `user`, `hostname` and `directory` as necessary. + +2. Deploy + +3. To allow SSH access from the nix-bitcoin node to the target node, either + use the remote node config below, or copy the contents of `$secretsDir/clightning-replication-ssh.pub` + to the `authorized_keys` file of `user` (or use `ssh-copy-id`). + +4. You can restrict the nix-bitcoin node's capabilities on the SSHFS target + using OpenSSH's builtin features, as detailed + [here](https://serverfault.com/questions/354615/allow-sftp-but-disallow-ssh). + + To implement this on NixOS, add the following to the NixOS configuration of + the SSHFS target node: + ```nix + systemd.tmpfiles.rules = [ + # Because this directory is chrooted by sshd, it must only be writable by user/group root + "d /var/backup/nb-replication 0755 root root - -" + "d /var/backup/nb-replication/writable 0700 nb-replication - - -" + ]; + + services.openssh = { + extraConfig = '' + Match user nb-replication + ChrootDirectory /var/backup/nb-replication + AllowTcpForwarding no + AllowAgentForwarding no + ForceCommand internal-sftp + PasswordAuthentication no + X11Forwarding no + ''; + }; + + users.users.nb-replication = { + isSystemUser = true; + group = "nb-replication"; + shell = "${pkgs.coreutils}/bin/false"; + openssh.authorizedKeys.keys = [ "" ]; + }; + users.groups.nb-replication = {}; + ``` + + With this setup, the corresponding `sshfs.destination` on the nix-bitcoin + node is `"nb-replication@hostname:writable"`. + +## Local directory target + +1. Add this to your `configuration.nix` + ```nix + services.clightning.replication = { + enable = true; + local.directory = "/var/backup/clightning"; + encrypt = true; + }; + ``` + + Leave out the `encrypt` line if you want to store data in + `local.directory` in plaintext. + +2. Deploy + +clightning will now replicate database files to `local.directory`. This +can be used to replicate to an external HDD by mounting it at path +`local.directory`. + +## Custom remote destination + +Follow the steps in section "Local directory target" above and mount a custom remote +destination (e.g., a NFS or SMB share) to `local.directory`.\ +You might want to disable `local.setupDirectory` in order to create the mount directory +yourself with custom permissions. + # Connect to RTL Normally you would connect to RTL via SSH tunneling with a command like this @@ -222,7 +320,6 @@ lndconnect-onion --host=mynode.org 5. Edit your deployment tool's configuration and change the node's address to `localhost` and the ssh port to ``. If you use krops as described in the [installation tutorial](./install.md), set `target = "localhost:";` in `krops/deploy.nix`. - 6. After deploying the new configuration, it will connect through the SSH tunnel you established in step iv. This also allows you to do more complex SSH setups that some deployment tools don't support. An example would be authenticating with [Trezor's SSH agent](https://github.com/romanz/trezor-agent), which provides extra security. # Initialize a Trezor for Bitcoin Core's Hardware Wallet Interface diff --git a/modules/clightning-replication.nix b/modules/clightning-replication.nix new file mode 100644 index 0000000..1cf58da --- /dev/null +++ b/modules/clightning-replication.nix @@ -0,0 +1,227 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + options.services.clightning.replication = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Enable live replication of the clightning database. + This prevents losing off-chain funds when the primary wallet file becomes + inaccessible. + + For setting the destination, you can either define option `sshfs.destination` + or `local.directory`. + + When `encrypt` is `false`, file `lightningd.sqlite3` is written to the destination. + When `encrypt` is `true`, directory `lightningd-db` is written to the destination. + It includes the encrypted database and gocryptfs metadata. + + See also: https://github.com/ElementsProject/lightning/blob/master/doc/BACKUP.md + ''; + }; + sshfs = { + destination = mkOption { + type = types.nullOr types.str; + default = null; + example = "user@10.0.0.1:directory"; + description = '' + The SSH destination for which a SSHFS will be mounted. + `directory` is relative to the home of `user`. + + A SSH key is automatically generated and stored in file + `$secretsDir/clightning-replication-ssh`. + The SSH server must allow logins via this key. + I.e., the `authorized_keys` file of `user` must contain + `$secretsDir/clightning-replication-ssh.pub`. + ''; + }; + port = mkOption { + type = types.port; + default = 22; + description = "SSH port of the remote server."; + }; + sshOptions = mkOption { + type = with types; listOf str; + default = [ "reconnect" "ServerAliveInterval=50" ]; + description = "SSH options used for mounting the SSHFS."; + }; + }; + local = { + directory = mkOption { + type = types.nullOr types.path; + default = null; + example = "/var/backup/clightning"; + description = '' + This option can be specified instead of `sshfs.destination` to enable + replication to a local directory. + + If `local.setupDirectory` is disabled, the directory + - must already exist when `clightning.service` (or `clightning-replication-mounts.service` + if `encrypt` is `true`) starts. + - must have write permissions for the `clightning` user. + + This option is also useful if you want to use a custom remote destination, + like a NFS or SMB share. + ''; + }; + setupDirectory = mkOption { + type = types.bool; + default = true; + description = '' + Create `local.directory` if it doesn't exist and set write permissions + for the `clightning` user. + ''; + }; + }; + encrypt = mkOption { + type = types.bool; + default = false; + description = '' + Whether to encrypt the replicated database with gocryptfs. + The encryption password is automatically generated and stored + in file `$secretsDir/clightning-replication-password`. + ''; + }; + }; + + cfg = config.services.clightning.replication; + inherit (config.services) clightning; + + secretsDir = config.nix-bitcoin.secretsDir; + network = config.services.bitcoind.makeNetworkName "bitcoin" "regtest"; + user = clightning.user; + group = clightning.group; + + useSshfs = cfg.sshfs.destination != null; + useMounts = useSshfs || cfg.encrypt; + + localDir = cfg.local.directory; + mountsDir = "/var/cache/clightning-replication"; + sshfsDir = "${mountsDir}/sshfs"; + plaintextDir = "${mountsDir}/plaintext"; + destDir = + if cfg.encrypt then + plaintextDir + else if useSshfs then + sshfsDir + else + localDir; +in { + inherit options; + + config = mkIf cfg.enable { + assertions = [ + { assertion = useSshfs || (localDir != null); + message = '' + services.clightning.replication: One of `sshfs.destination` or + `local.directory` must be set. + ''; + } + { assertion = !useSshfs || (localDir == null); + message = '' + services.clightning.replication: Only one of `sshfs.destination` and + `local.directory` must be set. + ''; + } + ]; + + environment.systemPackages = optionals cfg.encrypt [ pkgs.gocryptfs ]; + + systemd.tmpfiles.rules = optional (localDir != null && cfg.local.setupDirectory) + "d '${localDir}' 0770 ${user} ${group} - -"; + + services.clightning.wallet = let + mainDB = "${clightning.dataDir}/${network}/lightningd.sqlite3"; + replicaDB = "${destDir}/lightningd.sqlite3"; + in "sqlite3://${mainDB}:${replicaDB}"; + + systemd.services.clightning = { + bindsTo = mkIf useMounts [ "clightning-replication-mounts.service" ]; + serviceConfig.ReadWritePaths = [ + # We can't simply set `destDir` here because it might point to + # a FUSE mount. + # FUSE mounts can only be set up as `ReadWritePaths` by systemd when they + # are accessible by root. This would require FUSE-mounting with option + # `allow_other`. + (if useMounts then mountsDir else localDir) + ]; + }; + + systemd.services.clightning-replication-mounts = mkIf useMounts { + requiredBy = [ "clightning.service" ]; + before = [ "clightning.service" ]; + wants = [ "nix-bitcoin-secrets.target" ]; + after = [ "nix-bitcoin-secrets.target" ]; + path = [ + # Includes + # - The SUID-wrapped `fusermount` binary which enables FUSE + # for non-root users + # - The SUID-wrapped `mount` binary, used for unmounting + "/run/wrappers" + ] ++ optionals cfg.encrypt [ + # Includes `logger`, required by gocryptfs + pkgs.util-linux + ]; + + script = + optionalString useSshfs '' + mkdir -p ${sshfsDir} + ${pkgs.sshfs}/bin/sshfs ${cfg.sshfs.destination} -p ${toString cfg.sshfs.port} ${sshfsDir} \ + -o ${builtins.concatStringsSep "," ([ + "IdentityFile='${secretsDir}'/clightning-replication-ssh-key" + ] ++ cfg.sshfs.sshOptions)} + '' + + optionalString cfg.encrypt '' + cipherDir="${if useSshfs then sshfsDir else localDir}/lightningd-db" + mkdir -p "$cipherDir" ${plaintextDir} + gocryptfs=(${pkgs.gocryptfs}/bin/gocryptfs -passfile '${secretsDir}/clightning-replication-password') + # 1. init + if [[ ! -e $cipherDir/gocryptfs.conf ]]; then + "''${gocryptfs[@]}" -init "$cipherDir" + fi + # 2. mount + "''${gocryptfs[@]}" "$cipherDir" ${plaintextDir} + ''; + + postStop = + optionalString cfg.encrypt '' + umount ${plaintextDir} || true + '' + + optionalString useSshfs '' + umount ${sshfsDir} + ''; + + serviceConfig = { + StopPropagatedFrom = [ "clightning.service" ]; + CacheDirectory = "clightning-replication"; + CacheDirectoryMode = "770"; + User = user; + RemainAfterExit = "yes"; + Type = "oneshot"; + }; + }; + + nix-bitcoin = mkMerge [ + (mkIf useSshfs { + secrets.clightning-replication-ssh-key = { + user = user; + permissions = "400"; + }; + generateSecretsCmds.clightning-replication-ssh-key = '' + if [[ ! -f clightning-replication-ssh-key ]]; then + ${pkgs.openssh}/bin/ssh-keygen -t ed25519 -q -N "" -C "" -f clightning-replication-ssh-key + fi + ''; + }) + + (mkIf cfg.encrypt { + secrets.clightning-replication-password.user = user; + generateSecretsCmds.clightning-replication-password = '' + makePasswordSecret clightning-replication-password + ''; + }) + ]; + }; +} diff --git a/modules/clightning.nix b/modules/clightning.nix index fd6bc5a..612d57e 100644 --- a/modules/clightning.nix +++ b/modules/clightning.nix @@ -40,6 +40,15 @@ let default = "${cfg.dataDir}/${network}"; description = "The network data directory."; }; + wallet = mkOption { + type = types.nullOr types.str; + default = null; + example = "sqlite3:///var/lib/clightning/bitcoin/lightningd.sqlite3"; + description = '' + Wallet data scheme (sqlite3 or postgres) and location/connection + parameters, as fully qualified data source name. + ''; + }; extraConfig = mkOption { type = types.lines; default = ""; @@ -105,6 +114,7 @@ let bitcoin-rpcuser=${config.services.bitcoind.rpc.users.public.name} rpc-file-mode=0660 log-timestamps=false + ${optionalString (cfg.wallet != null) "wallet=${cfg.wallet}"} ${cfg.extraConfig} ''; diff --git a/modules/modules.nix b/modules/modules.nix index 08c20bc..fd39077 100644 --- a/modules/modules.nix +++ b/modules/modules.nix @@ -13,6 +13,7 @@ ./clightning.nix ./clightning-plugins ./clightning-rest.nix + ./clightning-replication.nix ./spark-wallet.nix ./lnd.nix ./lightning-loop.nix diff --git a/pkgs/default.nix b/pkgs/default.nix index 1b1cc7f..3dfbc14 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -4,7 +4,7 @@ in # Set default values for use without flakes { pkgs ? import { config = {}; overlays = []; } , pkgsUnstable ? import nixpkgsPinned.nixpkgs-unstable { - inherit (pkgs) system; + inherit (pkgs.stdenv) system; config = {}; overlays = []; } diff --git a/test/clightning-replication.nix b/test/clightning-replication.nix new file mode 100644 index 0000000..1a4c3bc --- /dev/null +++ b/test/clightning-replication.nix @@ -0,0 +1,153 @@ +# You can run this test via `run-tests.sh -s clightningReplication` + +let + nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs; +in +import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }: +with pkgs.lib; +let + keyDir = "${nixpkgs}/nixos/tests/initrd-network-ssh"; + keys = { + server = "${keyDir}/ssh_host_ed25519_key"; + client = "${keyDir}/id_ed25519"; + serverPub = readFile "${keys.server}.pub"; + clientPub = readFile "${keys.client}.pub"; + }; + + clientBaseConfig = { + imports = [ ../modules/modules.nix ]; + + nix-bitcoin.generateSecrets = true; + + services.clightning = { + enable = true; + replication.enable = true; + + # TODO-EXTERNAL: + # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s. + extraConfig = "disable-dns"; + }; + }; +in +{ + name = "clightning-replication"; + + nodes = let nodes = { + replicationLocal = { + imports = [ clientBaseConfig ]; + services.clightning.replication.local.directory = "/var/backup/clightning"; + }; + + replicationLocalEncrypted = { + imports = [ nodes.replicationLocal ]; + services.clightning.replication.encrypt = true; + }; + + replicationRemote = { + imports = [ clientBaseConfig ]; + nix-bitcoin.generateSecretsCmds.clightning-replication-ssh-key = mkForce '' + install -m 600 ${keys.client} clightning-replication-ssh-key + ''; + programs.ssh.knownHosts."server".publicKey = keys.serverPub; + services.clightning.replication.sshfs.destination = "nb-replication@server:writable"; + }; + + replicationRemoteEncrypted = { + imports = [ nodes.replicationRemote ]; + services.clightning.replication.encrypt = true; + }; + + server = { ... }: { + environment.etc."ssh-host-key" = { + source = keys.server; + mode = "400"; + }; + + services.openssh = { + enable = true; + extraConfig = '' + Match user nb-replication + ChrootDirectory /var/backup/nb-replication + AllowTcpForwarding no + AllowAgentForwarding no + ForceCommand internal-sftp + PasswordAuthentication no + X11Forwarding no + ''; + hostKeys = mkForce [ + { + path = "/etc/ssh-host-key"; + type = "ed25519"; + } + ]; + }; + + users.users.nb-replication = { + isSystemUser = true; + group = "nb-replication"; + shell = "${pkgs.coreutils}/bin/false"; + openssh.authorizedKeys.keys = [ keys.clientPub ]; + }; + users.groups.nb-replication = {}; + + systemd.tmpfiles.rules = [ + # Because this directory is chrooted by sshd, it must only be writable by user/group root + "d /var/backup/nb-replication 0755 root root - -" + "d /var/backup/nb-replication/writable 0700 nb-replication - - -" + ]; + }; + }; in nodes; + + testScript = { nodes, ... }: let + systems = builtins.concatStringsSep ", " + (mapAttrsToList (name: node: ''"${name}": "${node.config.system.build.toplevel}"'') nodes); + in '' + systems = { ${systems} } + + def switch_to_system(system): + cmd = f"{systems[system]}/bin/switch-to-configuration test >&2" + client.succeed(cmd) + + client = replicationLocal + + if not "is_interactive" in vars(): + client.start() + server.start() + + with subtest("local replication"): + client.wait_for_unit("clightning.service") + client.succeed("runuser -u clightning -- ls /var/backup/clightning/lightningd.sqlite3") + # No other user should be able to read the backup directory + client.fail("runuser -u bitcoin -- ls /var/backup/clightning") + + # If `switch_to_system` succeeds then all services, including clightning, + # have started successfully + switch_to_system("replicationLocalEncrypted") + with subtest("local replication encrypted"): + replica_db = "/var/cache/clightning-replication/plaintext/lightningd.sqlite3" + client.succeed(f"runuser -u clightning -- ls {replica_db}") + # No other user should be able to read the unencrypted files + client.fail(f"runuser -u bitcoin -- ls {replica_db}") + # A gocryptfs has been created + client.succeed("ls /var/backup/clightning/lightningd-db/gocryptfs.conf") + + server.wait_for_unit("sshd.service") + switch_to_system("replicationRemote") + with subtest("remote replication"): + replica_db = "/var/cache/clightning-replication/sshfs/lightningd.sqlite3" + client.succeed(f"runuser -u clightning -- ls {replica_db}") + # No other user should be able to read the unencrypted files + client.fail(f"runuser -u bitcoin -- ls {replica_db}") + # A clighting db exists on the server + server.succeed("ls /var/backup/nb-replication/writable/lightningd.sqlite3") + + switch_to_system("replicationRemoteEncrypted") + with subtest("remote replication encrypted"): + replica_db = "/var/cache/clightning-replication/plaintext/lightningd.sqlite3" + client.succeed(f"runuser -u clightning -- ls {replica_db}") + # No other user should be able to read the unencrypted files + client.fail(f"runuser -u bitcoin -- ls {replica_db}") + # A gocryptfs has been created on the server + server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf") + ''; +}) diff --git a/test/lib/make-test.nix b/test/lib/make-test.nix index 09f4b50..4c95a68 100644 --- a/test/lib/make-test.nix +++ b/test/lib/make-test.nix @@ -55,10 +55,29 @@ name: testConfig: container = { # The container name has a 11 char length limit containers.nb-test = { config, ... }: { - config = { - extra = config.config.test.container; - config = testConfig; - }; + imports = [ + { + config = { + extra = config.config.test.container; + config = testConfig; + }; + } + + # Enable FUSE inside the container when clightning replication + # is enabled. + # TODO-EXTERNAL: Remove this when + # https://github.com/systemd/systemd/issues/17607 + # has been resolved. This will also improve security. + ( + let + clightning = config.config.services.clightning; + in + lib.mkIf (clightning.enable && clightning.replication.enable) { + bindMounts."/dev/fuse" = { hostPath = "/dev/fuse"; }; + allowedDevices = [ { node = "/dev/fuse"; modifier = "rw"; } ]; + } + ) + ]; }; }; diff --git a/test/run-tests.sh b/test/run-tests.sh index 589fa94..d48be35 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -295,10 +295,11 @@ basic() { # All tests that only consist of building a nix derivation. # Their output is cached in /nix/store. buildable() { - basic + basic "$@" scenario=full buildTest "$@" scenario=regtest buildTest "$@" scenario=hardened buildTest "$@" + scenario=clightningReplication buildTest "$@" } examples() { diff --git a/test/tests.nix b/test/tests.nix index 9aac360..9c7773b 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -49,6 +49,8 @@ let }; tests.clightning = cfg.clightning.enable; + test.data.clightning-replication = cfg.clightning.replication.enable; + # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s. services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns"; test.data.clightning-plugins = let @@ -186,6 +188,11 @@ let tests.security = true; services.clightning.enable = true; + services.clightning.replication = { + enable = true; + encrypt = true; + local.directory = "/var/backup/clightning"; + }; test.features.clightningPlugins = true; services.rtl.enable = true; services.spark-wallet.enable = true; @@ -354,7 +361,12 @@ let }; makeTest' = import ./lib/make-test.nix pkgs; - tests = builtins.mapAttrs makeTest allScenarios; + tests = builtins.mapAttrs makeTest allScenarios // { + clightningReplication.vm = import ./clightning-replication.nix { + inherit pkgs; + inherit (pkgs.stdenv) system; + }; + }; getTest = name: tests.${name} or (makeTest name { services.${name}.enable = true; diff --git a/test/tests.py b/test/tests.py index b5509d7..1fb98e2 100644 --- a/test/tests.py +++ b/test/tests.py @@ -153,6 +153,14 @@ def _(): # This is a one-shot service, so this command only succeeds if the service succeeds succeed("systemctl start clightning-feeadjuster") + if test_data["clightning-replication"]: + replica_db = "/var/cache/clightning-replication/plaintext/lightningd.sqlite3" + succeed(f"runuser -u clightning -- ls {replica_db}") + # No other user should be able to read the unencrypted files + machine.fail(f"runuser -u bitcoin -- ls {replica_db}") + # A gocryptfs has been created + succeed("ls /var/backup/clightning/lightningd-db/gocryptfs.conf") + @test("lnd") def _(): assert_running("lnd")