simplify secrets file format

Each secret file to be deployed is now backed by one local file.
This simplifies 'setup-secrets' and the secret definitions.
Also, with the old format it was not possible to add new secrets
to secrets.nix in a simple way.

Old secrets are automatically converted to the new format when running
nix-shell.

Using the new option 'nix-bitcoin.secrets', secrets are now directly
defined by the services that use them.

modules/bitcoind.nix

@@ -307,5 +307,10 @@ in {
     };
     users.groups.${cfg.group} = {};
     users.groups.bitcoinrpc = {};
+
+    nix-bitcoin.secrets.bitcoin-rpcpassword = {
+      user = "bitcoin";
+      group = "bitcoinrpc";
+    };
   };
 }

modules/electrs.nix

@@ -103,8 +103,8 @@ in {
             listen ${toString config.services.electrs.nginxport} ssl;
             proxy_pass electrs;
 
-            ssl_certificate /secrets/nginx_cert;
+            ssl_certificate /secrets/nginx-cert;
-            ssl_certificate_key /secrets/nginx_key;
+            ssl_certificate_key /secrets/nginx-key;
             ssl_session_cache shared:SSL:1m;
             ssl_session_timeout 4h;
             ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
@@ -117,5 +117,12 @@ in {
       requires = [ "nix-bitcoin-secrets.target" ];
       after = [ "nix-bitcoin-secrets.target" ];
     };
+    nix-bitcoin.secrets = rec {
+      nginx-key = {
+        user = "nginx";
+        group = "root";
+      };
+      nginx-cert = nginx-key;
+    };
   };
 }

modules/lightning-charge.nix

@@ -30,7 +30,7 @@ in {
       requires = [ "clightning.service" ];
       after = [ "clightning.service" ];
       serviceConfig = {
-          EnvironmentFile = "/secrets/lightning-charge-api-token";
+          EnvironmentFile = "/secrets/lightning-charge-env";
           ExecStart = "${pkgs.nix-bitcoin.lightning-charge}/bin/charged -l ${config.services.clightning.dataDir} -d ${config.services.clightning.dataDir}/lightning-charge.db";
           # Unfortunately c-lightning doesn't allow setting the permissions of the rpc socket,
           # so this must run as the clightning user
@@ -42,5 +42,6 @@ in {
         // nix-bitcoin-services.nodejs
         // nix-bitcoin-services.allowTor;
     };
+    nix-bitcoin.secrets.lightning-charge-env.user = "clightning";
   };
 }

modules/liquid.nix

@@ -233,5 +233,6 @@ in {
       home = cfg.dataDir;
     };
     users.groups.${cfg.group} = {};
+    nix-bitcoin.secrets.liquid-rpcpassword.user = "liquid";
   };
 }

modules/lnd.nix

@@ -9,8 +9,8 @@ let
     datadir=${cfg.dataDir}
     logdir=${cfg.dataDir}/logs
     bitcoin.mainnet=1
-    tlscertpath=/secrets/lnd_cert
+    tlscertpath=/secrets/lnd-cert
-    tlskeypath=/secrets/lnd_key
+    tlskeypath=/secrets/lnd-key
 
     rpclisten=localhost:${toString cfg.rpcPort}
 
@@ -61,7 +61,7 @@ in {
       default = pkgs.writeScriptBin "lncli"
       # Switch user because lnd makes datadir contents readable by user only
       ''
-        exec sudo -u lnd ${pkgs.nix-bitcoin.lnd}/bin/lncli --tlscertpath /secrets/lnd_cert \
+        exec sudo -u lnd ${pkgs.nix-bitcoin.lnd}/bin/lncli --tlscertpath /secrets/lnd-cert \
           --macaroonpath '${cfg.dataDir}/chain/bitcoin/mainnet/admin.macaroon' "$@"
       '';
       description = "Binary to connect with the lnd instance.";
@@ -109,7 +109,7 @@ in {
           echo Create lnd seed
 
           ${pkgs.curl}/bin/curl -s \
-            --cacert /secrets/lnd_cert \
+            --cacert /secrets/lnd-cert \
             -X GET https://127.0.0.1:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /secrets/lnd-seed-mnemonic
         fi
 
@@ -117,7 +117,7 @@ in {
           echo Create lnd wallet
 
           ${pkgs.curl}/bin/curl -s --output /dev/null --show-error \
-            --cacert /secrets/lnd_cert \
+            --cacert /secrets/lnd-cert \
             -X POST -d "{\"wallet_password\": \"$(cat /secrets/lnd-wallet-password | tr -d '\n' | base64 -w0)\", \
             \"cipher_seed_mnemonic\": $(cat /secrets/lnd-seed-mnemonic | tr -d '\n')}" \
             https://127.0.0.1:8080/v1/initwallet
@@ -132,7 +132,7 @@ in {
 
           ${pkgs.curl}/bin/curl -s \
             -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${mainnetDir}/admin.macaroon')" \
-            --cacert /secrets/lnd_cert \
+            --cacert /secrets/lnd-cert \
             -X POST \
             -d "{\"wallet_password\": \"$(cat /secrets/lnd-wallet-password | tr -d '\n' | base64 -w0)\"}" \
             https://127.0.0.1:8080/v1/unlockwallet
@@ -151,5 +151,10 @@ in {
       home = cfg.dataDir;
     };
     users.groups.lnd = {};
+    nix-bitcoin.secrets = {
+      lnd-wallet-password.user = "lnd";
+      lnd-key.user = "lnd";
+      lnd-cert.user = "lnd";
+    };
   };
 }

modules/modules.nix

@@ -23,7 +23,7 @@ in {
     ./recurring-donations.nix
     ./hardware-wallets.nix
     ./lnd.nix
-    ./secrets/setup-secrets.nix
+    ./secrets/secrets.nix
   ];
 
   disabledModules = [ "services/networking/bitcoind.nix" ];

modules/nanopos.nix

@@ -58,7 +58,7 @@ in {
       requires = [ "lightning-charge.service" ];
       after = [ "lightning-charge.service" ];
       serviceConfig = {
-        EnvironmentFile = "/secrets/lightning-charge-api-token-for-nanopos";
+        EnvironmentFile = "/secrets/nanopos-env";
         ExecStart = "${pkgs.nix-bitcoin.nanopos}/bin/nanopos -y ${cfg.itemsFile} -p ${toString cfg.port} --show-bolt11";
 
         User = "nanopos";
@@ -73,5 +73,6 @@ in {
       group = "nanopos";
     };
     users.groups.nanopos = {};
+    nix-bitcoin.secrets.nanopos-env.user = "nanopos";
   };
 }

modules/secrets/make-secrets.nix

@@ -1,68 +0,0 @@
-{ secretsFile ? null, config ? null }:
-let
-  secrets = import secretsFile;
-  secretsDir = "/secrets/";
-  secret = { text ? null, keyFile ? null, user, group ? user }: {
-    inherit text keyFile user group;
-    destDir = secretsDir;
-    permissions = "0440";
-  };
-in rec {
-  allSecrets = {
-    bitcoin-rpcpassword = secret {
-      text = secrets.bitcoinrpcpassword;
-      user = "bitcoin";
-      group = "bitcoinrpc";
-    };
-    lnd-wallet-password = secret {
-      text = secrets.lnd-wallet-password;
-      user = "lnd";
-    };
-    lightning-charge-api-token = secret {
-      text = "API_TOKEN=" + secrets.lightning-charge-api-token;
-      user = "clightning";
-    };
-    # variable is called CHARGE_TOKEN instead of API_TOKEN
-    lightning-charge-api-token-for-nanopos = secret {
-      text = "CHARGE_TOKEN=" + secrets.lightning-charge-api-token;
-      user = "nanopos";
-    };
-    liquid-rpcpassword = secret {
-      text = secrets.liquidrpcpassword;
-      user = "liquid";
-    };
-    spark-wallet-login = secret {
-      text = "login=" + "spark-wallet:" + secrets.spark-wallet-password;
-      user = "clightning";
-    };
-    nginx_key = secret {
-      keyFile = toString ../../secrets/nginx.key;
-      user = "nginx";
-      group = "root";
-    };
-    nginx_cert = secret {
-      keyFile = toString ../../secrets/nginx.cert;
-      user = "nginx";
-      group = "root";
-    };
-    lnd_key = secret {
-      keyFile = toString ../../secrets/lnd.key;
-      user = "lnd";
-    };
-    lnd_cert = secret {
-      keyFile = toString ../../secrets/lnd.cert;
-      user = "lnd";
-    };
-  };
-
-  activeSecrets = let
-    secretsFor = service: attrs: if service.enable then attrs else {};
-  in with allSecrets;
-       (secretsFor config.services.bitcoind { inherit bitcoin-rpcpassword; })
-    // (secretsFor config.services.lnd { inherit lnd-wallet-password lnd_key lnd_cert; })
-    // (secretsFor config.services.lightning-charge { inherit lightning-charge-api-token; })
-    // (secretsFor config.services.nanopos { inherit lightning-charge-api-token-for-nanopos; })
-    // (secretsFor config.services.liquidd { inherit liquid-rpcpassword; })
-    // (secretsFor config.services.spark-wallet { inherit spark-wallet-login; })
-    // (secretsFor config.services.electrs { inherit nginx_key nginx_cert; });
-}

modules/secrets/secrets.nix

@@ -2,18 +2,41 @@
 
 with lib;
 let
+  cfg = config.nix-bitcoin;
   secretsDir = "/secrets/"; # TODO: make this an option
 
-  secrets = (import ./make-secrets.nix { inherit config; }).activeSecrets;
-
   setupSecrets = concatStrings (mapAttrsToList (n: v: ''
-    setupSecret ${n} ${v.user} ${v.group} ${v.permissions} ${optionalString (v.keyFile != null) (baseNameOf v.keyFile)}
+    setupSecret ${n} ${v.user} ${v.group} ${v.permissions} }
-  '') secrets);
+  '') cfg.secrets);
 in
 {
-  options.nix-bitcoin.setup-secrets = mkEnableOption "Set permissions for secrets generated by 'generate-secrets.sh'";
+  options.nix-bitcoin = {
+    secrets = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule (
+        { config, ... }: {
+          options = {
+            user = mkOption {
+              type = str;
+              default = "root";
+            };
+            group = mkOption {
+              type = str;
+              default = config.user;
+            };
+            permissions = mkOption {
+              type = str;
+              default = "0440";
+            };
+          };
+        }
+      ));
+    };
 
-  config = mkIf config.nix-bitcoin.setup-secrets {
+    setup-secrets = mkEnableOption "Set permissions for secrets generated by 'generate-secrets.sh'";
+  };
+
+  config = mkIf cfg.setup-secrets {
     systemd.targets.nix-bitcoin-secrets = {
       requires = [ "setup-secrets.service" ];
       after = [ "setup-secrets.service" ];
@@ -23,7 +46,7 @@ in
     #  - Create missing secrets that are composed of attrs from secrets.nix
     #  - Set owner and permissions for all used secrets
     #  - Make all other secrets accessible to root only
-    # For all steps make sure that no secrets are copied to the nix-store.
+    # For all steps make sure that no secrets are copied to the nix store.
     #
     systemd.services.setup-secrets = {
       serviceConfig = {
@@ -36,39 +59,15 @@ in
             user="$2"
             group="$3"
             permissions="$4"
-            srcFile="$5"
             if [[ ! -e $file ]]; then
-              if [[ $srcFile ]]; then
+              echo "Error: Secret file '$file' is missing"
-                if [[ ! -e $srcFile ]]; then
+              exit 1
-                  echo "Error: Secret source file '$srcFile' is missing"
-                  exit 1
-                fi
-                mv "$srcFile" "$file"
-              else
-                createFile "$file"
-              fi
             fi
             chown "$user:$group" "$file"
             chmod "$permissions" "$file"
             processedFiles+=("$file")
         }
 
-        createFile() {
-          file="$1"
-          # 'nix eval' requires filesystem or daemon access to a store even if nothing is built.
-          # Use a private store so that 'nix eval' always succeeds regardless of the
-          # execution environment, like a container.
-
-          # This tmp dir is automatically removed by systemd via PrivateTmp
-          [[ $store ]] || store="$(mktemp -d)"
-          secretsFile="$(realpath secrets.nix)" \
-          ${pkgs.nix}/bin/nix eval --raw --store "$store" "(
-            (import ${./make-secrets.nix} {
-              secretsFile = builtins.getEnv \"secretsFile\";
-            }).allSecrets.$file.text
-          )" > "$file"
-        }
-
         dir="${secretsDir}"
         if [[ ! -e $dir ]]; then
           echo "Error: Secrets dir '$dir' is missing"

modules/spark-wallet.nix

@@ -73,5 +73,6 @@ in {
         // nix-bitcoin-services.nodejs
         // nix-bitcoin-services.allowTor;
     };
+    nix-bitcoin.secrets.spark-wallet-login.user = "clightning";
   };
 }

network/network.nix

@@ -5,10 +5,11 @@
     { config, pkgs, lib, ... }: {
       imports = [ ../configuration.nix ];
 
-      deployment.keys = (import ../modules/secrets/make-secrets.nix {
+      deployment.keys = builtins.mapAttrs (n: v: {
-        inherit config;
+        keyFile = "${toString ../secrets}/${n}";
-        secretsFile = ../secrets/secrets.nix;
+        destDir = "/secrets/";
-      }).activeSecrets;
+        inherit (v) user group permissions;
+      }) config.nix-bitcoin.secrets;
 
       # nixops makes the secrets directory accessible only for users with group 'key'.
       # For compatibility with other deployment methods besides nixops, we forego the

pkgs/generate-secrets/generate-secrets.sh

@@ -1,43 +1,31 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 opensslConf=${1:-openssl.cnf}
-secretsFile=secrets.nix
 
-if [ ! -e "$secretsFile" ]; then
+makePasswordSecret() {
-    echo Write secrets to $secretsFile
+    [[ -e $1 ]] || apg -m 20 -x 20 -M Ncl -n 1 > "$1"
-    makepw="apg -m 20 -x 20 -M Ncl -n 1"
+}
-    {
-        echo \{
-        echo "  bitcoinrpcpassword = \"$($makepw)\";"
-        echo "  lnd-wallet-password = \"$($makepw)\";"
-        echo "  lightning-charge-api-token = \"$($makepw)\";"
-        echo "  liquidrpcpassword = \"$($makepw)\";"
-        echo "  spark-wallet-password = \"$($makepw)\";"
-        echo \}
-    } >> $secretsFile
-    echo Done
-else
-    echo $secretsFile already exists. Skipping.
-fi
 
-if [ ! -e nginx.key ] || [ ! -e nginx.cert ]; then
+makePasswordSecret bitcoin-rpcpassword
-    echo Generate Nginx Self-Signed Cert
+makePasswordSecret lnd-wallet-password
-    openssl genrsa -out nginx.key 2048
+makePasswordSecret liquid-rpcpassword
-    openssl req -new -key nginx.key -out nginx.csr -subj "/C=KN"
+makePasswordSecret lightning-charge-token
-    openssl x509 -req -days 1825 -in nginx.csr -signkey nginx.key -out nginx.cert
+makePasswordSecret spark-wallet-password
+
+[[ -e lightning-charge-env ]] || echo "API_TOKEN=$(cat lightning-charge-token)" > lightning-charge-env
+[[ -e nanopos-env          ]] || echo "CHARGE_TOKEN=$(cat lightning-charge-token)" > nanopos-env
+[[ -e spark-wallet-login   ]] || echo "login=spark-wallet:$(cat spark-wallet-password)" > spark-wallet-login
+
+if [[ ! -e nginx-key || ! -e nginx-cert ]]; then
+    openssl genrsa -out nginx-key 2048
+    openssl req -new -key nginx-key -out nginx.csr -subj "/C=KN"
+    openssl x509 -req -days 1825 -in nginx.csr -signkey nginx-key -out nginx-cert
     rm nginx.csr
-    echo Done
-else
-    echo Nginx Cert already exists. Skipping.
 fi
 
-if [ ! -e lnd.key ] || [ ! -e lnd.cert ]; then
+if [[ ! -e lnd-key || ! -e lnd-cert ]]; then
-    echo Generate LND compatible TLS Cert
+    openssl ecparam -genkey -name prime256v1 -out lnd-key
-    openssl ecparam -genkey -name prime256v1 -out lnd.key
+    openssl req -config $opensslConf -new -sha256 -key lnd-key -out lnd.csr -subj '/CN=localhost/O=lnd'
-    openssl req -config $opensslConf -new -sha256 -key lnd.key -out lnd.csr -subj '/CN=localhost/O=lnd'
+    openssl req -config $opensslConf -x509 -sha256 -days 1825 -key lnd-key -in lnd.csr -out lnd-cert
-    openssl req -config $opensslConf -x509 -sha256 -days 1825 -key lnd.key -in lnd.csr -out lnd.cert
     rm lnd.csr
-    echo Done
-else
-    echo LND cert already exists. Skipping.
 fi

pkgs/generate-secrets/update-and-generate.nix

@@ -0,0 +1,10 @@
+{ pkgs }: with pkgs;
+
+let
+  generate-secrets = callPackage ./. {};
+in
+writeScript "make-secrets" ''
+  # Update from old secrets format
+  [[ -e secrets.nix ]] && . ${./update-secrets.sh}
+  ${generate-secrets}
+''

pkgs/generate-secrets/update-secrets.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Update secrets from the old format to the current one where each secret
+# has a local source file.
+
+reportError() {
+    echo "Updating secrets failed. (Error in line $1)"
+    echo "The secret files have been moved to secrets/old-secrets"
+}
+trap 'reportError $LINENO' ERR
+
+echo "Updating old secrets to the current format."
+
+mkdir old-secrets
+# move all files into old-secrets
+shopt -s extglob dotglob
+mv !(old-secrets) old-secrets
+shopt -u dotglob
+
+secrets=$(cat old-secrets/secrets.nix)
+
+extractPassword() {
+    pwName="$1"
+    destFile="${2:-$pwName}"
+    echo "$secrets" | sed -nE "s/.*?$pwName = \"(.*?)\".*/\1/p" > "$destFile"
+}
+
+rename() {
+    old="old-secrets/$1"
+    if [[ -e $old ]]; then
+        cp "$old" "$2"
+    fi
+}
+
+extractPassword bitcoinrpcpassword bitcoin-rpcpassword
+extractPassword lnd-wallet-password
+extractPassword liquidrpcpassword liquid-rpcpassword
+extractPassword lightning-charge-api-token lightning-charge-token
+extractPassword spark-wallet-password
+
+rename nginx.key nginx-key
+rename nginx.cert nginx-cert
+rename lnd.key lnd-key
+rename lnd.cert lnd-cert
+
+rm -r old-secrets

shell.nix

@@ -7,7 +7,7 @@ stdenv.mkDerivation rec {
   name = "nix-bitcoin-environment";
 
   nixops19_09 = callPackage ./pkgs/nixops {};
-  generate-secrets = callPackage ./pkgs/generate-secrets {};
+  make-secrets = callPackage ./pkgs/generate-secrets/update-and-generate.nix {};
 
   buildInputs = [ nixops19_09 figlet ];
 
@@ -19,6 +19,6 @@ stdenv.mkDerivation rec {
     # keys already added to my ssh-agent.
     export SSH_AUTH_SOCK=""
     figlet "nix-bitcoin"
-    (mkdir -p secrets; cd secrets; ${generate-secrets})
+    (mkdir -p secrets; cd secrets; ${make-secrets})
   '';
 }