From c4d07707c40e3f33979e9aa8ac415ac29acd060a Mon Sep 17 00:00:00 2001 From: Arnaud Rebts Date: Thu, 20 Aug 2026 15:36:46 +0200 Subject: [PATCH 1/2] Fix sops-nix host decrypt: use age-converted pubkeys, not raw ssh-ed25519 k3s-sops-operator-seed.service failed on node1 with "Error getting data key: 0 successful groups required, got 0" when decrypting secrets/clusters/prd/sops-age-key.sops, even though node1's SSH host key was a correctly listed recipient and sops updatekeys/rotate reproduced the exact same failure every time. Root cause: sops's own "ssh-ed25519 ..." recipient support (filippo.io/age/agessh, meant for a human decrypting locally via SOPS_AGE_SSH_PRIVATE_KEY_FILE) and sops-nix's host-side decrypt (Mic92/ssh-to-age, used by sops-install-secrets) derive *different* X25519 keys from the same ed25519 SSH key. Confirmed reproducibly with a throwaway test keypair: ciphertext encrypted via filippo.io/age/agessh for a recipient never decrypts via the Mic92-derived identity for that same key. sops-nix's own README documents the correct pattern: convert a host's SSH key to its age1... form with ssh-to-age first, and use that string as the recipient, not the raw SSH pubkey. modules/flake/sops-config.nix's clusterMemberHostKeys was inserting a cluster-member host's raw SSH pubkey as an age recipient on secrets/clusters//sops-age-key.sops, which sops-nix on that host can never actually decrypt. provision-host-key.nix now also derives and commits each host's age-converted pubkey (ssh_host_ed25519_key.age-pub), and sops-config.nix uses that instead. Backfilled node1's age-pub, regenerated .sops.yaml, and rotated the cluster secret to the corrected recipient. Verified on node1: k3s-sops-operator-seed.service now succeeds. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01CezjyaVpC3FVPTUECp7cMR --- .sops.yaml | 4 +- modules/flake/provision-host-key.nix | 53 +++++++++++++------ modules/flake/sops-config.nix | 18 +++++-- secrets/clusters/prd/sops-age-key.sops | 16 +++--- .../hosts/node1/ssh_host_ed25519_key.age-pub | 1 + 5 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 secrets/hosts/node1/ssh_host_ed25519_key.age-pub diff --git a/.sops.yaml b/.sops.yaml index 85400b75..8b2966c9 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -22,9 +22,7 @@ creation_rules: AAAAC3NzaC1lZDI1NTE5AAAAIHIM3nsk3HxvEcplSqwynh9V2NzlYdI10mrR746SiJZb kid@fw13 - age1crlwx4k0eaq666fa8dqpq3k9rhkm8qvwf3ewuum8s8hmrzrm55fsl3tk8h - - ssh-ed25519 - AAAAC3NzaC1lZDI1NTE5AAAAILkFKxaxcAS/tQuS8Gfk8xaHSLDAm1VCPHo8M7umUdVJ - root@node1 + - age1jnjhgp7chadmgmph7qkms4gfvh4enf02vvmr4pcv4ks9n7ml2e4savuu8v path_regex: secrets/clusters/prd/.* - encrypted_regex: ^(data|stringData)$ key_groups: diff --git a/modules/flake/provision-host-key.nix b/modules/flake/provision-host-key.nix index ac10642c..67edca47 100644 --- a/modules/flake/provision-host-key.nix +++ b/modules/flake/provision-host-key.nix @@ -1,10 +1,21 @@ # nix run .#provision-host-key — generates an ed25519 SSH host key # locally, encrypts the private half with sops (binary mode — the payload is -# raw PEM bytes, not YAML/JSON structured data), and commits both the -# ciphertext and the plaintext public key to secrets/hosts//. Uses this -# repo's existing sops recipients (kid-vulkan/kid-fw13, .sops.yaml) — -# encryption only needs their public halves, already present in .sops.yaml, -# so this needs no new key material of its own to run. +# raw PEM bytes, not YAML/JSON structured data), and commits the ciphertext, +# the plaintext public key, and its age-converted form to +# secrets/hosts//. Uses this repo's existing sops recipients +# (kid-vulkan/kid-fw13, .sops.yaml) — encryption only needs their public +# halves, already present in .sops.yaml, so this needs no new key material +# of its own to run. +# +# The age-pub file matters because sops's own "ssh-ed25519 ..." recipient +# support (filippo.io/age/agessh, used when a human decrypts locally via +# SOPS_AGE_SSH_PRIVATE_KEY_FILE) and sops-nix's host-side decrypt +# (Mic92/ssh-to-age, used by sops-install-secrets on the host itself) derive +# *different* X25519 keys from the same ed25519 SSH key — ciphertext +# encrypted for the raw "ssh-ed25519 ..." string can never be decrypted by +# sops-nix on that host. Anything sops-nix itself must decrypt (modules/ +# flake/sops-config.nix's clusterMemberHostKeys) needs the host's converted +# age1... key as its recipient instead, matching sops-nix's own README. # # modules/flake/nixos-anywhere.nix consumes the committed key at install # time, injecting it via nixos-anywhere --extra-files so the host has its @@ -19,31 +30,39 @@ runtimeInputs = [ pkgs.openssh pkgs.sops + pkgs.ssh-to-age ]; text = '' host=''${1:?usage: provision-host-key } host_dir="secrets/hosts/$host" sops_file="$host_dir/ssh_host_ed25519_key.sops" pub_file="$host_dir/ssh_host_ed25519_key.pub" + age_pub_file="$host_dir/ssh_host_ed25519_key.age-pub" if [[ -f "$sops_file" ]]; then - echo "==> $host already has a committed key ($sops_file), skipping." - exit 0 - fi + echo "==> $host already has a committed key ($sops_file), skipping generation." + else + mkdir -p "$host_dir" + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' EXIT - mkdir -p "$host_dir" - tmp="$(mktemp -d)" - trap 'rm -rf "$tmp"' EXIT + echo "==> Generating ed25519 host key for $host..." + ssh-keygen -t ed25519 -N "" -f "$tmp/key" -C "root@$host" >/dev/null - echo "==> Generating ed25519 host key for $host..." - ssh-keygen -t ed25519 -N "" -f "$tmp/key" -C "root@$host" >/dev/null + echo "==> Encrypting private key with sops..." + sops --input-type binary --output-type binary --encrypt --output "$sops_file" "$tmp/key" - echo "==> Encrypting private key with sops..." - sops --input-type binary --output-type binary --encrypt --output "$sops_file" "$tmp/key" + cp "$tmp/key.pub" "$pub_file" - cp "$tmp/key.pub" "$pub_file" + echo "==> Committed: $sops_file, $pub_file" + fi + + if [[ ! -f "$age_pub_file" ]]; then + echo "==> Deriving age public key for $host..." + ssh-to-age -i "$pub_file" > "$age_pub_file" + echo "==> Committed: $age_pub_file" + fi - echo "==> Committed: $sops_file, $pub_file" echo "==> Run 'nix run .#write-sops-config' to add $host as an sops recipient." ''; }; diff --git a/modules/flake/sops-config.nix b/modules/flake/sops-config.nix index fe7be5e3..fe953cef 100644 --- a/modules/flake/sops-config.nix +++ b/modules/flake/sops-config.nix @@ -21,6 +21,7 @@ let secretsDir = ../../secrets; hostPubKeyPath = host: secretsDir + "/hosts/${host}/ssh_host_ed25519_key.pub"; + hostAgePubKeyPath = host: secretsDir + "/hosts/${host}/ssh_host_ed25519_key.age-pub"; hostNames = lib.unique ( lib.concatMap (system: builtins.attrNames (config.den.hosts.${system} or { })) config.systems @@ -56,11 +57,22 @@ let # (modules/den/aspects/services/k3s/sops-operator.nix) decrypts the # cluster's own sops-age key on each such host, using that host's own # persisted SSH key as its decryption identity, so this file needs each - # member host's pubkey as a recipient too, not just the cluster's own key. + # member host's key as a recipient too, not just the cluster's own key. + # + # Uses each host's age-converted pubkey (ssh_host_ed25519_key.age-pub, + # modules/flake/provision-host-key.nix), not its raw ssh-ed25519 pubkey: + # sops-nix's host-side decrypt (Mic92/ssh-to-age) and sops's own + # "ssh-ed25519 ..." recipient support (filippo.io/age/agessh, meant for a + # human decrypting locally) derive different X25519 keys from the same + # SSH key, so the raw pubkey would never actually be decryptable here. clusterMemberHostKeys = cluster: - map (host: lib.removeSuffix "\n" (builtins.readFile (hostPubKeyPath host))) ( - builtins.filter (host: (allHosts.${host}.k3s.clusterName or null) == cluster) provisionedHosts + map (host: lib.removeSuffix "\n" (builtins.readFile (hostAgePubKeyPath host))) ( + builtins.filter ( + host: + (allHosts.${host}.k3s.clusterName or null) == cluster + && builtins.pathExists (hostAgePubKeyPath host) + ) provisionedHosts ); clusterRule = cluster: { diff --git a/secrets/clusters/prd/sops-age-key.sops b/secrets/clusters/prd/sops-age-key.sops index 6e88c0e6..616b699c 100644 --- a/secrets/clusters/prd/sops-age-key.sops +++ b/secrets/clusters/prd/sops-age-key.sops @@ -1,26 +1,26 @@ { - "data": "ENC[AES256_GCM,data:uofmAwWxk7u2yrpMM8nBokD50xBKVjmnYRyF3kjeqZMtxSmq+DjhysCNWvcofo1ZBX1lcIBc77zz+X8DngaPMjwnHux2XBajio7oCJMKT72tInvhdeAX6FqKoWPxoujS69NVERb3/tI+GuINC3qmRrsPtF1i3bL3/HbHTQr5KrJYnhsJXtPD7avoHgcep+c8l5/szZBlhOIgZO/19rBx9t3iRD0ptykkT/Qot5qsJldoxM6+5Bz6Y2KSg5dZ,iv:1I+oohf8mqOc5S4BPzcVYtxWVE8iLx8GrQ25JW12btE=,tag:cdtvnJHQPTLCD+4XqdDOgA==,type:str]", + "data": "ENC[AES256_GCM,data:jFBYkq7vDOf/9Dc/CV26M8BHmJLxDNgh7RpEyeF6XlJic44TKtbKPY8TId18uapCG+v4tMgLz5SfVHnKNLGNyXjWZdC2KqC5mmHp9u5KQsp7+kyVwTmOJclEZ/vo/c88QLXSxpQ4Ujm8HBGA6vJ6Vt1V0acRpicZvmN15TxbjkFKIlyfPI3gRNIIoIXv75DbBZ8zzr145RO4Qeem5l3tPiLuILRxXdQ7HKjf0BqSOBKok7pH/UFlHFNjHYb/,iv:1I+oohf8mqOc5S4BPzcVYtxWVE8iLx8GrQ25JW12btE=,tag:7GREObE4lrGqkJbEnFv4MA==,type:str]", "sops": { "age": [ { - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFE1WkIwUSBrckxV\nV29KZ1Ruc0s0czhDSEU3ajg2QUdRUWt0TkpXVEFRQVMzQUYwS1NjCjlqKy9WQUph\nMHBZRGxVSGpuL28xczhaU2VWY1FvcUFXVC9QOVpuWHZLNDQKLS0tIElST09uUjE2\nOXByQkRKK2lna3hucWpqVVVDd3l4SmtFdEZDMHZhdHg1N3MK+1k8m4LXUSY/7bLo\nCEoJ0NRlRivVk3IzI59kGmRnb1U5Vshul3PDe9r4x0vJTRvJlPCC6+MQQKzSdx5/\nioTzNA==\n-----END AGE ENCRYPTED FILE-----\n", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFE1WkIwUSBDdUhp\nL1RXeHFpVzVGeS9FOC9qWW9VeHQ4TDF0YzI4UzE2OEk0OTh3Q2x3CmhVT20xNW9G\nWnAxaHJhd0dWY25GcWNGMTE1VXJjWWJjZVJJN1NUbFo0cFEKLS0tIDhLaVFJWW9W\nM2VuVlVSL2o5dVM0WHNVSm4vWEhwbTZSWGpRc2Fpa0V2U0kKWVKjWJa1wIJzCpoA\nKDxJKyvIBKFjAdJYFsQqen6BtWoMXcVF1Unma/kMxJDxVCiE/1DT1spkl3TgGWT6\n/o4azw==\n-----END AGE ENCRYPTED FILE-----\n", "recipient": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAcnmLrPeTJeKsasfU0qn4sP4lBNeOUgRG4iZDS8nyEo kid@vulkan" }, { - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IE1Wa1kxQSBmTmUr\nczJDbE5JeHVOdFVXbmdMWmRUeGg1K1VOVElqa0tqK0NkMWNFNXc4CnQrb3dYRU80\nRDZadVN5SDdsaGRzTmxBME1ENGpQR2FWNmJkYWFYRnFSWlEKLS0tIEN6RUJGcWM5\nUURRNXo5c2QvRHN4T2M4RVZsZ0wwT0lnU05zRHNhUGNMWjQKRlLWZ8/dYPaUapqI\nBpVA9b53HN5caxobHDK8I3SGSMItnYO+VuLenyF0P0cmksv1ekgE8n2rOIOuyfCi\nipO5jA==\n-----END AGE ENCRYPTED FILE-----\n", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IE1Wa1kxQSBlazFH\nT3czY2pEUE1zOUhwc0RuaFhCYnVHU055Q3hjaTVlOCtSQzVVdDFJCllWWlc4U2l5\nOTM5YUo5bDZPcktkczhTVWdHK3EzU0lmSzU3SUZmZjRBOTQKLS0tIHFSVHR6T0Zj\nOVM4ZXZibWlJYkUvcDJMWVhoK2RZS2NpWldEMUUyRGtjSnMK+4mZaO3ebKPcxHvk\nkXhebAwrgy08PcLxmyqNHwAbMUYQ5L2/nlszuQagbZ8ripbzVPlXHG3Ic4vWAMmf\nik7jWg==\n-----END AGE ENCRYPTED FILE-----\n", "recipient": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHIM3nsk3HxvEcplSqwynh9V2NzlYdI10mrR746SiJZb kid@fw13" }, { - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoVmVLNWFUWjhoM2ZqaHVy\nTzBoSmFleXhhR3YwYWNRSzRGQW51emt4V0FzCmlCbUJTMHlVek1IbE1jMVlUbVNO\nMDR0T3J0bWFtcFl1R0p6T3BibEc2Y0UKLS0tIFpHeFRuTWFzUkN5WGQ3N1NWRE90\nNG5TTS9TQW9IY0tMUWdWRzVvWE1HZUUK4CKj2hmwkfPZw1jl0mBZUzWx4B0ecIAo\nPLNCoWlThv6fxbmNVwIzX+kzeSNXWRMKLrVU564dDiO40KEvh6TSxA==\n-----END AGE ENCRYPTED FILE-----\n", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSdE53bVBWRnh0TlhZQ3Zo\nbU0vU2ZQNWV4aVZrTTRaNU15eVVGN3dEblYwCk5KemMxRTZpRjR3Zkc2ZU1SdjZo\nRW1vTHlWWThxa2x5N0krVjR5OVYwTzAKLS0tIFhSV3lDN0hmV3ZyNjBvbW9LTUw0\ndlpXWU05V0oyZnp4OWF1UGJHWHlkSHcKNJsnPIBcAAEk4WOCVeq+vrdmRUWyU5MB\nxJsLAcArRrqom7ZodFGYwpEKtYPht6856BmihNdzsmhEXfKZXvtlqw==\n-----END AGE ENCRYPTED FILE-----\n", "recipient": "age1crlwx4k0eaq666fa8dqpq3k9rhkm8qvwf3ewuum8s8hmrzrm55fsl3tk8h" }, { - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IG0wU09oQSBqTHFM\nKzYyd1h4d0Ura04yOFVuQjduenZHNDlmeWliM0lFVkI2RlM4Q0VVCnVMNTU0UVRt\nSElHNmY0MHNNNDRSbUFSRHpiT1VUbTVvNThpajBmZEF1SUkKLS0tIDFrM0ZURGQv\nSlhFNWQ5WldNY082dlNELzRUZjVwbmRIdHRLMWVVNVRBT2cKym8ZVzvgC6IVlOPW\nu01wJ5gHWeHe5i3JV7bGuG9H/QYd05wLKfL62vmr5asNsAR4EC42TS0MpM6neZHj\nYPdPrw==\n-----END AGE ENCRYPTED FILE-----\n", - "recipient": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILkFKxaxcAS/tQuS8Gfk8xaHSLDAm1VCPHo8M7umUdVJ root@node1" + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1azBNODFVa01vT0N4a1JQ\nQVl4TTZRQlN1OVFueThva1lYWXV6N0lnQVVjCjUrV2ZMemM5UEhCcUtUcEdQMEJ1\nZmsrcFRqaXQ3NHJWb1g1czJHTGo4QmsKLS0tIExvWE12cCtSUHVZVkptd1hNNUpO\naHVTOUdDZ2djTEhrbWJGdVR1UTBrK0kKLZFwv57ytMCvR9jW4l/Nn3l0zzpCs4iW\nFLLSJ+NeZMtzC9bmT+6p26J50vTQMVvCxcK1x0K2CAVp0mGam0jpHQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1jnjhgp7chadmgmph7qkms4gfvh4enf02vvmr4pcv4ks9n7ml2e4savuu8v" } ], - "lastmodified": "2026-08-17T17:28:34Z", - "mac": "ENC[AES256_GCM,data:65H0NTTRciYRPDTOKn/ImuXlBkKgUKJ0LUutkR8eBCtCEbD2TQ0YP3A0bbVwVKw/u2uyfKhG2ZTJpn8PMDqkV7jr0qj4bGCGF//MyZhFiINxux1rIibgyU0mlehn3ugRVSfxSgxieh6Ezsi7q+5fXjDTcYzzWgZl2hXPBEU9zyU=,iv:YZkIUACBm87Oymkc4rnziDX4Fj8fDmJrb512fuAFbu8=,tag:5dGRgber0HREJvgF78WbkQ==,type:str]", + "lastmodified": "2026-08-18T22:20:43Z", + "mac": "ENC[AES256_GCM,data:t2Y0T86owtw5Swo76aS/4hvrOu5KTQmZFNZsxyj8SoqYCV+JKp7RhTsRWZh3uUhWuPjXPKCejDEGD5hTgtBpPGjeIzxJ9sjU9urk1utmuZGvv4rS6Jfe7a/xYy0sX6NwPWKyt0tBidJCjDIVcX1FScGY2TGmqOPYb7/UgmhGgk4=,iv:ihLmVZxk2+fGkKifLqxCrdzgJSkfxkT3xqDChQWBJUw=,tag:2I9XpmF+UwvXXV8vjCSlXg==,type:str]", "unencrypted_suffix": "_unencrypted", "version": "3.13.3" } diff --git a/secrets/hosts/node1/ssh_host_ed25519_key.age-pub b/secrets/hosts/node1/ssh_host_ed25519_key.age-pub new file mode 100644 index 00000000..87ab6594 --- /dev/null +++ b/secrets/hosts/node1/ssh_host_ed25519_key.age-pub @@ -0,0 +1 @@ +age1jnjhgp7chadmgmph7qkms4gfvh4enf02vvmr4pcv4ks9n7ml2e4savuu8v From 633595a117e4a689f247d2a109b2d58795d80e12 Mon Sep 17 00:00:00 2001 From: Arnaud Rebts Date: Thu, 20 Aug 2026 15:45:48 +0200 Subject: [PATCH 2/2] Trim comments to the load-bearing fact only Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01CezjyaVpC3FVPTUECp7cMR --- modules/flake/provision-host-key.nix | 12 +++--------- modules/flake/sops-config.nix | 9 ++------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/modules/flake/provision-host-key.nix b/modules/flake/provision-host-key.nix index 67edca47..c4a42031 100644 --- a/modules/flake/provision-host-key.nix +++ b/modules/flake/provision-host-key.nix @@ -7,15 +7,9 @@ # halves, already present in .sops.yaml, so this needs no new key material # of its own to run. # -# The age-pub file matters because sops's own "ssh-ed25519 ..." recipient -# support (filippo.io/age/agessh, used when a human decrypts locally via -# SOPS_AGE_SSH_PRIVATE_KEY_FILE) and sops-nix's host-side decrypt -# (Mic92/ssh-to-age, used by sops-install-secrets on the host itself) derive -# *different* X25519 keys from the same ed25519 SSH key — ciphertext -# encrypted for the raw "ssh-ed25519 ..." string can never be decrypted by -# sops-nix on that host. Anything sops-nix itself must decrypt (modules/ -# flake/sops-config.nix's clusterMemberHostKeys) needs the host's converted -# age1... key as its recipient instead, matching sops-nix's own README. +# age-pub exists because sops's raw "ssh-ed25519 ..." recipients and +# sops-nix's host-side decrypt derive different X25519 keys from the same +# SSH key — sops-nix needs the age1... form as its recipient. # # modules/flake/nixos-anywhere.nix consumes the committed key at install # time, injecting it via nixos-anywhere --extra-files so the host has its diff --git a/modules/flake/sops-config.nix b/modules/flake/sops-config.nix index fe953cef..2afc86f9 100644 --- a/modules/flake/sops-config.nix +++ b/modules/flake/sops-config.nix @@ -58,13 +58,8 @@ let # cluster's own sops-age key on each such host, using that host's own # persisted SSH key as its decryption identity, so this file needs each # member host's key as a recipient too, not just the cluster's own key. - # - # Uses each host's age-converted pubkey (ssh_host_ed25519_key.age-pub, - # modules/flake/provision-host-key.nix), not its raw ssh-ed25519 pubkey: - # sops-nix's host-side decrypt (Mic92/ssh-to-age) and sops's own - # "ssh-ed25519 ..." recipient support (filippo.io/age/agessh, meant for a - # human decrypting locally) derive different X25519 keys from the same - # SSH key, so the raw pubkey would never actually be decryptable here. + # Uses each host's age-pub (provision-host-key.nix), not its raw + # ssh-ed25519 pubkey — sops-nix derives a different X25519 key from that. clusterMemberHostKeys = cluster: map (host: lib.removeSuffix "\n" (builtins.readFile (hostAgePubKeyPath host))) (