1= Fleet23++++4<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet temporary logo generated with midjourney" src="./docs/tmplogo.png" width="200px"></img></a></p>5++++67An NixOS cluster deployment tool.89== Advantages over existing configuration systems (NixOps/Morph)1011- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)12- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.13- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1415== Flake example1617{18 description = "My cluster configuration";19 inputs = {20 nixpkgs.url = "github:nixos/nixpkgs";21 fleet = {22 url = "github:CertainLach/fleet";23 inputs.nixpkgs.follows = "nixpkgs";24 };25 lanzaboote = {26 url = "github:nix-community/lanzaboote/v0.3.0";27 inputs.nixpkgs.follows = "nixpkgs";28 };29 };30 outputs = {31 nixpkgs,32 fleet,33 lanzaboote,34 ...35 }: {36 # TODO: This section of documentation needs to use flake-utils.37 formatter.x86_64-linux = let38 pkgs = import nixpkgs {system = "x86_64-linux";};39 in40 pkgs.alejandra;4142 devShell.x86_64-linux = let43 pkgs = import nixpkgs {44 system = "x86_64-linux";45 };46 in47 pkgs.mkShell {48 buildInputs = with pkgs; [49 fleet.packages.x86_64-linux.fleet50 ];51 };5253 # Single flake may contain multiple fleet configurations, default one is called... `default`54 fleetConfigurations.default = fleet.lib.fleetConfiguration {55 # nixpkgs used to build the systems56 inherit nixpkgs;57 # fleet wants to pass some data, like secrets, to do that - fleet writes all the encrypted secrets to fleet.nix58 # treat the contents of this file as implementation detail59 data = import ./fleet.nix;60 61 # globalModules section of fleet config declares modules, which are used for all configured nixos hosts.62 globalModules = [63 lanzaboote.nixosModules.lanzaboote64 ({65 config,66 lib,67 ...68 }: {69 # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.70 nix.registry.nixpkgs = {71 from = { id = "nixpkgs"; type = "indirect"; };72 flake = nixpkgs;73 exact = false;74 };75 })76 ];7778 # Those modules are used to configure all the machines in cluster at the same time, good example of global modules79 # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.80 #81 # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.82 modules = [83 ./wireguard84 # Multi-instancible modules example85 (import ./kubernetes {hosts = ["a" "b"];})86 (import ./kubernetes {hosts = ["c" "d"];})87 ];8889 # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...90 hosts.controlplane-1 = {91 # Every host has some system, for which the system configuration needs to be built92 system = "x86_64-linux";93 # And nixos modules94 modules = [95 ./controlplane-1/hardware-configuration.nix96 ./controlplane-1/configuration.nix97 # Configuration may also be specified inline, as in any nixos config.98 ({...}: {99 services.ray = {100 gpus = 4;101 cpus = 128;102 };103 })104 ];105 };106 };107 };108}109110== Secret generator example111112TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.113114=== Quickly run securely setup gitlab115116[source,nix]117----118{config, ...}: {119 secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {120 gitlab-initial-root = {121 generator = {mkPassword}: mkPassword {};122 } // ownership;123 gitlab-secret = {124 generator = {mkPassword}: mkPassword {};125 } // ownership;126 gitlab-otp = {127 generator = {mkPassword}: mkPassword {};128 } // ownership;129 gitlab-db = {130 generator = {mkPassword}: mkPassword {};131 } // ownership;132 gitlab-jws = {133 generator = {mkRsa}: mkRsa {};134 } // ownership;135 };136 services.gitlab = let secrets = config.secrets; in {137 enable = true;138 initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;139 secrets = {140 secretFile = secrets.gitlab-secret.secretPath;141 otpFile = secrets.gitlab-otp.secretPath;142 dbFile = secrets.gitlab-db.secretPath;143 jwsFile = secrets.gitlab-jws.secretPath;144 };145 };146}147----148149=== Securely initialize kubernetes secrets150151In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.152This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.153I want to build ansible-like script execution in fleet for this kind of tasks.154155[source,nix]156----157{...}: {158 159 nixpkgs.overlays = [160 (final: prev: let161 lib = final.lib;162 in {163 readKubernetesCa = {impureOn}:164 final.mkImpureSecretGenerator ''165 cd ~/ca166167 cert=kubernetes-intermediateCA.crt168169 expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")170 echo -n $expires_at > $out/expires_at171172 cat $cert > $out/public173 ''174 impureOn;175 mkKubernetesCert = {176 subj,177 sans ? [],178 impureOn,179 }:180 final.mkImpureSecretGenerator ''181 cd ~/ca182183 params=$(sudo mktemp)184 csr=$(sudo mktemp)185 cert=$(sudo mktemp)186 sudo openssl ecparam -genkey -name secp384r1 -out $params187 sudo openssl req -new -key $params \188 -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \189 ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \190 -out $csr191 sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert192193 expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")194 echo -n $expires_at > $out/expires_at195196 sudo cat $params | encrypt > $out/secret197 sudo cat $cert > $out/public198 ''199 impureOn;200 })201 ];202 203 204 205 environment.systemPackages = [206 pkgs.openssl.bin207 (pkgs.writeShellApplication {208 name = "hsms";209 text = ''210 set -eu211 export OPENSSL_CONF=${openssl-conf}212 213 HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})214 exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"215 '';216 })217 (pkgs.writeShellApplication {218 name = "hsmt";219 text = ''220 set -eu221 HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})222 exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"223 '';224 })225 ];226 227 228 sharedSecrets = {229 "ca.pem" = {230 231 regenerateOnOwnerAdded = false;232 233 expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];234 generator = {readKubernetesCa}:235 readKubernetesCa {236 impureOn = "[CENSORED]";237 };238 };239 "kube-admin.pem" = {240 regenerateOnOwnerAdded = false;241 expectedOwners = ["cluster-admin"];242 generator = {mkKubernetesCert}:243 mkKubernetesCert {244 subj = {245 CN = "admin";246 O = "system:masters";247 };248 impureOn = "[CENSORED]";249 };250 };251 "kube-apiserver.pem" = {252 253 254 255 256 257 258 259 regenerateOnOwnerAdded = true;260 expectedOwners = ["controlplane-1" "controlplane-2"];261 generator = {mkKubernetesCert}:262 mkKubernetesCert {263 inherit sans;264 subj.CN = "kubernetes";265 impureOn = "[CENSORED]";266 };267 };268}269----