`6899c702106b`

source

README.adoc9.3 KiBrendered source history
1= Fleet23++++4<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet temporary logo generated with midjourney" src="./docs/tmplogo.png" width="200px"></img></a></p>5++++67An NixOS cluster deployment tool.89== Advantages over existing configuration systems (NixOps/Morph)1011- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)12- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.13- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1415== Flake example1617[source,nix]18----19{20  description = "My cluster configuration";21  inputs = {22    nixpkgs.url = "github:nixos/nixpkgs";23    fleet = {24      url = "github:CertainLach/fleet";25      inputs.nixpkgs.follows = "nixpkgs";26    };27    flake-parts.url = "github:hercules-ci/flake-parts";28    lanzaboote = {29      url = "github:nix-community/lanzaboote/v0.3.0";30      inputs.nixpkgs.follows = "nixpkgs";31    };32  };33  outputs = inputs @ {34    nixpkgs,35    flake-parts,36    lanzaboote,37    ...38  }:39    flake-parts.lib.mkFlake {inherit inputs;} {40      imports = [inputs.fleet.flakeModules.default];4142      perSystem = {43        pkgs,44        system,45        ...46      }: {47        _module.args.pkgs = import nixpkgs {inherit system;};4849        formatter = pkgs.alejandra;50        devShells.default = pkgs.mkShell {51          packages = [inputs.fleet.packages.${system}.fleet];52        };53      };5455      # Single flake may contain multiple fleet configurations, default one is called... `default`56      fleetConfigurations.default = {57        # nixpkgs used to build the systems58        nixpkgs.buildUsing = nixpkgs;5960        # nixos option section of fleet config declares module, which is used for all configured nixos hosts.61        nixos.imports = [62          lanzaboote.nixosModules.lanzaboote63          {64            # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.65            nix.registry.nixpkgs = {66              from = {67                id = "nixpkgs";68                type = "indirect";69              };70              flake = nixpkgs;71              exact = false;72            };73          }74        ];7576        # Those modules are used to configure all the machines in cluster at the same time, good example of global modules77        # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.78        #79        # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.80        imports = [81          ./wireguard82          # Multi-instancible modules example83          (import ./kubernetes {hosts = ["a" "b"];})84          (import ./kubernetes {hosts = ["c" "d"];})85        ];8687        # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...88        hosts.controlplane-1 = {89          # Every host has some system, for which the system configuration needs to be built90          system = "x86_64-linux";91          # And nixos modules92          nixos.imports = [93            ./controlplane-1/hardware-configuration.nix94            ./controlplane-1/configuration.nix95            # Configuration may also be specified inline, as in any nixos config.96            {97              services.ray = {98                gpus = 4;99                cpus = 128;100              };101            }102          ];103        };104      };105    };106}107----108109== Secret generator example110111TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.112113=== Quickly run securely setup gitlab114115[source,nix]116----117{config, ...}: {118  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {119    gitlab-initial-root = {120      generator = {mkPassword}: mkPassword {};121    } // ownership;122    gitlab-secret = {123      generator = {mkPassword}: mkPassword {};124    } // ownership;125    gitlab-otp = {126      generator = {mkPassword}: mkPassword {};127    } // ownership;128    gitlab-db = {129      generator = {mkPassword}: mkPassword {};130    } // ownership;131    gitlab-jws = {132      generator = {mkRsa}: mkRsa {};133    } // ownership;134  };135  services.gitlab = let secrets = config.secrets; in {136    enable = true;137    initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;138    secrets = {139      secretFile = secrets.gitlab-secret.secretPath;140      otpFile = secrets.gitlab-otp.secretPath;141      dbFile = secrets.gitlab-db.secretPath;142      jwsFile = secrets.gitlab-jws.secretPath;143    };144  };145}146----147148=== Securely initialize kubernetes secrets149150In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.151This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.152I want to build ansible-like script execution in fleet for this kind of tasks.153154[source,nix]155----156{...}: {157  # First I define required secret generators:158  nixpkgs.overlays = [159    (final: prev: let160      lib = final.lib;161    in {162      readKubernetesCa = {impureOn}:163        final.mkImpureSecretGenerator ''164          cd ~/ca165166          cert=kubernetes-intermediateCA.crt167168          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")169          echo -n $expires_at > $out/expires_at170171          cat $cert > $out/public172        ''173        impureOn;174      mkKubernetesCert = {175        subj,176        sans ? [],177        impureOn,178      }:179        final.mkImpureSecretGenerator ''180          cd ~/ca181182          params=$(sudo mktemp)183          csr=$(sudo mktemp)184          cert=$(sudo mktemp)185          sudo openssl ecparam -genkey -name secp384r1 -out $params186          sudo openssl req -new -key $params \187            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \188            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \189            -out $csr190          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert191192          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")193          echo -n $expires_at > $out/expires_at194195          sudo cat $params | encrypt > $out/secret196          sudo cat $cert > $out/public197        ''198        impureOn;199    })200  ];201  # Those secret generators are impure, thus they are run in system environment.202  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.203  # I write a couple of scripts for executing openssl with HSM.204  environment.systemPackages = [205    pkgs.openssl.bin206    (pkgs.writeShellApplication {207      name = "hsms";208      text = ''209        set -eu210        export OPENSSL_CONF=${openssl-conf}211        # Yay, using secrets to generate secrets!212        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})213        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"214      '';215    })216    (pkgs.writeShellApplication {217      name = "hsmt";218      text = ''219        set -eu220        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})221        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"222      '';223    })224  ];225  # And finally, I have secrets, which are shared between machines.226  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.227  sharedSecrets = {228    "ca.pem" = {229      # This is just the public key, no need to regenerate it to change owner list230      regenerateOnOwnerAdded = false;231      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.232      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];233      generator = {readKubernetesCa}:234        readKubernetesCa {235          impureOn = "[CENSORED]";236        };237    };238    "kube-admin.pem" = {239      regenerateOnOwnerAdded = false;240      expectedOwners = ["cluster-admin"];241      generator = {mkKubernetesCert}:242        mkKubernetesCert {243          subj = {244            CN = "admin";245            O = "system:masters";246          };247          impureOn = "[CENSORED]";248        };249    };250    "kube-apiserver.pem" = {251      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.252      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...253      #254      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't255      # want for it to be able to pretend like it is a valid server.256      #257      # However, certificate revokation is complicated in my setup, and I can't show it here.258      regenerateOnOwnerAdded = true;259      expectedOwners = ["controlplane-1" "controlplane-2"];260      generator = {mkKubernetesCert}:261        mkKubernetesCert {262          inherit sans;263          subj.CN = "kubernetes";264          impureOn = "[CENSORED]";265        };266    };267}268----