`9e7bb975c3c5`

source

README.adoc9.1 KiBrendered source history
1= Fleet23++++4<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet temporary logo generated with midjourney" src="./docs/tmplogo.png" width="200px"></img></a></p>5++++67An NixOS cluster deployment tool.89== Advantages over existing configuration systems (NixOps/Morph)1011- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)12- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.13- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1415== Flake example1617[source,nix]18----19{20  description = "My cluster configuration";21  inputs = {22    nixpkgs.url = "github:nixos/nixpkgs";23    fleet = {24      url = "github:CertainLach/fleet";25      inputs.nixpkgs.follows = "nixpkgs";26    };27    flake-parts.url = "github:hercules-ci/flake-parts";28    lanzaboote = {29      url = "github:nix-community/lanzaboote/v0.3.0";30      inputs.nixpkgs.follows = "nixpkgs";31    };32  };33  outputs = inputs:34    inputs.flake-parts.lib.mkFlake {inherit inputs;} {35      imports = [inputs.fleet.flakeModules.default];3637      perSystem = {38        inputs',39        pkgs,40        system,41        ...42      }: {43        formatter = pkgs.alejandra;44        devShells.default =45          pkgs.mkShell {packages = [inputs'.fleet.packages.fleet];};46      };4748      # Single flake may contain multiple fleet configurations, default one is called... `default`49      fleetConfigurations.default = {50        # nixpkgs used to build the systems51        nixpkgs.buildUsing = inputs.nixpkgs;5253        # nixos option section of fleet config declares module, which is used for all configured nixos hosts.54        nixos = {55          imports = [inputs.lanzaboote.nixosModules.lanzaboote];5657          # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.58          nix.registry.nixpkgs = {59            from = {60              id = "nixpkgs";61              type = "indirect";62            };63            flake = inputs.nixpkgs;64            exact = false;65          };66        };6768        # Those modules are used to configure all the machines in cluster at the same time, good example of global modules69        # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.70        #71        # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.72        imports = [73          ./wireguard74          # Multi-instancible modules example75          (import ./kubernetes {hosts = ["a" "b"];})76          (import ./kubernetes {hosts = ["c" "d"];})77        ];7879        # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...80        hosts.controlplane-1 = {81          # Every host has some system, for which the system configuration needs to be built82          system = "x86_64-linux";83          nixos = {84            # And nixos modules85            imports = [86              ./controlplane-1/hardware-configuration.nix87              ./controlplane-1/configuration.nix88            ];89            # Configuration may also be specified inline, as in any nixos config.90            services.ray = {91              gpus = 4;92              cpus = 128;93            };94          };95        };96      };97    };98}99----100101== Secret generator example102103TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.104105=== Quickly run securely setup gitlab106107[source,nix]108----109{config, ...}: {110  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {111    gitlab-initial-root = {112      generator = {mkPassword}: mkPassword {};113    } // ownership;114    gitlab-secret = {115      generator = {mkPassword}: mkPassword {};116    } // ownership;117    gitlab-otp = {118      generator = {mkPassword}: mkPassword {};119    } // ownership;120    gitlab-db = {121      generator = {mkPassword}: mkPassword {};122    } // ownership;123    gitlab-jws = {124      generator = {mkRsa}: mkRsa {};125    } // ownership;126  };127  services.gitlab = let secrets = config.secrets; in {128    enable = true;129    initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;130    secrets = {131      secretFile = secrets.gitlab-secret.secretPath;132      otpFile = secrets.gitlab-otp.secretPath;133      dbFile = secrets.gitlab-db.secretPath;134      jwsFile = secrets.gitlab-jws.secretPath;135    };136  };137}138----139140=== Securely initialize kubernetes secrets141142In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.143This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.144I want to build ansible-like script execution in fleet for this kind of tasks.145146[source,nix]147----148{...}: {149  # First I define required secret generators:150  nixpkgs.overlays = [151    (final: prev: let152      lib = final.lib;153    in {154      readKubernetesCa = {impureOn}:155        final.mkImpureSecretGenerator ''156          cd ~/ca157158          cert=kubernetes-intermediateCA.crt159160          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")161          echo -n $expires_at > $out/expires_at162163          cat $cert > $out/public164        ''165        impureOn;166      mkKubernetesCert = {167        subj,168        sans ? [],169        impureOn,170      }:171        final.mkImpureSecretGenerator ''172          cd ~/ca173174          params=$(sudo mktemp)175          csr=$(sudo mktemp)176          cert=$(sudo mktemp)177          sudo openssl ecparam -genkey -name secp384r1 -out $params178          sudo openssl req -new -key $params \179            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \180            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \181            -out $csr182          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert183184          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")185          echo -n $expires_at > $out/expires_at186187          sudo cat $params | encrypt > $out/secret188          sudo cat $cert > $out/public189        ''190        impureOn;191    })192  ];193  # Those secret generators are impure, thus they are run in system environment.194  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.195  # I write a couple of scripts for executing openssl with HSM.196  environment.systemPackages = [197    pkgs.openssl.bin198    (pkgs.writeShellApplication {199      name = "hsms";200      text = ''201        set -eu202        export OPENSSL_CONF=${openssl-conf}203        # Yay, using secrets to generate secrets!204        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})205        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"206      '';207    })208    (pkgs.writeShellApplication {209      name = "hsmt";210      text = ''211        set -eu212        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})213        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"214      '';215    })216  ];217  # And finally, I have secrets, which are shared between machines.218  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.219  sharedSecrets = {220    "ca.pem" = {221      # This is just the public key, no need to regenerate it to change owner list222      regenerateOnOwnerAdded = false;223      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.224      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];225      generator = {readKubernetesCa}:226        readKubernetesCa {227          impureOn = "[CENSORED]";228        };229    };230    "kube-admin.pem" = {231      regenerateOnOwnerAdded = false;232      expectedOwners = ["cluster-admin"];233      generator = {mkKubernetesCert}:234        mkKubernetesCert {235          subj = {236            CN = "admin";237            O = "system:masters";238          };239          impureOn = "[CENSORED]";240        };241    };242    "kube-apiserver.pem" = {243      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.244      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...245      #246      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't247      # want for it to be able to pretend like it is a valid server.248      #249      # However, certificate revokation is complicated in my setup, and I can't show it here.250      regenerateOnOwnerAdded = true;251      expectedOwners = ["controlplane-1" "controlplane-2"];252      generator = {mkKubernetesCert}:253        mkKubernetesCert {254          inherit sans;255          subj.CN = "kubernetes";256          impureOn = "[CENSORED]";257        };258    };259}260----