`505f82ed3097`

source

README.adoc9.1 KiBrendered source history
1= Fleet23++++4<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet temporary logo generated with midjourney" src="./docs/tmplogo.png" width="200px"></img></a></p>5++++67An NixOS cluster deployment tool.89== Advantages over existing configuration systems (NixOps/Morph)1011- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)12- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.13- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1415== Flake example1617[source,nix]18----19{20  description = "My cluster configuration";21  inputs = {22    nixpkgs.url = "github:nixos/nixpkgs";23    fleet = {24      url = "github:CertainLach/fleet";25      inputs.nixpkgs.follows = "nixpkgs";26    };27    flake-parts.url = "github:hercules-ci/flake-parts";28    lanzaboote = {29      url = "github:nix-community/lanzaboote/v0.3.0";30      inputs.nixpkgs.follows = "nixpkgs";31    };32  };33  outputs = inputs: flake-parts.lib.mkFlake { inherit inputs; } {34    imports = [inputs.fleet.flakeModules.default];3536    perSystem = {pkgs, system, ...}: {37      _module.args.pkgs = import nixpkgs { inherit system; };3839      formatter = pkgs.alejandra;40      devShells.default = pkgs.mkShell {41        packages = [42          inputs.fleet.packages.${system}.fleet43        ];44      };45    };4647    # Single flake may contain multiple fleet configurations, default one is called... `default`48    fleetConfigurations.default = {49      # nixpkgs used to build the systems50      nixpkgs.buildUsing = nixpkgs;51      52      # nixos option section of fleet config declares module, which is used for all configured nixos hosts.53      nixos.imports = [54        lanzaboote.nixosModules.lanzaboote55        {56          # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.57          nix.registry.nixpkgs = {58            from = { id = "nixpkgs"; type = "indirect"; };59            flake = nixpkgs;60            exact = false;61          };62        }63      ];6465      # Those modules are used to configure all the machines in cluster at the same time, good example of global modules66      # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.67      #68      # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.69      imports = [70        ./wireguard71        # Multi-instancible modules example72        (import ./kubernetes {hosts = ["a" "b"];})73        (import ./kubernetes {hosts = ["c" "d"];})74      ];7576      # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...77      hosts.controlplane-1 = {78        # Every host has some system, for which the system configuration needs to be built79        system = "x86_64-linux";80        # And nixos modules81        nixos.imports = [82          ./controlplane-1/hardware-configuration.nix83          ./controlplane-1/configuration.nix84          # Configuration may also be specified inline, as in any nixos config.85          {86            services.ray = {87              gpus = 4;88              cpus = 128;89            };90          }91        ];92      };93    };94  };95}96----9798== Secret generator example99100TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.101102=== Quickly run securely setup gitlab103104[source,nix]105----106{config, ...}: {107  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {108    gitlab-initial-root = {109      generator = {mkPassword}: mkPassword {};110    } // ownership;111    gitlab-secret = {112      generator = {mkPassword}: mkPassword {};113    } // ownership;114    gitlab-otp = {115      generator = {mkPassword}: mkPassword {};116    } // ownership;117    gitlab-db = {118      generator = {mkPassword}: mkPassword {};119    } // ownership;120    gitlab-jws = {121      generator = {mkRsa}: mkRsa {};122    } // ownership;123  };124  services.gitlab = let secrets = config.secrets; in {125    enable = true;126    initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;127    secrets = {128      secretFile = secrets.gitlab-secret.secretPath;129      otpFile = secrets.gitlab-otp.secretPath;130      dbFile = secrets.gitlab-db.secretPath;131      jwsFile = secrets.gitlab-jws.secretPath;132    };133  };134}135----136137=== Securely initialize kubernetes secrets138139In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.140This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.141I want to build ansible-like script execution in fleet for this kind of tasks.142143[source,nix]144----145{...}: {146  # First I define required secret generators:147  nixpkgs.overlays = [148    (final: prev: let149      lib = final.lib;150    in {151      readKubernetesCa = {impureOn}:152        final.mkImpureSecretGenerator ''153          cd ~/ca154155          cert=kubernetes-intermediateCA.crt156157          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")158          echo -n $expires_at > $out/expires_at159160          cat $cert > $out/public161        ''162        impureOn;163      mkKubernetesCert = {164        subj,165        sans ? [],166        impureOn,167      }:168        final.mkImpureSecretGenerator ''169          cd ~/ca170171          params=$(sudo mktemp)172          csr=$(sudo mktemp)173          cert=$(sudo mktemp)174          sudo openssl ecparam -genkey -name secp384r1 -out $params175          sudo openssl req -new -key $params \176            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \177            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \178            -out $csr179          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert180181          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")182          echo -n $expires_at > $out/expires_at183184          sudo cat $params | encrypt > $out/secret185          sudo cat $cert > $out/public186        ''187        impureOn;188    })189  ];190  # Those secret generators are impure, thus they are run in system environment.191  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.192  # I write a couple of scripts for executing openssl with HSM.193  environment.systemPackages = [194    pkgs.openssl.bin195    (pkgs.writeShellApplication {196      name = "hsms";197      text = ''198        set -eu199        export OPENSSL_CONF=${openssl-conf}200        # Yay, using secrets to generate secrets!201        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})202        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"203      '';204    })205    (pkgs.writeShellApplication {206      name = "hsmt";207      text = ''208        set -eu209        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})210        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"211      '';212    })213  ];214  # And finally, I have secrets, which are shared between machines.215  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.216  sharedSecrets = {217    "ca.pem" = {218      # This is just the public key, no need to regenerate it to change owner list219      regenerateOnOwnerAdded = false;220      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.221      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];222      generator = {readKubernetesCa}:223        readKubernetesCa {224          impureOn = "[CENSORED]";225        };226    };227    "kube-admin.pem" = {228      regenerateOnOwnerAdded = false;229      expectedOwners = ["cluster-admin"];230      generator = {mkKubernetesCert}:231        mkKubernetesCert {232          subj = {233            CN = "admin";234            O = "system:masters";235          };236          impureOn = "[CENSORED]";237        };238    };239    "kube-apiserver.pem" = {240      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.241      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...242      #243      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't244      # want for it to be able to pretend like it is a valid server.245      #246      # However, certificate revokation is complicated in my setup, and I can't show it here.247      regenerateOnOwnerAdded = true;248      expectedOwners = ["controlplane-1" "controlplane-2"];249      generator = {mkKubernetesCert}:250        mkKubernetesCert {251          inherit sans;252          subj.CN = "kubernetes";253          impureOn = "[CENSORED]";254        };255    };256}257----