`trunk`

source

README.adoc9.0 KiBrendered source history
1++++2<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet logo" src="./docs/logo.svg" height="150"></img></a></p>3++++45An NixOS cluster deployment tool.67== Advantages over existing configuration systems (NixOps/Morph)89- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)10- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.11- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1213== Flake example1415[source,nix]16----17{18  description = "My cluster configuration";19  inputs = {20    nixpkgs.url = "github:nixos/nixpkgs";21    fleet = {22      url = "github:CertainLach/fleet";23      inputs.nixpkgs.follows = "nixpkgs";24    };25    flake-parts.url = "github:hercules-ci/flake-parts";26    lanzaboote = {27      url = "github:nix-community/lanzaboote/v0.3.0";28      inputs.nixpkgs.follows = "nixpkgs";29    };30  };31  outputs = inputs:32    inputs.flake-parts.lib.mkFlake {inherit inputs;} {33      imports = [inputs.fleet.flakeModules.default];3435      perSystem = {36        inputs',37        pkgs,38        system,39        ...40      }: {41        formatter = pkgs.alejandra;42        devShells.default =43          pkgs.mkShell {packages = [inputs'.fleet.packages.fleet];};44      };4546      # Single flake may contain multiple fleet configurations, default one is called... `default`47      fleetConfigurations.default = {48        # nixos option section of fleet config declares module, which is used for all configured nixos hosts.49        nixos = {50          imports = [inputs.lanzaboote.nixosModules.lanzaboote];5152          # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.53          nix.registry.nixpkgs = {54            from = {55              id = "nixpkgs";56              type = "indirect";57            };58            flake = inputs.nixpkgs;59            exact = false;60          };61        };6263        # Those modules are used to configure all the machines in cluster at the same time, good example of global modules64        # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.65        #66        # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.67        imports = [68          ./wireguard69          # Multi-instancible modules example70          (import ./kubernetes {hosts = ["a" "b"];})71          (import ./kubernetes {hosts = ["c" "d"];})72        ];7374        # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...75        hosts.controlplane-1 = {76          # Every host has some system, for which the system configuration needs to be built77          system = "x86_64-linux";78          nixos = {79            # And nixos modules80            imports = [81              ./controlplane-1/hardware-configuration.nix82              ./controlplane-1/configuration.nix83            ];84            # Configuration may also be specified inline, as in any nixos config.85            services.ray = {86              gpus = 4;87              cpus = 128;88            };89          };90        };91      };92    };93}94----9596== Secret generator example9798TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.99100=== Quickly run securely setup gitlab101102[source,nix]103----104{config, ...}: {105  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {106    gitlab-initial-root = {107      generator = {mkPassword}: mkPassword {};108    } // ownership;109    gitlab-secret = {110      generator = {mkPassword}: mkPassword {};111    } // ownership;112    gitlab-otp = {113      generator = {mkPassword}: mkPassword {};114    } // ownership;115    gitlab-db = {116      generator = {mkPassword}: mkPassword {};117    } // ownership;118    gitlab-jws = {119      generator = {mkRsa}: mkRsa {};120    } // ownership;121  };122  services.gitlab = let secrets = config.secrets; in {123    enable = true;124    initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;125    secrets = {126      secretFile = secrets.gitlab-secret.secretPath;127      otpFile = secrets.gitlab-otp.secretPath;128      dbFile = secrets.gitlab-db.secretPath;129      jwsFile = secrets.gitlab-jws.secretPath;130    };131  };132}133----134135=== Securely initialize kubernetes secrets136137In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.138This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.139I want to build ansible-like script execution in fleet for this kind of tasks.140141[source,nix]142----143{...}: {144  # First I define required secret generators:145  nixpkgs.overlays = [146    (final: prev: let147      lib = final.lib;148    in {149      readKubernetesCa = {impureOn}:150        final.mkImpureSecretGenerator ''151          cd ~/ca152153          cert=kubernetes-intermediateCA.crt154155          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")156          echo -n $expires_at > $out/expires_at157158          cat $cert > $out/public159        ''160        impureOn;161      mkKubernetesCert = {162        subj,163        sans ? [],164        impureOn,165      }:166        final.mkImpureSecretGenerator ''167          cd ~/ca168169          params=$(sudo mktemp)170          csr=$(sudo mktemp)171          cert=$(sudo mktemp)172          sudo openssl ecparam -genkey -name secp384r1 -out $params173          sudo openssl req -new -key $params \174            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \175            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \176            -out $csr177          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert178179          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")180          echo -n $expires_at > $out/expires_at181182          sudo cat $params | encrypt > $out/secret183          sudo cat $cert > $out/public184        ''185        impureOn;186    })187  ];188  # Those secret generators are impure, thus they are run in system environment.189  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.190  # I write a couple of scripts for executing openssl with HSM.191  environment.systemPackages = [192    pkgs.openssl.bin193    (pkgs.writeShellApplication {194      name = "hsms";195      text = ''196        set -eu197        export OPENSSL_CONF=${openssl-conf}198        # Yay, using secrets to generate secrets!199        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})200        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"201      '';202    })203    (pkgs.writeShellApplication {204      name = "hsmt";205      text = ''206        set -eu207        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})208        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"209      '';210    })211  ];212  # And finally, I have secrets, which are shared between machines.213  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.214  secrets = {215    "ca.pem" = {216      # This is just the public key, no need to regenerate it to change owner list217      regenerateOnOwnerAdded = false;218      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.219      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];220      generator = {readKubernetesCa}:221        readKubernetesCa {222          impureOn = "[CENSORED]";223        };224    };225    "kube-admin.pem" = {226      regenerateOnOwnerAdded = false;227      expectedOwners = ["cluster-admin"];228      generator = {mkKubernetesCert}:229        mkKubernetesCert {230          subj = {231            CN = "admin";232            O = "system:masters";233          };234          impureOn = "[CENSORED]";235        };236    };237    "kube-apiserver.pem" = {238      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.239      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...240      #241      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't242      # want for it to be able to pretend like it is a valid server.243      #244      # However, certificate revokation is complicated in my setup, and I can't show it here.245      regenerateOnOwnerAdded = true;246      expectedOwners = ["controlplane-1" "controlplane-2"];247      generator = {mkKubernetesCert}:248        mkKubernetesCert {249          inherit sans;250          subj.CN = "kubernetes";251          impureOn = "[CENSORED]";252        };253    };254}255----