`2a9ff813e781`

source

README.adoc9.4 KiBrendered source history
1= Fleet23++++4<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet temporary logo generated with midjourney" src="./docs/tmplogo.png" width="200px"></img></a></p>5++++67An NixOS cluster deployment tool.89== Advantages over existing configuration systems (NixOps/Morph)1011- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)12- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.13- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1415== Flake example1617[source,nix]18----19{20  description = "My cluster configuration";21  inputs = {22    nixpkgs.url = "github:nixos/nixpkgs";23    fleet = {24      url = "github:CertainLach/fleet";25      inputs.nixpkgs.follows = "nixpkgs";26    };27    lanzaboote = {28      url = "github:nix-community/lanzaboote/v0.3.0";29      inputs.nixpkgs.follows = "nixpkgs";30    };31  };32  outputs = {33    nixpkgs,34    fleet,35    lanzaboote,36    ...37  }: {38    # TODO: This section of documentation needs to use flake-utils.39    formatter.x86_64-linux = let40      pkgs = import nixpkgs {system = "x86_64-linux";};41    in42      pkgs.alejandra;4344    devShell.x86_64-linux = let45      pkgs = import nixpkgs {46        system = "x86_64-linux";47      };48    in49      pkgs.mkShell {50        buildInputs = with pkgs; [51          fleet.packages.x86_64-linux.fleet52        ];53      };5455    # Single flake may contain multiple fleet configurations, default one is called... `default`56    fleetConfigurations.default = fleet.lib.fleetConfiguration {57      # nixpkgs used to build the systems58      inherit nixpkgs;59      # fleet wants to pass some data, like secrets, to do that - fleet writes all the encrypted secrets to fleet.nix60      # treat the contents of this file as implementation detail61      data = import ./fleet.nix;62      63      # nixosModules section of fleet config declares modules, which are used for all configured nixos hosts.64      nixosModules = [65        lanzaboote.nixosModules.lanzaboote66        ({67          config,68          lib,69          ...70        }: {71          # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.72          nix.registry.nixpkgs = {73            from = { id = "nixpkgs"; type = "indirect"; };74            flake = nixpkgs;75            exact = false;76          };77        })78      ];7980      # Those modules are used to configure all the machines in cluster at the same time, good example of global modules81      # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.82      #83      # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.84      fleetModules = [85        ./wireguard86        # Multi-instancible modules example87        (import ./kubernetes {hosts = ["a" "b"];})88        (import ./kubernetes {hosts = ["c" "d"];})89      ];9091      # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...92      hosts.controlplane-1 = {93        # Every host has some system, for which the system configuration needs to be built94        system = "x86_64-linux";95        # And nixos modules96        nixosModules = [97          ./controlplane-1/hardware-configuration.nix98          ./controlplane-1/configuration.nix99          # Configuration may also be specified inline, as in any nixos config.100          ({...}: {101            services.ray = {102              gpus = 4;103              cpus = 128;104            };105          })106        ];107      };108    };109  };110}111----112113== Secret generator example114115TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.116117=== Quickly run securely setup gitlab118119[source,nix]120----121{config, ...}: {122  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {123    gitlab-initial-root = {124      generator = {mkPassword}: mkPassword {};125    } // ownership;126    gitlab-secret = {127      generator = {mkPassword}: mkPassword {};128    } // ownership;129    gitlab-otp = {130      generator = {mkPassword}: mkPassword {};131    } // ownership;132    gitlab-db = {133      generator = {mkPassword}: mkPassword {};134    } // ownership;135    gitlab-jws = {136      generator = {mkRsa}: mkRsa {};137    } // ownership;138  };139  services.gitlab = let secrets = config.secrets; in {140    enable = true;141    initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;142    secrets = {143      secretFile = secrets.gitlab-secret.secretPath;144      otpFile = secrets.gitlab-otp.secretPath;145      dbFile = secrets.gitlab-db.secretPath;146      jwsFile = secrets.gitlab-jws.secretPath;147    };148  };149}150----151152=== Securely initialize kubernetes secrets153154In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.155This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.156I want to build ansible-like script execution in fleet for this kind of tasks.157158[source,nix]159----160{...}: {161  # First I define required secret generators:162  nixpkgs.overlays = [163    (final: prev: let164      lib = final.lib;165    in {166      readKubernetesCa = {impureOn}:167        final.mkImpureSecretGenerator ''168          cd ~/ca169170          cert=kubernetes-intermediateCA.crt171172          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")173          echo -n $expires_at > $out/expires_at174175          cat $cert > $out/public176        ''177        impureOn;178      mkKubernetesCert = {179        subj,180        sans ? [],181        impureOn,182      }:183        final.mkImpureSecretGenerator ''184          cd ~/ca185186          params=$(sudo mktemp)187          csr=$(sudo mktemp)188          cert=$(sudo mktemp)189          sudo openssl ecparam -genkey -name secp384r1 -out $params190          sudo openssl req -new -key $params \191            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \192            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \193            -out $csr194          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert195196          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")197          echo -n $expires_at > $out/expires_at198199          sudo cat $params | encrypt > $out/secret200          sudo cat $cert > $out/public201        ''202        impureOn;203    })204  ];205  # Those secret generators are impure, thus they are run in system environment.206  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.207  # I write a couple of scripts for executing openssl with HSM.208  environment.systemPackages = [209    pkgs.openssl.bin210    (pkgs.writeShellApplication {211      name = "hsms";212      text = ''213        set -eu214        export OPENSSL_CONF=${openssl-conf}215        # Yay, using secrets to generate secrets!216        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})217        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"218      '';219    })220    (pkgs.writeShellApplication {221      name = "hsmt";222      text = ''223        set -eu224        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})225        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"226      '';227    })228  ];229  # And finally, I have secrets, which are shared between machines.230  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.231  sharedSecrets = {232    "ca.pem" = {233      # This is just the public key, no need to regenerate it to change owner list234      regenerateOnOwnerAdded = false;235      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.236      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];237      generator = {readKubernetesCa}:238        readKubernetesCa {239          impureOn = "[CENSORED]";240        };241    };242    "kube-admin.pem" = {243      regenerateOnOwnerAdded = false;244      expectedOwners = ["cluster-admin"];245      generator = {mkKubernetesCert}:246        mkKubernetesCert {247          subj = {248            CN = "admin";249            O = "system:masters";250          };251          impureOn = "[CENSORED]";252        };253    };254    "kube-apiserver.pem" = {255      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.256      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...257      #258      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't259      # want for it to be able to pretend like it is a valid server.260      #261      # However, certificate revokation is complicated in my setup, and I can't show it here.262      regenerateOnOwnerAdded = true;263      expectedOwners = ["controlplane-1" "controlplane-2"];264      generator = {mkKubernetesCert}:265        mkKubernetesCert {266          inherit sans;267          subj.CN = "kubernetes";268          impureOn = "[CENSORED]";269        };270    };271}272----