`trunk`

source

README.adoc9.3 KiBrendered source history
1++++2<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet logo" src="./docs/logo.svg" height="150"></img></a></p>3<p align="center">4  <a href="https://opencollective.com/jrsonnet"><img alt="opencollective" src="https://delta.rocks/badge/backers.svg"></img></a>5  <a href="./LICENSE"><img alt="license" src="https://delta.rocks/badge/License · MIT.svg"></img></a>6</p>7++++89An NixOS cluster deployment tool.1011== Advantages over existing configuration systems (NixOps/Morph)1213- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)14- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.15- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1617== Flake example1819[source,nix]20----21{22  description = "My cluster configuration";23  inputs = {24    nixpkgs.url = "github:nixos/nixpkgs";25    fleet = {26      url = "github:CertainLach/fleet";27      inputs.nixpkgs.follows = "nixpkgs";28    };29    flake-parts.url = "github:hercules-ci/flake-parts";30    lanzaboote = {31      url = "github:nix-community/lanzaboote/v0.3.0";32      inputs.nixpkgs.follows = "nixpkgs";33    };34  };35  outputs = inputs:36    inputs.flake-parts.lib.mkFlake {inherit inputs;} {37      imports = [inputs.fleet.flakeModules.default];3839      perSystem = {40        inputs',41        pkgs,42        system,43        ...44      }: {45        formatter = pkgs.alejandra;46        devShells.default =47          pkgs.mkShell {packages = [inputs'.fleet.packages.fleet];};48      };4950      # Single flake may contain multiple fleet configurations, default one is called... `default`51      fleetConfigurations.default = {52        # nixos option section of fleet config declares module, which is used for all configured nixos hosts.53        nixos = {54          imports = [inputs.lanzaboote.nixosModules.lanzaboote];5556          # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.57          nix.registry.nixpkgs = {58            from = {59              id = "nixpkgs";60              type = "indirect";61            };62            flake = inputs.nixpkgs;63            exact = false;64          };65        };6667        # Those modules are used to configure all the machines in cluster at the same time, good example of global modules68        # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.69        #70        # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.71        imports = [72          ./wireguard73          # Multi-instancible modules example74          (import ./kubernetes {hosts = ["a" "b"];})75          (import ./kubernetes {hosts = ["c" "d"];})76        ];7778        # Hosts attribute (may also be defined/extended using modules attribute) configures hosts...79        hosts.controlplane-1 = {80          # Every host has some system, for which the system configuration needs to be built81          system = "x86_64-linux";82          nixos = {83            # And nixos modules84            imports = [85              ./controlplane-1/hardware-configuration.nix86              ./controlplane-1/configuration.nix87            ];88            # Configuration may also be specified inline, as in any nixos config.89            services.ray = {90              gpus = 4;91              cpus = 128;92            };93          };94        };95      };96    };97}98----99100== Secret generator example101102TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.103104=== Quickly run securely setup gitlab105106[source,nix]107----108{config, ...}: {109  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {110    gitlab-initial-root = {111      generator = {mkPassword}: mkPassword {};112    } // ownership;113    gitlab-secret = {114      generator = {mkPassword}: mkPassword {};115    } // ownership;116    gitlab-otp = {117      generator = {mkPassword}: mkPassword {};118    } // ownership;119    gitlab-db = {120      generator = {mkPassword}: mkPassword {};121    } // ownership;122    gitlab-jws = {123      generator = {mkRsa}: mkRsa {};124    } // ownership;125  };126  services.gitlab = let secrets = config.secrets; in {127    enable = true;128    initialRootPasswordFile = secrets.gitlab-initial-root.secret.path;129    secrets = {130      secretFile = secrets.gitlab-secret.secret.path;131      otpFile = secrets.gitlab-otp.secret.path;132      dbFile = secrets.gitlab-db.secret.path;133      jwsFile = secrets.gitlab-jws.secret.path;134    };135  };136}137----138139=== Securely initialize kubernetes secrets140141In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.142This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.143I want to build ansible-like script execution in fleet for this kind of tasks.144145[source,nix]146----147{...}: {148  # First I define required secret generators:149  nixpkgs.overlays = [150    (final: prev: let151      lib = final.lib;152    in {153      readKubernetesCa = {impureOn}:154        final.mkImpureSecretGenerator ''155          cd ~/ca156157          cert=kubernetes-intermediateCA.crt158159          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")160          echo -n $expires_at > $out/expires_at161162          cat $cert > $out/public163        ''164        impureOn;165      mkKubernetesCert = {166        subj,167        sans ? [],168        impureOn,169      }:170        final.mkImpureSecretGenerator ''171          cd ~/ca172173          params=$(sudo mktemp)174          csr=$(sudo mktemp)175          cert=$(sudo mktemp)176          sudo openssl ecparam -genkey -name secp384r1 -out $params177          sudo openssl req -new -key $params \178            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \179            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \180            -out $csr181          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert182183          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")184          echo -n $expires_at > $out/expires_at185186          sudo cat $params | encrypt > $out/secret187          sudo cat $cert > $out/public188        ''189        impureOn;190    })191  ];192  # Those secret generators are impure, thus they are run in system environment.193  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.194  # I write a couple of scripts for executing openssl with HSM.195  environment.systemPackages = [196    pkgs.openssl.bin197    (pkgs.writeShellApplication {198      name = "hsms";199      text = ''200        set -eu201        export OPENSSL_CONF=${openssl-conf}202        # Yay, using secrets to generate secrets!203        HSM_PIN=$(cat ${config.secrets.hsm-pin.secret.path})204        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"205      '';206    })207    (pkgs.writeShellApplication {208      name = "hsmt";209      text = ''210        set -eu211        HSM_PIN=$(cat ${config.secrets.hsm-pin.secret.path})212        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"213      '';214    })215  ];216  # And finally, I have secrets, which are shared between machines.217  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.218  secrets = {219    "ca.pem" = {220      # This is just the public key, no need to regenerate it to change owner list221      regenerateOnOwnerAdded = false;222      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.223      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];224      generator = {readKubernetesCa}:225        readKubernetesCa {226          impureOn = "[CENSORED]";227        };228    };229    "kube-admin.pem" = {230      regenerateOnOwnerAdded = false;231      expectedOwners = ["cluster-admin"];232      generator = {mkKubernetesCert}:233        mkKubernetesCert {234          subj = {235            CN = "admin";236            O = "system:masters";237          };238          impureOn = "[CENSORED]";239        };240    };241    "kube-apiserver.pem" = {242      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.243      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...244      #245      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't246      # want for it to be able to pretend like it is a valid server.247      #248      # However, certificate revokation is complicated in my setup, and I can't show it here.249      regenerateOnOwnerAdded = true;250      expectedOwners = ["controlplane-1" "controlplane-2"];251      generator = {mkKubernetesCert}:252        mkKubernetesCert {253          inherit sans;254          subj.CN = "kubernetes";255          impureOn = "[CENSORED]";256        };257    };258}259----