`989a90dd7b97`

source

README.adoc6.4 KiBrendered source history
1= Fleet23++++4<p align="center"><a href="https://github.com/CertainLach/fleet"><img alt="fleet temporary logo generated with midjourney" src="./docs/tmplogo.png" width="200px"></img></a></p>5++++67An NixOS cluster deployment tool.89== Advantages over existing configuration systems (NixOps/Morph)1011- Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)12- Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.13- Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)1415== Secret generator example1617TODO:: This section should into some kind of fleet documentation... But as there is none, it is just left here as-is.1819=== Quickly run securely setup gitlab2021[source,nix]22----23{config, ...}: {24  secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {25    gitlab-initial-root = {26      generator = {mkPassword}: mkPassword {};27    } // ownership;28    gitlab-secret = {29      generator = {mkPassword}: mkPassword {};30    } // ownership;31    gitlab-otp = {32      generator = {mkPassword}: mkPassword {};33    } // ownership;34    gitlab-db = {35      generator = {mkPassword}: mkPassword {};36    } // ownership;37    gitlab-jws = {38      generator = {mkRsa}: mkRsa {};39    } // ownership;40  };41  services.gitlab = let secrets = config.secrets; in {42    enable = true;43    initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;44    secrets = {45      secretFile = secrets.gitlab-secret.secretPath;46      otpFile = secrets.gitlab-otp.secretPath;47      dbFile = secrets.gitlab-db.secretPath;48      jwsFile = secrets.gitlab-jws.secretPath;49    };50  };51}52----5354=== Securely initialize kubernetes secrets5556In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it.57This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there... But this is just the plan.58I want to build ansible-like script execution in fleet for this kind of tasks.5960[source,nix]61----62{...}: {63  # First I define required secret generators:64  nixpkgs.overlays = [65    (final: prev: let66      lib = final.lib;67    in {68      readKubernetesCa = {impureOn}:69        final.mkImpureSecretGenerator ''70          cd ~/ca7172          cert=kubernetes-intermediateCA.crt7374          expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")75          echo -n $expires_at > $out/expires_at7677          cat $cert > $out/public78        ''79        impureOn;80      mkKubernetesCert = {81        subj,82        sans ? [],83        impureOn,84      }:85        final.mkImpureSecretGenerator ''86          cd ~/ca8788          params=$(sudo mktemp)89          csr=$(sudo mktemp)90          cert=$(sudo mktemp)91          sudo openssl ecparam -genkey -name secp384r1 -out $params92          sudo openssl req -new -key $params \93            -subj "${lib.strings.concatStringsSep "" (lib.attrsets.mapAttrsToList (k: v: "/${k}=${v}") subj)}" \94            ${lib.optionalString (sans != []) "-addext \"subjectAltName = ${lib.strings.concatStringsSep "," sans}\""} \95            -out $csr96          sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert9798          expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")99          echo -n $expires_at > $out/expires_at100101          sudo cat $params | encrypt > $out/secret102          sudo cat $cert > $out/public103        ''104        impureOn;105    })106  ];107  # Those secret generators are impure, thus they are run in system environment.108  # Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.109  # I write a couple of scripts for executing openssl with HSM.110  environment.systemPackages = [111    pkgs.openssl.bin112    (pkgs.writeShellApplication {113      name = "hsms";114      text = ''115        set -eu116        export OPENSSL_CONF=${openssl-conf}117        # Yay, using secrets to generate secrets!118        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})119        exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"120      '';121    })122    (pkgs.writeShellApplication {123      name = "hsmt";124      text = ''125        set -eu126        HSM_PIN=$(cat ${config.secrets.hsm-pin.secretPath})127        exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"128      '';129    })130  ];131  # And finally, I have secrets, which are shared between machines.132  # Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.133  sharedSecrets = {134    "ca.pem" = {135      # This is just the public key, no need to regenerate it to change owner list136      regenerateOnOwnerAdded = false;137      # For secret regeneration/reencryption, we need to specify which machines SHOULD have it.138      expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];139      generator = {readKubernetesCa}:140        readKubernetesCa {141          impureOn = "[CENSORED]";142        };143    };144    "kube-admin.pem" = {145      regenerateOnOwnerAdded = false;146      expectedOwners = ["cluster-admin"];147      generator = {mkKubernetesCert}:148        mkKubernetesCert {149          subj = {150            CN = "admin";151            O = "system:masters";152          };153          impureOn = "[CENSORED]";154        };155    };156    "kube-apiserver.pem" = {157      # This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.158      # However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...159      #160      # In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't161      # want for it to be able to pretend like it is a valid server.162      #163      # However, certificate revokation is complicated in my setup, and I can't show it here.164      regenerateOnOwnerAdded = true;165      expectedOwners = ["controlplane-1" "controlplane-2"];166      generator = {mkKubernetesCert}:167        mkKubernetesCert {168          inherit sans;169          subj.CN = "kubernetes";170          impureOn = "[CENSORED]";171        };172    };173}174----