{
description = "My cluster configuration";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs";
fleet = {
url = "github:CertainLach/fleet";
inputs.nixpkgs.follows = "nixpkgs";
};
lanzaboote = {
url = "github:nix-community/lanzaboote/v0.3.0";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = {
nixpkgs,
fleet,
lanzaboote,
...
}: {
# TODO: This section of documentation needs to use flake-utils.
formatter.x86_64-linux = let
pkgs = import nixpkgs {system = "x86_64-linux";};
in
pkgs.alejandra;
devShell.x86_64-linux = let
pkgs = import nixpkgs {
system = "x86_64-linux";
};
in
pkgs.mkShell {
buildInputs = with pkgs; [
fleet.packages.x86_64-linux.fleet
];
};
# Single flake may contain multiple fleet configurations, default one is called... `default`
fleetConfigurations.default = fleet.lib.fleetConfiguration {
# nixpkgs used to build the systems
inherit nixpkgs;
# fleet wants to pass some data, like secrets, to do that - fleet writes all the encrypted secrets to fleet.nix
# treat the contents of this file as implementation detail
data = import ./fleet.nix;
# nixosModules section of fleet config declares modules, which are used for all configured nixos hosts.
nixosModules = [
lanzaboote.nixosModules.lanzaboote
({
config,
lib,
...
}: {
# Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.
nix.registry.nixpkgs = {
from = { id = "nixpkgs"; type = "indirect"; };
flake = nixpkgs;
exact = false;
};
})
];
# Those modules are used to configure all the machines in cluster at the same time, good example of global modules
# Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.
#
# Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.
fleetModules = [
./wireguard
# Multi-instancible modules example
(import ./kubernetes {hosts = ["a" "b"];})
(import ./kubernetes {hosts = ["c" "d"];})
];
# Hosts attribute (may also be defined/extended using modules attribute) configures hosts...
hosts.controlplane-1 = {
# Every host has some system, for which the system configuration needs to be built
system = "x86_64-linux";
# And nixos modules
nixosModules = [
./controlplane-1/hardware-configuration.nix
./controlplane-1/configuration.nix
# Configuration may also be specified inline, as in any nixos config.
({...}: {
services.ray = {
gpus = 4;
cpus = 128;
};
})
];
};
};
};
}difftreelog
feat lenient nixosModules type
in: trunk
8 files changed
README.adocdiffbeforeafterboth
An NixOS cluster deployment tool.
Advantages over existing configuration systems (NixOps/Morph)
-
Modules can configure multiple hosts at once (I.e for wireguard/kubernetes installation)
-
Secrets can be securely stored in Git (No one except target hosts can decrypt them), automatically regenerated, reencrypted, etc.
-
Automatic rollback on deployment failure, which will work, as long as system is passing initrd stage (So still be carefull with root filesystem mount)
Flake example
Secret generator example
- TODO
-
This section should into some kind of fleet documentation… But as there is none, it is just left here as-is.
Quickly run securely setup gitlab
{config, ...}: {
secrets = let ownership = { owner = "gitlab"; group = "gitlab"; }; in {
gitlab-initial-root = {
generator = {mkPassword}: mkPassword {};
} // ownership;
gitlab-secret = {
generator = {mkPassword}: mkPassword {};
} // ownership;
gitlab-otp = {
generator = {mkPassword}: mkPassword {};
} // ownership;
gitlab-db = {
generator = {mkPassword}: mkPassword {};
} // ownership;
gitlab-jws = {
generator = {mkRsa}: mkRsa {};
} // ownership;
};
services.gitlab = let secrets = config.secrets; in {
enable = true;
initialRootPasswordFile = secrets.gitlab-initial-root.secretPath;
secrets = {
secretFile = secrets.gitlab-secret.secretPath;
otpFile = secrets.gitlab-otp.secretPath;
dbFile = secrets.gitlab-db.secretPath;
jwsFile = secrets.gitlab-jws.secretPath;
};
};
}Securely initialize kubernetes secrets
In my homelab and clusters, I almost always have some sort of HSM, and to issue new kubernetes certs I directly connect to it. This setup should probably split into multiple steps, where I allow target machine to generate CSR, then copy it to the HSM machine, and then sign it there… But this is just the plan. I want to build ansible-like script execution in fleet for this kind of tasks.
{...}: {
# First I define required secret generators:
nixpkgs.overlays = [
(final: prev: let
lib = final.lib;
in {
readKubernetesCa = {impureOn}:
final.mkImpureSecretGenerator ''
cd ~/ca
cert=kubernetes-intermediateCA.crt
expires_at=$(openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")
echo -n $expires_at > $out/expires_at
cat $cert > $out/public
''
impureOn;
mkKubernetesCert = {
subj,
sans ? [],
impureOn,
}:
final.mkImpureSecretGenerator ''
cd ~/ca
params=$(sudo mktemp)
csr=$(sudo mktemp)
cert=$(sudo mktemp)
sudo openssl ecparam -genkey -name secp384r1 -out $params
sudo openssl req -new -key $params \
-subj "${lib.strings.concatStringsSep""(lib.attrsets.mapAttrsToList(kv"/${k}=${v}")subj)}" \
${lib.optionalString(sans!=[])"-addext \"subjectAltName = ${lib.strings.concatStringsSep","sans}\""} \
-out $csr
sudo hsms x509 -req -days 365 -in $csr -CA kubernetes-intermediateCA.crt -CAkey "pkcs11:object=[CENSORED] Kubernetes Intermediate CA;type=private" -CAcreateserial -copy_extensions copy -out $cert
expires_at=$(sudo openssl x509 -in $cert -noout -enddate | cut -d= -f2 | xargs -I{} date -u -d {} +"%Y-%m-%dT%H:%M:%S.%NZ")
echo -n $expires_at > $out/expires_at
sudo cat $params | encrypt > $out/secret
sudo cat $cert > $out/public
''
impureOn;
})
];
# Those secret generators are impure, thus they are run in system environment.
# Probably there needs to be a dedicated user for that kind of tasks, but this is my current setup, don't judge.
# I write a couple of scripts for executing openssl with HSM.
environment.systemPackages = [
pkgs.openssl.bin
(pkgs.writeShellApplication {
name = "hsms";
text = ''
set -eu
export OPENSSL_CONF=${openssl-conf}
# Yay, using secrets to generate secrets!
HSM_PIN=cat ${config.secrets.hsm-pin.secretPath})
exec ${pkgs.openssl}/bin/openssl "$@" -keyform=engine -CAkeyform=engine -engine=pkcs11 -passin=pass:"$HSM_PIN"
'';
})
(pkgs.writeShellApplication {
name = "hsmt";
text = ''
set -eu
HSM_PIN=cat ${config.secrets.hsm-pin.secretPath})
exec ${pkgs.opensc}/bin/pkcs11-tool -l --pin="$HSM_PIN" "$@"
'';
})
];
# And finally, I have secrets, which are shared between machines.
# Note that this example is somewhat wrong, as this goes not into the machine configuration, but to fleet configuration.
sharedSecrets = {
"ca.pem" = {
# This is just the public key, no need to regenerate it to change owner list
regenerateOnOwnerAdded = false;
# For secret regeneration/reencryption, we need to specify which machines SHOULD have it.
expectedOwners = ["controlplane-1" "controlplane-2" "worker-1" "worker-2"];
generator = {readKubernetesCa}:
readKubernetesCa {
impureOn = "[CENSORED]";
};
};
"kube-admin.pem" = {
regenerateOnOwnerAdded = false;
expectedOwners = ["cluster-admin"];
generator = {mkKubernetesCert}:
mkKubernetesCert {
subj = {
CN = "admin";
O = "system:masters";
};
impureOn = "[CENSORED]";
};
};
"kube-apiserver.pem" = {
# This secret depends on machine SANS, so if owner list has been changed, then we need to regenerate it.
# However, SANS dependency is in fact handled by secret seed, and secret is regenerated if the seed is changed...
#
# In this case regeneration is added as a half-assed security measure, as if apiserver is removed, we don't
# want for it to be able to pretend like it is a valid server.
#
# However, certificate revokation is complicated in my setup, and I can't show it here.
regenerateOnOwnerAdded = true;
expectedOwners = ["controlplane-1" "controlplane-2"];
generator = {mkKubernetesCert}:
mkKubernetesCert {
inherit sans;
subj.CN = "kubernetes";
impureOn = "[CENSORED]";
};
};
}flake.nixdiffbeforeafterboth--- a/flake.nix
+++ b/flake.nix
@@ -16,19 +16,18 @@
inputs.nixpkgs.follows = "nixpkgs";
};
};
- outputs = {
+ outputs = inputs @ {
self,
- rust-overlay,
flake-parts,
- nixpkgs,
- nixpkgs-stable-for-tests,
crane,
+ ...
}:
flake-parts.lib.mkFlake {
- # Not passing inputs through inputs for better visibility.
- inputs = {};
+ inherit inputs;
} {
- flake = {
+ flake = let
+ inherit (inputs.nixpkgs.lib) mapAttrs;
+ in {
lib = import ./lib {
fleetPkgsForPkgs = pkgs:
import ./pkgs {
@@ -45,11 +44,11 @@
'';
inventory = output: {
children =
- builtins.mapAttrs (configName: cluster: {
+ mapAttrs (configName: cluster: {
what = "fleet cluster configuration";
children =
- builtins.mapAttrs (hostName: host: {
+ mapAttrs (hostName: host: {
what = "host [${host.system}]";
})
cluster.config.hosts;
@@ -70,19 +69,20 @@
pkgs,
...
}: let
+ inherit (lib) mapAttrs' elem;
# Can also be built for darwin, through it is not usual to deploy nixos systems from macos machines.
# I have no hardware for such testing, thus only adding machines I actually have and use.
#
# It is not possible to deploy any host from armv6/armv7 hardware, and I don't think it even makes sense.
deployerSystems = ["aarch64-linux" "x86_64-linux"];
- deployerSystem = builtins.elem system deployerSystems;
+ deployerSystem = elem system deployerSystems;
lib = pkgs.lib;
rust = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
craneLib = (crane.mkLib pkgs).overrideToolchain rust;
in {
- _module.args.pkgs = import nixpkgs {
+ _module.args.pkgs = import inputs.nixpkgs {
inherit system;
- overlays = [(rust-overlay.overlays.default)];
+ overlays = [(inputs.rust-overlay.overlays.default)];
};
# Reference fleet package should be built with nightly rust, specified in rust-toolchain.toml.
packages = lib.mkIf deployerSystem (let
@@ -116,14 +116,14 @@
checks = let
packages = import ./pkgs {
inherit (pkgs) callPackage;
- craneLib = crane.mkLib (import nixpkgs {inherit system;});
+ craneLib = crane.mkLib pkgs;
};
packages-with-nixpkgs-stable = import ./pkgs {
inherit (pkgs) callPackage;
- craneLib = crane.mkLib (import nixpkgs-stable-for-tests {inherit system;});
+ craneLib = crane.mkLib (import inputs.nixpkgs-stable-for-tests {inherit system;});
};
prefixAttrs = prefix: attrs:
- nixpkgs.lib.attrsets.mapAttrs' (name: value: {
+ mapAttrs' (name: value: {
name = "${prefix}${name}";
value = value.overrideAttrs (prev: {
pname = "${prefix}${prev.pname}";
lib/fleetLib.nixdiffbeforeafterboth--- a/lib/fleetLib.nix
+++ b/lib/fleetLib.nix
@@ -2,8 +2,11 @@
{
nixpkgs,
hostNames,
-}:
-with nixpkgs.lib; rec {
+}: let
+ inherit (nixpkgs) lib;
+ inherit (lib) listToAttrs remove unique crossLists sort elemAt mkOptionType mkOverride optionalString;
+ inherit (lib.types) listOf coercedTo oneOf submodule;
+in rec {
hostsToAttrs = f:
listToAttrs (
map (name: {
@@ -34,6 +37,27 @@
then "${this}-${other}"
else "${other}-${this}";
+ types = rec {
+ anyModule = mkOptionType {
+ name = "submodule";
+ inherit (submodule {}) check;
+ merge = lib.options.mergeOneOption;
+ description = "Nixos module";
+ };
+ listOfAnyModuleStrict =
+ listOf anyModule;
+ listOfAnyModule =
+ coercedTo (oneOf [listOfAnyModuleStrict anyModule]) (
+ v:
+ if builtins.isAttrs v
+ then [v]
+ else if builtins.isFunction v
+ then [v]
+ else v
+ )
+ listOfAnyModuleStrict;
+ };
+
# mkDefault = mkOverride 1000
# For places, where fleet knows better than nixpkgs defaults.
mkFleetDefault = mkOverride 999;
modules/fleet/assertions.nixdiffbeforeafterboth--- a/modules/fleet/assertions.nix
+++ b/modules/fleet/assertions.nix
@@ -1,8 +1,10 @@
-{lib, ...}:
-with lib; {
+{lib, ...}: let
+ inherit (lib) mkOption;
+ inherit (lib.types) listOf unspecified str;
+in {
options = {
assertions = mkOption {
- type = types.listOf types.unspecified;
+ type = listOf unspecified;
internal = true;
default = [];
example = [
@@ -21,7 +23,7 @@
warnings = mkOption {
internal = true;
default = [];
- type = types.listOf types.str;
+ type = listOf str;
example = ["The `foo' service is deprecated and will go away soon!"];
description = ''
This option allows modules to show warnings to users during
modules/fleet/meta.nixdiffbeforeafterboth--- a/modules/fleet/meta.nix
+++ b/modules/fleet/meta.nix
@@ -4,58 +4,53 @@
config,
nixpkgs,
...
-}:
-with lib;
-with fleetLib; let
- hostModule = with types;
- {...} @ hostConfig: let
- hostName = hostConfig.config._module.args.name;
- in {
- options = {
- nixosModules = mkOption {
- type = listOf (mkOptionType {
- name = "submodule";
- inherit (submodule {}) check;
- merge = lib.options.mergeOneOption;
- description = "Nixos module";
- });
- description = "List of nixos modules";
- default = [];
- };
- system = mkOption {
- type = str;
- description = "Type of system";
- };
- encryptionKey = mkOption {
- type = str;
- description = "Encryption key";
- };
- nixosSystem = mkOption {
- type = unspecified;
- description = "Nixos configuration";
- };
- nixpkgs = mkOption {
- type = unspecified;
- description = "Nixpkgs override";
- default = nixpkgs;
- };
+}: let
+ inherit (fleetLib) hostsToAttrs mkFleetGeneratorDefault;
+ inherit (fleetLib.types) listOfAnyModule;
+ inherit (lib) mkOption mkOptionType;
+ inherit (lib.types) str unspecified attrsOf listOf submodule;
+ hostModule = {...} @ hostConfig: let
+ hostName = hostConfig.config._module.args.name;
+ in {
+ options = {
+ nixosModules = mkOption {
+ # Not too strict, but nixos module system will fix everything.
+ type =
+ listOfAnyModule;
+
+ description = "List of nixos modules";
+ default = [];
+ };
+ system = mkOption {
+ type = str;
+ description = "Type of system";
+ };
+ encryptionKey = mkOption {
+ type = str;
+ description = "Encryption key";
+ };
+ nixosSystem = mkOption {
+ type = unspecified;
+ description = "Nixos configuration";
+ };
+ nixpkgs = mkOption {
+ type = unspecified;
+ description = "Nixpkgs override";
+ default = nixpkgs;
};
- config = {
- nixosSystem = hostConfig.config.nixpkgs.lib.nixosSystem {
- inherit (hostConfig.config) system;
- modules = hostConfig.config.nixosModules;
- specialArgs = {
- inherit fleetLib;
- fleet = hostsToAttrs (host: config.hosts.${host}.nixosSystem.config);
- };
+ };
+ config = {
+ nixosSystem = hostConfig.config.nixpkgs.lib.nixosSystem {
+ inherit (hostConfig.config) system;
+ modules = hostConfig.config.nixosModules;
+ specialArgs = {
+ inherit fleetLib;
+ fleet = hostsToAttrs (host: config.hosts.${host}.nixosSystem.config);
};
- nixosModules = [
- ({...}: {
- networking.hostName = mkFleetGeneratorDefault hostName;
- })
- ];
};
+ nixosModules.networking.hostName = mkFleetGeneratorDefault hostName;
};
+ };
overlayType = mkOptionType {
name = "nixpkgs-overlay";
description = "nixpkgs overlay";
@@ -63,19 +58,14 @@
merge = lib.mergeOneOption;
};
in {
- options = with types; {
+ options = {
hosts = mkOption {
type = attrsOf (submodule hostModule);
default = {};
description = "Configurations of individual hosts";
};
nixosModules = mkOption {
- type = listOf (mkOptionType {
- name = "submodule";
- inherit (submodule {}) check;
- merge = lib.options.mergeOneOption;
- description = "Nixos modules";
- });
+ type = listOfAnyModule;
description = "Modules, which should be added to every system";
default = [];
};
@@ -89,9 +79,9 @@
nixosModules =
config.nixosModules
++ [
- ({...}: {
+ {
nixpkgs.overlays = config.overlays;
- })
+ }
];
});
nixosModules = import ../../nixos/modules/module-list.nix;
modules/fleet/secrets.nixdiffbeforeafterboth--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -3,11 +3,13 @@
fleetLib,
config,
...
-}:
-with lib;
-with fleetLib; let
- sharedSecret = with types; ({config, ...}: {
- freeformType = types.lazyAttrsOf unspecified;
+}: let
+ inherit (fleetLib) hostsToAttrs;
+ inherit (lib) mkOption mapAttrsToList mapAttrs filterAttrs concatStringsSep;
+ inherit (lib.types) lazyAttrsOf unspecified nullOr listOf str bool attrsOf submodule;
+
+ sharedSecret = {config, ...}: {
+ freeformType = lazyAttrsOf unspecified;
options = {
expectedOwners = mkOption {
type = nullOr (listOf str);
@@ -66,9 +68,9 @@
default = [];
};
};
- });
- hostSecret = with types; {
- freeformType = types.lazyAttrsOf unspecified;
+ };
+ hostSecret = {
+ freeformType = lazyAttrsOf unspecified;
options = {
createdAt = mkOption {
type = nullOr str;
@@ -81,7 +83,7 @@
};
};
in {
- options = with types; {
+ options = {
version = mkOption {
type = str;
default = "";
@@ -128,11 +130,7 @@
});
# TODO: Should this attribute be moved to `nixpkgs.overlays`?
overlays = [
- (final: prev: let
- lib = final.lib;
- inherit (lib) strings;
- inherit (strings) concatStringsSep;
- in {
+ (final: prev: {
mkSecretGenerators = {recipients}: rec {
# TODO: Merge both generators to one with consistent options syntax?
# Impure generator is built on local machine, then built closure is copied to remote machine,
nixos/meta.nixdiffbeforeafterboth--- a/nixos/meta.nix
+++ b/nixos/meta.nix
@@ -2,11 +2,13 @@
lib,
pkgs,
...
-}:
-with lib; {
- options = with types; {
+}: let
+ inherit (lib) mkOption;
+ inherit (lib.types) listOf str submodule;
+in {
+ options = {
nixpkgs.resolvedPkgs = mkOption {
- type = types.pkgs // {description = "nixpkgs.pkgs";};
+ type = lib.types.pkgs // {description = "nixpkgs.pkgs";};
description = "Value of pkgs";
};
tags = mkOption {
@@ -30,9 +32,6 @@
};
};
description = "Network definition of host";
- };
- buildTarget = mkOption {
- type = enum ["toplevel" "sd-image" "installation-cd"];
};
};
config = {
nixos/secrets.nixdiffbeforeafterboth--- a/nixos/secrets.nix
+++ b/nixos/secrets.nix
@@ -3,16 +3,17 @@
config,
pkgs,
...
-}:
-with lib; let
+}: let
inherit (lib.strings) hasPrefix removePrefix;
+ inherit (lib) mkOption mkOptionDefault mapAttrs stringAfter;
+ inherit (lib.types) submodule str attrsOf nullOr unspecified lazyAttrsOf;
plaintextPrefix = "<PLAINTEXT>";
plaintextNewlinePrefix = "<PLAINTEXT-NL>";
sysConfig = config;
secretPartType = secretName:
- types.submodule ({config, ...}: {
- options = with types; {
+ submodule ({config, ...}: {
+ options = {
raw = mkOption {
description = "Secret in fleet-specific undocumented format, do not use. Import from fleet.nix";
internal = true;
@@ -49,11 +50,11 @@
stablePath = mkOptionDefault "/run/secrets/${secretName}/${partName}";
};
});
- secretType = types.submodule ({config, ...}: let
+ secretType = submodule ({config, ...}: let
secretName = config._module.args.name;
in {
- freeformType = types.lazyAttrsOf (secretPartType secretName);
- options = with types; {
+ freeformType = lazyAttrsOf (secretPartType secretName);
+ options = {
shared = mkOption {
description = "Is this secret owned by this machine, or propagated from shared secrets";
default = false;
@@ -112,7 +113,7 @@
in {
options = {
secrets = mkOption {
- type = types.attrsOf secretType;
+ type = attrsOf secretType;
default = {};
description = "Host-local secrets";
};