git.delta.rocks / jrsonnet / refs/commits / cc4ecd613a27

difftreelog

refactor declare configuration using flake parts

Yaroslav Bolyukin2024-08-14parent: #f779c26.patch.diff
in: trunk

29 files changed

modifiedCargo.lockdiffbeforeafterboth
before · Cargo.lock
322 packageslockfile v3
after · Cargo.lock
322 packageslockfile v3
modifiedREADME.adocdiffbeforeafterboth
--- a/README.adoc
+++ b/README.adoc
@@ -24,44 +24,33 @@
       url = "github:CertainLach/fleet";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    flake-parts.url = "github:hercules-ci/flake-parts";
     lanzaboote = {
       url = "github:nix-community/lanzaboote/v0.3.0";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };
-  outputs = {
-    nixpkgs,
-    fleet,
-    lanzaboote,
-    ...
-  }: {
-    # TODO: This section of documentation needs to use flake-utils.
-    formatter.x86_64-linux = let
-      pkgs = import nixpkgs {system = "x86_64-linux";};
-    in
-      pkgs.alejandra;
+  outputs = inputs: flake-parts.lib.mkFlake { inherit inputs; } {
+    imports = [inputs.fleet.flakeModules.default];
 
-    devShell.x86_64-linux = let
-      pkgs = import nixpkgs {
-        system = "x86_64-linux";
-      };
-    in
-      pkgs.mkShell {
-        buildInputs = with pkgs; [
-          fleet.packages.x86_64-linux.fleet
+    perSystem = {pkgs, system, ...}: {
+      _module.args.pkgs = import nixpkgs { inherit system; };
+
+      formatter = pkgs.alejandra;
+      devShells.default = pkgs.mkShell {
+        packages = [
+          inputs.fleet.packages.${system}.fleet
         ];
       };
+    };
 
     # Single flake may contain multiple fleet configurations, default one is called... `default`
-    fleetConfigurations.default = fleet.lib.fleetConfiguration {
+    fleetConfigurations.default = {
       # nixpkgs used to build the systems
-      inherit nixpkgs;
-      # fleet wants to pass some data, like secrets, to do that - fleet writes all the encrypted secrets to fleet.nix
-      # treat the contents of this file as implementation detail
-      data = import ./fleet.nix;
+      nixpkgs.buildUsing = nixpkgs;
       
-      # nixosModules section of fleet config declares modules, which are used for all configured nixos hosts.
-      nixosModules = [
+      # nixos option section of fleet config declares module, which is used for all configured nixos hosts.
+      nixos.imports = [
         lanzaboote.nixosModules.lanzaboote
         {
           # Make `nix shell nixpkgs#thing` use the same nixpkgs, as used to build the system.
@@ -77,7 +66,7 @@
       # Is I.e wiring up the mesh VPN, or deploying kubernetes, or other things.
       #
       # Modules use the same semantics as standard nixos module system, they are just configuring all the hosts at once.
-      fleetModules = [
+      imports = [
         ./wireguard
         # Multi-instancible modules example
         (import ./kubernetes {hosts = ["a" "b"];})
@@ -89,7 +78,7 @@
         # Every host has some system, for which the system configuration needs to be built
         system = "x86_64-linux";
         # And nixos modules
-        nixosModules = [
+        nixos.imports = [
           ./controlplane-1/hardware-configuration.nix
           ./controlplane-1/configuration.nix
           # Configuration may also be specified inline, as in any nixos config.
modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -254,13 +254,8 @@
 	let host = config.host(&host).await?;
 	// let action = Action::from(self.subcommand.clone());
 	let fleet_config = &config.config_field;
-	let drv = nix_go!(
-		fleet_config.hosts[{ &host.name }]
-			.nixosSystem
-			.config
-			.system
-			.build[{ build_attr }]
-	);
+	let nixos = host.nixos_config().await?;
+	let drv = nix_go!(nixos.system.build[{ build_attr }]);
 	let outputs = drv.build().await.inspect_err(|_| {
 			if build_attr == "sdImage" {
 				info!("sd-image build failed");
@@ -335,6 +330,7 @@
 			let config = config.clone();
 			let span = info_span!("deploy", host = field::display(&host.name));
 			let hostname = host.name.clone();
+			let local_host = config.local_host();
 			// FIXME: Fix repl concurrency (see build-systems)
 			set.spawn_local(
 				(async move {
@@ -354,7 +350,10 @@
 							// at least for the first deployment, to provide trusted store key.
 							//
 							// It is much slower, yet doesn't require root on the deployer machine.
-							let mut sign = MyCommand::new("nix");
+							let Ok(mut sign) = local_host.cmd("nix").await else {
+								error!("failed to setup local");
+								return;
+							};
 							// Private key for host machine is registered in nix-sign.nix
 							sign.arg("store")
 								.arg("sign")
@@ -362,7 +361,7 @@
 								.arg("-r")
 								.arg(&built);
 							if let Err(e) = sign.sudo().run_nix().await {
-								warn!("Failed to sign store paths: {e}");
+								warn!("failed to sign store paths: {e}");
 							};
 						}
 						let mut tries = 0;
modifiedcmds/fleet/src/cmds/info.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/info.rs
+++ b/cmds/fleet/src/cmds/info.rs
@@ -38,9 +38,9 @@
 			InfoCmd::ListHosts { ref tagged } => {
 				'host: for host in config.list_hosts().await? {
 					if !tagged.is_empty() {
-						let config = &config.config_unchecked_field;
+						let config = &config.config_field;
 						let tags: Vec<String> =
-							nix_go_json!(config.hosts[{ host.name }].nixosSystem.config.tags);
+							nix_go_json!(config.hosts[{ host.name }].tags);
 						for tag in tagged {
 							if !tags.contains(tag) {
 								continue 'host;
modifiedcmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -598,7 +598,7 @@
 					return Ok(());
 				}
 
-				let config_field = &config.config_unchecked_field;
+				let config_field = &config.config_field;
 				let field = nix_go!(config_field.sharedSecrets[{ name }]);
 
 				let updated = update_owner_set(
@@ -623,7 +623,7 @@
 						.collect::<HashSet<_>>();
 					let shared_set = config.list_shared().into_iter().collect::<HashSet<_>>();
 					for missing in expected_shared_set.difference(&shared_set) {
-						let config_field = &config.config_unchecked_field;
+						let config_field = &config.config_field;
 						let secret = nix_go!(config_field.sharedSecrets[{ missing }]);
 						let expected_owners: Option<Vec<String>> =
 							nix_go_json!(secret.expectedOwners);
@@ -675,7 +675,7 @@
 				for name in &config.list_shared() {
 					info!("updating secret: {name}");
 					let data = config.shared_secret(name)?;
-					let config_field = &config.config_unchecked_field;
+					let config_field = &config.config_field;
 					let expected_owners: Vec<String> =
 						nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners);
 					if expected_owners.is_empty() {
modifiedcmds/fleet/src/command.rsdiffbeforeafterboth
--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -9,6 +9,8 @@
 use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};
 use tracing::debug;
 
+use crate::host::EscalationStrategy;
+
 fn escape_bash(input: &str, out: &mut String) {
 	const TO_ESCAPE: &str = "$ !\"#&'()*,;<>?[\\]^`{|}";
 	if input.chars().all(|c| !TO_ESCAPE.contains(c)) {
@@ -27,32 +29,51 @@
 fn ostoutf8(os: impl AsRef<OsStr>) -> String {
 	os.as_ref().to_str().expect("non-utf8 data").to_owned()
 }
-#[derive(Clone)]
+
+#[derive(Clone, Debug)]
 pub struct MyCommand {
 	command: String,
 	args: Vec<String>,
 	env: Vec<(String, String)>,
 	ssh_session: Option<Arc<Session>>,
+	escalation: EscalationStrategy,
+	escalate: bool,
 }
 impl MyCommand {
-	pub fn new_on(cmd: impl AsRef<OsStr>, session: Arc<Session>) -> Self {
+	pub fn new_on(
+		escalation: EscalationStrategy,
+		cmd: impl AsRef<OsStr>,
+		session: Arc<Session>,
+	) -> Self {
 		assert!(!cmd.as_ref().is_empty());
 		Self {
 			command: ostoutf8(cmd),
 			args: vec![],
 			env: vec![],
 			ssh_session: Some(session),
+			escalation,
+			escalate: false,
 		}
 	}
-	pub fn new(cmd: impl AsRef<OsStr>) -> Self {
+	pub fn new(escalation: EscalationStrategy, cmd: impl AsRef<OsStr>) -> Self {
 		assert!(!cmd.as_ref().is_empty());
 		Self {
 			command: ostoutf8(cmd),
 			args: vec![],
 			env: vec![],
 			ssh_session: None,
+			escalation,
+			escalate: false,
 		}
 	}
+	fn new_here(&self, cmd: impl AsRef<OsStr>) -> Self {
+		if let Some(ssh_session) = self.ssh_session.clone() {
+			Self::new_on(self.escalation, cmd, ssh_session)
+		} else {
+			Self::new(self.escalation, cmd)
+		}
+	}
+
 	fn into_args(self) -> Vec<String> {
 		let mut out = Vec::new();
 		if !self.env.is_empty() {
@@ -76,8 +97,7 @@
 		if self.env.is_empty() {
 			return self;
 		}
-		let mut out = Self::new("env");
-		out.ssh_session = self.ssh_session;
+		let mut out = self.new_here("env");
 		for (k, v) in self.env {
 			assert!(!k.contains('='));
 			out.arg(format!("{k}={v}"));
@@ -160,26 +180,46 @@
 		self
 	}
 	pub fn sudo(mut self) -> Self {
-		// TODO: Multiple escalation strategies.
-		// Maybe escalation should be moved to ConfigHost, to also support cases
-		// when there is no sudo on remote machine, but instead we can reconnect
-		// as root using ssh?
-		if std::env::var_os("NO_SUDO").is_some() {
-			let mut out = Self::new("su");
-			out.ssh_session = self.ssh_session.take();
-			out.arg("-c").arg(self.into_string());
-			out
-		} else {
-			let mut out = Self::new("sudo");
-			out.ssh_session = self.ssh_session.take();
-			out.args(self.into_args());
-			out
+		self.escalate = true;
+		self
+	}
+	fn wrap_sudo_if_needed(self) -> Self {
+		if !self.escalate {
+			return self;
+		}
+		match self.escalation {
+			EscalationStrategy::Su => {
+				let mut out = self.new_here("su");
+				out.arg("-c").arg(self.into_string());
+				out
+			}
+			EscalationStrategy::Sudo => {
+				let mut out = self.new_here("sudo");
+				out.args(self.into_args());
+				out
+			}
+			EscalationStrategy::Run0 => {
+				// run0 wants interactive authentication by default.
+				let mut run0 = self.new_here("run0");
+				let mut out = self.new_here("script");
+
+				// Red backgrounds messes with fleet formatting
+				run0.arg("--background=");
+				run0.args(self.into_args());
+
+				out.arg("-q");
+				out.arg("/dev/null");
+				out.arg("-c");
+				out.arg(run0.into_string());
+				dbg!(&out);
+				out
+			}
 		}
 	}
 
 	pub async fn run(self) -> Result<()> {
 		let str = self.clone().into_string();
-		let cmd = self.into_command_new()?;
+		let cmd = self.wrap_sudo_if_needed().into_command_new()?;
 		match cmd {
 			Either::Left(cmd) => run_nix_inner(str, cmd, &mut PlainHandler).await?,
 			Either::Right(cmd) => run_nix_inner_ssh(str, cmd, &mut PlainHandler).await?,
@@ -192,7 +232,7 @@
 	}
 	pub async fn run_bytes(self) -> Result<Vec<u8>> {
 		let str = self.clone().into_string();
-		let cmd = self.into_command_new()?;
+		let cmd = self.wrap_sudo_if_needed().into_command_new()?;
 		let v = match cmd {
 			Either::Left(cmd) => run_nix_inner_stdout(str, cmd, &mut PlainHandler).await?,
 			Either::Right(cmd) => run_nix_inner_stdout_ssh(str, cmd, &mut PlainHandler).await?,
@@ -200,17 +240,17 @@
 		Ok(v)
 	}
 
-	pub async fn run_nix_string(self) -> Result<String> {
+	pub async fn run_nix_string(mut self) -> Result<String> {
 		let str = self.clone().into_string();
-		let mut cmd = self.into_command();
-		cmd.arg("--log-format").arg("internal-json");
+		self.arg("--log-format").arg("internal-json");
+		let mut cmd = self.wrap_sudo_if_needed().into_command();
 		let bytes = run_nix_inner_stdout(str, cmd, &mut NixHandler::default()).await?;
 		Ok(String::from_utf8(bytes)?)
 	}
-	pub async fn run_nix(self) -> Result<()> {
+	pub async fn run_nix(mut self) -> Result<()> {
 		let str = self.clone().into_string();
-		let mut cmd = self.into_command();
-		cmd.arg("--log-format").arg("internal-json");
+		self.arg("--log-format").arg("internal-json");
+		let mut cmd = self.wrap_sudo_if_needed().into_command();
 		cmd.stdout(Stdio::inherit());
 		run_nix_inner(str, cmd, &mut NixHandler::default()).await
 	}
modifiedcmds/fleet/src/host.rsdiffbeforeafterboth
--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -1,5 +1,5 @@
 use std::{
-	cell::OnceCell,
+	cell::{LazyCell, OnceCell},
 	collections::BTreeMap,
 	env::current_dir,
 	ffi::{OsStr, OsString},
@@ -14,7 +14,7 @@
 use anyhow::{anyhow, bail, ensure, Context, Result};
 use clap::Parser;
 use fleet_shared::SecretData;
-use nix_eval::{nix_go, nix_go_json, NixSessionPool, Value};
+use nix_eval::{nix_go, nix_go_json, util::assert_warn, NixSessionPool, Value};
 use nom::{
 	bytes::complete::take_while1,
 	character::complete::char,
@@ -25,6 +25,7 @@
 use openssh::SessionBuilder;
 use serde::de::DeserializeOwned;
 use tempfile::NamedTempFile;
+use tracing::error;
 
 use crate::{
 	command::MyCommand,
@@ -39,8 +40,6 @@
 	pub nix_args: Vec<OsString>,
 	/// fleet_config.config
 	pub config_field: Value,
-	/// fleet_config.unchecked.config
-	pub config_unchecked_field: Value,
 
 	/// import nixpkgs {system = local};
 	pub default_pkgs: Value,
@@ -57,6 +56,13 @@
 	}
 }
 
+#[derive(Clone, Copy, Debug)]
+pub enum EscalationStrategy {
+	Sudo,
+	Run0,
+	Su,
+}
+
 pub struct ConfigHost {
 	config: Config,
 	pub name: String,
@@ -64,32 +70,57 @@
 	pub session: OnceLock<Arc<openssh::Session>>,
 	groups: OnceCell<Vec<String>>,
 
-	pub nixos_config: Option<Value>,
+	pub host_config: Option<Value>,
+	pub nixos_config: OnceCell<Value>,
 }
 impl ConfigHost {
+	pub async fn escalation_strategy(&self) -> Result<EscalationStrategy> {
+		// Prefer sudo, as run0 has some gotchas with polkit
+		// and too many repeating prompts.
+		if let Ok(_) = self.find_in_path("sudo").await {
+			return Ok(EscalationStrategy::Sudo);
+		}
+		if let Ok(_) = self.find_in_path("run0").await {
+			return Ok(EscalationStrategy::Run0);
+		}
+		Ok(EscalationStrategy::Su)
+	}
+	// TOCTOU is possible here in case if config is changed, but this case is not handled anywhere anyway,
+	// assuming getting tags always returns the same value.
 	pub async fn tags(&self) -> Result<Vec<String>> {
 		if let Some(v) = self.groups.get() {
 			return Ok(v.clone());
 		}
-		// TOCTOU is possible here in case if config is changed, but this case is not handled anywhere anyway,
-		// assuming getting tags always returns the same value.
-		let Some(nixos_config) = &self.nixos_config else {
+		let Some(host_config) = &self.host_config else {
 			return Ok(vec![]);
 		};
-		let tags: Vec<String> = nix_go_json!(nixos_config.tags);
+		let tags: Vec<String> = nix_go_json!(host_config.tags);
 
 		let _ = self.groups.set(tags.clone());
 
 		Ok(tags)
 	}
+	pub async fn nixos_config(&self) -> Result<Value> {
+		if let Some(v) = self.nixos_config.get() {
+			return Ok(v.clone());
+		}
+		let Some(host_config) = &self.host_config else {
+			bail!("local host has no nixos_config");
+		};
+		let nixos_config = nix_go!(host_config.nixos.config);
+		assert_warn("nixos config evaluation", &nixos_config).await?;
+
+		let _ = self.nixos_config.set(nixos_config.clone());
+
+		Ok(nixos_config)
+	}
 	async fn open_session(&self) -> Result<Arc<openssh::Session>> {
 		assert!(!self.local, "do not open ssh connection to local session");
 		// FIXME: TOCTOU
 		if let Some(session) = &self.session.get() {
 			return Ok((*session).clone());
 		};
-		let session = SessionBuilder::default();
-
+		let mut session = SessionBuilder::default();
 		let session = session
 			.connect(&self.name)
 			.await
@@ -129,6 +160,34 @@
 		let text = self.read_file_text(path).await?;
 		Ok(serde_json::from_str(&text)?)
 	}
+	pub async fn read_env(&self, env: &str) -> Result<String> {
+		let mut cmd = self.cmd("printenv").await?;
+		cmd.arg(env);
+		Ok(cmd.run_string().await?)
+	}
+	pub async fn find_in_path(&self, command: &str) -> Result<String> {
+		// // `which` is not a part of coreutils, and it might not exist on machine.
+		// let path = self.read_env("PATH").await?;
+		// // Assuming delimiter is :, we don't work with windows host, this check will be much
+		// // more sophisticated in remowt backend (and quicker, since actual PATH search will be done on remote machine)
+		// for ele in path.split(':') {
+		// 	let test_path = format!("{ele}/{cmd}");
+		// 	test -x etc
+		// }
+		// let mut cmd = self.cmd("printenv").await?;
+		// cmd.arg(env);
+		// Ok(cmd.run_string().await?)
+		// Assuming this is an environment issue if which doesn't exist, will be fixed with remowt.
+		let mut cmd = self
+			.cmd_escalation(
+				// Not used
+				EscalationStrategy::Su,
+				"which",
+			)
+			.await?;
+		cmd.arg(command);
+		cmd.run_string().await
+	}
 	pub async fn read_file_value<D: FromStr>(&self, path: impl AsRef<OsStr>) -> Result<D>
 	where
 		<D as FromStr>::Err: Display,
@@ -137,11 +196,19 @@
 		D::from_str(&text).map_err(|e| anyhow!("failed to parse value: {e}"))
 	}
 	pub async fn cmd(&self, cmd: impl AsRef<OsStr>) -> Result<MyCommand> {
+		self.cmd_escalation(self.escalation_strategy().await?, cmd)
+			.await
+	}
+	pub async fn cmd_escalation(
+		&self,
+		escalation: EscalationStrategy,
+		cmd: impl AsRef<OsStr>,
+	) -> Result<MyCommand> {
 		if self.local {
-			Ok(MyCommand::new(cmd))
+			Ok(MyCommand::new(escalation, cmd))
 		} else {
 			let session = self.open_session().await?;
-			Ok(MyCommand::new_on(cmd, session))
+			Ok(MyCommand::new_on(escalation, cmd, session))
 		}
 	}
 
@@ -181,7 +248,11 @@
 			// Path is located locally, thus already trusted.
 			return Ok(path.to_owned());
 		}
-		let mut nix = MyCommand::new("nix");
+		let mut nix = MyCommand::new(
+			// Not used
+			EscalationStrategy::Su,
+			"nix",
+		);
 		nix.arg("copy")
 			.arg("--substitute-on-destination")
 			.comparg("--to", format!("ssh-ng://{}", self.name))
@@ -210,9 +281,7 @@
 	}
 
 	pub async fn list_configured_secrets(&self) -> Result<Vec<String>> {
-		let Some(nixos) = &self.nixos_config else {
-			return Ok(vec![]);
-		};
+		let nixos = self.nixos_config().await?;
 		let secrets = nix_go!(nixos.secrets);
 		let mut out = Vec::new();
 		for name in secrets.list_fields().await? {
@@ -226,18 +295,14 @@
 		Ok(out)
 	}
 	pub async fn secret_field(&self, name: &str) -> Result<Value> {
-		let Some(nixos) = &self.nixos_config else {
-			bail!("host is virtual and has no secrets");
-		};
+		let nixos = self.nixos_config().await?;
 		Ok(nix_go!(nixos.secrets[{ name }]))
 	}
 
 	/// Packages for this host, resolved with nixpkgs overlays
 	pub async fn pkgs(&self) -> Result<Value> {
-		let Some(nixos) = &self.nixos_config else {
-			return Ok(self.config.default_pkgs.clone());
-		};
-		Ok(nix_go!(nixos.nixpkgs.resolvedPkgs))
+		let nixos = self.nixos_config().await?;
+		Ok(nix_go!(nixos._resolvedPkgs))
 	}
 }
 
@@ -317,7 +382,8 @@
 			name: "<virtual localhost>".to_owned(),
 			local: true,
 			session: OnceLock::new(),
-			nixos_config: None,
+			host_config: None,
+			nixos_config: OnceCell::new(),
 			groups: {
 				let cell = OnceCell::new();
 				let _ = cell.set(vec![]);
@@ -327,19 +393,22 @@
 	}
 
 	pub async fn host(&self, name: &str) -> Result<ConfigHost> {
-		let config = &self.config_unchecked_field;
-		let nixos_config = nix_go!(config.hosts[{ name }].nixosSystem.config);
+		let config = &self.config_field;
+		let host_config = nix_go!(config.hosts[{ name }]);
+
+
 		Ok(ConfigHost {
 			config: self.clone(),
 			name: name.to_owned(),
 			local: self.is_local(name),
 			session: OnceLock::new(),
-			nixos_config: Some(nixos_config),
+			host_config: Some(host_config),
+			nixos_config: OnceCell::new(),
 			groups: OnceCell::new(),
 		})
 	}
 	pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
-		let config = &self.config_unchecked_field;
+		let config = &self.config_field;
 		let names = nix_go!(config.hosts).list_fields().await?;
 		let mut out = vec![];
 		for name in names {
@@ -348,8 +417,8 @@
 		Ok(out)
 	}
 	pub async fn system_config(&self, host: &str) -> Result<Value> {
-		let fleet_field = &self.config_unchecked_field;
-		Ok(nix_go!(fleet_field.hosts[{ host }].nixosSystem.config))
+		let fleet_field = &self.config_field;
+		Ok(nix_go!(fleet_field.hosts[{ host }].nixos.config))
 	}
 
 	pub(super) fn data(&self) -> MutexGuard<FleetData> {
@@ -360,7 +429,7 @@
 	}
 	/// Shared secrets configured in fleet.nix or in flake
 	pub async fn list_configured_shared(&self) -> Result<Vec<String>> {
-		let config_field = &self.config_unchecked_field;
+		let config_field = &self.config_field;
 		Ok(nix_go!(config_field.sharedSecrets).list_fields().await?)
 	}
 	/// Shared secrets configured in fleet.nix
@@ -420,7 +489,7 @@
 		Ok(secret.clone())
 	}
 	pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
-		let config_field = &self.config_unchecked_field;
+		let config_field = &self.config_field;
 		Ok(nix_go_json!(
 			config_field.sharedSecrets[{ secret }].expectedOwners
 		))
@@ -525,26 +594,27 @@
 		}
 		let local_system = self.local_system.clone();
 
+		let mut fleet_data_path = directory.clone();
+		fleet_data_path.push("fleet.nix");
+		let bytes = std::fs::read_to_string(fleet_data_path)?;
+		let data: Mutex<FleetData> = nixlike::parse_str(&bytes)?;
+
 		let fleet_root = Value::binding(root_field, "fleetConfigurations").await?;
-		let fleet_field = nix_go!(fleet_root.default);
+		let fleet_field = nix_go!(fleet_root.default({ data }));
 
 		let config_field = nix_go!(fleet_field.config);
-		let config_unchecked_field = nix_go!(fleet_field.unchecked.config);
 
+		assert_warn("fleet config evaluation", &config_field).await?;
+
 		let import = nix_go!(builtins_field.import);
-		let overlays = nix_go!(config_unchecked_field.overlays);
-		let nixpkgs = nix_go!(fleet_field.nixpkgs | import);
+		let overlays = nix_go!(config_field.nixpkgs.overlays);
+		let nixpkgs = nix_go!(fleet_field.nixpkgs.buildUsing | import);
 
 		let default_pkgs = nix_go!(nixpkgs(Obj {
 			overlays,
 			system: { self.local_system.clone() },
 		}));
 
-		let mut fleet_data_path = directory.clone();
-		fleet_data_path.push("fleet.nix");
-		let bytes = std::fs::read_to_string(fleet_data_path)?;
-		let data = nixlike::parse_str(&bytes)?;
-
 		Ok(Config(Arc::new(FleetConfigInternals {
 			opts: self,
 			directory,
@@ -552,7 +622,6 @@
 			local_system,
 			nix_args,
 			config_field,
-			config_unchecked_field,
 			default_pkgs,
 		})))
 	}
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -58,7 +58,7 @@
 				path.push("file://");
 				path.push(entry.path());
 
-				let mut status = MyCommand::new("nix");
+				let mut status = config.local_host().cmd("nix").await?;
 				status.args(&config.nix_args);
 				status.arg("store").arg("prefetch-file").arg(path);
 				status.run_nix_string().instrument(span).await?;
modifiedcrates/nix-eval/Cargo.tomldiffbeforeafterboth
--- a/crates/nix-eval/Cargo.toml
+++ b/crates/nix-eval/Cargo.toml
@@ -5,6 +5,7 @@
 build = "build.rs"
 
 [dependencies]
+anyhow.workspace = true
 better-command.workspace = true
 futures = "0.3.30"
 itertools = "0.13.0"
modifiedcrates/nix-eval/src/lib.rsdiffbeforeafterboth
--- a/crates/nix-eval/src/lib.rs
+++ b/crates/nix-eval/src/lib.rs
@@ -17,6 +17,7 @@
 // Contains macros helpers
 #[doc(hidden)]
 pub mod macros;
+pub mod util;
 // #[allow(non_upper_case_globals, non_camel_case_types, non_snake_case)]
 // mod nix_raw {
 // 	include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
addedcrates/nix-eval/src/util.rsdiffbeforeafterboth
--- /dev/null
+++ b/crates/nix-eval/src/util.rs
@@ -0,0 +1,30 @@
+use anyhow::bail;
+use tracing::{debug, warn};
+use std::time::Instant;
+
+use crate::{nix_go_json, Value};
+
+pub async fn assert_warn(action: &str, val: &Value) -> anyhow::Result<()> {
+	let before_errors = Instant::now();
+	let errors: Vec<String> = nix_go_json!(val.errors);
+	debug!("errors evaluation took {:?}", before_errors.elapsed());
+	if !errors.is_empty() {
+		bail!(
+			"{action} failed with error{}{}",
+			(errors.len() != 1).then_some("s:\n- ").unwrap_or(": "),
+			errors.join("\n- "),
+		);
+	}
+
+	let before_errors = Instant::now();
+	let warnings: Vec<String> = nix_go_json!(val.warnings);
+	debug!("warnings evaluation took {:?}", before_errors.elapsed());
+	if !warnings.is_empty() {
+		warn!(
+			"{action} completed with warning{}{}",
+			(warnings.len() != 1).then_some("s:\n- ").unwrap_or(": "),
+			warnings.join("\n- "),
+		);
+	}
+	Ok(())
+}
modifiedcrates/nix-eval/src/value.rsdiffbeforeafterboth
--- a/crates/nix-eval/src/value.rs
+++ b/crates/nix-eval/src/value.rs
@@ -44,14 +44,14 @@
 				let v = nixlike::format_identifier(k.as_str());
 				write!(f, ".{v}")
 			}
-			Index::Apply(o) => {
-				write!(f, "<apply>({o})")
+			Index::Apply(_) => {
+				write!(f, "<apply>(...)")
 			}
 			Index::Expr(e) => {
 				write!(f, "[{}]", e.out)
 			}
-			Index::ExprApply(e) => {
-				write!(f, "<apply>({})", e.out)
+			Index::ExprApply(_) => {
+				write!(f, "<apply>(...)")
 			}
 			Index::Pipe(e) => {
 				write!(f, "<map>({})", e.out)
modifiedflake.nixdiffbeforeafterboth
--- a/flake.nix
+++ b/flake.nix
@@ -25,18 +25,23 @@
     flake-parts.lib.mkFlake {
       inherit inputs;
     } {
-      flake = let
-        inherit (inputs.nixpkgs.lib) mapAttrs;
-      in {
-        lib = import ./lib {
-          fleetPkgsForPkgs = pkgs:
-            import ./pkgs {
-              inherit (pkgs) callPackage;
-              craneLib = crane.mkLib pkgs;
-            };
-        };
+      flake = rec {
+        lib =
+          (import ./lib {
+            inherit (inputs.nixpkgs) lib;
+          })
+          // {
+            fleetConfiguration = throw "function-based interface is deprecated, use flake-parts syntax instead";
+          };
+        flakeModules.default = (import ./lib/flakePart.nix {
+          inherit crane;
+        });
+        flakeModule = flakeModules.default;
+
         # To be used with https://github.com/NixOS/nix/pull/8892
-        schemas = {
+        schemas = let
+          inherit (inputs.nixpkgs.lib) mapAttrs;
+        in {
           fleetConfigurations = {
             version = 1;
             doc = ''
@@ -69,7 +74,8 @@
         pkgs,
         ...
       }: let
-        inherit (lib) mapAttrs' elem;
+        inherit (lib.attrSets) mapAttrs';
+        inherit (lib.lists) elem;
         # Can also be built for darwin, through it is not usual to deploy nixos systems from macos machines.
         # I have no hardware for such testing, thus only adding machines I actually have and use.
         #
@@ -108,6 +114,7 @@
               pkg-config
               openssl
               bacon
+              nil
             ];
           };
         };
modifiedlib/default.nixdiffbeforeafterboth
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,60 +1,121 @@
-{fleetPkgsForPkgs}: {
-  fleetConfiguration = {
-    # TODO: Provide by fleet, instead of requesting user to provide it.
-    # This is not good that user needs to provide it, as it becomes a flake data, and fleet arbitrarily rewriting it
-    # always dirnets the flake. Instead, fleetConfiguration should return function, parameters of which should be filled
-    # by fleet itself, which is possible since fleet moving to nix repl execution.
-    data,
-    nixpkgs,
-    overlays ? [],
-    hosts,
-    fleetModules,
-    nixosModules ? [],
-    extraFleetLib ? {},
-  }: let
-    hostNames = nixpkgs.lib.attrNames hosts;
-    fleetLib =
-      (import ./fleetLib.nix {
-        inherit nixpkgs hostNames;
-      })
-      // extraFleetLib;
-  in let
-    root = nixpkgs.lib.evalModules {
-      modules =
-        (import ../modules/fleet/_modules.nix)
-        ++ [
-          data
-          ({...}: {
-            inherit nixosModules hosts;
-            overlays = [(final: prev: (fleetPkgsForPkgs final))] ++ overlays;
-          })
-        ]
-        ++ fleetModules;
-      specialArgs = {
-        inherit nixpkgs fleetLib;
+# Shared functions for fleet configuration, available as `fleet` module argument
+{lib}: let
+  inherit (lib.trivial) isFunction;
+  inherit (lib.options) mkOption mergeOneOption;
+  inherit (lib.modules) mkOverride;
+  inherit (lib.types) listOf submodule attrsOf mkOptionType;
+  inherit (lib.strings) optionalString;
+in rec {
+  types = {
+    overlay = mkOptionType {
+      name = "nixpkgs-overlay";
+      description = "nixpkgs overlay";
+      check = isFunction;
+      merge = mergeOneOption;
+    };
+    listOfOverlay = listOf types.overlay;
+
+    mkHostsType = module: attrsOf (submodule module);
+  };
+
+  options = {
+    mkHostsOption = module:
+      mkOption {
+        type = types.mkHostsType module;
       };
-    };
-    failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
-    checkedRoot =
-      if failedAssertions != []
-      then throw "Fleet failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
-      else nixpkgs.lib.showWarnings root.config.warnings root;
-    withData = {
-      root,
-      data,
-    }: {
-      config = root.config;
-    };
-    defaultData = withData {
-      inherit data;
-      root = checkedRoot;
-    };
-    uncheckedData = withData {inherit data root;};
-  in {
-    inherit nixpkgs overlays;
-    inherit (defaultData) config;
-    unchecked = {
-      inherit (uncheckedData) config;
-    };
   };
+
+  inherit (options) mkHostsOption;
+
+  modules = {
+    # mkDefault = mkOverride 1000
+    # For places, where fleet knows better than nixpkgs defaults.
+    mkFleetDefault = mkOverride 999;
+    # Some generators use mkDefault, but optionDefault is set by nixpkgs.
+    mkFleetGeneratorDefault = mkOverride 1001;
+  };
+
+  inherit (modules) mkFleetDefault mkFleetGeneratorDefault;
+
+  secrets = {
+    mkPassword = {size ? 32}: {
+      coreutils,
+      mkSecretGenerator,
+      ...
+    }:
+      mkSecretGenerator {
+        script = ''
+          mkdir $out
+          gh generate password -o $out/secret --size ${toString size}
+        '';
+      };
+
+    mkEd25519 = {
+      noEmbedPublic ? false,
+      encoding ? null,
+    }: {mkSecretGenerator, ...}:
+      mkSecretGenerator {
+        script = ''
+          mkdir $out
+          gh generate ed25519 -p $out/public -s $out/secret \
+            ${optionalString noEmbedPublic "--no-embed-public"} \
+            ${optionalString (encoding != null) "--encoding=${encoding}"}
+        '';
+      };
+
+    mkX25519 = {encoding ? null}: {mkSecretGenerator, ...}:
+      mkSecretGenerator {
+        script = ''
+          mkdir $out
+          gh generate x25519 -p $out/public -s $out/secret \
+            ${optionalString (encoding != null) "--encoding=${encoding}"}
+        '';
+      };
+
+    mkRsa = {size ? 4096}: {
+      openssl,
+      mkSecretGenerator,
+      ...
+    }:
+      mkSecretGenerator {
+        script = ''
+          mkdir $out
+
+          ${openssl}/bin/openssl genrsa -out rsa_private.key ${toString size}
+          ${openssl}/bin/openssl rsa -in rsa_private.key -pubout -out rsa_public.key
+
+          cat rsa_private.key | gh private -o $out/secret
+          cat rsa_public.key | gh public -o $out/public
+        '';
+      };
+
+    mkBytes = {
+      count ? 32,
+      encoding,
+      noNuls ? false,
+    }: {mkSecretGenerator, ...}:
+      mkSecretGenerator {
+        script = ''
+          mkdir $out
+          gh generate bytes --count=${toString count} --encoding=${encoding} -o $out/secret \
+            ${optionalString noNuls "--no-nuls"}
+        '';
+      };
+    mkHexBytes = {count ? 32}:
+      mkBytes {
+        inherit count;
+        encoding = "hex";
+      };
+    mkBase64Bytes = {count ? 32}:
+      mkBytes {
+        inherit count;
+        encoding = "base64";
+      };
+
+    # Wireguard
+    # mkWireguard = {}: mkX25519 {encoding = "base64";};
+    # mkWireguardPsk = {}: mkBase64Bytes {count = 32;};
+  };
+
+  inherit (secrets) mkPassword mkEd25519 mkX25519 mkRsa mkBytes mkHexBytes mkBase64Bytes;
 }
addedlib/flakePart.nixdiffbeforeafterboth
--- /dev/null
+++ b/lib/flakePart.nix
@@ -0,0 +1,70 @@
+{crane}: {
+  fleetLib,
+  lib,
+  config,
+  ...
+}: let
+  inherit (lib.options) mkOption;
+  inherit (lib.attrsets) mapAttrs;
+  inherit (lib.types) lazyAttrsOf deferredModule unspecified;
+  inherit (fleetLib.options) mkHostsOption;
+in {
+  options.fleetModules = mkOption {
+    type = lazyAttrsOf unspecified;
+    default = {};
+  };
+  options.fleetConfigurations = mkOption {
+    type = lazyAttrsOf deferredModule;
+    apply = nameToModule:
+      mapAttrs (
+        name: module: data: let
+          # To use user-provided nixpkgs, we first need to extract wanted nixpkgs attribute,
+          # to do that, evaluate all the modules with only needed option declared.
+          bootstrapEval = lib.evalModules {
+            modules = [
+              module
+              {
+                options.nixpkgs.buildUsing = mkOption {
+                  description = ''
+                    Nixpkgs to use for fleetConfiguration evaluation.
+                  '';
+                };
+                config._module.check = false;
+              }
+            ];
+          };
+          bootstrapNixpkgs = bootstrapEval.config.nixpkgs.buildUsing;
+          normalEval = bootstrapNixpkgs.lib.evalModules {
+            modules =
+              (import ../modules/fleet/_modules.nix)
+              ++ [
+                data
+                module
+                {
+                  options.hosts = mkHostsOption {
+                    nixos.nixpkgs.overlays = [
+                      (final: prev: {
+                        # FIXME: make this name not conflicting
+                        craneLib = crane.mkLib prev;
+                      })
+                    ];
+                  };
+                }
+              ];
+            specialArgs.fleetLib = import ../lib {
+              inherit (bootstrapNixpkgs) lib;
+            };
+          };
+        in
+          normalEval
+      )
+      nameToModule;
+  };
+  config = {
+    _module.args.fleetLib = import ../lib {inherit lib;};
+    flake.fleetConfigurations = config.fleetConfigurations;
+    flake.fleetModules = config.fleetModules;
+  };
+
+  _file = ./flakePart.nix;
+}
deletedlib/fleetLib.nixdiffbeforeafterboth
--- a/lib/fleetLib.nix
+++ /dev/null
@@ -1,144 +0,0 @@
-# Shared functions for fleet configuration, available as `fleet` module argument
-{
-  nixpkgs,
-  hostNames,
-}: let
-  inherit (nixpkgs) lib;
-  inherit (lib) listToAttrs remove unique crossLists sort elemAt mkOptionType mkOverride optionalString;
-  inherit (lib.types) listOf coercedTo oneOf submodule;
-in rec {
-  hostsToAttrs = f:
-    listToAttrs (
-      map (name: {
-        inherit name;
-        value = f name;
-      })
-      hostNames
-    );
-  hostsCartesian = remove null (
-    unique (
-      crossLists
-      (
-        a: b:
-          if a == b
-          then null
-          else hostsPair a b
-      ) [hostNames hostNames]
-    )
-  );
-  hostsPair = this: other: let
-    sorted = sort (a: b: a < b) [this other];
-  in {
-    a = elemAt sorted 0;
-    b = elemAt sorted 1;
-  };
-  hostPairName = this: other:
-    if this < other
-    then "${this}-${other}"
-    else "${other}-${this}";
-
-  types = rec {
-    anyModule = mkOptionType {
-      name = "submodule";
-      inherit (submodule {}) check;
-      merge = lib.options.mergeOneOption;
-      description = "Nixos module";
-    };
-    listOfAnyModuleStrict =
-      listOf anyModule;
-    listOfAnyModule =
-      coercedTo (oneOf [listOfAnyModuleStrict anyModule]) (
-        v:
-          if builtins.isAttrs v
-          then [v]
-          else if builtins.isFunction v
-          then [v]
-          else v
-      )
-      listOfAnyModuleStrict;
-  };
-
-  # mkDefault = mkOverride 1000
-  # For places, where fleet knows better than nixpkgs defaults.
-  mkFleetDefault = mkOverride 999;
-  # Some generators use mkDefault, but optionDefault is set by nixpkgs.
-  mkFleetGeneratorDefault = mkOverride 1001;
-
-  mkPassword = {size ? 32}: {
-    coreutils,
-    mkSecretGenerator,
-    ...
-  }:
-    mkSecretGenerator {
-      script = ''
-        mkdir $out
-        gh generate password -o $out/secret --size ${toString size}
-      '';
-    };
-
-  mkEd25519 = {
-    noEmbedPublic ? false,
-    encoding ? null,
-  }: {mkSecretGenerator, ...}:
-    mkSecretGenerator {
-      script = ''
-        mkdir $out
-        gh generate ed25519 -p $out/public -s $out/secret \
-          ${optionalString noEmbedPublic "--no-embed-public"} \
-          ${optionalString (encoding != null) "--encoding=${encoding}"}
-      '';
-    };
-
-  mkX25519 = {encoding ? null}: {mkSecretGenerator, ...}:
-    mkSecretGenerator {
-      script = ''
-        mkdir $out
-        gh generate x25519 -p $out/public -s $out/secret \
-          ${optionalString (encoding != null) "--encoding=${encoding}"}
-      '';
-    };
-
-  mkRsa = {size ? 4096}: {
-    openssl,
-    mkSecretGenerator,
-    ...
-  }:
-    mkSecretGenerator {
-      script = ''
-        mkdir $out
-
-        ${openssl}/bin/openssl genrsa -out rsa_private.key ${toString size}
-        ${openssl}/bin/openssl rsa -in rsa_private.key -pubout -out rsa_public.key
-
-        cat rsa_private.key | gh private -o $out/secret
-        cat rsa_public.key | gh public -o $out/public
-      '';
-    };
-
-  mkBytes = {
-    count ? 32,
-    encoding,
-    noNuls ? false,
-  }: {mkSecretGenerator, ...}:
-    mkSecretGenerator {
-      script = ''
-        mkdir $out
-        gh generate bytes --count=${toString count} --encoding=${encoding} -o $out/secret \
-          ${optionalString noNuls "--no-nuls"}
-      '';
-    };
-  mkHexBytes = {count ? 32}:
-    mkBytes {
-      inherit count;
-      encoding = "hex";
-    };
-  mkBase64Bytes = {count ? 32}:
-    mkBytes {
-      inherit count;
-      encoding = "base64";
-    };
-
-  # Wireguard
-  # mkWireguard = {}: mkX25519 {encoding = "base64";};
-  # mkWireguardPsk = {}: mkBase64Bytes {count = 32;};
-}
modifiedmodules/fleet/_modules.nixdiffbeforeafterboth
--- a/modules/fleet/_modules.nix
+++ b/modules/fleet/_modules.nix
@@ -1,5 +1,9 @@
 [
   ./assertions.nix
+  ./fleetLib.nix
+  ./hosts.nix
   ./meta.nix
+  ./nixos.nix
+  ./nixpkgs.nix
   ./secrets.nix
 ]
modifiedmodules/fleet/assertions.nixdiffbeforeafterboth
--- a/modules/fleet/assertions.nix
+++ b/modules/fleet/assertions.nix
@@ -1,6 +1,11 @@
-{lib, ...}: let
-  inherit (lib) mkOption;
+{
+  lib,
+  config,
+  ...
+}: let
+  inherit (lib.options) mkOption;
   inherit (lib.types) listOf unspecified str;
+  inherit (lib.lists) map filter;
 in {
   options = {
     assertions = mkOption {
@@ -30,6 +35,15 @@
         the evaluation of the system configuration.
       '';
     };
+    errors = mkOption {
+      type = listOf str;
+      internal = true;
+      description = ''
+        Similar to warnings, however build will fail if any error exists.
+      '';
+    };
   };
-  # impl of assertions is in <fleet/lib/default.nix>
+  config.errors =
+    map (v: v.message)
+    (filter (v: !v.assertion) config.assertions);
 }
addedmodules/fleet/fleetLib.nixdiffbeforeafterboth
--- /dev/null
+++ b/modules/fleet/fleetLib.nix
@@ -0,0 +1,9 @@
+{
+  lib,
+  config,
+  ...
+}: {
+  _module.args.fleetLib = import ../../lib {
+    inherit lib;
+  };
+}
addedmodules/fleet/hosts.nixdiffbeforeafterboth
--- /dev/null
+++ b/modules/fleet/hosts.nix
@@ -0,0 +1,40 @@
+{
+  lib,
+  fleetLib,
+  ...
+}: let
+  inherit (fleetLib.modules) mkFleetGeneratorDefault;
+  inherit (fleetLib.types) mkHostsType;
+  inherit (lib.options) mkOption;
+  inherit (lib.types) str listOf;
+in {
+  options = {
+    hosts = mkOption {
+      type = mkHostsType ({config, ...}: {
+        options = {
+          system = mkOption {
+            type = str;
+            description = "Type of the system.";
+          };
+          # TODO: This is part of fleet.nix, move it to separate toplevel data config option.
+          encryptionKey = mkOption {
+            type = str;
+            description = "Rage SSH encryption key for secrets.";
+          };
+          tags = mkOption {
+            type = listOf str;
+            description = "Host tag. In CLI, you can refer to all hosts having this tag using @tag syntax.";
+          };
+        };
+        config = {
+          nixos.networking.hostName = mkFleetGeneratorDefault config._module.args.name;
+          tags = ["all"];
+        };
+        _file = ./meta.nix;
+      });
+      default = {};
+      description = "Configurations of individual hosts";
+    };
+  };
+  _file = ./meta.nix;
+}
modifiedmodules/fleet/meta.nixdiffbeforeafterboth
--- a/modules/fleet/meta.nix
+++ b/modules/fleet/meta.nix
@@ -1,89 +1,8 @@
-{
-  lib,
-  fleetLib,
-  config,
-  nixpkgs,
-  ...
-}: let
-  inherit (fleetLib) hostsToAttrs mkFleetGeneratorDefault;
-  inherit (fleetLib.types) listOfAnyModule;
-  inherit (lib) mkOption mkOptionType;
-  inherit (lib.types) str unspecified attrsOf listOf submodule;
-  hostModule = {...} @ hostConfig: let
-    hostName = hostConfig.config._module.args.name;
-  in {
-    options = {
-      nixosModules = mkOption {
-        # Not too strict, but nixos module system will fix everything.
-        type =
-          listOfAnyModule;
-
-        description = "List of nixos modules";
-        default = [];
-      };
-      system = mkOption {
-        type = str;
-        description = "Type of system";
-      };
-      encryptionKey = mkOption {
-        type = str;
-        description = "Encryption key";
-      };
-      nixosSystem = mkOption {
-        type = unspecified;
-        description = "Nixos configuration";
-      };
-      nixpkgs = mkOption {
-        type = unspecified;
-        description = "Nixpkgs override";
-        default = nixpkgs;
-      };
-    };
-    config = {
-      nixosSystem = hostConfig.config.nixpkgs.lib.nixosSystem {
-        inherit (hostConfig.config) system;
-        modules = hostConfig.config.nixosModules;
-        specialArgs = {
-          inherit fleetLib;
-          fleet = hostsToAttrs (host: config.hosts.${host}.nixosSystem.config);
-        };
-      };
-      nixosModules.networking.hostName = mkFleetGeneratorDefault hostName;
-    };
-  };
-  overlayType = mkOptionType {
-    name = "nixpkgs-overlay";
-    description = "nixpkgs overlay";
-    check = lib.isFunction;
-    merge = lib.mergeOneOption;
-  };
+{lib, ...}: let
+  inherit (lib.modules) mkRemovedOptionModule;
 in {
-  options = {
-    hosts = mkOption {
-      type = attrsOf (submodule hostModule);
-      default = {};
-      description = "Configurations of individual hosts";
-    };
-    nixosModules = mkOption {
-      type = listOfAnyModule;
-      description = "Modules, which should be added to every system";
-      default = [];
-    };
-    overlays = mkOption {
-      default = [];
-      type = listOf overlayType;
-    };
-  };
-  config = {
-    hosts = hostsToAttrs (host: {
-      nixosModules =
-        config.nixosModules
-        ++ [
-          {
-            nixpkgs.overlays = config.overlays;
-          }
-        ];
-    });
-    nixosModules = import ../../nixos/modules/module-list.nix;
-  };
+  imports = [
+    (mkRemovedOptionModule ["fleetModules"] "replaced with imports.")
+    (mkRemovedOptionModule ["data"] "data is now provided by fleet itself, you can remove your import.")
+  ];
 }
addedmodules/fleet/nixos.nixdiffbeforeafterboth
--- /dev/null
+++ b/modules/fleet/nixos.nix
@@ -0,0 +1,55 @@
+{
+  lib,
+  fleetLib,
+  config,
+  ...
+}: let
+  inherit (lib.attrsets) mapAttrs;
+  inherit (lib.options) mkOption;
+  inherit (lib.types) deferredModule deferredModuleWith;
+  inherit (lib.modules) mkRemovedOptionModule;
+  inherit (fleetLib.options) mkHostsOption;
+
+  _file = ./nixos.nix;
+in {
+  options = {
+    nixos = mkOption {
+      description = ''
+        Nixos configuration for all hosts.
+      '';
+      type = deferredModule;
+    };
+    hosts = mkHostsOption (hostArgs: {
+      inherit _file;
+      options = {
+        nixos = mkOption {
+          description = ''
+            Nixos configuration for the current host.
+          '';
+          type = deferredModuleWith {
+            staticModules = import ../../nixos/modules/module-list.nix;
+          };
+          apply = module:
+            config.nixpkgs.buildUsing.lib.nixosSystem {
+              inherit (hostArgs.config) system;
+              modules = [module];
+            };
+        };
+      };
+      config = {
+        # imports = [
+        #   (mkRemovedOptionModule ["nixosModules"] "replaced with hosts.*.nixos.imports.")
+        # ];
+        nixos = {
+          imports = [
+            config.nixos
+          ];
+          config._module.args.fleet = mapAttrs (_: value: value.nixos.config) config.hosts;
+        };
+      };
+    });
+  };
+  imports = [
+    (mkRemovedOptionModule ["nixosModules"] "replaced with nixos.imports.")
+  ];
+}
addedmodules/fleet/nixpkgs.nixdiffbeforeafterboth
--- /dev/null
+++ b/modules/fleet/nixpkgs.nix
@@ -0,0 +1,58 @@
+{
+  lib,
+  fleetLib,
+  config,
+  ...
+}: let
+  inherit (lib.options) mkOption;
+  inherit (lib.types) path;
+  inherit (lib.modules) mkRemovedOptionModule;
+  inherit (fleetLib.options) mkHostsOption;
+  inherit (fleetLib.types) listOfOverlay;
+
+  _file = ./nixpkgs.lib;
+in {
+  options = {
+    nixpkgs = {
+      buildUsing = mkOption {
+        description = ''
+          Default nixpkgs to use for building the systems.
+        '';
+        type = path;
+      };
+      overlays = mkOption {
+        description = ''
+          Package overlays to apply for all the hosts, gets propagated into
+          `hosts.*.nixosModules.nixpkgs.overlays`.
+        '';
+        type = listOfOverlay;
+      };
+    };
+    hosts = mkHostsOption {
+      inherit _file;
+      options.nixpkgs.buildUsing = mkOption {
+        description = ''
+          Nixpkgs to use for building the system.
+
+          Note that this option is defined at the host level, not the nixosModules level,
+          nixosModules will be evaluated using this flake input.
+        '';
+        type = path;
+        default = config.nixpkgs.buildUsing;
+      };
+      # imports = [
+      # 	(mkRemovedOptionModule ["nixpkgs" "overlays"] "this option needs to be specified at nixosModules level")
+      # ];
+      config.nixos = {
+        inherit _file;
+        nixpkgs.overlays = config.nixpkgs.overlays;
+        imports = [
+          (mkRemovedOptionModule ["nixpkgs" "buildUsing"] "this option should be specified at the host level, not the nixosModules level")
+        ];
+      };
+    };
+  };
+  config.nixpkgs.overlays = [
+    (final: prev: import ../../pkgs {inherit (final) callPackage craneLib;})
+  ];
+}
modifiedmodules/fleet/secrets.nixdiffbeforeafterboth
--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -4,9 +4,12 @@
   config,
   ...
 }: let
-  inherit (fleetLib) hostsToAttrs;
-  inherit (lib) mkOption mapAttrsToList mapAttrs filterAttrs concatStringsSep;
+  inherit (fleetLib.options) mkHostsOption;
+  inherit (lib.options) mkOption;
   inherit (lib.types) lazyAttrsOf unspecified nullOr listOf str bool attrsOf submodule;
+  inherit (lib.lists) sort elem;
+  inherit (lib.attrsets) mapAttrsToList mapAttrs filterAttrs;
+  inherit (lib.strings) toJSON concatStringsSep;
 
   sharedSecret = {config, ...}: {
     freeformType = lazyAttrsOf unspecified;
@@ -82,11 +85,11 @@
       };
     };
   };
+  inherit (config) hostSecrets sharedSecrets;
 in {
   options = {
     version = mkOption {
       type = str;
-      default = "";
       internal = true;
     };
     sharedSecrets = mkOption {
@@ -100,36 +103,34 @@
       description = "Host secrets. Imported from fleet.nix";
       internal = true;
     };
+    hosts = mkHostsOption ({config, ...}: {
+      nixos = {
+        secrets = let
+          host = config._module.args.name;
+          processSecret = v:
+            (removeAttrs v ["createdAt" "expiresAt" "expectedOwners" "owners" "regenerateOnOwnerAdded" "regenerateOnOwnerRemoved"])
+            // {
+              shared = true;
+            };
+        in
+          (
+            mapAttrs (_: processSecret)
+            (filterAttrs (_: v: elem host v.owners) sharedSecrets)
+          )
+          // (mapAttrs (_: processSecret) (hostSecrets.${host} or {}));
+        _file = ./secrets.nix;
+      };
+    });
   };
   config = {
     assertions =
       mapAttrsToList
       (name: secret: {
-        assertion = secret.expectedOwners == null || builtins.sort (a: b: a < b) secret.owners == builtins.sort (a: b: a < b) secret.expectedOwners;
-        message = "Shared secret ${name} is expected to be encrypted for ${builtins.toJSON secret.expectedOwners}, but it is encrypted for ${builtins.toJSON secret.owners}. Run fleet secrets regenerate to fix";
+        assertion = secret.expectedOwners == null || sort (a: b: a < b) secret.owners == sort (a: b: a < b) secret.expectedOwners;
+        message = "Shared secret ${name} is expected to be encrypted for ${toJSON secret.expectedOwners}, but it is encrypted for ${toJSON secret.owners}. Run fleet secrets regenerate to fix";
       })
       config.sharedSecrets;
-    hosts = hostsToAttrs (host: {
-      nixosModules = let
-        # processPart
-        processSecret = v:
-          (removeAttrs v ["createdAt" "expiresAt" "expectedOwners" "owners" "regenerateOnOwnerAdded" "regenerateOnOwnerRemoved"])
-          // {
-            shared = true;
-          };
-      in [
-        {
-          secrets =
-            (
-              mapAttrs (_: processSecret)
-              (filterAttrs (_: v: builtins.elem host v.owners) config.sharedSecrets)
-            )
-            // (mapAttrs (_: processSecret) (config.hostSecrets.${host} or {}));
-        }
-      ];
-    });
-    # TODO: Should this attribute be moved to `nixpkgs.overlays`?
-    overlays = [
+    nixpkgs.overlays = [
       (final: prev: {
         mkSecretGenerators = {recipients}: rec {
           # TODO: Merge both generators to one with consistent options syntax?
addednixos/assertions.nixdiffbeforeafterboth
--- /dev/null
+++ b/nixos/assertions.nix
@@ -0,0 +1,24 @@
+# Similar module exists for fleet, however it also defines assertions and warnings,
+# which are already defined for nixos.
+{
+  lib,
+  config,
+  ...
+}: let
+  inherit (lib.options) mkOption;
+  inherit (lib.lists) map filter;
+  inherit (lib.types) listOf str;
+in {
+  options = {
+    errors = mkOption {
+      type = listOf str;
+      internal = true;
+      description = ''
+        Similar to warnings, however build will fail if any error exists.
+      '';
+    };
+  };
+  config.errors =
+    map (v: v.message)
+    (filter (v: !v.assertion) config.assertions);
+}
modifiednixos/meta.nixdiffbeforeafterboth
--- a/nixos/meta.nix
+++ b/nixos/meta.nix
@@ -3,18 +3,16 @@
   pkgs,
   ...
 }: let
-  inherit (lib) mkOption;
+  inherit (lib.options) mkOption;
   inherit (lib.types) listOf str submodule;
+  inherit (lib.modules) mkRemovedOptionModule;
 in {
   options = {
-    nixpkgs.resolvedPkgs = mkOption {
+    # TODO: Give a real name.
+    # Previously it was nixpkgs.resolvedPkgs, which was erroreously merged with nixpkgs override attribute.
+    _resolvedPkgs = mkOption {
       type = lib.types.pkgs // {description = "nixpkgs.pkgs";};
       description = "Value of pkgs";
-    };
-    tags = mkOption {
-      type = listOf str;
-      description = "Host tags";
-      default = [];
     };
     network = mkOption {
       type = submodule {
@@ -34,9 +32,11 @@
       description = "Network definition of host";
     };
   };
+  imports = [
+    (mkRemovedOptionModule ["tags"] "tags are now defined at the host level, not the nixos system level for fast filtering without evaluating unnecessary hosts.")
+  ];
   config = {
-    tags = ["all"];
     network = {};
-    nixpkgs.resolvedPkgs = pkgs;
+    _resolvedPkgs = pkgs;
   };
 }
modifiednixos/modules/module-list.nixdiffbeforeafterboth
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -1,4 +1,5 @@
 [
+  ../assertions.nix
   ../meta.nix
   ../secrets.nix
   ../rollback.nix
modifiednixos/nix-sign.nixdiffbeforeafterboth
--- a/nixos/nix-sign.nix
+++ b/nixos/nix-sign.nix
@@ -1,7 +1,12 @@
 # Required for nix copy in build_systems.rs
-{config, ...}: {
+{lib, config, ...}:
+let
+  inherit (lib.modules) mkIf;
+  hasPersistentHostname = config.networking.hostName != "";
+in
+{
   # https://github.com/NixOS/nix/issues/3023
-  systemd.services.generate-nix-cache-key = {
+  systemd.services.generate-nix-cache-key = mkIf hasPersistentHostname {
     wantedBy = ["multi-user.target"];
     serviceConfig.Type = "oneshot";
     path = [config.nix.package];
@@ -10,5 +15,5 @@
       nix-store --generate-binary-cache-key ${config.networking.hostName}-1 /etc/nix/private-key /etc/nix/public-key
     '';
   };
-  nix.settings.secret-key-files = "/etc/nix/private-key";
+  nix.settings.secret-key-files = mkIf hasPersistentHostname "/etc/nix/private-key";
 }
modifiednixos/secrets.nixdiffbeforeafterboth
--- a/nixos/secrets.nix
+++ b/nixos/secrets.nix
@@ -5,7 +5,11 @@
   ...
 }: let
   inherit (lib.strings) hasPrefix removePrefix;
-  inherit (lib) mkOption mkOptionDefault mapAttrs stringAfter;
+  inherit (lib.stringsWithDeps) stringAfter;
+  inherit (lib.options) mkOption;
+  inherit (lib.lists) optional;
+  inherit (lib.attrsets) mapAttrs;
+  inherit (lib.modules) mkOptionDefault mkIf;
   inherit (lib.types) submodule str attrsOf nullOr unspecified lazyAttrsOf;
   plaintextPrefix = "<PLAINTEXT>";
   plaintextNewlinePrefix = "<PLAINTEXT-NL>";
@@ -110,6 +114,7 @@
       builtins.toJSON (mapAttrs (_: processSecret)
         config.secrets);
   };
+  useSysusers = (config.systemd ? sysusers && config.systemd.sysusers.enable) || (config ? userborn && config.userborn.enable);
 in {
   options = {
     secrets = mkOption {
@@ -120,21 +125,44 @@
   };
   config = {
     environment.systemPackages = [pkgs.fleet-install-secrets];
+
+    systemd.services.fleet-install-secrets = mkIf useSysusers {
+      wantedBy = ["sysinit.target"];
+      after = ["systemd-sysusers.service"];
+      restartTriggers = [
+        secretsFile
+      ];
+      aliases = [
+        "sops-install-secrets"
+        "agenix-install-secrets"
+      ];
+
+      unitConfig.DefaultDependencies = false;
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "${pkgs.fleet-install-secrets}/bin/fleet-install-secrets install ${secretsFile}";
+      };
+    };
     system.activationScripts.decryptSecrets =
-      stringAfter (
-        [
-          # secrets are owned by user/group, thus we need to refer to those
-          "users"
-          "groups"
-          "specialfs"
-        ]
-        # nixos-impermanence compatibility: secrets are encrypted by host-key,
-        # but with impermanence we expect that the host-key is installed by
-        # persist-file activation script.
-        ++ (lib.optional (config.system.activationScripts ? "persist-files") "persist-files")
-      ) ''
-        1>&2 echo "setting up secrets"
-        ${pkgs.fleet-install-secrets}/bin/fleet-install-secrets install ${secretsFile}
-      '';
+      mkIf (!useSysusers)
+      (
+        stringAfter (
+          [
+            # secrets are owned by user/group, thus we need to refer to those
+            "users"
+            "groups"
+            "specialfs"
+          ]
+          # nixos-impermanence compatibility: secrets are encrypted by host-key,
+          # but with impermanence we expect that the host-key is installed by
+          # persist-file activation script.
+          ++ (optional (config.system.activationScripts ? "persist-files") "persist-files")
+        ) ''
+          1>&2 echo "setting up secrets"
+          ${pkgs.fleet-install-secrets}/bin/fleet-install-secrets install ${secretsFile}
+        ''
+      );
   };
 }