git.delta.rocks / fleet / refs/commits / 2438e2b594b6

difftreelog

feat remote_derivation modes

vkqwypzmYaroslav Bolyukin2026-06-19parent: #28094c9.patch.diff
in: push-kyumtlkprzyo

2 files changed

modifiedcrates/fleet-base/src/host.rsdiffbeforeafterboth
before · crates/fleet-base/src/host.rs
1use std::{2	collections::{BTreeMap, BTreeSet, HashSet},3	future::Future,4	io::Write,5	ops::Deref,6	path::PathBuf,7	pin::Pin,8	str::FromStr,9	sync::{Arc, OnceLock},10};1112use anyhow::{Context, Result, anyhow, bail, ensure};13use camino::{Utf8Path, Utf8PathBuf};14use chrono::{DateTime, Utc};15use fleet_shared::SecretData;16use nix_eval::{Store, Value, drv::DrvGraph, eval_store, nix_go, nix_go_json, util::assert_warn};17use remowt_client::{AgentBundle, Remowt};18use remowt_endpoints::fs::FsClient;19use remowt_fleet::NixClient;20use remowt_link_shared::BifConfig;21use remowt_ui_prompt::auto::AutoPrompter;22use remowt_ui_prompt::bifrost::PromptEndpoints;23use remowt_ui_prompt::{PrependSourcePrompter, Source};24use tempfile::NamedTempFile;25use tokio::{sync::OnceCell, task::spawn_blocking};26use tracing::{info, warn};2728use crate::fleetdata::{29	FleetData, FleetSecretData, FleetSecretDistribution, FleetSecretPart, SecretOwner,30};3132pub struct FleetConfigInternals {33	pub prefer_identities: BTreeSet<SecretOwner>,34	pub now: DateTime<Utc>,3536	/// Fleet project directory, containing fleet.nix file.37	pub directory: PathBuf,38	/// builtins.currentSystem39	pub local_system: String,40	pub data: Arc<FleetData>,41	/// fleet_config.config42	pub config_field: Value,43	/// flake.output44	pub flake_outputs: Value,45	// TODO: Remove with connectivity refactor46	pub localhost: String,4748	/// import nixpkgs {system = local};49	pub default_pkgs: Value,50	/// inputs.nixpkgs51	pub nixpkgs: Value,5253	pub local_host: OnceLock<Arc<ConfigHost>>,54}5556// TODO: Make field not pub57#[derive(Clone)]58pub struct Config(pub Arc<FleetConfigInternals>);5960impl Deref for Config {61	type Target = FleetConfigInternals;6263	fn deref(&self) -> &Self::Target {64		&self.065	}66}6768#[derive(Clone, PartialEq, Copy, Debug)]69pub enum DeployKind {70	/// NixOS => NixOS managed by fleet71	UpgradeToFleet,72	/// NixOS managed by fleet => NixOS managed by fleet73	Fleet,74	/// Remote host has /mnt, /mnt/boot mounted,75	/// generated config is added to fleet configuration.76	NixosInstall,77	/// Remote host has some system and nix installed in multi-user mode (/nix is owned by root),78	/// generated config is added to fleet configuration,79	/// and /etc/NIXOS_LUSTRATE exists, fleet will perform the rest.80	NixosLustrate,81}8283impl FromStr for DeployKind {84	type Err = anyhow::Error;85	fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {86		match s {87			"upgrade-to-fleet" => Ok(Self::UpgradeToFleet),88			"fleet" => Ok(Self::Fleet),89			"nixos-install" => Ok(Self::NixosInstall),90			"nixos-lustrate" => Ok(Self::NixosLustrate),91			v => bail!(92				"unknown deploy_kind: {v}; expected on of \"upgrade-to-fleet\", \"fleet\", \"nixos-install\", \"nixos-lustrate\""93			),94		}95	}96}97pub struct ConfigHost {98	config: Config,99	pub name: String,100	groups: OnceLock<Vec<String>>,101102	// TODO: Both of those values are taken from host opts, there should be a cleaner way to specify it103	deploy_kind: OnceCell<DeployKind>,104	session_destination: OnceLock<String>,105	legacy_ssh_store: OnceLock<bool>,106107	pub host_config: Option<Value>,108	pub nixos_config: OnceLock<Value>,109	pub nixos_unchecked_config: OnceLock<Value>,110	pub pkgs_override: Option<Value>,111112	// TODO: Move command helpers away with connectivity refactor113	pub local: bool,114	pub remowt: OnceLock<Remowt>,115	nix_store: OnceLock<Arc<Store>>,116	nix_plugin: OnceCell<()>,117}118119const NIX_PLUGIN_ID: u16 = 2;120121fn agents_dir() -> Result<PathBuf> {122	std::env::var_os("REMOWT_AGENTS_DIR")123		.map(PathBuf::from)124		.or_else(|| option_env!("REMOWT_AGENTS_DIR").map(PathBuf::from))125		.ok_or_else(|| {126			anyhow!("no remowt-agents bundle; set REMOWT_AGENTS_DIR to a remowt-agents output")127		})128}129130fn agent_bundle() -> Result<AgentBundle> {131	AgentBundle::from_dir(agents_dir()?)132}133134enum RemoteDerivationMode {135	CopySigned,136	Copy,137	Attic,138	Cachix,139}140141impl ConfigHost {142	pub fn set_session_destination(&self, dest: String) {143		self.session_destination144			.set(dest)145			.expect("session destination is already set")146	}147	pub fn set_deploy_kind(&self, kind: DeployKind) {148		self.deploy_kind149			.set(kind)150			.expect("deploy kind is already set");151	}152	pub fn set_legacy_ssh_store(&self, legacy: bool) {153		self.legacy_ssh_store154			.set(legacy)155			.expect("legacy ssh store is already set")156	}157	pub async fn deploy_kind(&self) -> Result<DeployKind> {158		self.deploy_kind.get_or_try_init(|| async {159			let remowt = self.remowt().await?;160			let fs = remowt.endpoints::<FsClient<_>>();161			let is_fleet_managed = match fs.file_exists(Utf8PathBuf::from("/etc/FLEET_HOST")).await {162				Ok(v) => v,163				Err(e) => {164					bail!("failed to query remote system kind: {e}");165				}166			};167			if !is_fleet_managed {168				bail!(169					"{}",170					indoc::indoc! {"171					host is not marked as managed by fleet172					if you're not trying to lustrate/install system from scratch,173					you should either174						1. manually create /etc/FLEET_HOST file on the target host,175						2. use ?deploy_kind=fleet host argument if you're upgrading from older version of fleet176						3. use ?deploy_kind=upgrade_to_fleet if you're upgrading from plain nixos to fleet-managed nixos177					for installation use ?deploy_kind=nixos_install / ?deploy_kind=nixos_lustrate 178				"}179				);180			}181			Ok(DeployKind::Fleet)182		}).await.copied()183	}184	async fn connection(&self) -> Result<Remowt> {185		if let Some(conn) = self.remowt.get() {186			return Ok(conn.clone());187		}188		let bundle = agent_bundle()?;189		let conn = if self.local {190			Remowt::connect_local(&bundle, "remowt-fleet".to_owned())191				.await192				.context("starting local remowt agent")?193		} else {194			let dest = self195				.session_destination196				.get()197				.cloned()198				.unwrap_or_else(|| self.name.clone());199			Remowt::connect(&dest, &bundle, "remowt-fleet".to_owned())200				.await201				.map_err(|e| anyhow!("remowt error while connecting to {}: {e:#?}", self.name))?202		};203		PromptEndpoints(PrependSourcePrompter {204			prompter: AutoPrompter::new().await,205			source: if self.local {206				vec![]207			} else {208				vec![Source(std::borrow::Cow::Owned(format!(209					"ssh host: {}",210					self.name211				)))]212			},213			description: "".to_owned(),214		})215		.register_endpoints(&mut conn.rpc());216		let _ = self.remowt.set(conn);217		Ok(self.remowt.get().expect("just set").clone())218	}219220	/// Client for this host's unprivileged agent.221	pub async fn remowt(&self) -> Result<Remowt> {222		self.connection().await223	}224225	pub async fn nix_client(&self) -> Result<NixClient<BifConfig>> {226		let remowt = self.remowt().await?;227		let plugin_id = self.ensure_nix_plugin().await?;228		Ok(remowt.plugin_endpoints(plugin_id))229	}230	pub fn ensure_nix_plugin(&self) -> Pin<Box<dyn Future<Output = Result<u16>> + Send + '_>> {231		Box::pin(async {232			self.nix_plugin233				.get_or_try_init(|| async {234					let pkgs = self.pkgs()?;235					let name = "remowt-plugin-fleet";236					let plugin = nix_go!(pkgs[{ name }]);237					let built = plugin238						.build("out")239						.context("failed to build the fleet nix plugin")?;240					let copied = self241						.remote_derivation(&built)242						.await243						.context("failed to copy the fleet nix plugin to the host store")?;244					let bin = copied.join("bin/remowt-plugin-fleet");245					self.remowt()246						.await?247						.run0_load_plugin_path(NIX_PLUGIN_ID, bin.as_str())248						.await249						.context("failed to load the fleet nix plugin")?;250					anyhow::Ok(())251				})252				.await?;253			Ok(NIX_PLUGIN_ID)254		})255	}256257	async fn nix_store(&self) -> Result<Arc<Store>> {258		if let Some(store) = self.nix_store.get() {259			return Ok(store.clone());260		}261		let conn = self.connection().await?;262		let socket = match self.deploy_kind().await? {263			DeployKind::NixosInstall => {264				remowt_fleet::nix_store_socket(conn, "/mnt?require-sigs=false").await?265			}266			_ => remowt_fleet::nix_store_socket(conn, "auto").await?,267		};268		let uri = format!("unix://{}", socket.display());269		let store = Arc::new(Store::open(&uri)?);270		let _ = self.nix_store.set(store);271		Ok(self.nix_store.get().expect("just set").clone())272	}273274	pub async fn decrypt(&self, data: SecretData) -> Result<Vec<u8>> {275		ensure!(data.encrypted, "secret is not encrypted");276		let remowt = self.remowt().await?;277		let mut cmd = remowt.cmd("fleet-install-secrets");278		cmd.arg("decrypt").eqarg("--secret", data.to_string());279		let encoded = cmd280			.sudo()281			.run_string()282			.await283			.context("failed to call remote host for decrypt")?;284		let data: SecretData = encoded.parse().map_err(|e| anyhow!("{e}"))?;285		ensure!(!data.encrypted, "secret came out encrypted");286		Ok(data.data)287	}288	pub async fn reencrypt_distribution(289		&self,290		data: &FleetSecretDistribution,291		targets: BTreeSet<SecretOwner>,292		now: DateTime<Utc>,293	) -> Result<FleetSecretDistribution> {294		let mut parts = BTreeMap::new();295		for (part_name, part) in &data.secret.parts {296			parts.insert(297				part_name.clone(),298				if part.raw.encrypted {299					FleetSecretPart {300						raw: self.reencrypt(part.raw.clone(), targets.clone()).await?,301					}302				} else {303					part.clone()304				},305			);306		}307		let secret = FleetSecretData {308			created_at: data.secret.created_at,309			expires_at: data.secret.expires_at,310			generation_data: data.secret.generation_data.clone(),311			parts,312		};313		Ok(FleetSecretDistribution::new(targets, secret, now))314	}315	pub async fn reencrypt(316		&self,317		data: SecretData,318		targets: BTreeSet<SecretOwner>,319	) -> Result<SecretData> {320		let remowt = self.remowt().await?;321		ensure!(data.encrypted, "secret is not encrypted");322		let mut cmd = remowt.cmd("fleet-install-secrets");323		cmd.arg("reencrypt").eqarg("--secret", data.to_string());324		for target in targets {325			let key = self.config.key(&target).await?;326			cmd.eqarg("--targets", key);327		}328		let encoded = cmd329			.sudo()330			.run_string()331			.await332			.context("failed to call remote host for decrypt")?;333		let data: SecretData = encoded.parse().map_err(|e| anyhow!("{e}"))?;334		ensure!(data.encrypted, "secret came out not encrypted");335		Ok(data)336	}337	/// Returns path for futureproofing, as path might change i.e on conversion to CA338	pub async fn remote_derivation(&self, path: impl AsRef<Utf8Path>) -> Result<Utf8PathBuf> {339		let path = path.as_ref().to_owned();340		if self.local {341			// Path is located locally, thus already trusted.342			return Ok(path);343		}344		let sign: Pin<Box<dyn Future<Output = Result<()>> + Send>> = {345			let path = path.clone();346			let graph = DrvGraph::resolve(&eval_store(), &path)347				.context("failed to resolve graph to be uploaded")?;348			info!("signing {} paths", graph.nodes.len());349350			Box::pin(async move {351				let local = self.config.local_host();352				let plugin_id = local.ensure_nix_plugin().await?;353				let nix = local354					.remowt()355					.await?356					.plugin_endpoints::<remowt_fleet::NixClient<_>>(plugin_id);357				nix.sign_closure(path, Utf8PathBuf::from("/etc/nix/private-key"))358					.await359					.map_err(|e| anyhow!("{e:?}"))?360					.map_err(|e| anyhow!("{e}"))?;361				Ok(())362			})363		};364		if let Err(e) = sign.await {365			warn!("failed to sign store paths: {e}");366		}367		let store = self.nix_store().await?;368		let eval_store = eval_store();369		{370			let path = path.clone();371			spawn_blocking(move || eval_store.copy_to(&store, path.as_ref()))372				.await373				.expect("copy_to panicked")374				.context("copying closure to remote store")?;375		}376		Ok(path)377	}378}379380struct HostSecretDefinition(Value);381382impl ConfigHost {383	// TOCTOU is possible here in case if config is changed, but this case is not handled anywhere anyway,384	// assuming getting tags always returns the same value.385	pub fn tags(&self) -> Result<Vec<String>> {386		if let Some(v) = self.groups.get() {387			return Ok(v.clone());388		}389		let Some(host_config) = &self.host_config else {390			return Ok(vec![]);391		};392		let tags: Vec<String> = nix_go_json!(host_config.tags);393394		let _ = self.groups.set(tags.clone());395396		Ok(tags)397	}398	pub fn nixos_config(&self) -> Result<Value> {399		if let Some(v) = self.nixos_config.get() {400			return Ok(v.clone());401		}402		let Some(host_config) = &self.host_config else {403			bail!("local host has no nixos_config");404		};405		let nixos_config = nix_go!(host_config.nixos.config);406		assert_warn("nixos config evaluation", &nixos_config)?;407408		let _ = self.nixos_config.set(nixos_config.clone());409410		Ok(nixos_config)411	}412	pub fn nixos_unchecked_config(&self) -> Result<Value> {413		if let Some(v) = self.nixos_unchecked_config.get() {414			return Ok(v.clone());415		}416		let Some(host_config) = &self.host_config else {417			bail!("local host has no nixos_config");418		};419		let nixos_config = nix_go!(host_config.nixos_unchecked.config);420421		let _ = self.nixos_unchecked_config.set(nixos_config.clone());422423		Ok(nixos_config)424	}425426	pub fn list_defined_secrets(&self) -> Result<Vec<String>> {427		let nixos = self.nixos_unchecked_config()?;428		let secrets = nix_go!(nixos.secrets);429		secrets.list_fields()430	}431432	/// Packages for this host, resolved with nixpkgs overlays433	pub fn pkgs(&self) -> Result<Value> {434		if let Some(value) = &self.pkgs_override {435			return Ok(value.clone());436		}437		let Some(host_config) = &self.host_config else {438			bail!("local host has no host_config");439		};440		// TODO: Should nixos.options be cached?441		Ok(nix_go!(host_config.nixos.options._module.args.value.pkgs))442	}443}444445#[derive(Clone)]446pub struct SharedSecretDefinition(Value);447impl SharedSecretDefinition {448	pub fn expected_owners(&self) -> Result<BTreeSet<SecretOwner>> {449		let secret = &self.0;450		Ok(nix_go_json!(secret.expectedOwners))451	}452	pub fn allow_different(&self) -> Result<bool> {453		let secret = &self.0;454		Ok(nix_go_json!(secret.allowDifferent))455	}456	pub fn regenerate_on_owner_added(&self) -> Result<bool> {457		let secret = &self.0;458		Ok(nix_go_json!(secret.regenerateOnOwnerAdded))459	}460	pub fn regenerate_on_owner_removed(&self) -> Result<bool> {461		let secret = &self.0;462		Ok(nix_go_json!(secret.regenerateOnOwnerRemoved))463	}464	pub fn generator(&self) -> Result<Value> {465		let secret = &self.0;466		Ok(nix_go!(secret.generator))467	}468}469470impl Config {471	pub fn tagged_hostnames(&self, tag: &str) -> Result<Vec<String>> {472		let config = &self.config_field;473		let tagged: Vec<String> = nix_go_json!(config.taggedWith[{ tag }]);474		Ok(tagged)475	}476	pub fn expand_owner_set(&self, owners: Vec<String>) -> Result<BTreeSet<String>> {477		let mut out = BTreeSet::new();478		for owner in owners {479			if let Some(tag) = owner.strip_prefix('@') {480				let hosts = self.tagged_hostnames(tag)?;481				out.extend(hosts);482			} else {483				out.insert(owner);484			}485		}486		Ok(out)487	}488	pub fn local_host(&self) -> Arc<ConfigHost> {489		self.local_host490			.get_or_init(|| {491				Arc::new(ConfigHost {492					config: self.clone(),493					name: "<virtual localhost>".to_owned(),494					host_config: None,495					nixos_config: OnceLock::new(),496					nixos_unchecked_config: OnceLock::new(),497					groups: {498						let cell = OnceLock::new();499						let _ = cell.set(vec![]);500						cell501					},502					pkgs_override: Some(self.default_pkgs.clone()),503504					local: true,505					remowt: OnceLock::new(),506					nix_store: OnceLock::new(),507					nix_plugin: OnceCell::new(),508					deploy_kind: OnceCell::new(),509					session_destination: OnceLock::new(),510					legacy_ssh_store: OnceLock::new(),511				})512			})513			.clone()514	}515516	pub fn preferred_hosts(517		&self,518		filter: impl Fn(&str) -> bool,519	) -> Result<impl Iterator<Item = Result<ConfigHost>>> {520		let prefer = self521			.prefer_identities522			.iter()523			.filter_map(|v| v.as_host())524			.collect::<HashSet<_>>();525		let config = &self.config_field;526		let mut names = nix_go!(config.hosts).list_fields()?;527		names.retain(|s| filter(s));528		names.sort_by_key(|h| prefer.contains(h.as_str()));529530		Ok(names.into_iter().map(|h| self.host(&h)))531	}532533	pub fn host(&self, name: &str) -> Result<ConfigHost> {534		let config = &self.config_field;535		let host_config = nix_go!(config.hosts[{ name }]);536537		Ok(ConfigHost {538			config: self.clone(),539			name: name.to_owned(),540			host_config: Some(host_config),541			nixos_config: OnceLock::new(),542			nixos_unchecked_config: OnceLock::new(),543			groups: OnceLock::new(),544			pkgs_override: None,545546			// TODO: Remove with connectivit refactor547			local: self.localhost == name,548			remowt: OnceLock::new(),549			nix_store: OnceLock::new(),550			nix_plugin: OnceCell::new(),551			deploy_kind: OnceCell::new(),552			session_destination: OnceLock::new(),553			legacy_ssh_store: OnceLock::new(),554		})555	}556	pub fn list_hosts(&self) -> Result<Vec<ConfigHost>> {557		let config = &self.config_field;558		let names = nix_go!(config.hosts).list_fields()?;559		let mut out = vec![];560		for name in names {561			out.push(self.host(&name)?);562		}563		Ok(out)564	}565	// TODO: Replace usages with .host().nixos_config566	pub fn system_config(&self, host: &str) -> Result<Value> {567		let fleet_field = &self.config_field;568		Ok(nix_go!(fleet_field.hosts[{ host }].nixos.config))569	}570571	pub fn secret_definition(&self, secret: &str) -> Result<Option<SharedSecretDefinition>> {572		let config = &self.config_field;573		let shared_secrets = nix_go!(config.secrets);574		if !shared_secrets.has_field(secret)? {575			return Ok(None);576		}577		Ok(Some(SharedSecretDefinition(nix_go!(578			shared_secrets[secret]579		))))580	}581582	pub fn save(&self) -> Result<()> {583		let mut tempfile = NamedTempFile::new_in(self.directory.clone()).context("failed to create updated version of fleet.nix in the same directory as original.\nDo you have write access to it? Access only to the fleet.nix won't be enough, the directory is used for atomic overwrite operation.\nIt is not recommended to use fleet by root anyway, move fleet project to your home directory.")?;584		let data = nixlike::serialize(&*self.data)?;585		tempfile.write_all(586			format!(587				"# This file contains fleet state and shouldn't be edited by hand\n\n{data}\n\n# vim: ts=2 et nowrap\n"588			)589			.as_bytes(),590		)?;591		let mut fleet_data_path = self.directory.clone();592		fleet_data_path.push("fleet.nix");593		tempfile.persist(fleet_data_path)?;594		Ok(())595	}596}
after · crates/fleet-base/src/host.rs
1use std::{2	collections::{BTreeMap, BTreeSet, HashSet},3	future::Future,4	io::Write,5	ops::Deref,6	path::PathBuf,7	pin::Pin,8	str::FromStr,9	sync::{Arc, OnceLock},10};1112use anyhow::{Context, Result, anyhow, bail, ensure};13use camino::{Utf8Path, Utf8PathBuf};14use chrono::{DateTime, Utc};15use fleet_shared::SecretData;16use nix_eval::{Store, Value, drv::DrvGraph, eval_store, nix_go, nix_go_json, util::assert_warn};17use remowt_client::{AgentBundle, Remowt};18use remowt_endpoints::fs::FsClient;19use remowt_fleet::NixClient;20use remowt_link_shared::BifConfig;21use remowt_ui_prompt::auto::AutoPrompter;22use remowt_ui_prompt::bifrost::PromptEndpoints;23use remowt_ui_prompt::{PrependSourcePrompter, Source};24use tempfile::NamedTempFile;25use tokio::{sync::OnceCell, task::spawn_blocking};26use tracing::{info, instrument, warn};2728use crate::fleetdata::{29	FleetData, FleetSecretData, FleetSecretDistribution, FleetSecretPart, SecretOwner,30};3132pub struct FleetConfigInternals {33	pub prefer_identities: BTreeSet<SecretOwner>,34	pub now: DateTime<Utc>,3536	/// Fleet project directory, containing fleet.nix file.37	pub directory: PathBuf,38	/// builtins.currentSystem39	pub local_system: String,40	pub data: Arc<FleetData>,41	/// fleet_config.config42	pub config_field: Value,43	/// flake.output44	pub flake_outputs: Value,45	// TODO: Remove with connectivity refactor46	pub localhost: String,4748	/// import nixpkgs {system = local};49	pub default_pkgs: Value,50	/// inputs.nixpkgs51	pub nixpkgs: Value,5253	pub local_host: OnceLock<Arc<ConfigHost>>,54}5556// TODO: Make field not pub57#[derive(Clone)]58pub struct Config(pub Arc<FleetConfigInternals>);5960impl Deref for Config {61	type Target = FleetConfigInternals;6263	fn deref(&self) -> &Self::Target {64		&self.065	}66}6768#[derive(Clone, PartialEq, Copy, Debug)]69pub enum DeployKind {70	/// NixOS => NixOS managed by fleet71	UpgradeToFleet,72	/// NixOS managed by fleet => NixOS managed by fleet73	Fleet,74	/// Remote host has /mnt, /mnt/boot mounted,75	/// generated config is added to fleet configuration.76	NixosInstall,77	/// Remote host has some system and nix installed in multi-user mode (/nix is owned by root),78	/// generated config is added to fleet configuration,79	/// and /etc/NIXOS_LUSTRATE exists, fleet will perform the rest.80	NixosLustrate,81}8283impl FromStr for DeployKind {84	type Err = anyhow::Error;85	fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {86		match s {87			"upgrade-to-fleet" => Ok(Self::UpgradeToFleet),88			"fleet" => Ok(Self::Fleet),89			"nixos-install" => Ok(Self::NixosInstall),90			"nixos-lustrate" => Ok(Self::NixosLustrate),91			v => bail!(92				"unknown deploy_kind: {v}; expected on of \"upgrade-to-fleet\", \"fleet\", \"nixos-install\", \"nixos-lustrate\""93			),94		}95	}96}97pub struct ConfigHost {98	config: Config,99	pub name: String,100	groups: OnceLock<Vec<String>>,101102	// TODO: Both of those values are taken from host opts, there should be a cleaner way to specify it103	deploy_kind: OnceCell<DeployKind>,104	session_destination: OnceLock<String>,105	legacy_ssh_store: OnceLock<bool>,106107	pub host_config: Option<Value>,108	pub nixos_config: OnceLock<Value>,109	pub nixos_unchecked_config: OnceLock<Value>,110	pub pkgs_override: Option<Value>,111112	// TODO: Move command helpers away with connectivity refactor113	pub local: bool,114	pub remowt: OnceCell<Remowt>,115	nix_store: OnceCell<Arc<Store>>,116	nix_plugin: OnceCell<()>,117}118119const NIX_PLUGIN_ID: u16 = 2;120121fn agents_dir() -> Result<PathBuf> {122	std::env::var_os("REMOWT_AGENTS_DIR")123		.map(PathBuf::from)124		.or_else(|| option_env!("REMOWT_AGENTS_DIR").map(PathBuf::from))125		.ok_or_else(|| {126			anyhow!("no remowt-agents bundle; set REMOWT_AGENTS_DIR to a remowt-agents output")127		})128}129130fn agent_bundle() -> Result<AgentBundle> {131	AgentBundle::from_dir(agents_dir()?)132}133134enum RemoteDerivationMode {135	/// Closure is (pre)signed, remote host is expected to trust our signatures.136	CopySigned,137	/// Closure is not signed, using escalated nix daemon to bypass signature verification.138	CopyPrivileged,139	/// Closure is uploaded to attic, remote host is expected to trust its signature.140	Attic { name: String },141}142143impl ConfigHost {144	pub fn set_session_destination(&self, dest: String) {145		self.session_destination146			.set(dest)147			.expect("session destination is already set")148	}149	pub fn set_deploy_kind(&self, kind: DeployKind) {150		self.deploy_kind151			.set(kind)152			.expect("deploy kind is already set");153	}154	pub fn set_legacy_ssh_store(&self, legacy: bool) {155		self.legacy_ssh_store156			.set(legacy)157			.expect("legacy ssh store is already set")158	}159	pub async fn deploy_kind(&self) -> Result<DeployKind> {160		self.deploy_kind.get_or_try_init(|| async {161			let remowt = self.remowt().await?;162			let fs = remowt.endpoints::<FsClient<_>>();163			let is_fleet_managed = match fs.file_exists(Utf8PathBuf::from("/etc/FLEET_HOST")).await {164				Ok(v) => v,165				Err(e) => {166					bail!("failed to query remote system kind: {e}");167				}168			};169			if !is_fleet_managed {170				bail!(171					"{}",172					indoc::indoc! {"173					host is not marked as managed by fleet174					if you're not trying to lustrate/install system from scratch,175					you should either176						1. manually create /etc/FLEET_HOST file on the target host,177						2. use ?deploy_kind=fleet host argument if you're upgrading from older version of fleet178						3. use ?deploy_kind=upgrade_to_fleet if you're upgrading from plain nixos to fleet-managed nixos179					for installation use ?deploy_kind=nixos_install / ?deploy_kind=nixos_lustrate 180				"}181				);182			}183			Ok(DeployKind::Fleet)184		}).await.copied()185	}186	async fn new_remowt(&self) -> Result<Remowt> {187		let bundle = agent_bundle()?;188		let conn = if self.local {189			Remowt::connect_local(&bundle, "remowt-fleet".to_owned())190				.await191				.context("starting local remowt agent")?192		} else {193			let dest = self194				.session_destination195				.get()196				.cloned()197				.unwrap_or_else(|| self.name.clone());198			Remowt::connect(&dest, &bundle, "remowt-fleet".to_owned())199				.await200				.map_err(|e| anyhow!("remowt error while connecting to {}: {e:#?}", self.name))?201		};202		PromptEndpoints(PrependSourcePrompter {203			prompter: AutoPrompter::new().await,204			source: if self.local {205				vec![]206			} else {207				vec![Source(std::borrow::Cow::Owned(format!(208					"ssh host: {}",209					self.name210				)))]211			},212			description: "".to_owned(),213		})214		.register_endpoints(&mut conn.rpc());215		Ok(conn)216	}217218	/// Client for this host's unprivileged agent.219	pub async fn remowt(&self) -> Result<Remowt> {220		self.remowt221			.get_or_try_init(|| self.new_remowt())222			.await223			.cloned()224	}225226	pub async fn nix_client(&self) -> Result<NixClient<BifConfig>> {227		let remowt = self.remowt().await?;228		let plugin_id = self.ensure_nix_plugin().await?;229		Ok(remowt.plugin_endpoints(plugin_id))230	}231	pub fn ensure_nix_plugin(&self) -> Pin<Box<dyn Future<Output = Result<u16>> + Send + '_>> {232		Box::pin(async {233			self.nix_plugin234				.get_or_try_init(|| async {235					let pkgs = self.pkgs()?;236					let name = "remowt-plugin-fleet";237					let plugin = nix_go!(pkgs[{ name }]);238					let built = plugin239						.build("out")240						.context("failed to build the fleet nix plugin")?;241					let copied = self242						.remote_derivation(&built)243						.await244						.context("failed to copy the fleet nix plugin to the host store")?;245					let bin = copied.join("bin/remowt-plugin-fleet");246					self.remowt()247						.await?248						.run0_load_plugin_path(NIX_PLUGIN_ID, bin.as_str())249						.await250						.context("failed to load the fleet nix plugin")?;251					anyhow::Ok(())252				})253				.await?;254			Ok(NIX_PLUGIN_ID)255		})256	}257258	async fn new_nix_store(&self) -> Result<Store> {259		let conn = self.remowt().await?;260		let socket = match self.deploy_kind().await? {261			DeployKind::NixosInstall => {262				remowt_fleet::nix_store_socket(conn, "/mnt?require-sigs=false").await?263			}264			_ => remowt_fleet::nix_store_socket(conn, "auto").await?,265		};266		// TODO: Utf8Path267		let uri = format!("unix://{}", socket.display());268		Store::open(&uri)269	}270	async fn nix_store(&self) -> Result<Arc<Store>> {271		self.nix_store272			.get_or_try_init(async || Ok(Arc::new(self.new_nix_store().await?)))273			.await274			.cloned()275	}276277	pub async fn decrypt(&self, data: SecretData) -> Result<Vec<u8>> {278		ensure!(data.encrypted, "secret is not encrypted");279		let remowt = self.remowt().await?;280		let mut cmd = remowt.cmd("fleet-install-secrets");281		cmd.arg("decrypt").eqarg("--secret", data.to_string());282		let encoded = cmd283			.sudo()284			.run_string()285			.await286			.context("failed to call remote host for decrypt")?;287		let data: SecretData = encoded.parse().map_err(|e| anyhow!("{e}"))?;288		ensure!(!data.encrypted, "secret came out encrypted");289		Ok(data.data)290	}291	pub async fn reencrypt_distribution(292		&self,293		data: &FleetSecretDistribution,294		targets: BTreeSet<SecretOwner>,295		now: DateTime<Utc>,296	) -> Result<FleetSecretDistribution> {297		let mut parts = BTreeMap::new();298		for (part_name, part) in &data.secret.parts {299			parts.insert(300				part_name.clone(),301				if part.raw.encrypted {302					FleetSecretPart {303						raw: self.reencrypt(part.raw.clone(), targets.clone()).await?,304					}305				} else {306					part.clone()307				},308			);309		}310		let secret = FleetSecretData {311			created_at: data.secret.created_at,312			expires_at: data.secret.expires_at,313			generation_data: data.secret.generation_data.clone(),314			parts,315		};316		Ok(FleetSecretDistribution::new(targets, secret, now))317	}318	pub async fn reencrypt(319		&self,320		data: SecretData,321		targets: BTreeSet<SecretOwner>,322	) -> Result<SecretData> {323		let remowt = self.remowt().await?;324		ensure!(data.encrypted, "secret is not encrypted");325		let mut cmd = remowt.cmd("fleet-install-secrets");326		cmd.arg("reencrypt").eqarg("--secret", data.to_string());327		for target in targets {328			let key = self.config.key(&target).await?;329			cmd.eqarg("--targets", key);330		}331		let encoded = cmd332			.sudo()333			.run_string()334			.await335			.context("failed to call remote host for decrypt")?;336		let data: SecretData = encoded.parse().map_err(|e| anyhow!("{e}"))?;337		ensure!(data.encrypted, "secret came out not encrypted");338		Ok(data)339	}340	/// Returns path for futureproofing, as path might change i.e on conversion to CA341	#[instrument(skip(self))]342	pub async fn remote_derivation(&self, path: &Utf8Path) -> Result<Utf8PathBuf> {343		let path = path.to_owned();344		if self.local {345			// Path is located locally, thus already trusted.346			return Ok(path);347		}348		let sign: Pin<Box<dyn Future<Output = Result<()>> + Send>> = {349			let path = path.clone();350			let graph = DrvGraph::resolve(&eval_store(), &path)351				.context("failed to resolve graph to be uploaded")?;352			info!("signing {} paths", graph.nodes.len());353354			Box::pin(async move {355				let local = self.config.local_host();356				let plugin_id = local.ensure_nix_plugin().await?;357				let nix = local358					.remowt()359					.await?360					.plugin_endpoints::<remowt_fleet::NixClient<_>>(plugin_id);361				nix.sign_closure(path, Utf8PathBuf::from("/etc/nix/private-key"))362					.await363					.map_err(|e| anyhow!("{e:?}"))?364					.map_err(|e| anyhow!("{e}"))?;365				Ok(())366			})367		};368		if let Err(e) = sign.await {369			warn!("failed to sign store paths: {e}");370		}371		let store = self.nix_store().await?;372		let eval_store = eval_store();373		{374			let path = path.clone();375			spawn_blocking(move || eval_store.copy_to(&store, path.as_ref()))376				.await377				.expect("copy_to panicked")378				.context("copying closure to remote store")?;379		}380		Ok(path)381	}382}383384struct HostSecretDefinition(Value);385386impl ConfigHost {387	// TOCTOU is possible here in case if config is changed, but this case is not handled anywhere anyway,388	// assuming getting tags always returns the same value.389	pub fn tags(&self) -> Result<Vec<String>> {390		if let Some(v) = self.groups.get() {391			return Ok(v.clone());392		}393		let Some(host_config) = &self.host_config else {394			return Ok(vec![]);395		};396		let tags: Vec<String> = nix_go_json!(host_config.tags);397398		let _ = self.groups.set(tags.clone());399400		Ok(tags)401	}402	pub fn nixos_config(&self) -> Result<Value> {403		if let Some(v) = self.nixos_config.get() {404			return Ok(v.clone());405		}406		let Some(host_config) = &self.host_config else {407			bail!("local host has no nixos_config");408		};409		let nixos_config = nix_go!(host_config.nixos.config);410		assert_warn("nixos config evaluation", &nixos_config)?;411412		let _ = self.nixos_config.set(nixos_config.clone());413414		Ok(nixos_config)415	}416	pub fn nixos_unchecked_config(&self) -> Result<Value> {417		if let Some(v) = self.nixos_unchecked_config.get() {418			return Ok(v.clone());419		}420		let Some(host_config) = &self.host_config else {421			bail!("local host has no nixos_config");422		};423		let nixos_config = nix_go!(host_config.nixos_unchecked.config);424425		let _ = self.nixos_unchecked_config.set(nixos_config.clone());426427		Ok(nixos_config)428	}429430	pub fn list_defined_secrets(&self) -> Result<Vec<String>> {431		let nixos = self.nixos_unchecked_config()?;432		let secrets = nix_go!(nixos.secrets);433		secrets.list_fields()434	}435436	/// Packages for this host, resolved with nixpkgs overlays437	pub fn pkgs(&self) -> Result<Value> {438		if let Some(value) = &self.pkgs_override {439			return Ok(value.clone());440		}441		let Some(host_config) = &self.host_config else {442			bail!("local host has no host_config");443		};444		// TODO: Should nixos.options be cached?445		Ok(nix_go!(host_config.nixos.options._module.args.value.pkgs))446	}447}448449#[derive(Clone)]450pub struct SharedSecretDefinition(Value);451impl SharedSecretDefinition {452	pub fn expected_owners(&self) -> Result<BTreeSet<SecretOwner>> {453		let secret = &self.0;454		Ok(nix_go_json!(secret.expectedOwners))455	}456	pub fn allow_different(&self) -> Result<bool> {457		let secret = &self.0;458		Ok(nix_go_json!(secret.allowDifferent))459	}460	pub fn regenerate_on_owner_added(&self) -> Result<bool> {461		let secret = &self.0;462		Ok(nix_go_json!(secret.regenerateOnOwnerAdded))463	}464	pub fn regenerate_on_owner_removed(&self) -> Result<bool> {465		let secret = &self.0;466		Ok(nix_go_json!(secret.regenerateOnOwnerRemoved))467	}468	pub fn generator(&self) -> Result<Value> {469		let secret = &self.0;470		Ok(nix_go!(secret.generator))471	}472}473474impl Config {475	pub fn tagged_hostnames(&self, tag: &str) -> Result<Vec<String>> {476		let config = &self.config_field;477		let tagged: Vec<String> = nix_go_json!(config.taggedWith[{ tag }]);478		Ok(tagged)479	}480	pub fn expand_owner_set(&self, owners: Vec<String>) -> Result<BTreeSet<String>> {481		let mut out = BTreeSet::new();482		for owner in owners {483			if let Some(tag) = owner.strip_prefix('@') {484				let hosts = self.tagged_hostnames(tag)?;485				out.extend(hosts);486			} else {487				out.insert(owner);488			}489		}490		Ok(out)491	}492	pub fn local_host(&self) -> Arc<ConfigHost> {493		self.local_host494			.get_or_init(|| {495				Arc::new(ConfigHost {496					config: self.clone(),497					name: "<virtual localhost>".to_owned(),498					host_config: None,499					nixos_config: OnceLock::new(),500					nixos_unchecked_config: OnceLock::new(),501					groups: {502						let cell = OnceLock::new();503						let _ = cell.set(vec![]);504						cell505					},506					pkgs_override: Some(self.default_pkgs.clone()),507508					local: true,509					remowt: OnceCell::new(),510					nix_store: OnceCell::new(),511					nix_plugin: OnceCell::new(),512					deploy_kind: OnceCell::new(),513					session_destination: OnceLock::new(),514					legacy_ssh_store: OnceLock::new(),515				})516			})517			.clone()518	}519520	pub fn preferred_hosts(521		&self,522		filter: impl Fn(&str) -> bool,523	) -> Result<impl Iterator<Item = Result<ConfigHost>>> {524		let prefer = self525			.prefer_identities526			.iter()527			.filter_map(|v| v.as_host())528			.collect::<HashSet<_>>();529		let config = &self.config_field;530		let mut names = nix_go!(config.hosts).list_fields()?;531		names.retain(|s| filter(s));532		names.sort_by_key(|h| prefer.contains(h.as_str()));533534		Ok(names.into_iter().map(|h| self.host(&h)))535	}536537	pub fn host(&self, name: &str) -> Result<ConfigHost> {538		let config = &self.config_field;539		let host_config = nix_go!(config.hosts[{ name }]);540541		Ok(ConfigHost {542			config: self.clone(),543			name: name.to_owned(),544			host_config: Some(host_config),545			nixos_config: OnceLock::new(),546			nixos_unchecked_config: OnceLock::new(),547			groups: OnceLock::new(),548			pkgs_override: None,549550			// TODO: Remove with connectivit refactor551			local: self.localhost == name,552			remowt: OnceCell::new(),553			nix_store: OnceCell::new(),554			nix_plugin: OnceCell::new(),555			deploy_kind: OnceCell::new(),556			session_destination: OnceLock::new(),557			legacy_ssh_store: OnceLock::new(),558		})559	}560	pub fn list_hosts(&self) -> Result<Vec<ConfigHost>> {561		let config = &self.config_field;562		let names = nix_go!(config.hosts).list_fields()?;563		let mut out = vec![];564		for name in names {565			out.push(self.host(&name)?);566		}567		Ok(out)568	}569	// TODO: Replace usages with .host().nixos_config570	pub fn system_config(&self, host: &str) -> Result<Value> {571		let fleet_field = &self.config_field;572		Ok(nix_go!(fleet_field.hosts[{ host }].nixos.config))573	}574575	pub fn secret_definition(&self, secret: &str) -> Result<Option<SharedSecretDefinition>> {576		let config = &self.config_field;577		let shared_secrets = nix_go!(config.secrets);578		if !shared_secrets.has_field(secret)? {579			return Ok(None);580		}581		Ok(Some(SharedSecretDefinition(nix_go!(582			shared_secrets[secret]583		))))584	}585586	pub fn save(&self) -> Result<()> {587		let mut tempfile = NamedTempFile::new_in(self.directory.clone()).context("failed to create updated version of fleet.nix in the same directory as original.\nDo you have write access to it? Access only to the fleet.nix won't be enough, the directory is used for atomic overwrite operation.\nIt is not recommended to use fleet by root anyway, move fleet project to your home directory.")?;588		let data = nixlike::serialize(&*self.data)?;589		tempfile.write_all(590			format!(591				"# This file contains fleet state and shouldn't be edited by hand\n\n{data}\n\n# vim: ts=2 et nowrap\n"592			)593			.as_bytes(),594		)?;595		let mut fleet_data_path = self.directory.clone();596		fleet_data_path.push("fleet.nix");597		tempfile.persist(fleet_data_path)?;598		Ok(())599	}600}
modifiedpkgs/default.nixdiffbeforeafterboth
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -7,7 +7,7 @@
   remowt-agents-bundle = callPackage ./remowt-agents-bundle.nix { inherit craneLib; };
 in
 {
-  fleet = callPackage ./fleet.nix { inherit craneLib inputs; };
+  fleet = callPackage ./fleet.nix { inherit craneLib inputs remowt-agents-bundle; };
   fleet-install-secrets = callPackage ./fleet-install-secrets.nix { inherit craneLib; };
   fleet-generator-helper = callPackage ./fleet-generator-helper.nix { inherit craneLib; };