git.delta.rocks / jrsonnet / refs/commits / f5a8281dad2c

--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1055,6 +1055,7 @@
  "shlex",
  "tabled",
  "tempfile",
+ "thiserror 2.0.17",
  "time",
  "tokio",
  "tokio-util",
@@ -1087,6 +1088,7 @@
  "serde_json",
  "tabled",
  "tempfile",
+ "thiserror 2.0.17",
  "time",
  "tokio",
  "tokio-util",

--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -47,6 +47,7 @@
 nom = "8.0.0"
 opentelemetry = "0.30.0"
 opentelemetry_sdk = "0.30.0"
+thiserror.workspace = true
 tracing-indicatif = { version = "0.3", optional = true }
 tracing-opentelemetry = "0.31.0"
 

--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -2,17 +2,18 @@
 	collections::{BTreeMap, BTreeSet, HashSet},
 	io::{self, Read, Write, stdin, stdout},
 	path::PathBuf,
-	slice,
 };
 
-use age::Recipient;
 use anyhow::{Context, Result, anyhow, bail, ensure};
 use chrono::{DateTime, Utc};
 use clap::Parser;
 use fleet_base::{
-	fleetdata::{FleetSecret, FleetSecretPart, FleetSharedSecret, encrypt_secret_data},
+	fleetdata::{
+		FleetHostSecret, FleetSecretData, FleetSecretPart, FleetSharedSecret, encrypt_secret_data,
+	},
 	host::Config,
 	opts::FleetOpts,
+	secret::{Expectations, RegenerationReason, SharedSecretDefinition, secret_needs_regeneration},
 };
 use fleet_shared::SecretData;
 use nix_eval::{NixType, Value, nix_go, nix_go_json};
@@ -144,76 +145,55 @@
 		#[clap(short = 'p', long, default_value = "secret")]
 		part: String,
 	},
-}
-
-fn secret_needs_regeneration(
-	secret: &FleetSecret,
-	expected_generation_data: &serde_json::Value,
-) -> bool {
-	let data_is_expected = secret.generation_data == *expected_generation_data;
-	// TODO: Leeway?
-	let expired = secret
-		.expires_at
-		.map(|expiration| expiration < Utc::now())
-		.unwrap_or(false);
-	expired || !data_is_expected
 }
 
 #[allow(clippy::too_many_arguments)]
-#[tracing::instrument(skip(config, secret, field, prefer_identities))]
+#[tracing::instrument(skip(config, secret, definition, prefer_identities))]
 async fn maybe_regenerate_shared_secret(
 	secret_name: &str,
 	config: &Config,
 	mut secret: FleetSharedSecret,
-	field: Value,
-	expected_owners: &[String],
-	expected_generation_data: serde_json::Value,
+	definition: SharedSecretDefinition,
 	prefer_identities: &[String],
+	expectations: &Expectations,
 ) -> Result<FleetSharedSecret> {
-	let original_set = secret.owners.clone();
+	let reason = secret_needs_regeneration(&secret.secret, &secret.owners, expectations);
+	let value = definition.inner();
 
-	let set = original_set.iter().collect::<BTreeSet<_>>();
-	let expected_set = expected_owners.iter().collect::<BTreeSet<_>>();
-
-	let regeneration_required =
-		secret_needs_regeneration(&secret.secret, &expected_generation_data);
-
-	if set == expected_set && !regeneration_required {
-		info!("no need to update owner list, it is already correct");
-		return Ok(secret);
-	}
-
-	let should_regenerate = if regeneration_required {
-		info!("secret has its generation data changed, regeneration is required");
-		true
-	} else if set.difference(&expected_set).next().is_some() {
-		// TODO: Remove this warning for revokable secrets.
-		warn!(
-			"host was removed from secret owners, but until this host rebuild, the secret will still be stored on it."
-		);
-		nix_go_json!(field.regenerateOnOwnerRemoved)
-	} else if expected_set.difference(&set).next().is_some() {
-		nix_go_json!(field.regenerateOnOwnerAdded)
-	} else {
-		false
+	let (should_reencrypt, reason) = match reason {
+		Some(RegenerationReason::OwnersAdded(_)) => {
+			// Secret always needs to be reencrypted for new owners to be able to read it
+			(
+				true,
+				if nix_go_json!(value.regenerateOnOwnerAdded) {
+					reason
+				} else {
+					None
+				},
+			)
+		}
+		Some(RegenerationReason::OwnersRemoved(_)) => {
+			// No need to reencrypt, we can just leave stanzas in place.
+			if nix_go_json!(value.regenerateOnOwnerRemoved) {
+				(true, reason)
+			} else {
+				(false, None)
+			}
+		}
+		Some(_) => (true, reason),
+		None => (false, None),
 	};
 
-	if should_regenerate {
-		info!("secret needs to be regenerated");
-		let generated = generate_shared(
-			config,
-			secret_name,
-			field,
-			expected_owners.to_vec(),
-			expected_generation_data,
-		)
-		.await?;
+	if let Some(reason) = reason {
+		info!("secret needs to be regenerated: {reason}");
+		let generated = generate_shared(config, secret_name, definition, expectations).await?;
 		Ok(generated)
-	} else {
+	} else if should_reencrypt {
+		info!("secret needs to be reencrypted");
 		let identity_holder = if !prefer_identities.is_empty() {
 			prefer_identities
 				.iter()
-				.find(|i| original_set.iter().any(|s| s == *i))
+				.find(|i| secret.owners.iter().any(|s| s == *i))
 		} else {
 			secret.owners.first()
 		};
@@ -228,12 +208,16 @@
 			}
 			let host = config.host(identity_holder).await?;
 			let encrypted = host
-				.reencrypt(part.raw.clone(), expected_owners.to_vec())
+				.reencrypt(
+					part.raw.clone(),
+					expectations.owners.iter().cloned().collect(),
+				)
 				.await?;
 			part.raw = encrypted;
 		}
-
-		secret.owners = expected_owners.to_vec();
+		secret.owners = expectations.owners.clone();
+		Ok(secret)
+	} else {
 		Ok(secret)
 	}
 }
@@ -250,8 +234,8 @@
 	_display_name: &str,
 	_secret: Value,
 	_default_generator: Value,
-	_owners: &[String],
-) -> Result<FleetSecret> {
+	_expectations: &Expectations,
+) -> Result<FleetSecretData> {
 	bail!("pure generators are broken for now")
 }
 async fn generate_impure(
@@ -259,9 +243,8 @@
 	_display_name: &str,
 	secret: Value,
 	default_generator: Value,
-	expected_owners: &[String],
-	expected_generation_data: serde_json::Value,
-) -> Result<FleetSecret> {
+	expectations: &Expectations,
+) -> Result<FleetSecretData> {
 	let generator = nix_go!(secret.generator);
 	let on: Option<String> = nix_go_json!(default_generator.impureOn);
 
@@ -276,12 +259,11 @@
 	let mk_secret_generators = nix_go!(on_pkgs.mkSecretGenerators);
 
 	let mut recipients = Vec::new();
-	for owner in expected_owners {
+	for owner in &expectations.owners {
 		let key = config.key(owner).await?;
 		recipients.push(key);
 	}
 	let generators = nix_go!(mk_secret_generators(Obj { recipients }));
-	// FIXME: Apparently, // operator is slow in nix
 	let pkgs_and_generators = on_pkgs.attrs_update(generators)?;
 
 	let call_package = nix_go!(nixpkgs.lib.callPackageWith(pkgs_and_generators));
@@ -331,20 +313,26 @@
 	let created_at = host.read_file_value(format!("{out}/created_at")).await?;
 	let expires_at = host.read_file_value(format!("{out}/expires_at")).await.ok();
 
-	Ok(FleetSecret {
+	let new_data = FleetSecretData {
 		created_at,
 		expires_at,
 		parts,
-		generation_data: expected_generation_data,
-	})
+		generation_data: expectations.generation_data.clone(),
+	};
+
+	if let Some(reason) = secret_needs_regeneration(&new_data, &expectations.owners, expectations) {
+		bail!("newly generated secret needs to be regenerated: {reason}")
+	}
+
+	Ok(new_data)
 }
+
 async fn generate(
 	config: &Config,
 	display_name: &str,
 	secret: Value,
-	expected_owners: &[String],
-	expected_generation_data: serde_json::Value,
-) -> Result<FleetSecret> {
+	expectations: &Expectations,
+) -> Result<FleetSecretData> {
 	let generator = nix_go!(secret.generator);
 	// Can't properly check on nix module system level
 	{
@@ -388,8 +376,7 @@
 				display_name,
 				secret,
 				default_generator,
-				expected_owners,
-				expected_generation_data,
+				expectations,
 			)
 			.await
 		}
@@ -399,7 +386,7 @@
 				display_name,
 				secret,
 				default_generator,
-				expected_owners,
+				expectations,
 			)
 			.await
 		}
@@ -408,21 +395,14 @@
 async fn generate_shared(
 	config: &Config,
 	display_name: &str,
-	secret: Value,
-	expected_owners: Vec<String>,
-	expected_generation_data: serde_json::Value,
+	secret: SharedSecretDefinition,
+	expectations: &Expectations,
 ) -> Result<FleetSharedSecret> {
 	// let owners: Vec<String> = nix_go_json!(secret.expectedOwners);
 	Ok(FleetSharedSecret {
-		secret: generate(
-			config,
-			display_name,
-			secret,
-			&expected_owners,
-			expected_generation_data,
-		)
-		.await?,
-		owners: expected_owners,
+		managed: Some(true),
+		secret: generate(config, display_name, secret.inner(), expectations).await?,
+		owners: expectations.owners.clone(),
 	})
 }
 
@@ -457,11 +437,11 @@
 }
 
 fn parse_machines(
-	initial: Vec<String>,
+	initial: BTreeSet<String>,
 	machines: Option<Vec<String>>,
 	mut add_machines: Vec<String>,
 	mut remove_machines: Vec<String>,
-) -> Result<Vec<String>> {
+) -> Result<BTreeSet<String>> {
 	if machines.is_none() && add_machines.is_empty() && remove_machines.is_empty() {
 		bail!("no operation");
 	}
@@ -470,7 +450,6 @@
 	let mut target_machines = initial;
 	info!("Currently encrypted for {initial_machines:?}");
 
-	// ensure!(machines.is_some() || !add_machines.is_empty() || )
 	if let Some(machines) = machines {
 		ensure!(
 			add_machines.is_empty() && remove_machines.is_empty(),
@@ -487,20 +466,13 @@
 	}
 
 	for machine in &remove_machines {
-		let mut removed = false;
-		while let Some(pos) = target_machines.iter().position(|m| m == machine) {
-			target_machines.swap_remove(pos);
-			removed = true;
-		}
-		if !removed {
+		if !target_machines.remove(machine) {
 			warn!("secret is not enabled for {machine}");
 		}
 	}
 	for machine in &add_machines {
-		if target_machines.iter().any(|m| m == machine) {
+		if !target_machines.insert(machine.to_owned()) {
 			warn!("secret is already added to {machine}");
-		} else {
-			target_machines.push(machine.to_owned());
 		}
 	}
 	if !remove_machines.is_empty() {
@@ -527,7 +499,7 @@
 				}
 			}
 			Secret::AddShared {
-				mut machines,
+				machines,
 				name,
 				force,
 				public,
@@ -537,25 +509,32 @@
 				re_add,
 				part: part_name,
 			} => {
+				let mut machines: BTreeSet<String> = machines.into_iter().collect();
 				// TODO: Forbid updating secrets with set expectedOwners (= not user-managed).
 
-				let exists = config.has_shared(&name);
-				if exists && !force && !re_add {
-					bail!("secret already defined");
-				}
-				if re_add {
-					// Fixme: use clap to limit this usage
-					ensure!(!force, "--force and --readd are not compatible");
-					ensure!(exists, "secret doesn't exists");
-					ensure!(
-						machines.is_empty(),
-						"you can't use machines argument for --readd"
-					);
-					let shared = config.shared_secret(&name)?;
-					machines = shared.owners;
-				}
+				if let Some(old_shared) = config.shared_secret(&name)? {
+					if !force && !re_add {
+						bail!("secret already defined");
+					};
+					if old_shared.managed.unwrap_or(false) {
+						bail!("secret is marked as managed, should not be updated manually");
+					};
+					if re_add {
+						// Fixme: use clap to limit this usage
+						ensure!(!force, "--force and --readd are not compatible");
+						ensure!(
+							machines.is_empty(),
+							"you can't use machines argument for --readd"
+						);
+						machines = old_shared.owners;
+					}
+				} else if re_add {
+					bail!("secret doesn't exists");
+				};
 
-				let recipients = config.recipients(machines.clone()).await?;
+				let recipients = config
+					.recipients(machines.iter().cloned().collect())
+					.await?;
 
 				let mut parts = BTreeMap::new();
 
@@ -563,9 +542,8 @@
 				io::stdin().read_to_end(&mut input)?;
 
 				if !input.is_empty() {
-					let encrypted =
-						encrypt_secret_data(recipients.iter().map(|r| r as &dyn Recipient), input)
-							.ok_or_else(|| anyhow!("no recipients provided"))?;
+					let encrypted = encrypt_secret_data(recipients.iter(), input)
+						.ok_or_else(|| anyhow!("no recipients provided"))?;
 					parts.insert(part_name, FleetSecretPart { raw: encrypted });
 				}
 
@@ -576,8 +554,9 @@
 				config.replace_shared(
 					name,
 					FleetSharedSecret {
+						managed: Some(false),
 						owners: machines,
-						secret: FleetSecret {
+						secret: FleetSecretData {
 							created_at: Utc::now(),
 							expires_at,
 							parts,
@@ -607,34 +586,47 @@
 						.host_secret(&machine, &name)
 						.context("failed to read existing secret for --merge")?
 				} else {
-					FleetSecret {
-						created_at: Utc::now(),
-						expires_at: None,
-						parts: BTreeMap::new(),
-						generation_data: serde_json::Value::Null,
+					FleetHostSecret {
+						managed: Some(false),
+						secret: FleetSecretData {
+							created_at: Utc::now(),
+							expires_at: None,
+							parts: BTreeMap::new(),
+							generation_data: serde_json::Value::Null,
+						},
 					}
 				};
+				if out.managed.unwrap_or(false) {
+					bail!("secret is managed by fleet and should not be updated manually");
+				}
+				out.managed = Some(false);
 
 				if let Some(secret) = parse_secret().await? {
 					let recipient = config.recipient(&machine).await?;
-					let encrypted = encrypt_secret_data([&recipient as &dyn Recipient], secret)
-						.expect("recipient provided");
+					let encrypted =
+						encrypt_secret_data([&recipient], secret).expect("recipient provided");
 					if out
+						.secret
 						.parts
 						.insert(part_name.clone(), FleetSecretPart { raw: encrypted })
 						.is_some() && !replace
 					{
-						bail!("part {part_name:?} is already defined");
+						bail!(
+							"part {part_name:?} is already defined, use --replace if you wish to replace it"
+						);
 					}
 				}
 
 				if let Some(public) = parse_public(public, public_file).await? {
 					if out
+						.secret
 						.parts
 						.insert(public_name.clone(), FleetSecretPart { raw: public })
 						.is_some() && !replace
 					{
-						bail!("part {public_name:?} is already defined");
+						bail!(
+							"part {public_name:?} is already defined, use --replace if you wish to replace it"
+						);
 					}
 				};
 
@@ -647,7 +639,7 @@
 				part: part_name,
 			} => {
 				let secret = config.host_secret(&machine, &name)?;
-				let Some(secret) = secret.parts.get(&part_name) else {
+				let Some(secret) = secret.secret.parts.get(&part_name) else {
 					bail!("no part {part_name} in secret {name}");
 				};
 				let data = if secret.raw.encrypted {
@@ -664,7 +656,9 @@
 				part: part_name,
 				prefer_identities,
 			} => {
-				let secret = config.shared_secret(&name)?;
+				let Some(secret) = config.shared_secret(&name)? else {
+					bail!("secret doesn't exists");
+				};
 				let Some(part) = secret.secret.parts.get(&part_name) else {
 					bail!("no part {part_name} in secret {name}");
 				};
@@ -695,7 +689,9 @@
 			} => {
 				// TODO: Forbid updating secrets with set expectedOwners (= not user-managed).
 
-				let secret = config.shared_secret(&name)?;
+				let Some(secret) = config.shared_secret(&name)? else {
+					bail!("secret doesn't exists");
+				};
 				if secret.secret.parts.values().all(|v| !v.raw.encrypted) {
 					bail!("no secret");
 				}
@@ -714,20 +710,16 @@
 					return Ok(());
 				}
 
-				let config_field = &config.config_field;
-				let name_clone = name.clone();
-				let field = nix_go!(config_field.sharedSecrets[name_clone]);
-				let expected_generation_data = nix_go_json!(field.expectedGenerationData);
+				let definition = config.shared_secret_definition(&name)?;
+				let expectations = definition.expectations()?;
 
 				let updated = maybe_regenerate_shared_secret(
 					&name,
 					config,
 					secret,
-					field,
-					&target_machines,
-					expected_generation_data,
+					definition,
 					&prefer_identities,
-					// None,
+					&expectations,
 				)
 				.await?;
 				config.replace_shared(name, updated);
@@ -737,36 +729,26 @@
 				skip_hosts,
 			} => {
 				info!("checking for secrets to regenerate");
+				let expected_shared_set = config
+					.list_configured_shared()
+					.await?
+					.into_iter()
+					.collect::<HashSet<_>>();
 				let stored_shared_set = config.list_shared().into_iter().collect::<HashSet<_>>();
 				{
 					// Generate missing shared
 					let _span = info_span!("shared").entered();
-					let expected_shared_set = config
-						.list_configured_shared()
-						.await?
-						.into_iter()
-						.collect::<HashSet<_>>();
 					for missing in expected_shared_set.difference(&stored_shared_set) {
-						let config_field = &config.config_field;
-						let secret = nix_go!(config_field.sharedSecrets[{ missing }]);
-						let expected_generation_data: serde_json::Value =
-							nix_go_json!(secret.expectedGenerationData);
-						let expected_owners: Option<Vec<String>> =
-							nix_go_json!(secret.expectedOwners);
-						let Some(expected_owners) = expected_owners else {
-							// Can't generate this missing secret, as it has no defined owners.
+						let definition = config.shared_secret_definition(missing)?;
+						if !definition.is_managed()? {
+							info!("skipping unmanaged secret: {missing}");
 							continue;
-						};
+						}
+						let expectations = definition.expectations()?;
 						info!("generating secret: {missing}");
-						let shared = generate_shared(
-							config,
-							missing,
-							secret,
-							expected_owners,
-							expected_generation_data,
-						)
-						.in_current_span()
-						.await?;
+						let shared = generate_shared(config, missing, definition, &expectations)
+							.in_current_span()
+							.await?;
 						config.replace_shared(missing.to_string(), shared)
 					}
 				}
@@ -778,26 +760,22 @@
 
 						let _span = info_span!("host", host = host.name).entered();
 						let expected_set = host
-							.list_configured_secrets()
-							.in_current_span()
-							.await?
+							.list_defined_secrets()?
 							.into_iter()
 							.collect::<HashSet<_>>();
 						let stored_set = config
 							.list_secrets(&host.name)
 							.into_iter()
 							.collect::<HashSet<_>>();
-						for missing in expected_set.difference(&stored_set) {
-							info!("generating secret: {missing}");
-							let secret = host.secret_field(missing).in_current_span().await?;
-							let expected_generation_data =
-								nix_go_json!(secret.expectedGenerationData);
+						for missing_secret in expected_set.difference(&stored_set) {
+							info!("generating missing secret: {missing_secret}");
+							let definition = host.secret_definition(missing_secret)?;
+							let expectations = definition.expectations()?;
 							let generated = match generate(
 								config,
-								missing,
-								secret,
-								slice::from_ref(&host.name),
-								expected_generation_data,
+								missing_secret,
+								definition.inner(),
+								&expectations,
 							)
 							.in_current_span()
 							.await
@@ -808,21 +786,27 @@
 									continue;
 								}
 							};
-							config.insert_secret(&host.name, missing.to_string(), generated)
+							config.insert_secret(
+								&host.name,
+								missing_secret.to_string(),
+								FleetHostSecret {
+									managed: Some(true),
+									secret: generated,
+								},
+							)
 						}
-						for name in stored_set {
-							info!("updating secret: {name}");
-							let data = config.host_secret(&host.name, &name)?;
-							let secret = host.secret_field(&name).in_current_span().await?;
-							let expected_generation_data =
-								nix_go_json!(secret.expectedGenerationData);
-							if secret_needs_regeneration(&data, &expected_generation_data) {
+						for known_secret in stored_set.intersection(&expected_set) {
+							info!("updating secret: {known_secret}");
+							let data = config.host_secret(&host.name, known_secret)?;
+							let definition = host.secret_definition(known_secret)?;
+							let expectations = definition.expectations()?;
+							if let Some(regen_reason) = data.needs_regeneration(&expectations) {
+								info!("needs regeneration: {regen_reason}");
 								let generated = match generate(
 									config,
-									&name,
-									secret,
-									slice::from_ref(&host.name),
-									expected_generation_data,
+									known_secret,
+									definition.inner(),
+									&expectations,
 								)
 								.in_current_span()
 								.await
@@ -833,43 +817,44 @@
 										continue;
 									}
 								};
-								config.insert_secret(&host.name, name.to_string(), generated)
+								config.insert_secret(
+									&host.name,
+									known_secret.to_string(),
+									FleetHostSecret {
+										managed: Some(true),
+										secret: generated,
+									},
+								)
 							}
 						}
+						for removed_secret in stored_set.difference(&expected_set) {
+							info!("removing secret: {removed_secret}");
+							config.remove_secret(&host.name, removed_secret);
+						}
 					}
 				}
-				let mut to_remove = Vec::new();
-				for name in &stored_shared_set {
-					info!("updating secret: {name}");
-					let data = config.shared_secret(name)?;
-					let config_field = &config.config_field;
-					let expected_owners: Option<Vec<String>> =
-						nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners);
-					let Some(expected_owners) = expected_owners else {
-						warn!("secret was removed from fleet config: {name}, removing from data");
-						to_remove.push(name.to_string());
-						continue;
-					};
+				for known_secret in stored_shared_set.intersection(&expected_shared_set) {
+					info!("updating shared secret: {known_secret}");
+					let data = config.shared_secret(known_secret)?.expect("exists");
 
-					let secret = nix_go!(config_field.sharedSecrets[{ name }]);
-					let expected_generation_data = nix_go_json!(secret.expectedGenerationData);
+					let definition = config.shared_secret_definition(known_secret)?;
+					let expectations = definition.expectations()?;
 					config.replace_shared(
-						name.to_owned(),
+						known_secret.to_owned(),
 						maybe_regenerate_shared_secret(
-							name,
+							known_secret,
 							config,
 							data,
-							secret,
-							&expected_owners,
-							expected_generation_data,
+							definition,
 							&prefer_identities,
-							// None,
+							&expectations,
 						)
 						.await?,
 					);
 				}
-				for k in to_remove {
-					config.remove_shared(&k);
+				for removed_secret in stored_shared_set.difference(&expected_shared_set) {
+					info!("removing shared secret: {removed_secret}");
+					config.remove_shared(removed_secret);
 				}
 			}
 			Secret::List {} => {
@@ -885,13 +870,14 @@
 				let mut table = vec![];
 				for name in configured.iter().cloned() {
 					let config = config.clone();
-					let expected_owners = config.shared_secret_expected_owners(&name).await?;
-					let data = config.shared_secret(&name)?;
+					let data = config.shared_secret(&name)?.expect("exists");
+					let definition = config.shared_secret_definition(&name)?;
+					let expectations = definition.expectations()?;
 					let owners = data
 						.owners
 						.iter()
 						.map(|o| {
-							if expected_owners.contains(o) {
+							if expectations.owners.contains(o) {
 								o.green().to_string()
 							} else {
 								o.red().to_string()
@@ -912,7 +898,7 @@
 				add,
 			} => {
 				let secret = config.host_secret(&machine, &name)?;
-				if let Some(data) = secret.parts.get(&part) {
+				if let Some(data) = secret.secret.parts.get(&part) {
 					let host = config.host(&machine).await?;
 					let secret = host.decrypt(data.raw.clone()).await?;
 					String::from_utf8(secret).context("secret is not utf8")?

--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -27,7 +27,7 @@
 use tracing::{Instrument, error, info, info_span};
 #[cfg(feature = "indicatif")]
 use tracing_indicatif::IndicatifLayer;
-use tracing_subscriber::{EnvFilter, fmt::format::Format, prelude::*};
+use tracing_subscriber::{EnvFilter, prelude::*};
 
 #[derive(Parser)]
 struct Prefetch {}

--- a/crates/fleet-base/Cargo.toml
+++ b/crates/fleet-base/Cargo.toml
@@ -24,6 +24,7 @@
 serde_json = "1.0.140"
 tabled = "0.20.0"
 tempfile.workspace = true
+thiserror.workspace = true
 time = { version = "0.3.41", features = ["parsing"] }
 tokio.workspace = true
 tokio-util = "0.7.15"

1use std::{1use std::{
2	collections::BTreeMap,2	collections::{BTreeMap, BTreeSet},
3	io::{self, Cursor},3	io::{self, Cursor},
4};4};
55
13use serde::{Deserialize, Serialize, de::Error};13use serde::{Deserialize, Serialize, de::Error};
14use serde_json::Value;14use serde_json::Value;
15
16use crate::secret::{Expectations, RegenerationReason, secret_needs_regeneration};
1517
16#[derive(Serialize, Deserialize, Default)]18#[derive(Serialize, Deserialize, Default)]
17#[serde(rename_all = "camelCase")]19#[serde(rename_all = "camelCase")]
75	pub shared_secrets: BTreeMap<String, FleetSharedSecret>,77	pub shared_secrets: BTreeMap<String, FleetSharedSecret>,
76	#[serde(default)]78	#[serde(default)]
77	#[serde(skip_serializing_if = "BTreeMap::is_empty")]79	#[serde(skip_serializing_if = "BTreeMap::is_empty")]
78	pub host_secrets: BTreeMap<String, BTreeMap<String, FleetSecret>>,80	pub host_secrets: BTreeMap<String, BTreeMap<String, FleetHostSecret>>,
7981
80	// extra_name => anything82	// extra_name => anything
81	#[serde(default)]83	#[serde(default)]
82	#[serde(skip_serializing_if = "BTreeMap::is_empty")]84	#[serde(skip_serializing_if = "BTreeMap::is_empty")]
83	pub extra: BTreeMap<String, Value>,85	pub extra: BTreeMap<String, Value>,
84}86}
85
86#[derive(Serialize, Deserialize, Clone)]
87#[serde(rename_all = "camelCase")]
88#[must_use]
89pub struct FleetSharedSecret {
90	pub owners: Vec<String>,
91	#[serde(flatten)]
92	pub secret: FleetSecret,
93}
9487
95/// Returns None if recipients.is_empty()88/// Returns None if recipients.is_empty()
96pub fn encrypt_secret_data<'a>(89pub fn encrypt_secret_data<'r>(
97	recipients: impl IntoIterator<Item = &'a dyn Recipient>,90	recipients: impl IntoIterator<Item = &'r Box<dyn Recipient>>,
98	data: Vec<u8>,91	data: Vec<u8>,
99) -> Option<SecretData> {92) -> Option<SecretData> {
100	let mut encrypted = vec![];93	let mut encrypted = vec![];
101	let mut encryptor = age::Encryptor::with_recipients(recipients.into_iter())94	let mut encryptor = age::Encryptor::with_recipients(recipients.into_iter().map(|v| &**v))
102		.ok()?95		.ok()?
103		.wrap_output(&mut encrypted)96		.wrap_output(&mut encrypted)
104		.expect("in memory write");97		.expect("in memory write");
118#[derive(Serialize, Deserialize, Clone)]111#[derive(Serialize, Deserialize, Clone)]
119#[serde(rename_all = "camelCase")]112#[serde(rename_all = "camelCase")]
120#[must_use]113#[must_use]
121pub struct FleetSecret {114pub struct FleetSecretData {
122	#[serde(default = "Utc::now")]115	#[serde(default = "Utc::now")]
123	pub created_at: DateTime<Utc>,116	pub created_at: DateTime<Utc>,
124	#[serde(default)]117	#[serde(default)]
133	pub generation_data: Value,126	pub generation_data: Value,
134}127}
128
129#[derive(Serialize, Deserialize, Clone)]
130#[serde(rename_all = "camelCase")]
131#[must_use]
132pub struct FleetHostSecret {
133	#[serde(default)]
134	#[serde(skip_serializing_if = "Option::is_none")]
135	pub managed: Option<bool>,
136	#[serde(flatten)]
137	pub secret: FleetSecretData,
138}
139impl FleetHostSecret {
140	pub fn needs_regeneration(&self, expectations: &Expectations) -> Option<RegenerationReason> {
141		secret_needs_regeneration(&self.secret, &expectations.owners, expectations)
142	}
143}
144
145#[derive(Serialize, Deserialize, Clone)]
146#[serde(rename_all = "camelCase")]
147#[must_use]
148pub struct FleetSharedSecret {
149	#[serde(default)]
150	#[serde(skip_serializing_if = "Option::is_none")]
151	pub managed: Option<bool>,
152	pub owners: BTreeSet<String>,
153	#[serde(flatten)]
154	pub secret: FleetSecretData,
155}
135156

--- a/crates/fleet-base/src/host.rs
+++ b/crates/fleet-base/src/host.rs
@@ -22,7 +22,8 @@
 
 use crate::{
 	command::MyCommand,
-	fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
+	fleetdata::{FleetData, FleetHostSecret, FleetSharedSecret},
+	secret::{HostSecretDefinition, SharedSecretDefinition},
 };
 
 pub struct FleetConfigInternals {
@@ -234,7 +235,7 @@
 		let is_fleet_managed = match self.file_exists("/etc/FLEET_HOST").await {
 			Ok(v) => v,
 			Err(e) => {
-				bail!("failed to query remote system kind: {}", e);
+				bail!("failed to query remote system kind: {e}");
 			}
 		};
 		if !is_fleet_managed {
@@ -501,7 +502,7 @@
 
 		Ok(nixos_config)
 	}
-	pub async fn nixos_unchecked_config(&self) -> Result<Value> {
+	pub fn nixos_unchecked_config(&self) -> Result<Value> {
 		if let Some(v) = self.nixos_unchecked_config.get() {
 			return Ok(v.clone());
 		}
@@ -515,23 +516,17 @@
 		Ok(nixos_config)
 	}
 
-	pub async fn list_configured_secrets(&self) -> Result<Vec<String>> {
-		let nixos = self.nixos_unchecked_config().await?;
+	pub fn list_defined_secrets(&self) -> Result<Vec<String>> {
+		let nixos = self.nixos_unchecked_config()?;
 		let secrets = nix_go!(nixos.secrets);
-		let mut out = Vec::new();
-		for name in secrets.list_fields()? {
-			let secret = secrets.get_field(&name).context("getting secret")?;
-			let is_shared: bool = nix_go_json!(secret.shared);
-			if is_shared {
-				continue;
-			}
-			out.push(name);
-		}
-		Ok(out)
+		secrets.list_fields()
 	}
-	pub async fn secret_field(&self, name: &str) -> Result<Value> {
-		let nixos = self.nixos_unchecked_config().await?;
-		Ok(nix_go!(nixos.secrets[{ name }]))
+	pub fn secret_definition(&self, name: &str) -> Result<HostSecretDefinition> {
+		let nixos = self.nixos_unchecked_config()?;
+		Ok(HostSecretDefinition(
+			self.name.clone(),
+			nix_go!(nixos.secrets[{ name }]),
+		))
 	}
 
 	/// Packages for this host, resolved with nixpkgs overlays
@@ -648,10 +643,19 @@
 
 	pub fn list_secrets(&self, host: &str) -> Vec<String> {
 		let data = self.data();
-		let Some(secrets) = data.host_secrets.get(host) else {
-			return Vec::new();
-		};
-		secrets.keys().cloned().collect()
+		let mut out = data
+			.host_secrets
+			.get(host)
+			.map(|s| s.keys().cloned().collect::<Vec<String>>())
+			.unwrap_or_default();
+
+		for (name, shared) in data.shared_secrets.iter() {
+			if shared.owners.contains(host) {
+				out.push(name.clone());
+			}
+		}
+
+		out
 	}
 
 	pub fn has_secret(&self, host: &str, secret: &str) -> bool {
@@ -661,34 +665,44 @@
 		};
 		host_secrets.contains_key(secret)
 	}
-	pub fn insert_secret(&self, host: &str, secret: String, value: FleetSecret) {
+	pub fn insert_secret(&self, host: &str, secret: String, value: FleetHostSecret) {
 		let mut data = self.data_mut();
 		let host_secrets = data.host_secrets.entry(host.to_owned()).or_default();
 		host_secrets.insert(secret, value);
 	}
+	pub fn remove_secret(&self, host: &str, secret: &str) {
+		let mut data = self.data_mut();
+		let host_secrets = data.host_secrets.entry(host.to_owned()).or_default();
+		host_secrets.remove(secret);
+	}
 
-	pub fn host_secret(&self, host: &str, secret: &str) -> Result<FleetSecret> {
+	pub fn host_secret(&self, host: &str, secret: &str) -> Result<FleetHostSecret> {
 		let data = self.data();
-		let Some(host_secrets) = data.host_secrets.get(host) else {
-			bail!("no secrets for machine {host}");
+		if let Some(host_secrets) = data.host_secrets.get(host) {
+			if let Some(secret) = host_secrets.get(secret) {
+				return Ok(secret.clone());
+			}
 		};
-		let Some(secret) = host_secrets.get(secret) else {
+		let Some(shared) = data.shared_secrets.get(secret) else {
 			bail!("machine {host} has no secret {secret}");
 		};
-		Ok(secret.clone())
+		if !shared.owners.contains(host) {
+			bail!("shared secret {secret} is not owned by {host}");
+		};
+		Ok(FleetHostSecret {
+			managed: shared.managed,
+			secret: shared.secret.clone(),
+		})
 	}
-	pub fn shared_secret(&self, secret: &str) -> Result<FleetSharedSecret> {
+	pub fn shared_secret(&self, secret: &str) -> Result<Option<FleetSharedSecret>> {
 		let data = self.data();
-		let Some(secret) = data.shared_secrets.get(secret) else {
-			bail!("no shared secret {secret}");
-		};
-		Ok(secret.clone())
+		Ok(data.shared_secrets.get(secret).cloned())
 	}
-	pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
+	pub fn shared_secret_definition(&self, secret: &str) -> Result<SharedSecretDefinition> {
 		let config_field = &self.config_field;
-		Ok(nix_go_json!(
-			config_field.sharedSecrets[{ secret }].expectedOwners
-		))
+		Ok(SharedSecretDefinition(nix_go!(
+			config_field.sharedSecrets[{ secret }]
+		)))
 	}
 
 	// TODO: Should this be something modifiable from other processes?

--- a/crates/fleet-base/src/keys.rs
+++ b/crates/fleet-base/src/keys.rs
@@ -39,12 +39,14 @@
 		}
 	}
 	/// Insecure, requires root
-	pub async fn recipient(&self, host: &str) -> anyhow::Result<impl Recipient + use<>> {
+	pub async fn recipient(&self, host: &str) -> anyhow::Result<Box<dyn Recipient>> {
 		let key = self.key(host).await?;
-		age::ssh::Recipient::from_str(&key).map_err(|e| anyhow!("parse recipient error: {:?}", e))
+		age::ssh::Recipient::from_str(&key)
+			.map_err(|e| anyhow!("parse recipient error: {e:?}"))
+			.map(|v| Box::new(v) as Box<dyn Recipient>)
 	}
 
-	pub async fn recipients(&self, hosts: Vec<String>) -> Result<Vec<impl Recipient + use<>>> {
+	pub async fn recipients(&self, hosts: Vec<String>) -> Result<Vec<Box<dyn Recipient>>> {
 		let hosts = self.expand_owner_set(hosts).await?;
 		futures::stream::iter(hosts.iter())
 			.then(|m| self.recipient(m.as_ref()))

--- a/crates/fleet-base/src/lib.rs
+++ b/crates/fleet-base/src/lib.rs
@@ -4,3 +4,4 @@
 pub mod host;
 mod keys;
 pub mod opts;
+pub mod secret;

--- a/crates/nix-eval/src/lib.rs
+++ b/crates/nix-eval/src/lib.rs
@@ -729,10 +729,13 @@
 
 		with_default_context(|c, es| unsafe { get_list_byidx(c, self.0, es, v as u32) }).map(Self)
 	}
-	pub fn attrs_update(self, other: Value/*, ignore_errors: bool*/) -> Result<Self> {
+	pub fn attrs_update(self, other: Value /*, ignore_errors: bool*/) -> Result<Self> {
 		let attrs_update_fn = Self::eval("a: b: a // b")?;
 
-		attrs_update_fn.call(self)?.call(other).context("attrs update")
+		attrs_update_fn
+			.call(self)?
+			.call(other)
+			.context("attrs update")
 	}
 	pub fn get_field(&self, name: impl AsFieldName) -> Result<Self> {
 		if !matches!(self.type_of(), NixType::Attrs) {
@@ -840,6 +843,9 @@
 	pub fn is_function(&self) -> bool {
 		self.functor_kind().is_some()
 	}
+	pub fn is_null(&self) -> bool {
+		matches!(self.type_of(), NixType::Null)
+	}
 }
 
 impl From<String> for Value {

--- a/flake.nix
+++ b/flake.nix
@@ -181,7 +181,7 @@
                 inputs'.nix.packages.nix-fetchers-c
                 inputs'.nix.packages.nix-store-c
 
-                (rage.overrideAttrs {cargoFeatures = ["plugin"];})
+                (rage.overrideAttrs { cargoFeatures = [ "plugin" ]; })
               ];
               environment.PROTOC = "${pkgs.protobuf}/bin/protoc";
             };

--- a/modules/extras/tf.nix
+++ b/modules/extras/tf.nix
@@ -38,17 +38,19 @@
       # will be somehow processed by fleet tf.
       sensitive = true;
     };
-    fleetConfigurations.default = {config, ...}: {
-      options.data = mkDataOption {
-        # host => hostData
-        options.extra.terraformHosts = mkOption {
-          default = { };
-          type = attrsOf (attrsOf unspecified);
-          description = "Hosts data provided by fleet tf";
+    fleetConfigurations.default =
+      { config, ... }:
+      {
+        options.data = mkDataOption {
+          # host => hostData
+          options.extra.terraformHosts = mkOption {
+            default = { };
+            type = attrsOf (attrsOf unspecified);
+            description = "Hosts data provided by fleet tf";
+          };
         };
+        config.hosts = config.data.extra.terraformHosts;
       };
-      config.hosts = config.data.extra.terraformHosts;
-    };
 
     perSystem.imports = [ ./tf-bootstrap.nix ];
   };

--- a/modules/nixos/secrets.nix
+++ b/modules/nixos/secrets.nix
@@ -6,11 +6,11 @@
   ...
 }:
 let
-  inherit (builtins) hashString;
+  inherit (builtins) hashString elemAt length toJSON filter;
   inherit (lib.stringsWithDeps) stringAfter;
   inherit (lib.options) mkOption literalExpression;
   inherit (lib.lists) optional;
-  inherit (lib.attrsets) mapAttrs;
+  inherit (lib.attrsets) mapAttrs mapAttrsToList;
   inherit (lib.modules) mkIf;
   inherit (lib.types)
     submodule
@@ -22,10 +22,29 @@
     uniq
     functionTo
     package
+    listOf
     ;
   inherit (fleetLib.strings) decodeRawSecret;
 
   sysConfig = config;
+  secretPartDataType = submodule {
+    options = {
+      raw = mkOption {
+        type = str;
+        internal = true;
+        description = "Encoded & Encrypted secret part data, passed from fleet.nix";
+      };
+    };
+  };
+  secretDataType = submodule {
+    freeformType = lazyAttrsOf secretPartDataType;
+    options = {
+      shared = mkOption {
+        description = "Is this secret owned by this machine, or propagated from shared secrets";
+        default = false;
+      };
+    };
+  };
   secretPartType =
     secretName:
     submodule (
@@ -35,11 +54,6 @@
       in
       {
         options = {
-          raw = mkOption {
-            type = str;
-            internal = true;
-            description = "Encoded & Encrypted secret part data, passed from fleet.nix";
-          };
           hash = mkOption {
             type = str;
             description = "Hash of secret in encoded format";
@@ -50,34 +64,50 @@
           };
           stablePath = mkOption {
             type = str;
-            description = "Path to secret part, incorporating data hash (thus it will be updated on secret change)";
+            description = "Path to secret part, stable path (users are expected to watch for file changes/re-read secret on demand)";
           };
           data = mkOption {
             type = str;
             description = "Secret public data (only available for plaintext)";
           };
         };
-        config = {
-          hash = hashString "sha1" config.raw;
-          data = decodeRawSecret config.raw;
-          path = "/run/secrets/${secretName}/${config.hash}-${partName}";
-          stablePath = "/run/secrets/${secretName}/${partName}";
-        };
+        config =
+          let
+            raw = sysConfig.data.secrets.${secretName}.${partName}.raw;
+          in
+          {
+            hash = hashString "sha1" raw;
+            data = decodeRawSecret raw;
+            path = "/run/secrets/${secretName}/${config.hash}-${partName}";
+            stablePath = "/run/secrets/${secretName}/${partName}";
+          };
       }
     );
   secretType = submodule (
-    { config, ... }:
+    {
+      config,
+      loc,
+      options,
+      ...
+    }:
     let
-      secretName = config._module.args.name;
+      secretName =
+        # Due to config definition for freeformType, we can't just use _module.args due to infinite recursion, instead
+        # extract the secret name the ugly way...
+        let
+          saLoc = options._module.specialArgs.loc;
+          comp = elemAt saLoc;
+        in
+        assert
+          (length saLoc == 2 ||
+          length saLoc == 4 &&
+          comp 0 == "secrets" && comp 2 == "_module" && comp 3 == "specialArgs") ||
+          throw "Unexpected module structure ${toJSON saLoc}";
+        if length saLoc == 2 then "documentation generator stub" else comp 1;
     in
     {
       freeformType = lazyAttrsOf (secretPartType secretName);
       options = {
-        shared = mkOption {
-          description = "Is this secret owned by this machine, or propagated from shared secrets";
-          default = false;
-        };
-
         generator = mkOption {
           type = uniq (nullOr (functionTo package));
           description = "Derivation to evaluate for secret generation";
@@ -104,18 +134,30 @@
           description = "Data that gets embedded into secret part";
           default = null;
         };
+        expectedPrivateParts = mkOption {
+          type = listOf str;
+          default = [ ];
+          description = "List of parts that are expected to be encrypted";
+        };
+        expectedPublicParts = mkOption {
+          type = listOf str;
+          default = [ ];
+          description = "List of parts that are expected to be public";
+        };
       };
+      config = mapAttrs (_: _: { }) (removeAttrs (sysConfig.data.secrets.${secretName} or {}) [ "shared" ]);
     }
   );
-  processPart = part: {
-    inherit (part) raw path stablePath;
+  processPart = secretName: partName: part: {
+    inherit (part) path stablePath;
+    raw = config.data.secrets.${secretName}.${partName}.raw;
   };
   processSecret =
-    secret:
+    secretName: secret:
     {
       inherit (secret) group mode owner;
     }
-    // (mapAttrs (_: processPart) (
+    // (mapAttrs (processPart secretName) (
       removeAttrs secret [
         "shared"
         "generator"
@@ -123,11 +165,14 @@
         "group"
         "owner"
         "expectedGenerationData"
+        "expectedPrivateParts"
+        "expectedPublicParts"
       ]
     ));
+  secretsData = (mapAttrs (processSecret) config.secrets);
   secretsFile = pkgs.writeTextFile {
     name = "secrets.json";
-    text = builtins.toJSON (mapAttrs (_: processSecret) config.secrets);
+    text = toJSON secretsData;
   };
   useSysusers =
     (config.systemd ? sysusers && config.systemd.sysusers.enable)
@@ -135,15 +180,38 @@
 in
 {
   options = {
+    data.secrets = mkOption {
+      type = attrsOf secretDataType;
+      default = { };
+      description = "Host-local secret data";
+    };
     secrets = mkOption {
       type = attrsOf secretType;
       default = { };
       description = "Host-local secrets";
     };
+    system.secretsData = mkOption {
+      type = unspecified;
+      default = {};
+      description = "secrets.json contents";
+    };
   };
   config = {
+    system = {inherit secretsData;};
     environment.systemPackages = [ pkgs.fleet-install-secrets ];
 
+    warnings = filter (v: v!=null) (mapAttrsToList (
+      name: secret:
+      if
+        secret.expectedPrivateParts == [ ]
+        && secret.expectedPublicParts == [ ]
+        && !(config.data.secrets.${name} or { shared = false; }).shared
+      then
+        "Secret ${name} has no expected parts defined, this is deprecated for better visibility"
+      else
+        null
+    ) config.secrets);
+
     systemd.services.fleet-install-secrets = mkIf useSysusers {
       wantedBy = [ "sysinit.target" ];
       after = [ "systemd-sysusers.service" ];

--- a/modules/secrets-data.nix
+++ b/modules/secrets-data.nix
@@ -151,8 +151,9 @@
           toJSON (config.data.sharedSecrets.${name} or { owners = [ ]; }).owners
         }. Run fleet secrets regenerate to fix";
       }) config.sharedSecrets)
+
       ++ (mapAttrsToList (name: secret: {
-        # TODO: Same aassertion should be in host secrets
+        # TODO: Same assertion should be in host secrets
         assertion =
           (config.data.sharedSecrets.${name} or { generationData = null; }).generationData
           == secret.expectedGenerationData;
@@ -160,6 +161,5 @@
           toJSON (config.data.sharedSecrets.${name} or { generationData = null; }).generationData
         }. Run fleet secrets regenerate to fix";
       }) config.sharedSecrets);
-    sharedSecrets = mapAttrs (_: _: { }) config.data.sharedSecrets;
   };
 }

--- a/modules/secrets.nix
+++ b/modules/secrets.nix
@@ -69,6 +69,16 @@
           description = "Contextual metadata embedded within the secret part value";
           default = null;
         };
+        expectedPrivateParts = mkOption {
+          type = listOf str;
+          default = [ ];
+          description = "List of parts that are expected to be encrypted";
+        };
+        expectedPublicParts = mkOption {
+          type = listOf str;
+          default = [ ];
+          description = "List of parts that are expected to be public";
+        };
       };
     };
 in
@@ -81,16 +91,25 @@
     };
   };
   config = {
-    hosts = mapAttrs (_: secretMap: {
-      nixos.secrets = mapAttrs (
-        _: s:
-        removeAttrs s [
-          "createdAt"
-          "expiresAt"
-          "generationData"
-        ]
-      ) secretMap;
-    }) config.data.hostSecrets;
+    hosts = mapAttrs (
+      _: secretMap:
+      let
+        partsOf =
+          s:
+          removeAttrs s [
+            "createdAt"
+            "expiresAt"
+            "generationData"
+          ];
+
+      in
+      {
+        nixos.data.secrets = mapAttrs (_: s: partsOf s) secretMap;
+        # nixos.secrets = mapAttrs (
+        #   _: s: mapAttrs (_: _: {}) (partsOf s)
+        # ) secretMap;
+      }
+    ) config.data.hostSecrets;
     nixpkgs.overlays = [
       (final: prev: {
         mkSecretGenerators =