From f5a8281dad2cfb22c293b4e4d1475ee89f611466 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sun, 26 Oct 2025 06:45:36 +0000 Subject: [PATCH] feat: expected secret parts --- --- a/Cargo.lock +++ b/Cargo.lock @@ -1055,6 +1055,7 @@ "shlex", "tabled", "tempfile", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -1087,6 +1088,7 @@ "serde_json", "tabled", "tempfile", + "thiserror 2.0.17", "time", "tokio", "tokio-util", --- a/cmds/fleet/Cargo.toml +++ b/cmds/fleet/Cargo.toml @@ -47,6 +47,7 @@ nom = "8.0.0" opentelemetry = "0.30.0" opentelemetry_sdk = "0.30.0" +thiserror.workspace = true tracing-indicatif = { version = "0.3", optional = true } tracing-opentelemetry = "0.31.0" --- a/cmds/fleet/src/cmds/secrets/mod.rs +++ b/cmds/fleet/src/cmds/secrets/mod.rs @@ -2,17 +2,18 @@ collections::{BTreeMap, BTreeSet, HashSet}, io::{self, Read, Write, stdin, stdout}, path::PathBuf, - slice, }; -use age::Recipient; use anyhow::{Context, Result, anyhow, bail, ensure}; use chrono::{DateTime, Utc}; use clap::Parser; use fleet_base::{ - fleetdata::{FleetSecret, FleetSecretPart, FleetSharedSecret, encrypt_secret_data}, + fleetdata::{ + FleetHostSecret, FleetSecretData, FleetSecretPart, FleetSharedSecret, encrypt_secret_data, + }, host::Config, opts::FleetOpts, + secret::{Expectations, RegenerationReason, SharedSecretDefinition, secret_needs_regeneration}, }; use fleet_shared::SecretData; use nix_eval::{NixType, Value, nix_go, nix_go_json}; @@ -144,76 +145,55 @@ #[clap(short = 'p', long, default_value = "secret")] part: String, }, -} - -fn secret_needs_regeneration( - secret: &FleetSecret, - expected_generation_data: &serde_json::Value, -) -> bool { - let data_is_expected = secret.generation_data == *expected_generation_data; - // TODO: Leeway? - let expired = secret - .expires_at - .map(|expiration| expiration < Utc::now()) - .unwrap_or(false); - expired || !data_is_expected } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(skip(config, secret, field, prefer_identities))] +#[tracing::instrument(skip(config, secret, definition, prefer_identities))] async fn maybe_regenerate_shared_secret( secret_name: &str, config: &Config, mut secret: FleetSharedSecret, - field: Value, - expected_owners: &[String], - expected_generation_data: serde_json::Value, + definition: SharedSecretDefinition, prefer_identities: &[String], + expectations: &Expectations, ) -> Result { - let original_set = secret.owners.clone(); + let reason = secret_needs_regeneration(&secret.secret, &secret.owners, expectations); + let value = definition.inner(); - let set = original_set.iter().collect::>(); - let expected_set = expected_owners.iter().collect::>(); - - let regeneration_required = - secret_needs_regeneration(&secret.secret, &expected_generation_data); - - if set == expected_set && !regeneration_required { - info!("no need to update owner list, it is already correct"); - return Ok(secret); - } - - let should_regenerate = if regeneration_required { - info!("secret has its generation data changed, regeneration is required"); - true - } else if set.difference(&expected_set).next().is_some() { - // TODO: Remove this warning for revokable secrets. - warn!( - "host was removed from secret owners, but until this host rebuild, the secret will still be stored on it." - ); - nix_go_json!(field.regenerateOnOwnerRemoved) - } else if expected_set.difference(&set).next().is_some() { - nix_go_json!(field.regenerateOnOwnerAdded) - } else { - false + let (should_reencrypt, reason) = match reason { + Some(RegenerationReason::OwnersAdded(_)) => { + // Secret always needs to be reencrypted for new owners to be able to read it + ( + true, + if nix_go_json!(value.regenerateOnOwnerAdded) { + reason + } else { + None + }, + ) + } + Some(RegenerationReason::OwnersRemoved(_)) => { + // No need to reencrypt, we can just leave stanzas in place. + if nix_go_json!(value.regenerateOnOwnerRemoved) { + (true, reason) + } else { + (false, None) + } + } + Some(_) => (true, reason), + None => (false, None), }; - if should_regenerate { - info!("secret needs to be regenerated"); - let generated = generate_shared( - config, - secret_name, - field, - expected_owners.to_vec(), - expected_generation_data, - ) - .await?; + if let Some(reason) = reason { + info!("secret needs to be regenerated: {reason}"); + let generated = generate_shared(config, secret_name, definition, expectations).await?; Ok(generated) - } else { + } else if should_reencrypt { + info!("secret needs to be reencrypted"); let identity_holder = if !prefer_identities.is_empty() { prefer_identities .iter() - .find(|i| original_set.iter().any(|s| s == *i)) + .find(|i| secret.owners.iter().any(|s| s == *i)) } else { secret.owners.first() }; @@ -228,12 +208,16 @@ } let host = config.host(identity_holder).await?; let encrypted = host - .reencrypt(part.raw.clone(), expected_owners.to_vec()) + .reencrypt( + part.raw.clone(), + expectations.owners.iter().cloned().collect(), + ) .await?; part.raw = encrypted; } - - secret.owners = expected_owners.to_vec(); + secret.owners = expectations.owners.clone(); + Ok(secret) + } else { Ok(secret) } } @@ -250,8 +234,8 @@ _display_name: &str, _secret: Value, _default_generator: Value, - _owners: &[String], -) -> Result { + _expectations: &Expectations, +) -> Result { bail!("pure generators are broken for now") } async fn generate_impure( @@ -259,9 +243,8 @@ _display_name: &str, secret: Value, default_generator: Value, - expected_owners: &[String], - expected_generation_data: serde_json::Value, -) -> Result { + expectations: &Expectations, +) -> Result { let generator = nix_go!(secret.generator); let on: Option = nix_go_json!(default_generator.impureOn); @@ -276,12 +259,11 @@ let mk_secret_generators = nix_go!(on_pkgs.mkSecretGenerators); let mut recipients = Vec::new(); - for owner in expected_owners { + for owner in &expectations.owners { let key = config.key(owner).await?; recipients.push(key); } let generators = nix_go!(mk_secret_generators(Obj { recipients })); - // FIXME: Apparently, // operator is slow in nix let pkgs_and_generators = on_pkgs.attrs_update(generators)?; let call_package = nix_go!(nixpkgs.lib.callPackageWith(pkgs_and_generators)); @@ -331,20 +313,26 @@ let created_at = host.read_file_value(format!("{out}/created_at")).await?; let expires_at = host.read_file_value(format!("{out}/expires_at")).await.ok(); - Ok(FleetSecret { + let new_data = FleetSecretData { created_at, expires_at, parts, - generation_data: expected_generation_data, - }) + generation_data: expectations.generation_data.clone(), + }; + + if let Some(reason) = secret_needs_regeneration(&new_data, &expectations.owners, expectations) { + bail!("newly generated secret needs to be regenerated: {reason}") + } + + Ok(new_data) } + async fn generate( config: &Config, display_name: &str, secret: Value, - expected_owners: &[String], - expected_generation_data: serde_json::Value, -) -> Result { + expectations: &Expectations, +) -> Result { let generator = nix_go!(secret.generator); // Can't properly check on nix module system level { @@ -388,8 +376,7 @@ display_name, secret, default_generator, - expected_owners, - expected_generation_data, + expectations, ) .await } @@ -399,7 +386,7 @@ display_name, secret, default_generator, - expected_owners, + expectations, ) .await } @@ -408,21 +395,14 @@ async fn generate_shared( config: &Config, display_name: &str, - secret: Value, - expected_owners: Vec, - expected_generation_data: serde_json::Value, + secret: SharedSecretDefinition, + expectations: &Expectations, ) -> Result { // let owners: Vec = nix_go_json!(secret.expectedOwners); Ok(FleetSharedSecret { - secret: generate( - config, - display_name, - secret, - &expected_owners, - expected_generation_data, - ) - .await?, - owners: expected_owners, + managed: Some(true), + secret: generate(config, display_name, secret.inner(), expectations).await?, + owners: expectations.owners.clone(), }) } @@ -457,11 +437,11 @@ } fn parse_machines( - initial: Vec, + initial: BTreeSet, machines: Option>, mut add_machines: Vec, mut remove_machines: Vec, -) -> Result> { +) -> Result> { if machines.is_none() && add_machines.is_empty() && remove_machines.is_empty() { bail!("no operation"); } @@ -470,7 +450,6 @@ let mut target_machines = initial; info!("Currently encrypted for {initial_machines:?}"); - // ensure!(machines.is_some() || !add_machines.is_empty() || ) if let Some(machines) = machines { ensure!( add_machines.is_empty() && remove_machines.is_empty(), @@ -487,20 +466,13 @@ } for machine in &remove_machines { - let mut removed = false; - while let Some(pos) = target_machines.iter().position(|m| m == machine) { - target_machines.swap_remove(pos); - removed = true; - } - if !removed { + if !target_machines.remove(machine) { warn!("secret is not enabled for {machine}"); } } for machine in &add_machines { - if target_machines.iter().any(|m| m == machine) { + if !target_machines.insert(machine.to_owned()) { warn!("secret is already added to {machine}"); - } else { - target_machines.push(machine.to_owned()); } } if !remove_machines.is_empty() { @@ -527,7 +499,7 @@ } } Secret::AddShared { - mut machines, + machines, name, force, public, @@ -537,25 +509,32 @@ re_add, part: part_name, } => { + let mut machines: BTreeSet = machines.into_iter().collect(); // TODO: Forbid updating secrets with set expectedOwners (= not user-managed). - let exists = config.has_shared(&name); - if exists && !force && !re_add { - bail!("secret already defined"); - } - if re_add { - // Fixme: use clap to limit this usage - ensure!(!force, "--force and --readd are not compatible"); - ensure!(exists, "secret doesn't exists"); - ensure!( - machines.is_empty(), - "you can't use machines argument for --readd" - ); - let shared = config.shared_secret(&name)?; - machines = shared.owners; - } + if let Some(old_shared) = config.shared_secret(&name)? { + if !force && !re_add { + bail!("secret already defined"); + }; + if old_shared.managed.unwrap_or(false) { + bail!("secret is marked as managed, should not be updated manually"); + }; + if re_add { + // Fixme: use clap to limit this usage + ensure!(!force, "--force and --readd are not compatible"); + ensure!( + machines.is_empty(), + "you can't use machines argument for --readd" + ); + machines = old_shared.owners; + } + } else if re_add { + bail!("secret doesn't exists"); + }; - let recipients = config.recipients(machines.clone()).await?; + let recipients = config + .recipients(machines.iter().cloned().collect()) + .await?; let mut parts = BTreeMap::new(); @@ -563,9 +542,8 @@ io::stdin().read_to_end(&mut input)?; if !input.is_empty() { - let encrypted = - encrypt_secret_data(recipients.iter().map(|r| r as &dyn Recipient), input) - .ok_or_else(|| anyhow!("no recipients provided"))?; + let encrypted = encrypt_secret_data(recipients.iter(), input) + .ok_or_else(|| anyhow!("no recipients provided"))?; parts.insert(part_name, FleetSecretPart { raw: encrypted }); } @@ -576,8 +554,9 @@ config.replace_shared( name, FleetSharedSecret { + managed: Some(false), owners: machines, - secret: FleetSecret { + secret: FleetSecretData { created_at: Utc::now(), expires_at, parts, @@ -607,34 +586,47 @@ .host_secret(&machine, &name) .context("failed to read existing secret for --merge")? } else { - FleetSecret { - created_at: Utc::now(), - expires_at: None, - parts: BTreeMap::new(), - generation_data: serde_json::Value::Null, + FleetHostSecret { + managed: Some(false), + secret: FleetSecretData { + created_at: Utc::now(), + expires_at: None, + parts: BTreeMap::new(), + generation_data: serde_json::Value::Null, + }, } }; + if out.managed.unwrap_or(false) { + bail!("secret is managed by fleet and should not be updated manually"); + } + out.managed = Some(false); if let Some(secret) = parse_secret().await? { let recipient = config.recipient(&machine).await?; - let encrypted = encrypt_secret_data([&recipient as &dyn Recipient], secret) - .expect("recipient provided"); + let encrypted = + encrypt_secret_data([&recipient], secret).expect("recipient provided"); if out + .secret .parts .insert(part_name.clone(), FleetSecretPart { raw: encrypted }) .is_some() && !replace { - bail!("part {part_name:?} is already defined"); + bail!( + "part {part_name:?} is already defined, use --replace if you wish to replace it" + ); } } if let Some(public) = parse_public(public, public_file).await? { if out + .secret .parts .insert(public_name.clone(), FleetSecretPart { raw: public }) .is_some() && !replace { - bail!("part {public_name:?} is already defined"); + bail!( + "part {public_name:?} is already defined, use --replace if you wish to replace it" + ); } }; @@ -647,7 +639,7 @@ part: part_name, } => { let secret = config.host_secret(&machine, &name)?; - let Some(secret) = secret.parts.get(&part_name) else { + let Some(secret) = secret.secret.parts.get(&part_name) else { bail!("no part {part_name} in secret {name}"); }; let data = if secret.raw.encrypted { @@ -664,7 +656,9 @@ part: part_name, prefer_identities, } => { - let secret = config.shared_secret(&name)?; + let Some(secret) = config.shared_secret(&name)? else { + bail!("secret doesn't exists"); + }; let Some(part) = secret.secret.parts.get(&part_name) else { bail!("no part {part_name} in secret {name}"); }; @@ -695,7 +689,9 @@ } => { // TODO: Forbid updating secrets with set expectedOwners (= not user-managed). - let secret = config.shared_secret(&name)?; + let Some(secret) = config.shared_secret(&name)? else { + bail!("secret doesn't exists"); + }; if secret.secret.parts.values().all(|v| !v.raw.encrypted) { bail!("no secret"); } @@ -714,20 +710,16 @@ return Ok(()); } - let config_field = &config.config_field; - let name_clone = name.clone(); - let field = nix_go!(config_field.sharedSecrets[name_clone]); - let expected_generation_data = nix_go_json!(field.expectedGenerationData); + let definition = config.shared_secret_definition(&name)?; + let expectations = definition.expectations()?; let updated = maybe_regenerate_shared_secret( &name, config, secret, - field, - &target_machines, - expected_generation_data, + definition, &prefer_identities, - // None, + &expectations, ) .await?; config.replace_shared(name, updated); @@ -737,36 +729,26 @@ skip_hosts, } => { info!("checking for secrets to regenerate"); + let expected_shared_set = config + .list_configured_shared() + .await? + .into_iter() + .collect::>(); let stored_shared_set = config.list_shared().into_iter().collect::>(); { // Generate missing shared let _span = info_span!("shared").entered(); - let expected_shared_set = config - .list_configured_shared() - .await? - .into_iter() - .collect::>(); for missing in expected_shared_set.difference(&stored_shared_set) { - let config_field = &config.config_field; - let secret = nix_go!(config_field.sharedSecrets[{ missing }]); - let expected_generation_data: serde_json::Value = - nix_go_json!(secret.expectedGenerationData); - let expected_owners: Option> = - nix_go_json!(secret.expectedOwners); - let Some(expected_owners) = expected_owners else { - // Can't generate this missing secret, as it has no defined owners. + let definition = config.shared_secret_definition(missing)?; + if !definition.is_managed()? { + info!("skipping unmanaged secret: {missing}"); continue; - }; + } + let expectations = definition.expectations()?; info!("generating secret: {missing}"); - let shared = generate_shared( - config, - missing, - secret, - expected_owners, - expected_generation_data, - ) - .in_current_span() - .await?; + let shared = generate_shared(config, missing, definition, &expectations) + .in_current_span() + .await?; config.replace_shared(missing.to_string(), shared) } } @@ -778,26 +760,22 @@ let _span = info_span!("host", host = host.name).entered(); let expected_set = host - .list_configured_secrets() - .in_current_span() - .await? + .list_defined_secrets()? .into_iter() .collect::>(); let stored_set = config .list_secrets(&host.name) .into_iter() .collect::>(); - for missing in expected_set.difference(&stored_set) { - info!("generating secret: {missing}"); - let secret = host.secret_field(missing).in_current_span().await?; - let expected_generation_data = - nix_go_json!(secret.expectedGenerationData); + for missing_secret in expected_set.difference(&stored_set) { + info!("generating missing secret: {missing_secret}"); + let definition = host.secret_definition(missing_secret)?; + let expectations = definition.expectations()?; let generated = match generate( config, - missing, - secret, - slice::from_ref(&host.name), - expected_generation_data, + missing_secret, + definition.inner(), + &expectations, ) .in_current_span() .await @@ -808,21 +786,27 @@ continue; } }; - config.insert_secret(&host.name, missing.to_string(), generated) + config.insert_secret( + &host.name, + missing_secret.to_string(), + FleetHostSecret { + managed: Some(true), + secret: generated, + }, + ) } - for name in stored_set { - info!("updating secret: {name}"); - let data = config.host_secret(&host.name, &name)?; - let secret = host.secret_field(&name).in_current_span().await?; - let expected_generation_data = - nix_go_json!(secret.expectedGenerationData); - if secret_needs_regeneration(&data, &expected_generation_data) { + for known_secret in stored_set.intersection(&expected_set) { + info!("updating secret: {known_secret}"); + let data = config.host_secret(&host.name, known_secret)?; + let definition = host.secret_definition(known_secret)?; + let expectations = definition.expectations()?; + if let Some(regen_reason) = data.needs_regeneration(&expectations) { + info!("needs regeneration: {regen_reason}"); let generated = match generate( config, - &name, - secret, - slice::from_ref(&host.name), - expected_generation_data, + known_secret, + definition.inner(), + &expectations, ) .in_current_span() .await @@ -833,43 +817,44 @@ continue; } }; - config.insert_secret(&host.name, name.to_string(), generated) + config.insert_secret( + &host.name, + known_secret.to_string(), + FleetHostSecret { + managed: Some(true), + secret: generated, + }, + ) } } + for removed_secret in stored_set.difference(&expected_set) { + info!("removing secret: {removed_secret}"); + config.remove_secret(&host.name, removed_secret); + } } } - let mut to_remove = Vec::new(); - for name in &stored_shared_set { - info!("updating secret: {name}"); - let data = config.shared_secret(name)?; - let config_field = &config.config_field; - let expected_owners: Option> = - nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners); - let Some(expected_owners) = expected_owners else { - warn!("secret was removed from fleet config: {name}, removing from data"); - to_remove.push(name.to_string()); - continue; - }; + for known_secret in stored_shared_set.intersection(&expected_shared_set) { + info!("updating shared secret: {known_secret}"); + let data = config.shared_secret(known_secret)?.expect("exists"); - let secret = nix_go!(config_field.sharedSecrets[{ name }]); - let expected_generation_data = nix_go_json!(secret.expectedGenerationData); + let definition = config.shared_secret_definition(known_secret)?; + let expectations = definition.expectations()?; config.replace_shared( - name.to_owned(), + known_secret.to_owned(), maybe_regenerate_shared_secret( - name, + known_secret, config, data, - secret, - &expected_owners, - expected_generation_data, + definition, &prefer_identities, - // None, + &expectations, ) .await?, ); } - for k in to_remove { - config.remove_shared(&k); + for removed_secret in stored_shared_set.difference(&expected_shared_set) { + info!("removing shared secret: {removed_secret}"); + config.remove_shared(removed_secret); } } Secret::List {} => { @@ -885,13 +870,14 @@ let mut table = vec![]; for name in configured.iter().cloned() { let config = config.clone(); - let expected_owners = config.shared_secret_expected_owners(&name).await?; - let data = config.shared_secret(&name)?; + let data = config.shared_secret(&name)?.expect("exists"); + let definition = config.shared_secret_definition(&name)?; + let expectations = definition.expectations()?; let owners = data .owners .iter() .map(|o| { - if expected_owners.contains(o) { + if expectations.owners.contains(o) { o.green().to_string() } else { o.red().to_string() @@ -912,7 +898,7 @@ add, } => { let secret = config.host_secret(&machine, &name)?; - if let Some(data) = secret.parts.get(&part) { + if let Some(data) = secret.secret.parts.get(&part) { let host = config.host(&machine).await?; let secret = host.decrypt(data.raw.clone()).await?; String::from_utf8(secret).context("secret is not utf8")? --- a/cmds/fleet/src/main.rs +++ b/cmds/fleet/src/main.rs @@ -27,7 +27,7 @@ use tracing::{Instrument, error, info, info_span}; #[cfg(feature = "indicatif")] use tracing_indicatif::IndicatifLayer; -use tracing_subscriber::{EnvFilter, fmt::format::Format, prelude::*}; +use tracing_subscriber::{EnvFilter, prelude::*}; #[derive(Parser)] struct Prefetch {} --- a/crates/fleet-base/Cargo.toml +++ b/crates/fleet-base/Cargo.toml @@ -24,6 +24,7 @@ serde_json = "1.0.140" tabled = "0.20.0" tempfile.workspace = true +thiserror.workspace = true time = { version = "0.3.41", features = ["parsing"] } tokio.workspace = true tokio-util = "0.7.15" --- a/crates/fleet-base/src/fleetdata.rs +++ b/crates/fleet-base/src/fleetdata.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, io::{self, Cursor}, }; @@ -13,6 +13,8 @@ use serde::{Deserialize, Serialize, de::Error}; use serde_json::Value; +use crate::secret::{Expectations, RegenerationReason, secret_needs_regeneration}; + #[derive(Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct HostData { @@ -75,30 +77,21 @@ pub shared_secrets: BTreeMap, #[serde(default)] #[serde(skip_serializing_if = "BTreeMap::is_empty")] - pub host_secrets: BTreeMap>, + pub host_secrets: BTreeMap>, // extra_name => anything #[serde(default)] #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub extra: BTreeMap, -} - -#[derive(Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -#[must_use] -pub struct FleetSharedSecret { - pub owners: Vec, - #[serde(flatten)] - pub secret: FleetSecret, } /// Returns None if recipients.is_empty() -pub fn encrypt_secret_data<'a>( - recipients: impl IntoIterator, +pub fn encrypt_secret_data<'r>( + recipients: impl IntoIterator>, data: Vec, ) -> Option { let mut encrypted = vec![]; - let mut encryptor = age::Encryptor::with_recipients(recipients.into_iter()) + let mut encryptor = age::Encryptor::with_recipients(recipients.into_iter().map(|v| &**v)) .ok()? .wrap_output(&mut encrypted) .expect("in memory write"); @@ -118,7 +111,7 @@ #[derive(Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] #[must_use] -pub struct FleetSecret { +pub struct FleetSecretData { #[serde(default = "Utc::now")] pub created_at: DateTime, #[serde(default)] @@ -132,3 +125,31 @@ #[serde(skip_serializing_if = "Value::is_null")] pub generation_data: Value, } + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +#[must_use] +pub struct FleetHostSecret { + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub managed: Option, + #[serde(flatten)] + pub secret: FleetSecretData, +} +impl FleetHostSecret { + pub fn needs_regeneration(&self, expectations: &Expectations) -> Option { + secret_needs_regeneration(&self.secret, &expectations.owners, expectations) + } +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +#[must_use] +pub struct FleetSharedSecret { + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub managed: Option, + pub owners: BTreeSet, + #[serde(flatten)] + pub secret: FleetSecretData, +} --- a/crates/fleet-base/src/host.rs +++ b/crates/fleet-base/src/host.rs @@ -22,7 +22,8 @@ use crate::{ command::MyCommand, - fleetdata::{FleetData, FleetSecret, FleetSharedSecret}, + fleetdata::{FleetData, FleetHostSecret, FleetSharedSecret}, + secret::{HostSecretDefinition, SharedSecretDefinition}, }; pub struct FleetConfigInternals { @@ -234,7 +235,7 @@ let is_fleet_managed = match self.file_exists("/etc/FLEET_HOST").await { Ok(v) => v, Err(e) => { - bail!("failed to query remote system kind: {}", e); + bail!("failed to query remote system kind: {e}"); } }; if !is_fleet_managed { @@ -501,7 +502,7 @@ Ok(nixos_config) } - pub async fn nixos_unchecked_config(&self) -> Result { + pub fn nixos_unchecked_config(&self) -> Result { if let Some(v) = self.nixos_unchecked_config.get() { return Ok(v.clone()); } @@ -515,23 +516,17 @@ Ok(nixos_config) } - pub async fn list_configured_secrets(&self) -> Result> { - let nixos = self.nixos_unchecked_config().await?; + pub fn list_defined_secrets(&self) -> Result> { + let nixos = self.nixos_unchecked_config()?; let secrets = nix_go!(nixos.secrets); - let mut out = Vec::new(); - for name in secrets.list_fields()? { - let secret = secrets.get_field(&name).context("getting secret")?; - let is_shared: bool = nix_go_json!(secret.shared); - if is_shared { - continue; - } - out.push(name); - } - Ok(out) + secrets.list_fields() } - pub async fn secret_field(&self, name: &str) -> Result { - let nixos = self.nixos_unchecked_config().await?; - Ok(nix_go!(nixos.secrets[{ name }])) + pub fn secret_definition(&self, name: &str) -> Result { + let nixos = self.nixos_unchecked_config()?; + Ok(HostSecretDefinition( + self.name.clone(), + nix_go!(nixos.secrets[{ name }]), + )) } /// Packages for this host, resolved with nixpkgs overlays @@ -648,10 +643,19 @@ pub fn list_secrets(&self, host: &str) -> Vec { let data = self.data(); - let Some(secrets) = data.host_secrets.get(host) else { - return Vec::new(); - }; - secrets.keys().cloned().collect() + let mut out = data + .host_secrets + .get(host) + .map(|s| s.keys().cloned().collect::>()) + .unwrap_or_default(); + + for (name, shared) in data.shared_secrets.iter() { + if shared.owners.contains(host) { + out.push(name.clone()); + } + } + + out } pub fn has_secret(&self, host: &str, secret: &str) -> bool { @@ -661,34 +665,44 @@ }; host_secrets.contains_key(secret) } - pub fn insert_secret(&self, host: &str, secret: String, value: FleetSecret) { + pub fn insert_secret(&self, host: &str, secret: String, value: FleetHostSecret) { let mut data = self.data_mut(); let host_secrets = data.host_secrets.entry(host.to_owned()).or_default(); host_secrets.insert(secret, value); } + pub fn remove_secret(&self, host: &str, secret: &str) { + let mut data = self.data_mut(); + let host_secrets = data.host_secrets.entry(host.to_owned()).or_default(); + host_secrets.remove(secret); + } - pub fn host_secret(&self, host: &str, secret: &str) -> Result { + pub fn host_secret(&self, host: &str, secret: &str) -> Result { let data = self.data(); - let Some(host_secrets) = data.host_secrets.get(host) else { - bail!("no secrets for machine {host}"); + if let Some(host_secrets) = data.host_secrets.get(host) { + if let Some(secret) = host_secrets.get(secret) { + return Ok(secret.clone()); + } }; - let Some(secret) = host_secrets.get(secret) else { + let Some(shared) = data.shared_secrets.get(secret) else { bail!("machine {host} has no secret {secret}"); }; - Ok(secret.clone()) + if !shared.owners.contains(host) { + bail!("shared secret {secret} is not owned by {host}"); + }; + Ok(FleetHostSecret { + managed: shared.managed, + secret: shared.secret.clone(), + }) } - pub fn shared_secret(&self, secret: &str) -> Result { + pub fn shared_secret(&self, secret: &str) -> Result> { let data = self.data(); - let Some(secret) = data.shared_secrets.get(secret) else { - bail!("no shared secret {secret}"); - }; - Ok(secret.clone()) + Ok(data.shared_secrets.get(secret).cloned()) } - pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result> { + pub fn shared_secret_definition(&self, secret: &str) -> Result { let config_field = &self.config_field; - Ok(nix_go_json!( - config_field.sharedSecrets[{ secret }].expectedOwners - )) + Ok(SharedSecretDefinition(nix_go!( + config_field.sharedSecrets[{ secret }] + ))) } // TODO: Should this be something modifiable from other processes? --- a/crates/fleet-base/src/keys.rs +++ b/crates/fleet-base/src/keys.rs @@ -39,12 +39,14 @@ } } /// Insecure, requires root - pub async fn recipient(&self, host: &str) -> anyhow::Result> { + pub async fn recipient(&self, host: &str) -> anyhow::Result> { let key = self.key(host).await?; - age::ssh::Recipient::from_str(&key).map_err(|e| anyhow!("parse recipient error: {:?}", e)) + age::ssh::Recipient::from_str(&key) + .map_err(|e| anyhow!("parse recipient error: {e:?}")) + .map(|v| Box::new(v) as Box) } - pub async fn recipients(&self, hosts: Vec) -> Result>> { + pub async fn recipients(&self, hosts: Vec) -> Result>> { let hosts = self.expand_owner_set(hosts).await?; futures::stream::iter(hosts.iter()) .then(|m| self.recipient(m.as_ref())) --- a/crates/fleet-base/src/lib.rs +++ b/crates/fleet-base/src/lib.rs @@ -4,3 +4,4 @@ pub mod host; mod keys; pub mod opts; +pub mod secret; --- /dev/null +++ b/crates/fleet-base/src/secret.rs @@ -0,0 +1,136 @@ +use std::collections::BTreeSet; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use nix_eval::{Value, nix_go, nix_go_json}; + +use crate::fleetdata::FleetSecretData; + +#[derive(Debug)] +pub struct Expectations { + pub owners: BTreeSet, + pub generation_data: serde_json::Value, + pub public_parts: BTreeSet, + pub private_parts: BTreeSet, +} + +pub struct HostSecretDefinition(pub(crate) String, pub(crate) Value); +impl HostSecretDefinition { + pub fn is_managed(&self) -> Result { + let value = &self.1; + Ok(!nix_go!(value.generator).is_null()) + } + pub fn expectations(&self) -> Result { + let value = &self.1; + Ok(Expectations { + owners: BTreeSet::from([self.0.clone()]), + generation_data: nix_go_json!(value.expectedGenerationData), + public_parts: nix_go_json!(value.expectedPublicParts), + private_parts: nix_go_json!(value.expectedPrivateParts), + }) + } + pub fn inner(&self) -> Value { + self.1.clone() + } +} + +pub struct SharedSecretDefinition(pub(crate) Value); +impl SharedSecretDefinition { + pub fn is_managed(&self) -> Result { + let value = &self.0; + Ok(!nix_go!(value.generator).is_null()) + } + pub fn expectations(&self) -> Result { + let value = &self.0; + Ok(Expectations { + owners: nix_go_json!(value.expectedOwners), + generation_data: nix_go_json!(value.expectedGenerationData), + public_parts: nix_go_json!(value.expectedPublicParts), + private_parts: nix_go_json!(value.expectedPrivateParts), + }) + } + pub fn inner(&self) -> Value { + self.0.clone() + } +} + +#[derive(thiserror::Error, Debug)] +pub enum RegenerationReason { + #[error("owners added: {0:?}")] + OwnersAdded(BTreeSet), + #[error("owners added: {0:?}")] + OwnersRemoved(BTreeSet), + #[error("unexpected generation data, expected: {expected:?}, found: {found:?}")] + GenerationData { + expected: serde_json::Value, + found: serde_json::Value, + }, + #[error("unexpected part list, expected: {expected:?}, found: {found:?}")] + PartList { + expected: BTreeSet, + found: BTreeSet, + }, + #[error("part {0} is expected to be encrypted")] + ExpectedPrivate(String), + #[error("part {0} is not expected to be encrypted")] + ExpectedPublic(String), + #[error("secret is expired at {0}")] + Expired(DateTime), +} + +pub fn secret_needs_regeneration( + secret: &FleetSecretData, + owners: &BTreeSet, + expectations: &Expectations, +) -> Option { + if !owners.is_empty() { + let added: BTreeSet = expectations.owners.difference(owners).cloned().collect(); + if !added.is_empty() { + return Some(RegenerationReason::OwnersAdded(added)); + } + + let removed: BTreeSet = owners.difference(&expectations.owners).cloned().collect(); + if !removed.is_empty() { + return Some(RegenerationReason::OwnersRemoved(removed)); + } + } + + if secret.generation_data != expectations.generation_data { + return Some(RegenerationReason::GenerationData { + expected: expectations.generation_data.clone(), + found: secret.generation_data.clone(), + }); + } + + if !expectations.public_parts.is_empty() || !expectations.private_parts.is_empty() { + let expected: BTreeSet = expectations + .public_parts + .union(&expectations.private_parts) + .cloned() + .collect(); + let found: BTreeSet = secret.parts.keys().cloned().collect(); + + if found != expected { + return Some(RegenerationReason::PartList { expected, found }); + } + + for (name, value) in secret.parts.iter() { + if value.raw.encrypted { + if !expectations.private_parts.contains(name) { + return Some(RegenerationReason::ExpectedPrivate(name.clone())); + } + } else if !expectations.public_parts.contains(name) { + return Some(RegenerationReason::ExpectedPublic(name.clone())); + } + } + } + + if let Some(expiration) = secret.expires_at { + // TODO: Leeway? + if expiration < Utc::now() { + return Some(RegenerationReason::Expired(expiration)); + } + } + + None +} --- a/crates/nix-eval/src/lib.rs +++ b/crates/nix-eval/src/lib.rs @@ -729,10 +729,13 @@ with_default_context(|c, es| unsafe { get_list_byidx(c, self.0, es, v as u32) }).map(Self) } - pub fn attrs_update(self, other: Value/*, ignore_errors: bool*/) -> Result { + pub fn attrs_update(self, other: Value /*, ignore_errors: bool*/) -> Result { let attrs_update_fn = Self::eval("a: b: a // b")?; - attrs_update_fn.call(self)?.call(other).context("attrs update") + attrs_update_fn + .call(self)? + .call(other) + .context("attrs update") } pub fn get_field(&self, name: impl AsFieldName) -> Result { if !matches!(self.type_of(), NixType::Attrs) { @@ -840,6 +843,9 @@ pub fn is_function(&self) -> bool { self.functor_kind().is_some() } + pub fn is_null(&self) -> bool { + matches!(self.type_of(), NixType::Null) + } } impl From for Value { --- a/flake.nix +++ b/flake.nix @@ -181,7 +181,7 @@ inputs'.nix.packages.nix-fetchers-c inputs'.nix.packages.nix-store-c - (rage.overrideAttrs {cargoFeatures = ["plugin"];}) + (rage.overrideAttrs { cargoFeatures = [ "plugin" ]; }) ]; environment.PROTOC = "${pkgs.protobuf}/bin/protoc"; }; --- a/modules/extras/tf.nix +++ b/modules/extras/tf.nix @@ -38,17 +38,19 @@ # will be somehow processed by fleet tf. sensitive = true; }; - fleetConfigurations.default = {config, ...}: { - options.data = mkDataOption { - # host => hostData - options.extra.terraformHosts = mkOption { - default = { }; - type = attrsOf (attrsOf unspecified); - description = "Hosts data provided by fleet tf"; + fleetConfigurations.default = + { config, ... }: + { + options.data = mkDataOption { + # host => hostData + options.extra.terraformHosts = mkOption { + default = { }; + type = attrsOf (attrsOf unspecified); + description = "Hosts data provided by fleet tf"; + }; }; + config.hosts = config.data.extra.terraformHosts; }; - config.hosts = config.data.extra.terraformHosts; - }; perSystem.imports = [ ./tf-bootstrap.nix ]; }; --- a/modules/nixos/secrets.nix +++ b/modules/nixos/secrets.nix @@ -6,11 +6,11 @@ ... }: let - inherit (builtins) hashString; + inherit (builtins) hashString elemAt length toJSON filter; inherit (lib.stringsWithDeps) stringAfter; inherit (lib.options) mkOption literalExpression; inherit (lib.lists) optional; - inherit (lib.attrsets) mapAttrs; + inherit (lib.attrsets) mapAttrs mapAttrsToList; inherit (lib.modules) mkIf; inherit (lib.types) submodule @@ -22,10 +22,29 @@ uniq functionTo package + listOf ; inherit (fleetLib.strings) decodeRawSecret; sysConfig = config; + secretPartDataType = submodule { + options = { + raw = mkOption { + type = str; + internal = true; + description = "Encoded & Encrypted secret part data, passed from fleet.nix"; + }; + }; + }; + secretDataType = submodule { + freeformType = lazyAttrsOf secretPartDataType; + options = { + shared = mkOption { + description = "Is this secret owned by this machine, or propagated from shared secrets"; + default = false; + }; + }; + }; secretPartType = secretName: submodule ( @@ -35,11 +54,6 @@ in { options = { - raw = mkOption { - type = str; - internal = true; - description = "Encoded & Encrypted secret part data, passed from fleet.nix"; - }; hash = mkOption { type = str; description = "Hash of secret in encoded format"; @@ -50,34 +64,50 @@ }; stablePath = mkOption { type = str; - description = "Path to secret part, incorporating data hash (thus it will be updated on secret change)"; + description = "Path to secret part, stable path (users are expected to watch for file changes/re-read secret on demand)"; }; data = mkOption { type = str; description = "Secret public data (only available for plaintext)"; }; }; - config = { - hash = hashString "sha1" config.raw; - data = decodeRawSecret config.raw; - path = "/run/secrets/${secretName}/${config.hash}-${partName}"; - stablePath = "/run/secrets/${secretName}/${partName}"; - }; + config = + let + raw = sysConfig.data.secrets.${secretName}.${partName}.raw; + in + { + hash = hashString "sha1" raw; + data = decodeRawSecret raw; + path = "/run/secrets/${secretName}/${config.hash}-${partName}"; + stablePath = "/run/secrets/${secretName}/${partName}"; + }; } ); secretType = submodule ( - { config, ... }: + { + config, + loc, + options, + ... + }: let - secretName = config._module.args.name; + secretName = + # Due to config definition for freeformType, we can't just use _module.args due to infinite recursion, instead + # extract the secret name the ugly way... + let + saLoc = options._module.specialArgs.loc; + comp = elemAt saLoc; + in + assert + (length saLoc == 2 || + length saLoc == 4 && + comp 0 == "secrets" && comp 2 == "_module" && comp 3 == "specialArgs") || + throw "Unexpected module structure ${toJSON saLoc}"; + if length saLoc == 2 then "documentation generator stub" else comp 1; in { freeformType = lazyAttrsOf (secretPartType secretName); options = { - shared = mkOption { - description = "Is this secret owned by this machine, or propagated from shared secrets"; - default = false; - }; - generator = mkOption { type = uniq (nullOr (functionTo package)); description = "Derivation to evaluate for secret generation"; @@ -104,18 +134,30 @@ description = "Data that gets embedded into secret part"; default = null; }; + expectedPrivateParts = mkOption { + type = listOf str; + default = [ ]; + description = "List of parts that are expected to be encrypted"; + }; + expectedPublicParts = mkOption { + type = listOf str; + default = [ ]; + description = "List of parts that are expected to be public"; + }; }; + config = mapAttrs (_: _: { }) (removeAttrs (sysConfig.data.secrets.${secretName} or {}) [ "shared" ]); } ); - processPart = part: { - inherit (part) raw path stablePath; + processPart = secretName: partName: part: { + inherit (part) path stablePath; + raw = config.data.secrets.${secretName}.${partName}.raw; }; processSecret = - secret: + secretName: secret: { inherit (secret) group mode owner; } - // (mapAttrs (_: processPart) ( + // (mapAttrs (processPart secretName) ( removeAttrs secret [ "shared" "generator" @@ -123,11 +165,14 @@ "group" "owner" "expectedGenerationData" + "expectedPrivateParts" + "expectedPublicParts" ] )); + secretsData = (mapAttrs (processSecret) config.secrets); secretsFile = pkgs.writeTextFile { name = "secrets.json"; - text = builtins.toJSON (mapAttrs (_: processSecret) config.secrets); + text = toJSON secretsData; }; useSysusers = (config.systemd ? sysusers && config.systemd.sysusers.enable) @@ -135,15 +180,38 @@ in { options = { + data.secrets = mkOption { + type = attrsOf secretDataType; + default = { }; + description = "Host-local secret data"; + }; secrets = mkOption { type = attrsOf secretType; default = { }; description = "Host-local secrets"; }; + system.secretsData = mkOption { + type = unspecified; + default = {}; + description = "secrets.json contents"; + }; }; config = { + system = {inherit secretsData;}; environment.systemPackages = [ pkgs.fleet-install-secrets ]; + warnings = filter (v: v!=null) (mapAttrsToList ( + name: secret: + if + secret.expectedPrivateParts == [ ] + && secret.expectedPublicParts == [ ] + && !(config.data.secrets.${name} or { shared = false; }).shared + then + "Secret ${name} has no expected parts defined, this is deprecated for better visibility" + else + null + ) config.secrets); + systemd.services.fleet-install-secrets = mkIf useSysusers { wantedBy = [ "sysinit.target" ]; after = [ "systemd-sysusers.service" ]; --- a/modules/secrets-data.nix +++ b/modules/secrets-data.nix @@ -151,8 +151,9 @@ toJSON (config.data.sharedSecrets.${name} or { owners = [ ]; }).owners }. Run fleet secrets regenerate to fix"; }) config.sharedSecrets) + ++ (mapAttrsToList (name: secret: { - # TODO: Same aassertion should be in host secrets + # TODO: Same assertion should be in host secrets assertion = (config.data.sharedSecrets.${name} or { generationData = null; }).generationData == secret.expectedGenerationData; @@ -160,6 +161,5 @@ toJSON (config.data.sharedSecrets.${name} or { generationData = null; }).generationData }. Run fleet secrets regenerate to fix"; }) config.sharedSecrets); - sharedSecrets = mapAttrs (_: _: { }) config.data.sharedSecrets; }; } --- a/modules/secrets.nix +++ b/modules/secrets.nix @@ -69,6 +69,16 @@ description = "Contextual metadata embedded within the secret part value"; default = null; }; + expectedPrivateParts = mkOption { + type = listOf str; + default = [ ]; + description = "List of parts that are expected to be encrypted"; + }; + expectedPublicParts = mkOption { + type = listOf str; + default = [ ]; + description = "List of parts that are expected to be public"; + }; }; }; in @@ -81,16 +91,25 @@ }; }; config = { - hosts = mapAttrs (_: secretMap: { - nixos.secrets = mapAttrs ( - _: s: - removeAttrs s [ - "createdAt" - "expiresAt" - "generationData" - ] - ) secretMap; - }) config.data.hostSecrets; + hosts = mapAttrs ( + _: secretMap: + let + partsOf = + s: + removeAttrs s [ + "createdAt" + "expiresAt" + "generationData" + ]; + + in + { + nixos.data.secrets = mapAttrs (_: s: partsOf s) secretMap; + # nixos.secrets = mapAttrs ( + # _: s: mapAttrs (_: _: {}) (partsOf s) + # ) secretMap; + } + ) config.data.hostSecrets; nixpkgs.overlays = [ (final: prev: { mkSecretGenerators = -- gitstuff