git.delta.rocks / jrsonnet / refs/commits / feb74e5a3cbb

1use std::{2	collections::{BTreeMap, BTreeSet, HashSet},3	io::{self, stdin, stdout, Read, Write},4	path::PathBuf,5};67use anyhow::{anyhow, bail, ensure, Context, Result};8use chrono::{DateTime, Utc};9use clap::Parser;10use fleet_base::{11	fleetdata::{encrypt_secret_data, FleetSecret, FleetSecretPart, FleetSharedSecret},12	host::Config,13	opts::FleetOpts,14};15use fleet_shared::SecretData;16use nix_eval::{nix_go, nix_go_json, NixBuildBatch, Value};17use owo_colors::OwoColorize;18use serde::Deserialize;19use tabled::{Table, Tabled};20use tokio::fs::read;21use tracing::{error, info, info_span, warn, Instrument};2223#[derive(Parser)]24pub enum Secret {25	/// Force load host keys for all defined hosts26	ForceKeys,27	/// Add secret, data should be provided in stdin28	AddShared {29		/// Secret name30		name: String,31		/// Secret owners32		#[clap(long, short)]33		machines: Vec<String>,34		/// Override secret if already present35		#[clap(long)]36		force: bool,37		/// Secret public part38		#[clap(long)]39		public: Option<String>,40		/// Load public part from specified file41		#[clap(long)]42		public_file: Option<PathBuf>,4344		/// Create a notification on secret expiration45		#[clap(long)]46		expires_at: Option<DateTime<Utc>>,4748		/// Secret with this name already exists, override its value while keeping the same owners.49		#[clap(long)]50		re_add: bool,5152		/// How to name public secret part53		#[clap(long, short = 'p', default_value = "public")]54		public_part: String,55		/// How to name private secret part56		#[clap(short = 's', long, default_value = "secret")]57		part: String,58	},59	/// Add secret, data should be provided in stdin60	Add {61		/// Secret name62		name: String,63		/// Secret owner64		#[clap(short = 'm', long)]65		machine: String,66		/// Replace secret if already present67		#[clap(long)]68		replace: bool,69		/// Add new parts to existing secret70		#[clap(long)]71		merge: bool,72		/// Secret public part73		#[clap(long)]74		public: Option<String>,75		/// Load public part from specified file76		#[clap(long)]77		public_file: Option<PathBuf>,7879		/// How to name public secret part80		#[clap(short = 'p', long, default_value = "public")]81		public_part: String,82		/// How to name private secret part83		#[clap(short = 's', long, default_value = "secret")]84		part: String,85	},86	/// Read secret from remote host, requires sudo on said host87	Read {88		name: String,89		#[clap(short = 'm', long)]90		machine: String,9192		/// Which private secret part to read93		#[clap(short = 'p', long, default_value = "secret")]94		part: String,95	},96	UpdateShared {97		name: String,9899		#[clap(short = 'm', long)]100		machine: Option<Vec<String>>,101102		#[clap(long)]103		add_machine: Vec<String>,104		#[clap(long)]105		remove_machine: Vec<String>,106107		/// Which host should we use to decrypt108		#[clap(long)]109		prefer_identities: Vec<String>,110	},111	Regenerate {112		/// Which host should we use to decrypt, in case if reencryption is required, without113		/// regeneration114		#[clap(long)]115		prefer_identities: Vec<String>,116	},117	List {},118	Edit {119		name: String,120		#[clap(short = 'm', long)]121		machine: String,122123		#[clap(long)]124		add: bool,125126		/// Which private secret part to read127		#[clap(short = 'p', long, default_value = "secret")]128		part: String,129	},130}131132#[tracing::instrument(skip(config, secret, field, prefer_identities, batch))]133async fn update_owner_set(134	secret_name: &str,135	config: &Config,136	mut secret: FleetSharedSecret,137	field: Value,138	updated_set: &[String],139	prefer_identities: &[String],140	batch: Option<NixBuildBatch>,141) -> Result<FleetSharedSecret> {142	let original_set = secret.owners.clone();143144	let set = original_set.iter().collect::<BTreeSet<_>>();145	let expected_set = updated_set.iter().collect::<BTreeSet<_>>();146147	if set == expected_set {148		info!("no need to update owner list, it is already correct");149		return Ok(secret);150	}151152	let should_regenerate = if set.difference(&expected_set).next().is_some() {153		// TODO: Remove this warning for revokable secrets.154		warn!("host was removed from secret owners, but until this host rebuild, the secret will still be stored on it.");155		nix_go_json!(field.regenerateOnOwnerRemoved)156	} else if expected_set.difference(&set).next().is_some() {157		nix_go_json!(field.regenerateOnOwnerAdded)158	} else {159		false160	};161162	if should_regenerate {163		info!("secret is owner-dependent, will regenerate");164		let generated = generate_shared(config, secret_name, field, updated_set.to_vec(), batch).await?;165		Ok(generated)166	} else {167		drop(batch);168		let identity_holder = if !prefer_identities.is_empty() {169			prefer_identities170				.iter()171				.find(|i| original_set.iter().any(|s| s == *i))172		} else {173			secret.owners.first()174		};175		let Some(identity_holder) = identity_holder else {176			bail!("no available holder found");177		};178179		for (part_name, part) in secret.secret.parts.iter_mut() {180			let _span = info_span!("part reencryption", part_name);181			if !part.raw.encrypted {182				continue;183			}184			let host = config.host(identity_holder).await?;185			let encrypted = host186				.reencrypt(part.raw.clone(), updated_set.to_vec())187				.await?;188			part.raw = encrypted;189		}190191		secret.owners = updated_set.to_vec();192		Ok(secret)193	}194}195196#[derive(Deserialize)]197#[serde(rename_all = "camelCase")]198enum GeneratorKind {199	Impure,200	Pure,201}202203async fn generate_pure(204	_config: &Config,205	_display_name: &str,206	_secret: Value,207	_default_generator: Value,208	_owners: &[String],209) -> Result<FleetSecret> {210	bail!("pure generators are broken for now")211}212async fn generate_impure(213	config: &Config,214	_display_name: &str,215	secret: Value,216	default_generator: Value,217	owners: &[String],218	batch: Option<NixBuildBatch>,219) -> Result<FleetSecret> {220	let generator = nix_go!(secret.generator);221	let on: Option<String> = nix_go_json!(default_generator.impureOn);222223	let host = if let Some(on) = &on {224		config.host(on).await?225	} else {226		config.local_host()227	};228	let on_pkgs = host.pkgs().await?;229	let call_package = nix_go!(on_pkgs.callPackage);230	let mk_secret_generators = nix_go!(on_pkgs.mkSecretGenerators);231232	let mut recipients = Vec::new();233	for owner in owners {234		let key = config.key(owner).await?;235		recipients.push(key);236	}237	let generators = nix_go!(mk_secret_generators(Obj { recipients }));238239	let generator = nix_go!(call_package(generator)(generators));240241	let generator = generator.build_maybe_batch(batch).await?;242	let generator = generator243		.get("out")244		.ok_or_else(|| anyhow!("missing generateImpure out"))?;245	let generator = host.remote_derivation(generator).await?;246247	let out_parent = host.mktemp_dir().await?;248	let out = format!("{out_parent}/out");249250	let mut gen = host.cmd(generator).await?;251	gen.env("out", &out);252	if on.is_none() {253		// This path is local, thus we can feed `OsString` directly to env var... But I don't think that's necessary to handle.254		let project_path: String = config255			.directory256			.clone()257			.into_os_string()258			.into_string()259			.map_err(|s| anyhow!("fleet project path is not utf-8: {s:?}"))?;260		gen.env("FLEET_PROJECT", project_path);261	}262	gen.run().await.context("impure generator")?;263264	{265		let marker = host.read_file_text(format!("{out}/marker")).await?;266		ensure!(marker == "SUCCESS", "generation not succeeded");267	}268269	let mut parts = BTreeMap::new();270	for part in host.read_dir(&out).await? {271		if part == "created_at" || part == "expired_at" || part == "marker" {272			continue;273		}274		let contents: SecretData = host275			.read_file_text(format!("{out}/{part}"))276			.await?277			.parse()278			.map_err(|e| anyhow!("failed to decode secret {out:?} part {part:?}: {e}"))?;279		parts.insert(part.to_owned(), FleetSecretPart { raw: contents });280	}281282	let created_at = host.read_file_value(format!("{out}/created_at")).await?;283	let expires_at = host.read_file_value(format!("{out}/expires_at")).await.ok();284285	Ok(FleetSecret {286		created_at,287		expires_at,288		parts,289	})290}291async fn generate(292	config: &Config,293	display_name: &str,294	secret: Value,295	owners: &[String],296	batch: Option<NixBuildBatch>,297) -> Result<FleetSecret> {298	let generator = nix_go!(secret.generator);299	// Can't properly check on nix module system level300	{301		let gen_ty = generator.type_of().await?;302		if gen_ty == "null" {303			bail!("secret has no generator defined, can't automatically generate it.");304		}305		if gen_ty != "lambda" {306			bail!("generator should be lambda, got {gen_ty}");307		}308	}309	let default_pkgs = &config.default_pkgs;310	let default_call_package = nix_go!(default_pkgs.callPackage);311	let default_mk_secret_generators = nix_go!(default_pkgs.mkSecretGenerators);312	// Generators provide additional information in passthru, to access313	// passthru we should call generator, but information about where this generator is supposed to build314	// is located in passthru... Thus evaluating generator on host.315	//316	// Maybe it is also possible to do some magic with __functor?317	//318	// I don't want to make modules always responsible for additional secret data anyway,319	// so it should be in derivation, and not in the secret data itself.320	let generators = nix_go!(default_mk_secret_generators(Obj {321		recipients: <Vec<String>>::new(),322	}));323	let default_generator = nix_go!(default_call_package(generator)(generators));324325	let kind: GeneratorKind = nix_go_json!(default_generator.generatorKind);326327	match kind {328		GeneratorKind::Impure => {329			generate_impure(330				config,331				display_name,332				secret,333				default_generator,334				owners,335				batch,336			)337			.await338		}339		GeneratorKind::Pure => {340			generate_pure(config, display_name, secret, default_generator, owners).await341		}342	}343}344async fn generate_shared(345	config: &Config,346	display_name: &str,347	secret: Value,348	expected_owners: Vec<String>,349	batch: Option<NixBuildBatch>,350) -> Result<FleetSharedSecret> {351	// let owners: Vec<String> = nix_go_json!(secret.expectedOwners);352	Ok(FleetSharedSecret {353		secret: generate(config, display_name, secret, &expected_owners, batch).await?,354		owners: expected_owners,355	})356}357358async fn parse_public(359	public: Option<String>,360	public_file: Option<PathBuf>,361) -> Result<Option<SecretData>> {362	Ok(match (public, public_file) {363		(Some(v), None) => Some(SecretData {364			data: v.into(),365			encrypted: false,366		}),367		(None, Some(v)) => Some(SecretData {368			data: read(v).await?,369			encrypted: false,370		}),371		(Some(_), Some(_)) => {372			bail!("only public or public_file should be set")373		}374		(None, None) => None,375	})376}377378async fn parse_secret() -> Result<Option<Vec<u8>>> {379	let mut input = vec![];380	stdin().read_to_end(&mut input)?;381	if input.is_empty() {382		Ok(None)383	} else {384		Ok(Some(input))385	}386}387388fn parse_machines(389	initial: Vec<String>,390	machines: Option<Vec<String>>,391	mut add_machines: Vec<String>,392	mut remove_machines: Vec<String>,393) -> Result<Vec<String>> {394	if machines.is_none() && add_machines.is_empty() && remove_machines.is_empty() {395		bail!("no operation");396	}397398	let initial_machines = initial.clone();399	let mut target_machines = initial;400	info!("Currently encrypted for {initial_machines:?}");401402	// ensure!(machines.is_some() || !add_machines.is_empty() || )403	if let Some(machines) = machines {404		ensure!(405			add_machines.is_empty() && remove_machines.is_empty(),406			"can't combine --machines and --add-machines/--remove-machines"407		);408		let target = initial_machines.iter().collect::<HashSet<_>>();409		let source = machines.iter().collect::<HashSet<_>>();410		for removed in target.difference(&source) {411			remove_machines.push((*removed).clone());412		}413		for added in source.difference(&target) {414			add_machines.push((*added).clone());415		}416	}417418	for machine in &remove_machines {419		let mut removed = false;420		while let Some(pos) = target_machines.iter().position(|m| m == machine) {421			target_machines.swap_remove(pos);422			removed = true;423		}424		if !removed {425			warn!("secret is not enabled for {machine}");426		}427	}428	for machine in &add_machines {429		if target_machines.iter().any(|m| m == machine) {430			warn!("secret is already added to {machine}");431		} else {432			target_machines.push(machine.to_owned());433		}434	}435	if !remove_machines.is_empty() {436		// TODO: maybe force secret regeneration?437		// Not that useful without revokation.438		warn!("secret will not be regenerated for removed machines, and until host rebuild, they will still possess the ability to decode secret");439	}440	Ok(target_machines)441}442impl Secret {443	pub async fn run(self, config: &Config, opts: &FleetOpts) -> Result<()> {444		match self {445			Secret::ForceKeys => {446				for host in config.list_hosts().await? {447					if opts.should_skip(&host).await? {448						continue;449					}450					config.key(&host.name).await?;451				}452			}453			Secret::AddShared {454				mut machines,455				name,456				force,457				public,458				public_part: public_name,459				public_file,460				expires_at,461				re_add,462				part: part_name,463			} => {464				// TODO: Forbid updating secrets with set expectedOwners (= not user-managed).465466				let exists = config.has_shared(&name);467				if exists && !force && !re_add {468					bail!("secret already defined");469				}470				if re_add {471					// Fixme: use clap to limit this usage472					ensure!(!force, "--force and --readd are not compatible");473					ensure!(exists, "secret doesn't exists");474					ensure!(475						machines.is_empty(),476						"you can't use machines argument for --readd"477					);478					let shared = config.shared_secret(&name)?;479					machines = shared.owners;480				}481482				let recipients = config.recipients(machines.clone()).await?;483484				let mut parts = BTreeMap::new();485486				let mut input = vec![];487				io::stdin().read_to_end(&mut input)?;488489				if !input.is_empty() {490					let encrypted = encrypt_secret_data(recipients, input)491						.ok_or_else(|| anyhow!("no recipients provided"))?;492					parts.insert(part_name, FleetSecretPart { raw: encrypted });493				}494495				if let Some(public) = parse_public(public, public_file).await? {496					parts.insert(public_name, FleetSecretPart { raw: public });497				}498499				config.replace_shared(500					name,501					FleetSharedSecret {502						owners: machines,503						secret: FleetSecret {504							created_at: Utc::now(),505							expires_at,506							parts,507						},508					},509				);510			}511			Secret::Add {512				machine,513				name,514				replace,515				merge,516				public,517				public_part: public_name,518				public_file,519				part: part_name,520			} => {521				if config.has_secret(&machine, &name) && !replace && !merge {522					bail!("secret already defined.\nUse --replace to override, or --merge to add new parts to existing secret");523				}524525				let mut out = if merge && !replace {526					config527						.host_secret(&machine, &name)528						.context("failed to read existing secret for --merge")?529				} else {530					FleetSecret {531						created_at: Utc::now(),532						expires_at: None,533						parts: BTreeMap::new(),534					}535				};536537				if let Some(secret) = parse_secret().await? {538					let recipient = config.recipient(&machine).await?;539					let encrypted =540						encrypt_secret_data(vec![recipient], secret).expect("recipient provided");541					if out542						.parts543						.insert(part_name.clone(), FleetSecretPart { raw: encrypted })544						.is_some() && !replace545					{546						bail!("part {part_name:?} is already defined");547					}548				}549550				if let Some(public) = parse_public(public, public_file).await? {551					if out552						.parts553						.insert(public_name.clone(), FleetSecretPart { raw: public })554						.is_some() && !replace555					{556						bail!("part {public_name:?} is already defined");557					}558				};559560				config.insert_secret(&machine, name, out);561			}562			#[allow(clippy::await_holding_refcell_ref)]563			Secret::Read {564				name,565				machine,566				part: part_name,567			} => {568				let secret = config.host_secret(&machine, &name)?;569				let Some(secret) = secret.parts.get(&part_name) else {570					bail!("no part {part_name} in secret {name}");571				};572				let data = if secret.raw.encrypted {573					let host = config.host(&machine).await?;574					host.decrypt(secret.raw.clone()).await?575				} else {576					secret.raw.data.clone()577				};578579				stdout().write_all(&data)?;580			}581			Secret::UpdateShared {582				name,583				machine,584				add_machine,585				remove_machine,586				prefer_identities,587			} => {588				// TODO: Forbid updating secrets with set expectedOwners (= not user-managed).589590				let secret = config.shared_secret(&name)?;591				if secret.secret.parts.values().all(|v| !v.raw.encrypted) {592					bail!("no secret");593				}594595				let initial_machines = secret.owners.clone();596				let target_machines = parse_machines(597					initial_machines.clone(),598					machine,599					add_machine,600					remove_machine,601				)?;602603				if target_machines.is_empty() {604					info!("no machines left for secret, removing it");605					config.remove_shared(&name);606					return Ok(());607				}608609				let config_field = &config.config_field;610				let field = nix_go!(config_field.sharedSecrets[{ name }]);611612				let updated = update_owner_set(613					&name,614					config,615					secret,616					field,617					&target_machines,618					&prefer_identities,619					None,620				)621				.await?;622				config.replace_shared(name, updated);623			}624			Secret::Regenerate { prefer_identities } => {625				info!("checking for secrets to regenerate");626				{627					let shared_batch = None;628					let _span = info_span!("shared").entered();629					let expected_shared_set = config630						.list_configured_shared()631						.await?632						.into_iter()633						.collect::<HashSet<_>>();634					let shared_set = config.list_shared().into_iter().collect::<HashSet<_>>();635					for missing in expected_shared_set.difference(&shared_set) {636						let config_field = &config.config_field;637						let secret = nix_go!(config_field.sharedSecrets[{ missing }]);638						let expected_owners: Option<Vec<String>> =639							nix_go_json!(secret.expectedOwners);640						let Some(expected_owners) = expected_owners else {641							// TODO: Might still need to regenerate642							continue;643						};644						info!("generating secret: {missing}");645						let shared = generate_shared(646							config,647							missing,648							secret,649							expected_owners,650							shared_batch.clone(),651						)652						.in_current_span()653						.await?;654						config.replace_shared(missing.to_string(), shared)655					}656				}657				let hosts_batch = None;658				for host in config.list_hosts().await? {659					if opts.should_skip(&host).await? {660						continue;661					}662663					let _span = info_span!("host", host = host.name).entered();664					let expected_set = host665						.list_configured_secrets()666						.in_current_span()667						.await?668						.into_iter()669						.collect::<HashSet<_>>();670					let stored_set = config671						.list_secrets(&host.name)672						.into_iter()673						.collect::<HashSet<_>>();674					for missing in expected_set.difference(&stored_set) {675						info!("generating secret: {missing}");676						let secret = host.secret_field(missing).in_current_span().await?;677						let generated = match generate(678							config,679							missing,680							secret,681							&[host.name.clone()],682							hosts_batch.clone(),683						)684						.in_current_span()685						.await686						{687							Ok(v) => v,688							Err(e) => {689								error!("{e:?}");690								continue;691							}692						};693						config.insert_secret(&host.name, missing.to_string(), generated)694					}695				}696				let mut to_remove = Vec::new();697				for name in &config.list_shared() {698					info!("updating secret: {name}");699					let data = config.shared_secret(name)?;700					let config_field = &config.config_field;701					let expected_owners: Vec<String> =702						nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners);703					if expected_owners.is_empty() {704						warn!("secret was removed from fleet config: {name}, removing from data");705						to_remove.push(name.to_string());706						continue;707					}708709					let secret = nix_go!(config_field.sharedSecrets[{ name }]);710					config.replace_shared(711						name.to_owned(),712						update_owner_set(713							name,714							config,715							data,716							secret,717							&expected_owners,718							&prefer_identities,719							None,720						)721						.await?,722					);723				}724				for k in to_remove {725					config.remove_shared(&k);726				}727			}728			Secret::List {} => {729				let _span = info_span!("loading secrets").entered();730				let configured = config.list_configured_shared().await?;731				#[derive(Tabled)]732				struct SecretDisplay {733					#[tabled(rename = "Name")]734					name: String,735					#[tabled(rename = "Owners")]736					owners: String,737				}738				let mut table = vec![];739				for name in configured.iter().cloned() {740					let config = config.clone();741					let expected_owners = config.shared_secret_expected_owners(&name).await?;742					let data = config.shared_secret(&name)?;743					let owners = data744						.owners745						.iter()746						.map(|o| {747							if expected_owners.contains(o) {748								o.green().to_string()749							} else {750								o.red().to_string()751							}752						})753						.collect::<Vec<_>>();754					table.push(SecretDisplay {755						owners: owners.join(", "),756						name,757					})758				}759				info!("loaded\n{}", Table::new(table).to_string())760			}761			Secret::Edit {762				name,763				machine,764				part,765				add,766			} => {767				let secret = config.host_secret(&machine, &name)?;768				if let Some(data) = secret.parts.get(&part) {769					let host = config.host(&machine).await?;770					let secret = host.decrypt(data.raw.clone()).await?;771					String::from_utf8(secret).context("secret is not utf8")?772				} else if add {773					String::new()774				} else {775					bail!("part {part} not found in secret {name}. Did you mean to `--add` it?");776				};777			}778		}779		Ok(())780	}781}782783/*784async fn edit_temp_file(785	builder: tempfile::Builder<'_, '_>,786	r: Vec<u8>,787	header: &str,788	comment: &str,789) -> Result<(Vec<u8>, Option<String>), anyhow::Error> {790	if !stdin().is_tty() {791		// TODO: Also try to open /dev/tty directly?792		bail!("stdin is not tty, can't open editor");793	}794795	use std::fmt::Write;796	let mut file = builder.tempfile()?;797798	let mut full_header = String::new();799	let mut had = false;800	for line in header.trim_end().lines() {801		had = true;802		writeln!(&mut full_header, "{comment}{line}")?;803	}804	if had {805		writeln!(&mut full_header, "{}", comment.trim_end())?;806	}807	writeln!(808		&mut full_header,809		"{comment}Do not touch this header! It will be removed automatically"810	)?;811812	file.write_all(full_header.as_bytes())?;813	file.write_all(&r)?;814815	let abs_path = file.into_temp_path();816	let editor = std::env::var_os("VISUAL")817		.or_else(|| std::env::var_os("EDITOR"))818		.unwrap_or_else(|| "vi".into());819	let editor_args = shlex::bytes::split(editor.as_encoded_bytes())820		.ok_or_else(|| anyhow!("EDITOR env var has wrong syntax"))?;821	let editor_args = editor_args822		.into_iter()823		.map(|v| {824			// Only ASCII subsequences are replaced825			unsafe { OsString::from_encoded_bytes_unchecked(v) }826		})827		.collect_vec();828	let Some((editor, args)) = editor_args.split_first() else {829		bail!("EDITOR env var has no command");830	};831	let mut command = Command::new(editor);832	command.args(args);833834	let path_arg = abs_path.canonicalize()?;835836	// TODO: Save full state, using tcget/_getmode/_setmode837	let was_raw = terminal::is_raw_mode_enabled()?;838	terminal::enable_raw_mode()?;839840	let status = command.arg(path_arg).status().await;841842	if !was_raw {843		terminal::disable_raw_mode()?;844	}845846	let success = match status {847		Ok(s) => s.success(),848		Err(e) if e.kind() == io::ErrorKind::NotFound => {849			bail!("editor not found")850		}851		Err(e) => bail!("editor spawn error: {e}"),852	};853854	let mut file = std::fs::read(&abs_path).context("read editor output")?;855	let Some(v) = file.strip_prefix(full_header.as_bytes()) else {856		todo!();857	};858	todo!();859860	// Ok((success, abs_path))861}862*/