git.delta.rocks / jrsonnet / refs/commits / eee08d567ee4

difftreelog

refactor remote command management

Yaroslav Bolyukin2023-06-09parent: #837e795.patch.diff
in: trunk

8 files changed

modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
after · cmds/fleet/src/cmds/build_systems.rs
1use std::{env::current_dir, time::Duration};23use crate::command::MyCommand;4use crate::host::Config;5use anyhow::Result;6use clap::Parser;7use tokio::{task::LocalSet, time::sleep};8use tracing::{error, field, info, info_span, warn, Instrument};910#[derive(Parser, Clone)]11pub struct BuildSystems {12	/// Do not continue on error13	#[clap(long)]14	fail_fast: bool,15	/// Run builds as sudo16	#[clap(long)]17	privileged_build: bool,18	#[clap(subcommand)]19	subcommand: Subcommand,20}2122enum UploadAction {23	Test,24	Boot,25	Switch,26}27impl UploadAction {28	fn name(&self) -> &'static str {29		match self {30			UploadAction::Test => "test",31			UploadAction::Boot => "boot",32			UploadAction::Switch => "switch",33		}34	}3536	pub(crate) fn should_switch_profile(&self) -> bool {37		matches!(self, Self::Switch | Self::Boot)38	}39	pub(crate) fn should_activate(&self) -> bool {40		matches!(self, Self::Switch | Self::Test)41	}42}4344enum PackageAction {45	SdImage,46	InstallationCd,47}48impl PackageAction {49	fn build_attr(&self) -> String {50		match self {51			PackageAction::SdImage => "sdImage".to_owned(),52			PackageAction::InstallationCd => "installationCd".to_owned(),53		}54	}55}5657enum Action {58	Upload { action: Option<UploadAction> },59	Package(PackageAction),60}61impl Action {62	fn build_attr(&self) -> String {63		match self {64			Action::Upload { .. } => "toplevel".to_owned(),65			Action::Package(p) => p.build_attr(),66		}67	}68}6970impl From<Subcommand> for Action {71	fn from(s: Subcommand) -> Self {72		match s {73			Subcommand::Upload => Self::Upload { action: None },74			Subcommand::Test => Self::Upload {75				action: Some(UploadAction::Test),76			},77			Subcommand::Boot => Self::Upload {78				action: Some(UploadAction::Boot),79			},80			Subcommand::Switch => Self::Upload {81				action: Some(UploadAction::Switch),82			},83			Subcommand::SdImage => Self::Package(PackageAction::SdImage),84			Subcommand::InstallationCd => Self::Package(PackageAction::InstallationCd),85		}86	}87}8889#[derive(Parser, Clone)]90enum Subcommand {91	/// Upload, but do not switch92	Upload,93	/// Upload + switch to built system until reboot94	Test,95	/// Upload + switch to built system after reboot96	Boot,97	/// Upload + test + boot98	Switch,99100	/// Build SD .img image101	SdImage,102	/// Build an installation cd ISO image103	InstallationCd,104}105106impl BuildSystems {107	async fn build_task(self, config: Config, host: String) -> Result<()> {108		info!("building");109		let action = Action::from(self.subcommand.clone());110		let built = {111			let dir = tempfile::tempdir()?;112			dir.path().to_owned()113		};114115		let mut nix_build = MyCommand::new("nix");116		nix_build117			.args([118				"build",119				"--impure",120				"--json",121				// "--show-trace",122				"--no-link",123			])124			.comparg("--out-link", &built)125			.arg(126				config.configuration_attr_name(&format!(127					"buildSystems.{}.{host}",128					action.build_attr()129				)),130			)131			.args(&config.nix_args);132133		if self.privileged_build {134			nix_build = nix_build.sudo();135		}136137		nix_build.run_nix().await.map_err(|e| {138			if action.build_attr() == "sdImage" {139				info!("sd-image build failed");140				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");141				info!("This module was automatically imported before, but was removed for better customization")142			}143			e144		})?;145		let built = std::fs::canonicalize(built)?;146147		match action {148			Action::Upload { action } => {149				if !config.is_local(&host) {150					info!("uploading system closure");151					let mut tries = 0;152					loop {153						let mut nix = MyCommand::new("nix");154						nix.arg("copy")155							.comparg("--to", format!("ssh://root@{host}"))156							.arg(&built);157						match nix.run_nix().await {158							Ok(()) => break,159							Err(e) if tries < 3 => {160								tries += 1;161								warn!("Copy failure ({}/3): {}", tries, e);162								sleep(Duration::from_millis(5000)).await;163							}164							Err(e) => return Err(e),165						}166					}167				}168				if let Some(action) = action {169					if action.should_switch_profile() {170						info!("switching generation");171						let mut cmd = MyCommand::new("nix-env");172						cmd.comparg("--profile", "/nix/var/nix/profiles/system")173							.comparg("--set", &built);174						config.run_on(&host, cmd, true).await?;175					}176					if action.should_activate() {177						info!("executing activation script");178						let mut switch_script = built.clone();179						switch_script.push("bin");180						switch_script.push("switch-to-configuration");181						let mut cmd = MyCommand::new(switch_script);182						cmd.arg(action.name());183						config.run_on(&host, cmd, true).await?;184					}185				}186			}187			Action::Package(PackageAction::SdImage) => {188				let mut out = current_dir()?;189				out.push(format!("sd-image-{}", host));190191				info!("building sd image to {:?}", out);192				let mut nix_build = MyCommand::new("nix");193				nix_build194					.args(["build", "--impure", "--no-link"])195					.comparg("--out-link", &out)196					.arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))197					.args(&config.nix_args);198				if !self.fail_fast {199					nix_build.arg("--keep-going");200				}201				if self.privileged_build {202					nix_build = nix_build.sudo();203				}204205				nix_build.run_nix().await?;206			}207			Action::Package(PackageAction::InstallationCd) => {208				let mut out = current_dir()?;209				out.push(format!("installation-cd-{}", host));210211				info!("building sd image to {:?}", out);212				let mut nix_build = MyCommand::new("nix");213				nix_build214					.args(["build", "--impure", "--no-link"])215					.comparg("--out-link", &out)216					.arg(217						config.configuration_attr_name(&format!(218							"buildSystems.installationCd.{}",219							host,220						)),221					)222					.args(&config.nix_args);223				if !self.fail_fast {224					nix_build.arg("--keep-going");225				}226				if self.privileged_build {227					nix_build = nix_build.sudo();228				}229230				nix_build.run_nix().await?;231			}232		};233		Ok(())234	}235236	pub async fn run(self, config: &Config) -> Result<()> {237		let hosts = config.list_hosts().await?;238		let set = LocalSet::new();239		let this = &self;240		for host in hosts.iter() {241			if config.should_skip(host) {242				continue;243			}244			let config = config.clone();245			let host = host.clone();246			let this = this.clone();247			let span = info_span!("deployment", host = field::display(&host));248			set.spawn_local(249				(async move {250					match this.build_task(config, host).await {251						Ok(_) => {}252						Err(e) => {253							error!("failed to deploy host: {}", e)254						}255					}256				})257				.instrument(span),258			);259		}260		set.await;261		Ok(())262	}263}
modifiedcmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -2,18 +2,16 @@
 	fleetdata::{FleetSecret, FleetSharedSecret},
 	host::Config,
 };
-use age::{Decryptor, Encryptor};
 use anyhow::{bail, ensure, Context, Result};
 use clap::Parser;
 use futures::{StreamExt, TryStreamExt};
 use std::{
 	collections::HashSet,
-	io::{self, Cursor, Read, Write},
-	iter,
+	io::{self, Cursor, Read},
 	path::PathBuf,
 };
 use tokio::fs::read_to_string;
-use tracing::{info, warn};
+use tracing::{info, warn, error};
 
 #[derive(Parser)]
 pub enum Secrets {
@@ -313,6 +311,7 @@
 				}
 				let mut to_remove = Vec::new();
 				for name in &config.list_shared() {
+					info!("updating secret: {name}");
 					let mut data = config.shared_secret(name)?;
 					let expected_owners: Vec<String> = config
 						.shared_config_attr(&format!("sharedSecrets.\"{name}\".expectedOwners"))
@@ -326,14 +325,12 @@
 					let expected_set = expected_owners.iter().collect::<HashSet<_>>();
 					let should_remove = set.difference(&expected_set).next().is_some();
 					if set != expected_set {
-						warn!("reconfiguring owners for {name}");
-						let generator: Option<String> = config
-							.shared_config_attr(&format!("sharedSecrets.\"{name}\".generator"))
+						let owner_dependent: bool = config
+							.shared_config_attr(&format!("sharedSecrets.\"{name}\".ownerDependent"))
 							.await?;
-						// TODO: if !.owner_dependent
-						if let Some(str) = generator {
-							todo!("regenerate")
-						} else {
+						if !owner_dependent {
+							warn!("reencrypting secret '{name}' for new owner set");
+							// TODO: force regeneration
 							if should_remove {
 								warn!("secret will not be regenerated for removed machines, and until host rebuild, they will still possess the ability to decode secret");
 							}
@@ -367,7 +364,16 @@
 							data.secret.secret = encrypted;
 							data.owners = expected_owners;
 							config.replace_shared(name.to_owned(), data);
+						} else if let Some(generator) = config
+							.shared_config_attr::<Option<String>>(&format!("sharedSecrets.\"{name}\".generator"))
+							.await?
+						{
+							todo!("regenerate secret {name} with {generator}");
+						} else {
+							error!("secret '{name}' should be regenerated manually");
 						}
+					} else {
+						info!("secret data is ok")
 					}
 				}
 				for k in to_remove {
modifiedcmds/fleet/src/command.rsdiffbeforeafterboth
--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -1,4 +1,4 @@
-use std::{ffi::OsStr, process::Stdio};
+use std::{ffi::OsStr, process::Stdio, task::Poll};
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
@@ -7,20 +7,292 @@
 	de::{DeserializeOwned, Visitor},
 	Deserialize,
 };
-use tokio::{process::Command, select};
+use tokio::{io::AsyncRead, process::Command, select};
 use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};
 use tracing::{info, warn};
 
+fn escape_bash(input: &str, out: &mut String) {
+	const TO_ESCAPE: &str = "$ !\"#&'()*,;<>?[\\]^`{|}";
+	if input.chars().all(|c| !TO_ESCAPE.contains(c)) {
+		out.push_str(input);
+		return;
+	}
+	out.push('\'');
+	for (i, v) in input.split('\'').enumerate() {
+		if i != 0 {
+			out.push_str("'\"'\"'");
+		}
+		out.push_str(v);
+	}
+	out.push('\'');
+}
+fn ostoutf8(os: impl AsRef<OsStr>) -> String {
+	os.as_ref().to_str().expect("non-utf8 data").to_owned()
+}
+#[derive(Clone)]
+pub struct MyCommand {
+	command: String,
+	args: Vec<String>,
+	env: Vec<(String, String)>,
+}
+impl MyCommand {
+	pub fn new(cmd: impl AsRef<OsStr>) -> Self {
+		assert!(!cmd.as_ref().is_empty());
+		Self {
+			command: ostoutf8(cmd),
+			args: vec![],
+			env: vec![],
+		}
+	}
+	fn into_args(self) -> Vec<String> {
+		let mut out = Vec::new();
+		if !self.env.is_empty() {
+			out.push("env".to_owned());
+			for (k, v) in self.env {
+				assert!(!k.contains("="));
+				out.push(format!("{k}={v}"));
+			}
+		}
+		out.push(self.command);
+		out.extend(self.args.into_iter());
+		out
+	}
+	fn into_string(self) -> String {
+		let mut out = String::new();
+		if !self.env.is_empty() {
+			out.push_str("env");
+			for (k, v) in self.env {
+				out.push(' ');
+				assert!(!k.contains("="));
+				escape_bash(&k, &mut out);
+				out.push('=');
+				escape_bash(&v, &mut out);
+			}
+		}
+		if !out.is_empty() {
+			out.push(' ');
+		}
+		escape_bash(&self.command, &mut out);
+		for arg in self.args {
+			out.push(' ');
+			escape_bash(&arg, &mut out);
+		}
+		out
+	}
+	fn into_command(self) -> Command {
+		let mut out = Command::new(self.command);
+		out.args(self.args);
+		for (k, v) in self.env {
+			out.env(k, v);
+		}
+		out
+	}
+	pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
+		let arg = arg.as_ref();
+		self.args.push(ostoutf8(arg));
+		self
+	}
+	pub fn eqarg(&mut self, arg: impl AsRef<OsStr>, value: impl AsRef<OsStr>) -> &mut Self {
+		let arg = arg.as_ref();
+		let value = value.as_ref();
+		let arg = ostoutf8(arg);
+		let value = ostoutf8(value);
+		self.arg(format!("{arg}={value}"));
+		self
+	}
+	pub fn comparg(&mut self, arg: impl AsRef<OsStr>, value: impl AsRef<OsStr>) -> &mut Self {
+		self.arg(arg);
+		self.arg(value);
+		self
+	}
+	pub fn args<V: AsRef<OsStr>>(&mut self, args: impl IntoIterator<Item = V>) -> &mut Self {
+		for arg in args.into_iter() {
+			let arg = arg.as_ref();
+			self.args.push(ostoutf8(arg));
+		}
+		self
+	}
+	pub fn sudo(self) -> Self {
+		let mut out = Self::new("sudo");
+		out.args(self.into_args());
+		out
+	}
+	pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {
+		let mut out = Self::new("ssh");
+		out.arg(on).arg("--");
+		out.arg(self.into_string());
+		out
+	}
+
+	pub async fn run(self) -> Result<()> {
+		let str = self.clone().into_string();
+		info!("running {str}");
+		let mut cmd = self.into_command();
+		cmd.inherit_stdio();
+		let out = cmd.spawn()?.wait_with_output().await?;
+		if !out.status.success() {
+			anyhow::bail!("command '{}' failed with status {}", str, out.status);
+		}
+		Ok(())
+	}
+	pub async fn run_string(self) -> Result<String> {
+		let str = self.clone().into_string();
+		info!("running {str}");
+		let mut cmd = self.into_command();
+		cmd.inherit_stdio();
+		cmd.stdout(Stdio::piped());
+		let out = cmd.spawn()?.wait_with_output().await?;
+		if !out.status.success() {
+			anyhow::bail!("command '{}' failed with status {}", str, out.status);
+		}
+		Ok(String::from_utf8(out.stdout)?)
+	}
+	pub async fn run_nix_json<T: DeserializeOwned>(self) -> Result<T> {
+		let str = self.run_nix_string().await?;
+		serde_json::from_str(&str).with_context(|| format!("{:?}", str))
+	}
+
+	pub async fn run_nix_string(self) -> Result<String> {
+		let str = self.clone().into_string();
+		let mut cmd = self.into_command();
+		cmd.stdout(Stdio::piped());
+		run_nix_inner(str, cmd).await.map(|v| v.unwrap())
+	}
+	pub async fn run_nix(self) -> Result<()> {
+		let str = self.clone().into_string();
+		let mut cmd = self.into_command();
+		cmd.stdout(Stdio::inherit());
+		run_nix_inner(str, cmd).await.map(|v| {
+			assert!(v.is_none());
+		})
+	}
+}
+
+struct EmptyAsyncRead;
+impl AsyncRead for EmptyAsyncRead {
+	fn poll_read(
+		self: std::pin::Pin<&mut Self>,
+		_cx: &mut std::task::Context<'_>,
+		_buf: &mut tokio::io::ReadBuf<'_>,
+	) -> Poll<std::io::Result<()>> {
+		Poll::Pending
+	}
+}
+
+async fn run_nix_inner(str: String, mut cmd: Command) -> Result<Option<String>> {
+	info!("running {str}");
+	cmd.arg("--log-format").arg("internal-json");
+	cmd.stderr(Stdio::piped());
+	let mut child = cmd.spawn()?;
+	let mut stderr = child.stderr.take().unwrap();
+	let stdout = child.stdout.take();
+	let wants_stdout = stdout.is_some();
+	let mut err = FramedRead::new(&mut stderr, LinesCodec::new());
+	let mut out: Box<dyn AsyncRead + Unpin> = stdout
+		.map(|s| Box::new(s) as Box<dyn AsyncRead + Unpin>)
+		.unwrap_or_else(|| Box::new(EmptyAsyncRead));
+	let mut out = FramedRead::new(&mut out, BytesCodec::new());
+
+	// while let Some(line) = read.next().await? {}
+
+	let mut out_buf = if wants_stdout { Some(vec![]) } else { None };
+	loop {
+		select! {
+			e = err.next() => {
+				if let Some(e) = e {
+					let e = e?;
+					if let Some(e) = e.strip_prefix("@nix ") {
+
+						let log: NixLog = match serde_json::from_str(e) {
+							Ok(l) => l,
+							Err(err) => {
+								warn!("failed to parse nix log line {:?}: {}", e, err);
+								continue;
+							},
+						};
+						match log {
+							NixLog::Msg { msg, raw_msg, .. } => {
+								if !(msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m Git tree '") && msg.ends_with("' is dirty"))
+									&& !msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m not writing modified lock file of flake")
+									&& msg != "\u{1b}[35;1mwarning:\u{1b}[0m \u{1b}[31;1merror:\u{1b}[0m SQLite database '\u{1b}[35;1m/nix/var/nix/db/db.sqlite\u{1b}[0m' is busy" {
+									if let Some(raw_msg) = raw_msg {
+										info!(target: "nix", "{raw_msg}\n{msg}")
+									}else {
+										info!(target: "nix", "{msg}")
+
+									}
+								}
+							},
+							NixLog::Start { ref fields, typ, .. } if typ == 105 && !fields.is_empty() => {
+								if let [LogField::String(drv), ..] = &fields[..] {
+									info!(target: "nix","building {}", drv)
+								} else {
+									warn!("bad build log: {:?}", log)
+								}
+							},
+							NixLog::Start { ref fields, typ, .. } if typ == 100 && fields.len() >= 3 => {
+								if let [LogField::String(drv), LogField::String(from), LogField::String(to), ..] = &fields[..] {
+									info!(target: "nix","copying {} {} -> {}", drv, from, to)
+								} else {
+									warn!("bad copy log: {:?}", log)
+								}
+							},
+							NixLog::Start { text, typ, .. } if typ == 0 || typ == 102 || typ == 103 || typ == 104 => {
+								if !text.is_empty() && text != "querying info about missing paths" && text != "copying 0 paths" {
+									info!(target: "nix", "{}", text)
+								}
+							},
+							NixLog::Start { text, level: 0, typ: 108, .. } if text.is_empty() => {
+								// Cache lookup? Coupled with copy log
+							},
+							NixLog::Start { text, level: 4, typ: 109, .. } if text.starts_with("querying info about ") => {
+								// Cache lookup
+							}
+							NixLog::Start { text, level: 4, typ: 101, .. } if text.starts_with("downloading ") => {
+								// NAR downloading, coupled with copy log
+							}
+							NixLog::Start { text, level: 1, typ: 111, .. } if text.starts_with("waiting for a machine to build ") => {
+								// Useless repeating notification about build
+							}
+							NixLog::Start { text, level: 3, typ: 111, .. } if text.starts_with("resolved derivation:  ") => {
+								// CA resolved
+							}
+							NixLog::Stop { .. } => {},
+							NixLog::Result { .. } => {},
+							_ => warn!("unknown log: {:?}", log)
+						};
+					} else {
+						warn!(target="nix","unknown: {}", e)
+					}
+				}
+			},
+			o = out.next() => {
+				if let Some(o) = o {
+					out_buf.as_mut().expect("stdout == wants_stdout").extend_from_slice(&o?);
+				}
+			},
+			code = child.wait() => {
+				let code = code?;
+				if !code.success() {
+					anyhow::bail!("command '{str}' failed with status {}", code);
+				}
+				break;
+			}
+		}
+	}
+
+	Ok(out_buf.map(String::from_utf8).transpose()?)
+}
+
 #[async_trait]
 pub trait CommandExt {
-	async fn run_nix(&mut self) -> Result<()>;
-	async fn run_nix_json<T: DeserializeOwned>(&mut self) -> Result<T>;
-	async fn run_nix_string(&mut self) -> Result<String>;
-	async fn run(&mut self) -> Result<()>;
-	async fn run_json<T: DeserializeOwned>(&mut self) -> Result<T>;
-	async fn run_string(&mut self) -> Result<String>;
+	// async fn run_nix(&mut self) -> Result<()>;
+	// async fn run_nix_json<T: DeserializeOwned>(&mut self) -> Result<T>;
+	// async fn run_nix_string(&mut self) -> Result<String>;
+	// async fn run(&mut self) -> Result<()>;
+	// async fn run_json<T: DeserializeOwned>(&mut self) -> Result<T>;
+	// async fn run_string(&mut self) -> Result<String>;
 	fn inherit_stdio(&mut self) -> &mut Self;
-	fn ssh_on(host: impl AsRef<OsStr>, command: impl AsRef<OsStr>) -> Self;
 }
 
 #[derive(Debug)]
@@ -91,170 +363,9 @@
 
 #[async_trait]
 impl CommandExt for Command {
-	async fn run_nix(&mut self) -> Result<()> {
-		self.run_nix_string().await.map(|_| ())
-	}
-	async fn run_nix_json<T: DeserializeOwned>(&mut self) -> Result<T> {
-		let str = self.run_nix_string().await?;
-		serde_json::from_str(&str).with_context(|| format!("{:?}", str))
-	}
-
-	async fn run_nix_string(&mut self) -> Result<String> {
-		self.arg("--log-format").arg("internal-json");
-		self.stderr(Stdio::piped());
-		self.stdout(Stdio::piped());
-		let mut child = self.spawn()?;
-		let mut stderr = child.stderr.take().unwrap();
-		let mut stdout = child.stdout.take().unwrap();
-		let mut err = FramedRead::new(&mut stderr, LinesCodec::new());
-		let mut out = FramedRead::new(&mut stdout, BytesCodec::new());
-
-		// while let Some(line) = read.next().await? {}
-
-		let mut out_buf = vec![];
-		loop {
-			select! {
-				e = err.next() => {
-					if let Some(e) = e {
-						let e = e?;
-						if let Some(e) = e.strip_prefix("@nix ") {
-
-							let log: NixLog = match serde_json::from_str(e) {
-								Ok(l) => l,
-								Err(err) => {
-									warn!("failed to parse nix log line {:?}: {}", e, err);
-									continue;
-								},
-							};
-							match log {
-								NixLog::Msg { msg, raw_msg, .. } => {
-									if !(msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m Git tree '") && msg.ends_with("' is dirty"))
-										&& !msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m not writing modified lock file of flake")
-										&& msg != "\u{1b}[35;1mwarning:\u{1b}[0m \u{1b}[31;1merror:\u{1b}[0m SQLite database '\u{1b}[35;1m/nix/var/nix/db/db.sqlite\u{1b}[0m' is busy" {
-										if let Some(raw_msg) = raw_msg {
-											info!(target: "nix", "{raw_msg}\n{msg}")
-										}else {
-											info!(target: "nix", "{msg}")
-
-										}
-									}
-								},
-								NixLog::Start { ref fields, typ, .. } if typ == 105 && !fields.is_empty() => {
-									if let [LogField::String(drv), ..] = &fields[..] {
-										info!(target: "nix","building {}", drv)
-									} else {
-										warn!("bad build log: {:?}", log)
-									}
-								},
-								NixLog::Start { ref fields, typ, .. } if typ == 100 && fields.len() >= 3 => {
-									if let [LogField::String(drv), LogField::String(from), LogField::String(to), ..] = &fields[..] {
-										info!(target: "nix","copying {} {} -> {}", drv, from, to)
-									} else {
-										warn!("bad copy log: {:?}", log)
-									}
-								},
-								NixLog::Start { text, typ, .. } if typ == 0 || typ == 102 || typ == 103 || typ == 104 => {
-									if !text.is_empty() && text != "querying info about missing paths" && text != "copying 0 paths" {
-										info!(target: "nix", "{}", text)
-									}
-								},
-								NixLog::Start { text, level: 0, typ: 108, .. } if text.is_empty() => {
-									// Cache lookup? Coupled with copy log
-								},
-								NixLog::Start { text, level: 4, typ: 109, .. } if text.starts_with("querying info about ") => {
-									// Cache lookup
-								}
-								NixLog::Start { text, level: 4, typ: 101, .. } if text.starts_with("downloading ") => {
-									// NAR downloading, coupled with copy log
-								}
-								NixLog::Start { text, level: 1, typ: 111, .. } if text.starts_with("waiting for a machine to build ") => {
-									// Useless repeating notification about build
-								}
-								NixLog::Start { text, level: 3, typ: 111, .. } if text.starts_with("resolved derivation:  ") => {
-									// CA resolved
-								}
-								NixLog::Stop { .. } => {},
-								NixLog::Result { .. } => {},
-								_ => warn!("unknown log: {:?}", log)
-							};
-						} else {
-							warn!(target="nix","unknown: {}", e)
-						}
-					}
-				},
-				o = out.next() => {
-					if let Some(o) = o {
-						out_buf.extend_from_slice(&o?);
-					}
-				},
-				code = child.wait() => {
-					let code = code?;
-					if !code.success() {
-						anyhow::bail!("command ({:?}) failed with status {}", self, code);
-					}
-					break;
-				}
-			}
-		}
-
-		Ok(String::from_utf8(out_buf)?)
-	}
-
 	fn inherit_stdio(&mut self) -> &mut Self {
 		self.stderr(Stdio::inherit());
+		self.stdout(Stdio::inherit());
 		self
-	}
-
-	async fn run(&mut self) -> Result<()> {
-		self.stderr(Stdio::piped());
-		self.stdout(Stdio::piped());
-		let mut child = self.spawn()?;
-		let mut stderr = child.stderr.take().unwrap();
-		let mut stdout = child.stdout.take().unwrap();
-		let mut err = FramedRead::new(&mut stderr, LinesCodec::new());
-		let mut out = FramedRead::new(&mut stdout, LinesCodec::new());
-		loop {
-			select! {
-				e = err.next() => {
-					if let Some(e) = e {
-						warn!("{}", e?);
-					}
-				},
-				o = out.next() => {
-					if let Some(o) = o {
-						info!("{}", o?);
-					}
-				},
-				code = child.wait() => {
-					let code = code?;
-					if !code.success() {
-						anyhow::bail!("command ({:?}) failed with status {}", self, code);
-					}
-					break;
-				}
-			}
-		}
-		Ok(())
-	}
-
-	async fn run_json<T: DeserializeOwned>(&mut self) -> Result<T> {
-		let str = self.run_string().await?;
-		serde_json::from_str(&str).with_context(|| format!("{:?}", str))
-	}
-
-	async fn run_string(&mut self) -> Result<String> {
-		self.inherit_stdio();
-		self.stdout(Stdio::piped());
-		let out = self.spawn()?.wait_with_output().await?;
-		if !out.status.success() {
-			anyhow::bail!("command ({:?}) failed with status {}", self, out.status);
-		}
-		Ok(String::from_utf8(out.stdout)?)
-	}
-
-	fn ssh_on(host: impl AsRef<OsStr>, command: impl AsRef<OsStr>) -> Self {
-		let mut cmd = Command::new("ssh");
-		cmd.arg(host).arg("--").arg(command);
-		cmd
 	}
 }
modifiedcmds/fleet/src/host.rsdiffbeforeafterboth
--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -1,7 +1,7 @@
 use std::{
 	cell::{Ref, RefCell, RefMut},
 	env::current_dir,
-	ffi::{OsStr, OsString},
+	ffi::OsString,
 	io::Write,
 	ops::Deref,
 	path::PathBuf,
@@ -12,10 +12,9 @@
 use clap::{ArgGroup, Parser};
 use serde::de::DeserializeOwned;
 use tempfile::NamedTempFile;
-use tokio::process::Command;
 
 use crate::{
-	command::CommandExt,
+	command::MyCommand,
 	fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
 };
 
@@ -52,24 +51,24 @@
 		self.opts.localhost.as_ref().map(|s| s as &str) == Some(host)
 	}
 
-	pub fn command_on(&self, host: &str, program: impl AsRef<OsStr>, sudo: bool) -> Command {
-		if self.is_local(host) {
-			if sudo {
-				let mut cmd = Command::new("sudo");
-				cmd.arg(program);
-				cmd
-			} else {
-				Command::new(program)
-			}
-		} else {
-			let mut cmd = Command::new("ssh");
-			cmd.arg(host).arg("--");
-			if sudo {
-				cmd.arg("sudo");
-			}
-			cmd.arg(program);
-			cmd
+	pub async fn run_on(&self, host: &str, mut command: MyCommand, sudo: bool) -> Result<()> {
+		if sudo {
+			command = command.sudo();
+		}
+		if !self.is_local(host) {
+			command = command.ssh(host);
+		}
+		command.run().await
+	}
+	#[must_use]
+	pub async fn run_string_on(&self, host: &str, mut command: MyCommand, sudo: bool) -> Result<String> {
+		if sudo {
+			command = command.sudo();
+		}
+		if !self.is_local(host) {
+			command = command.ssh(host);
 		}
+		command.run_string().await
 	}
 
 	pub fn configuration_attr_name(&self, name: &str) -> OsString {
@@ -83,36 +82,36 @@
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<String>> {
-		Command::new("nix")
-			.arg("eval")
+		let mut cmd = MyCommand::new("nix");
+		cmd.arg("eval")
 			.arg(self.configuration_attr_name("configuredHosts"))
 			.args(["--apply", "builtins.attrNames", "--json", "--show-trace"])
-			.args(&self.nix_args)
-			.run_nix_json()
+			.args(&self.nix_args);
+		cmd.run_nix_json()
 			.await
 	}
 	pub async fn shared_config_attr<T: DeserializeOwned>(&self, attr: &str) -> Result<T> {
-		Command::new("nix")
-			.arg("eval")
+		let mut cmd = MyCommand::new("nix");
+		cmd.arg("eval")
 			.arg(self.configuration_attr_name(&format!("configUnchecked.{}", attr)))
 			.args(["--json", "--show-trace"])
-			.args(&self.nix_args)
-			.run_nix_json()
+			.args(&self.nix_args);
+		cmd.run_nix_json()
 			.await
 	}
 	pub async fn shared_config_attr_names(&self, attr: &str) -> Result<Vec<String>> {
-		Command::new("nix")
-			.arg("eval")
+		let mut cmd = MyCommand::new("nix");
+		cmd.arg("eval")
 			.arg(self.configuration_attr_name(&format!("configUnchecked.{}", attr)))
 			.args(["--apply", "builtins.attrNames"])
 			.args(["--json", "--show-trace"])
-			.args(&self.nix_args)
-			.run_nix_json()
+			.args(&self.nix_args);
+		cmd.run_nix_json()
 			.await
 	}
 	pub async fn config_attr<T: DeserializeOwned>(&self, host: &str, attr: &str) -> Result<T> {
-		Command::new("nix")
-			.arg("eval")
+		let mut cmd = MyCommand::new("nix");
+		cmd.arg("eval")
 			.arg(
 				self.configuration_attr_name(&format!(
 					"configuredSystems.{}.config.{}",
@@ -120,8 +119,8 @@
 				)),
 			)
 			.args(["--json", "--show-trace"])
-			.args(&self.nix_args)
-			.run_nix_json()
+			.args(&self.nix_args);
+		cmd.run_nix_json()
 			.await
 	}
 
@@ -171,23 +170,20 @@
 
 	pub async fn decrypt_on_host(&self, host: &str, data: Vec<u8>) -> Result<Vec<u8>>{
 		let data = z85::encode(&data);
-		let encoded = self.command_on(host, "fleet-install-secrets", true)
-			.arg("decrypt")
-			.arg("--secret")
-			.arg(data).run_string().await.context("failed to call remote host for decrypt")?.trim().to_owned();
+		let mut cmd = MyCommand::new("fleet-install-secrets");
+		cmd.arg("decrypt").eqarg("--secret", data);
+		cmd = cmd.sudo().ssh(host);
+		let encoded = cmd.run_string().await.context("failed to call remote host for decrypt")?.trim().to_owned();
 		Ok(z85::decode(encoded).context("bad encoded data? outdated host?")?)
 	}
 	pub async fn reencrypt_on_host(&self, host: &str, data: Vec<u8>, targets: Vec<String>) -> Result<Vec<u8>>{
 		let data = z85::encode(&data);
-		let mut recmd = self.command_on(host, "fleet-install-secrets", true);
-		recmd
-			.arg("reencrypt")
-			.arg("--secret")
-			.arg(format!("\"{}\"", data.replace('$', "\\$")));
+		let mut recmd = MyCommand::new("fleet-install-secrets");
+		recmd.arg("reencrypt").eqarg("--secret",data);
 		for target in targets {
-			recmd.arg("--targets");
-			recmd.arg(format!("\"{target}\""));
+			recmd.eqarg("--targets", target);
 		}
+		recmd = recmd.sudo().ssh(host);
 		let encoded = recmd.run_string().await.context("failed to call remote host for decrypt")?.trim().to_owned();
 		Ok(z85::decode(encoded).context("bad encoded data? outdated host?")?)
 	}
modifiedcmds/fleet/src/keys.rsdiffbeforeafterboth
--- a/cmds/fleet/src/keys.rs
+++ b/cmds/fleet/src/keys.rs
@@ -1,6 +1,7 @@
 use std::str::FromStr;
 
-use crate::{command::CommandExt, host::Config};
+use crate::command::MyCommand;
+use crate::host::Config;
 use anyhow::{anyhow, Result};
 use tracing::warn;
 
@@ -26,11 +27,9 @@
 			Ok(key)
 		} else {
 			warn!("Loading key for {}", host);
-			let key = self
-				.command_on(host, "cat", false)
-				.arg("/etc/ssh/ssh_host_ed25519_key.pub")
-				.run_string()
-				.await?;
+			let mut cmd = MyCommand::new("cat");
+			cmd.arg("/etc/ssh/ssh_host_ed25519_key.pub");
+			let key = self.run_string_on(host, cmd, false).await?;
 			self.update_key(host, key.clone());
 			Ok(key)
 		}
modifiedcmds/install-secrets/src/main.rsdiffbeforeafterboth
--- a/cmds/install-secrets/src/main.rs
+++ b/cmds/install-secrets/src/main.rs
@@ -250,7 +250,7 @@
 
 			if plaintext {
 				let s = String::from_utf8(decrypted).context("output is not utf8")?;
-				print!("{}", s);
+				print!("{s}");
 			} else {
 				println!("{}", SecretWrapper(decrypted));
 			}
modifiedmodules/fleet/secrets.nixdiffbeforeafterboth
--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -2,15 +2,6 @@
 let
   sharedSecret = with types; {
     options = {
-      owners = mkOption {
-        type = listOf str;
-        description = ''
-          For which owners this secret is currently encrypted,
-          if not matches expectedOwners - then this secret is considered outdated, and
-          should be regenerated/reencrypted
-        '';
-        default = [ ];
-      };
       expectedOwners = mkOption {
         type = listOf str;
         description = ''
@@ -25,12 +16,38 @@
         description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted";
       };
       generator = mkOption {
-        type = nullOr package;
-        description = ''
-          Derivation to execute for secret generation
+        type = nullOr (submodule {
+          packages = mkOption {
+            type = attrsOf package;
+            description = ''
+              Derivation to execute for shared secret generation (key = system).
+              This derivation should produce directory, with exactly two files:
+                - publicData
+                - encryptedSecretData
+
+              If null - secret value may only be created manually.
+            '';
+          };
+          expectedData = mkOption {
+            type = types.unspecified;
+            description = "Data expected to be used for secret generation, if doesn't match specified - secret should be regenerated";
+          };
+          dependencies = mkOption {
+            type = listOf str;
+            description = ''
+              List of secrets, on which this secret depends.
 
-          If null - may only be created manually
-        '';
+              During generation, generator command will be ran on host, which already has specified secrets generated.
+            '';
+            default = [];
+          };
+          data = mkOption {
+            type = types.unspecified;
+            description = "Data used for secret generation. Imported from fleet.nix";
+            default = null;
+            internal = true;
+          };
+        });
         default = null;
       };
       expireIn = mkOption {
@@ -38,15 +55,28 @@
         description = "Time in hours, in which this secret should be regenerated";
         default = null;
       };
+
+      owners = mkOption {
+        type = listOf str;
+        description = ''
+          For which owners this secret is currently encrypted,
+          if not matches expectedOwners - then this secret is considered outdated, and
+          should be regenerated/reencrypted.
+
+          Imported from fleet.nix
+        '';
+        default = [ ];
+      };
       public = mkOption {
         type = nullOr str;
-        description = "Secret public data";
+        description = "Secret public data. Imported from fleet.nix";
         default = null;
       };
       secret = mkOption {
         type = nullOr str;
-        description = "Encrypted secret data";
+        description = "Encrypted secret data. Imported from fleet.nix";
         default = null;
+        internal = true;
       };
     };
   };
modifiednixos/secrets.nixdiffbeforeafterboth
--- a/nixos/secrets.nix
+++ b/nixos/secrets.nix
@@ -5,14 +5,14 @@
 let
   sysConfig = config;
   secretType = types.submodule ({ config, ... }: {
-    config = rec {
-      stableSecretPath = mkOptionDefault "/run/secrets/secret-stable-${config._module.args.name}";
-      secretPath = mkOptionDefault "/run/secrets/secret-${config.secretHash}-${config._module.args.name}";
-      secretHash = mkOptionDefault (if config.secret != null then (builtins.hashString "sha1" config.secret) else "<missingno>");
+    config = let secretName = config._module.args.name; in rec {
+      stableSecretPath = mkOptionDefault "/run/secrets/secret-stable-${secretName}";
+      secretPath = mkOptionDefault "/run/secrets/secret-${config.secretHash}-${secretName}";
+      secretHash = mkOptionDefault (if config.secret != null then (builtins.hashString "sha1" config.secret) else throw "secret is not defined for secret ${secretName}");
 
-      stablePublicPath = mkOptionDefault "/run/secrets/public-stable-${config._module.args.name}";
-      publicPath = mkOptionDefault "/run/secrets/public-${config.publicHash}-${config._module.args.name}";
-      publicHash = mkOptionDefault (if config.public != null then (builtins.hashString "sha1" config.public) else "<missingno>");
+      stablePublicPath = mkOptionDefault "/run/secrets/public-stable-${secretName}";
+      publicPath = mkOptionDefault "/run/secrets/public-${config.publicHash}-${secretName}";
+      publicHash = mkOptionDefault (if config.public != null then (builtins.hashString "sha1" config.public) else throw "public is not defined for secret ${secretName}");
     };
     options = {
       public = mkOption {
@@ -77,7 +77,13 @@
   });
   secretsFile = pkgs.writeTextFile {
     name = "secrets.json";
-    text = builtins.toJSON config.secrets;
+    text = builtins.toJSON (mapAttrs (_: value: rec {
+      inherit (value) group mode owner secret public;
+      publicPath = if public != null then value.publicPath else "/missingno";
+      stablePublicPath = if public != null then value.stablePublicPath else "/missingno";
+      secretPath = if secret != null then value.secretPath else "/missingno";
+      stableSecretPath = if secret != null then value.stableSecretPath else "/missingno";
+    }) config.secrets);
   };
 in
 {