git.delta.rocks / jrsonnet / refs/commits / aac7f3854e2a

difftreelog

fix(fleet-cmd) include the required package for tab completions (#5)

Petr Portnov | PROgrm_JARvis2024-06-02parent: #ad7852d.patch.diff
in: trunk
* fix(fleet-cmd): include the required package for tab completions

* style(fleet-cmd): reformat automatically

9 files changed

modifiedcmds/fleet/src/better_nix_eval.rsdiffbeforeafterboth
--- a/cmds/fleet/src/better_nix_eval.rs
+++ b/cmds/fleet/src/better_nix_eval.rs
@@ -1,25 +1,24 @@
 //! Wrapper around nix repl, which allows to work on nix code, without relying on
 //! nix libexpr. I mean, nix libexpr is good, but until it has no C bindings, this is the royal PITA.
 
-use std::collections::HashMap;
-use std::ffi::{OsStr, OsString};
-use std::fmt::{self, Display};
-use std::path::PathBuf;
-use std::process::Stdio;
-use std::sync::{Arc, OnceLock};
+use std::{
+	collections::HashMap,
+	ffi::{OsStr, OsString},
+	fmt::{self, Display},
+	path::PathBuf,
+	process::Stdio,
+	sync::{Arc, OnceLock},
+};
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
 use better_command::{ClonableHandler, Handler, NixHandler, NoopHandler};
 use futures::StreamExt;
 use itertools::Itertools;
-use serde::de::DeserializeOwned;
-use serde::{Deserialize, Serialize};
-use tokio::io::AsyncWriteExt;
-use tokio::process::{ChildStderr, ChildStdin, ChildStdout, Command};
-use tokio::select;
-use tokio::sync::{mpsc, oneshot, Mutex};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use tokio::{
+	io::AsyncWriteExt,
+	process::{ChildStderr, ChildStdin, ChildStdout, Command},
+	select,
+	sync::{mpsc, oneshot, Mutex},
+};
 use tracing::{debug, error, warn, Level};
-
-
-
-
modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
before · cmds/fleet/src/cmds/build_systems.rs
1use std::os::unix::fs::symlink;2use std::path::PathBuf;3use std::{env::current_dir, time::Duration};45use crate::command::MyCommand;6use crate::host::{Config, ConfigHost};7use anyhow::{anyhow, Result};8use clap::{Parser, ValueEnum};9use itertools::Itertools as _;10use nix_eval::nix_go;11use tokio::{task::LocalSet, time::sleep};12use tracing::{error, field, info, info_span, warn, Instrument};1314#[derive(Parser)]15pub struct Deploy {16	/// Disable automatic rollback17	#[clap(long)]18	disable_rollback: bool,19	/// Action to execute after system is built20	action: DeployAction,21}2223#[derive(ValueEnum, Clone, Copy)]24enum DeployAction {25	/// Upload derivation, but do not execute the update.26	Upload,27	/// Upload and execute the activation script, old version will be used after reboot.28	Test,29	/// Upload and set as current system profile, but do not execute activation script.30	Boot,31	/// Upload, set current profile, and execute activation script.32	Switch,33}3435impl DeployAction {36	pub(crate) fn name(&self) -> Option<&'static str> {37		match self {38			DeployAction::Upload => None,39			DeployAction::Test => Some("test"),40			DeployAction::Boot => Some("boot"),41			DeployAction::Switch => Some("switch"),42		}43	}44	pub(crate) fn should_switch_profile(&self) -> bool {45		matches!(self, Self::Switch | Self::Boot)46	}47	pub(crate) fn should_activate(&self) -> bool {48		matches!(self, Self::Switch | Self::Test)49	}50	pub(crate) fn should_create_rollback_marker(&self) -> bool {51		// Upload does nothing on the target machine, other than uploading the closure.52		// In boot case we want to have rollback marker prepared, so that the system may rollback itself on the next boot.53		!matches!(self, Self::Upload)54	}55	pub(crate) fn should_schedule_rollback_run(&self) -> bool {56		matches!(self, Self::Switch | Self::Test)57	}58}5960#[derive(Parser, Clone)]61pub struct BuildSystems {62	/// Attribute to build. Systems are deployed from "toplevel" attr, well-known used attributes63	/// are "sdImage"/"isoImage", and your configuration may include any other build attributes.64	#[clap(long, default_value = "toplevel")]65	build_attr: String,66}6768struct Generation {69	id: u32,70	current: bool,71	datetime: String,72}73async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {74	let mut cmd = host.cmd("nix-env").await?;75	cmd.comparg("--profile", "/nix/var/nix/profiles/system")76		.arg("--list-generations");77	// Sudo is required due to --list-generations acquiring lock on the profile.78	let data = cmd.sudo().run_string().await?;79	let generations = data80		.split('\n')81		.map(|e| e.trim())82		.filter(|&l| !l.is_empty())83		.filter_map(|g| {84			let gen: Option<Generation> = try {85				let mut parts = g.split_whitespace();86				let id = parts.next()?;87				let id: u32 = id.parse().ok()?;88				let date = parts.next()?;89				let time = parts.next()?;90				let current = if let Some(current) = parts.next() {91					if current == "(current)" {92						Some(true)93					} else {94						None95					}96				} else {97					Some(false)98				};99				let current = current?;100				if parts.next().is_some() {101					warn!("unexpected text after generation: {g}");102				}103				Generation {104					id,105					current,106					datetime: format!("{date} {time}"),107				}108			};109			if gen.is_none() {110				warn!("bad generation: {g}")111			}112			gen113		})114		.collect::<Vec<_>>();115	let current = generations116		.into_iter()117		.filter(|g| g.current)118		.at_most_one()119		.map_err(|_e| anyhow!("bad list-generations output"))?120		.ok_or_else(|| anyhow!("failed to find generation"))?;121	Ok(current)122}123124async fn deploy_task(125	action: DeployAction,126	host: &ConfigHost,127	built: PathBuf,128	disable_rollback: bool,129) -> Result<()> {130	let mut failed = false;131	// TODO: Lockfile, to prevent concurrent system switch?132	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback133	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to134	// unit name conflict in systemd-run135	// This code is tied to rollback.nix136	if !disable_rollback && action.should_create_rollback_marker() {137		let _span = info_span!("preparing").entered();138		info!("preparing for rollback");139		let generation = get_current_generation(host).await?;140		info!(141			"rollback target would be {} {}",142			generation.id, generation.datetime143		);144		{145			let mut cmd = host.cmd("sh").await?;146			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));147			if let Err(e) = cmd.sudo().run().await {148				error!("failed to set rollback marker: {e}");149				failed = true;150			}151		}152		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.153		// Kicking it on manually will work best.154		//155		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will156		// only allow one instance of it.157158		// TODO: We should also watch how this process is going.159		// After running this command, we have less than 3 minutes to deploy everything,160		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.161		// Anyway, reboot will still help in this case.162		if action.should_schedule_rollback_run() {163			let mut cmd = host.cmd("systemd-run").await?;164			cmd.comparg("--on-active", "3min")165				.comparg("--unit", "rollback-watchdog-run")166				.arg("systemctl")167				.arg("start")168				.arg("rollback-watchdog.service");169			if let Err(e) = cmd.sudo().run().await {170				error!("failed to schedule rollback run: {e}");171				failed = true;172			}173		}174	}175176	if action.should_switch_profile() && !failed {177		info!("switching generation");178		let mut cmd = host.cmd("nix-env").await?;179		cmd.comparg("--profile", "/nix/var/nix/profiles/system")180			.comparg("--set", &built);181		if let Err(e) = cmd.sudo().run().await {182			error!("failed to switch generation: {e}");183			failed = true;184		}185	}186187	// FIXME: Connection might be disconnected after activation run188189	if action.should_activate() && !failed {190		let _span = info_span!("activating").entered();191		info!("executing activation script");192		let mut switch_script = built.clone();193		switch_script.push("bin");194		switch_script.push("switch-to-configuration");195		let mut cmd = host.cmd(switch_script).in_current_span().await?;196		cmd.arg(action.name().expect("upload.should_activate == false"));197		if let Err(e) = cmd.sudo().run().in_current_span().await {198			error!("failed to activate: {e}");199			failed = true;200		}201	}202	if action.should_create_rollback_marker() {203		if !disable_rollback {204			if failed {205				if action.should_schedule_rollback_run() {206					info!("executing rollback");207					if let Err(e) = host208						.systemctl_start("rollback-watchdog.service")209						.instrument(info_span!("rollback"))210						.await211					{212						error!("failed to trigger rollback: {e}")213					}214				}215			} else {216				info!("trying to mark upgrade as successful");217				if let Err(e) = host218					.rm_file("/etc/fleet_rollback_marker", true)219					.in_current_span()220					.await221				{222					error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")223				}224			}225			info!("disarming watchdog, just in case");226			if let Err(_e) = host.systemctl_stop("rollback-watchdog.timer").await {227				// It is ok, if there was no reboot - then timer might not be running.228			}229			if action.should_schedule_rollback_run() {230				if let Err(e) = host.systemctl_stop("rollback-watchdog-run.timer").await {231					error!("failed to disarm rollback run: {e}");232				}233			}234		} else if let Err(_e) = host235			.rm_file("/etc/fleet_rollback_marker", true)236			.in_current_span()237			.await238		{239			// Marker might not exist, yet better try to remove it.240		}241	}242	Ok(())243}244245async fn build_task(config: Config, host: String, build_attr: &str) -> Result<PathBuf> {246	info!("building");247	let host = config.host(&host).await?;248	// let action = Action::from(self.subcommand.clone());249	let fleet_config = &config.config_field;250	let drv = nix_go!(251		fleet_config.hosts[{ &host.name }]252			.nixosSystem253			.config254			.system255			.build[{ build_attr }]256	);257	let outputs = drv.build().await.map_err(|e| {258			if build_attr == "sdImage" {259				info!("sd-image build failed");260				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");261			}262			e263		})?;264	let out_output = outputs265		.get("out")266		.ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;267268	Ok(out_output.clone())269}270271impl BuildSystems {272	pub async fn run(self, config: &Config) -> Result<()> {273		let hosts = config.list_hosts().await?;274		let set = LocalSet::new();275		let build_attr = self.build_attr.clone();276		for host in hosts.into_iter() {277			if config.should_skip(&host.name) {278				continue;279			}280			let config = config.clone();281			let span = info_span!("build", host = field::display(&host.name));282			let hostname = host.name;283			let build_attr = build_attr.clone();284			// FIXME: Since the introduction of better-nix-eval,285			// due to single repl used for builds, hosts are waiting for each other to build,286			// instead of building concurrently.287			//288			// Open multiple repls?289			//290			// Create build batcher, which will behave similar to golangs291			// WaitGroup, and start executing once all the build tasks are scheduled?292			// This also allows to cleanup build output, as there will be no longer293			// "waiting for remote machine" messages in the cases when one package is needed for294			// multiple hosts.295			set.spawn_local(296				(async move {297					let built = match build_task(config, hostname.clone(), &build_attr).await {298						Ok(path) => path,299						Err(e) => {300							error!("failed to deploy host: {}", e);301							return;302						}303					};304					// TODO: Handle error305					let mut out = current_dir().expect("cwd exists");306					out.push(format!("built-{}", hostname));307308					info!("linking iso image to {:?}", out);309					if let Err(e) = symlink(built, out) {310						error!("failed to symlink: {e}")311					}312				})313				.instrument(span),314			);315		}316		set.await;317		Ok(())318	}319}320321impl Deploy {322	pub async fn run(self, config: &Config) -> Result<()> {323		let hosts = config.list_hosts().await?;324		let set = LocalSet::new();325		for host in hosts.into_iter() {326			if config.should_skip(&host.name) {327				continue;328			}329			let config = config.clone();330			let span = info_span!("deploy", host = field::display(&host.name));331			let hostname = host.name.clone();332			// FIXME: Fix repl concurrency (see build-systems)333			set.spawn_local(334				(async move {335					let built = match build_task(config.clone(), hostname.clone(), "toplevel").await336					{337						Ok(path) => path,338						Err(e) => {339							error!("failed to deploy host: {}", e);340							return;341						}342					};343					if !config.is_local(&hostname) {344						info!("uploading system closure");345						{346							// TODO: Move to remote_derivation method.347							// Alternatively, nix store make-content-addressed can be used,348							// at least for the first deployment, to provide trusted store key.349							//350							// It is much slower, yet doesn't require root on the deployer machine.351							let mut sign = MyCommand::new("nix");352							// Private key for host machine is registered in nix-sign.nix353							sign.arg("store")354								.arg("sign")355								.comparg("--key-file", "/etc/nix/private-key")356								.arg("-r")357								.arg(&built);358							if let Err(e) = sign.sudo().run_nix().await {359								warn!("Failed to sign store paths: {e}");360							};361						}362						let mut tries = 0;363						loop {364							match host.remote_derivation(&built).await {365								Ok(remote) => {366									assert!(remote == built, "CA derivations aren't implemented");367									break;368								}369								Err(e) if tries < 3 => {370									tries += 1;371									warn!("copy failure ({}/3): {}", tries, e);372									sleep(Duration::from_millis(5000)).await;373								}374								Err(e) => {375									error!("upload failed: {e}");376									return;377								}378							}379						}380					}381					if let Err(e) =382						deploy_task(self.action, &host, built, self.disable_rollback).await383					{384						error!("activation failed: {e}");385					}386				})387				.instrument(span),388			);389		}390		set.await;391		Ok(())392	}393}
after · cmds/fleet/src/cmds/build_systems.rs
1use std::{env::current_dir, os::unix::fs::symlink, path::PathBuf, time::Duration};23use anyhow::{anyhow, Result};4use clap::{Parser, ValueEnum};5use itertools::Itertools as _;6use nix_eval::nix_go;7use tokio::{task::LocalSet, time::sleep};8use tracing::{error, field, info, info_span, warn, Instrument};910use crate::{11	command::MyCommand,12	host::{Config, ConfigHost},13};1415#[derive(Parser)]16pub struct Deploy {17	/// Disable automatic rollback18	#[clap(long)]19	disable_rollback: bool,20	/// Action to execute after system is built21	action: DeployAction,22}2324#[derive(ValueEnum, Clone, Copy)]25enum DeployAction {26	/// Upload derivation, but do not execute the update.27	Upload,28	/// Upload and execute the activation script, old version will be used after reboot.29	Test,30	/// Upload and set as current system profile, but do not execute activation script.31	Boot,32	/// Upload, set current profile, and execute activation script.33	Switch,34}3536impl DeployAction {37	pub(crate) fn name(&self) -> Option<&'static str> {38		match self {39			DeployAction::Upload => None,40			DeployAction::Test => Some("test"),41			DeployAction::Boot => Some("boot"),42			DeployAction::Switch => Some("switch"),43		}44	}45	pub(crate) fn should_switch_profile(&self) -> bool {46		matches!(self, Self::Switch | Self::Boot)47	}48	pub(crate) fn should_activate(&self) -> bool {49		matches!(self, Self::Switch | Self::Test)50	}51	pub(crate) fn should_create_rollback_marker(&self) -> bool {52		// Upload does nothing on the target machine, other than uploading the closure.53		// In boot case we want to have rollback marker prepared, so that the system may rollback itself on the next boot.54		!matches!(self, Self::Upload)55	}56	pub(crate) fn should_schedule_rollback_run(&self) -> bool {57		matches!(self, Self::Switch | Self::Test)58	}59}6061#[derive(Parser, Clone)]62pub struct BuildSystems {63	/// Attribute to build. Systems are deployed from "toplevel" attr, well-known used attributes64	/// are "sdImage"/"isoImage", and your configuration may include any other build attributes.65	#[clap(long, default_value = "toplevel")]66	build_attr: String,67}6869struct Generation {70	id: u32,71	current: bool,72	datetime: String,73}74async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {75	let mut cmd = host.cmd("nix-env").await?;76	cmd.comparg("--profile", "/nix/var/nix/profiles/system")77		.arg("--list-generations");78	// Sudo is required due to --list-generations acquiring lock on the profile.79	let data = cmd.sudo().run_string().await?;80	let generations = data81		.split('\n')82		.map(|e| e.trim())83		.filter(|&l| !l.is_empty())84		.filter_map(|g| {85			let gen: Option<Generation> = try {86				let mut parts = g.split_whitespace();87				let id = parts.next()?;88				let id: u32 = id.parse().ok()?;89				let date = parts.next()?;90				let time = parts.next()?;91				let current = if let Some(current) = parts.next() {92					if current == "(current)" {93						Some(true)94					} else {95						None96					}97				} else {98					Some(false)99				};100				let current = current?;101				if parts.next().is_some() {102					warn!("unexpected text after generation: {g}");103				}104				Generation {105					id,106					current,107					datetime: format!("{date} {time}"),108				}109			};110			if gen.is_none() {111				warn!("bad generation: {g}")112			}113			gen114		})115		.collect::<Vec<_>>();116	let current = generations117		.into_iter()118		.filter(|g| g.current)119		.at_most_one()120		.map_err(|_e| anyhow!("bad list-generations output"))?121		.ok_or_else(|| anyhow!("failed to find generation"))?;122	Ok(current)123}124125async fn deploy_task(126	action: DeployAction,127	host: &ConfigHost,128	built: PathBuf,129	disable_rollback: bool,130) -> Result<()> {131	let mut failed = false;132	// TODO: Lockfile, to prevent concurrent system switch?133	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback134	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to135	// unit name conflict in systemd-run136	// This code is tied to rollback.nix137	if !disable_rollback && action.should_create_rollback_marker() {138		let _span = info_span!("preparing").entered();139		info!("preparing for rollback");140		let generation = get_current_generation(host).await?;141		info!(142			"rollback target would be {} {}",143			generation.id, generation.datetime144		);145		{146			let mut cmd = host.cmd("sh").await?;147			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));148			if let Err(e) = cmd.sudo().run().await {149				error!("failed to set rollback marker: {e}");150				failed = true;151			}152		}153		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.154		// Kicking it on manually will work best.155		//156		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will157		// only allow one instance of it.158159		// TODO: We should also watch how this process is going.160		// After running this command, we have less than 3 minutes to deploy everything,161		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.162		// Anyway, reboot will still help in this case.163		if action.should_schedule_rollback_run() {164			let mut cmd = host.cmd("systemd-run").await?;165			cmd.comparg("--on-active", "3min")166				.comparg("--unit", "rollback-watchdog-run")167				.arg("systemctl")168				.arg("start")169				.arg("rollback-watchdog.service");170			if let Err(e) = cmd.sudo().run().await {171				error!("failed to schedule rollback run: {e}");172				failed = true;173			}174		}175	}176177	if action.should_switch_profile() && !failed {178		info!("switching generation");179		let mut cmd = host.cmd("nix-env").await?;180		cmd.comparg("--profile", "/nix/var/nix/profiles/system")181			.comparg("--set", &built);182		if let Err(e) = cmd.sudo().run().await {183			error!("failed to switch generation: {e}");184			failed = true;185		}186	}187188	// FIXME: Connection might be disconnected after activation run189190	if action.should_activate() && !failed {191		let _span = info_span!("activating").entered();192		info!("executing activation script");193		let mut switch_script = built.clone();194		switch_script.push("bin");195		switch_script.push("switch-to-configuration");196		let mut cmd = host.cmd(switch_script).in_current_span().await?;197		cmd.arg(action.name().expect("upload.should_activate == false"));198		if let Err(e) = cmd.sudo().run().in_current_span().await {199			error!("failed to activate: {e}");200			failed = true;201		}202	}203	if action.should_create_rollback_marker() {204		if !disable_rollback {205			if failed {206				if action.should_schedule_rollback_run() {207					info!("executing rollback");208					if let Err(e) = host209						.systemctl_start("rollback-watchdog.service")210						.instrument(info_span!("rollback"))211						.await212					{213						error!("failed to trigger rollback: {e}")214					}215				}216			} else {217				info!("trying to mark upgrade as successful");218				if let Err(e) = host219					.rm_file("/etc/fleet_rollback_marker", true)220					.in_current_span()221					.await222				{223					error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")224				}225			}226			info!("disarming watchdog, just in case");227			if let Err(_e) = host.systemctl_stop("rollback-watchdog.timer").await {228				// It is ok, if there was no reboot - then timer might not be running.229			}230			if action.should_schedule_rollback_run() {231				if let Err(e) = host.systemctl_stop("rollback-watchdog-run.timer").await {232					error!("failed to disarm rollback run: {e}");233				}234			}235		} else if let Err(_e) = host236			.rm_file("/etc/fleet_rollback_marker", true)237			.in_current_span()238			.await239		{240			// Marker might not exist, yet better try to remove it.241		}242	}243	Ok(())244}245246async fn build_task(config: Config, host: String, build_attr: &str) -> Result<PathBuf> {247	info!("building");248	let host = config.host(&host).await?;249	// let action = Action::from(self.subcommand.clone());250	let fleet_config = &config.config_field;251	let drv = nix_go!(252		fleet_config.hosts[{ &host.name }]253			.nixosSystem254			.config255			.system256			.build[{ build_attr }]257	);258	let outputs = drv.build().await.map_err(|e| {259			if build_attr == "sdImage" {260				info!("sd-image build failed");261				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");262			}263			e264		})?;265	let out_output = outputs266		.get("out")267		.ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;268269	Ok(out_output.clone())270}271272impl BuildSystems {273	pub async fn run(self, config: &Config) -> Result<()> {274		let hosts = config.list_hosts().await?;275		let set = LocalSet::new();276		let build_attr = self.build_attr.clone();277		for host in hosts.into_iter() {278			if config.should_skip(&host.name) {279				continue;280			}281			let config = config.clone();282			let span = info_span!("build", host = field::display(&host.name));283			let hostname = host.name;284			let build_attr = build_attr.clone();285			// FIXME: Since the introduction of better-nix-eval,286			// due to single repl used for builds, hosts are waiting for each other to build,287			// instead of building concurrently.288			//289			// Open multiple repls?290			//291			// Create build batcher, which will behave similar to golangs292			// WaitGroup, and start executing once all the build tasks are scheduled?293			// This also allows to cleanup build output, as there will be no longer294			// "waiting for remote machine" messages in the cases when one package is needed for295			// multiple hosts.296			set.spawn_local(297				(async move {298					let built = match build_task(config, hostname.clone(), &build_attr).await {299						Ok(path) => path,300						Err(e) => {301							error!("failed to deploy host: {}", e);302							return;303						}304					};305					// TODO: Handle error306					let mut out = current_dir().expect("cwd exists");307					out.push(format!("built-{}", hostname));308309					info!("linking iso image to {:?}", out);310					if let Err(e) = symlink(built, out) {311						error!("failed to symlink: {e}")312					}313				})314				.instrument(span),315			);316		}317		set.await;318		Ok(())319	}320}321322impl Deploy {323	pub async fn run(self, config: &Config) -> Result<()> {324		let hosts = config.list_hosts().await?;325		let set = LocalSet::new();326		for host in hosts.into_iter() {327			if config.should_skip(&host.name) {328				continue;329			}330			let config = config.clone();331			let span = info_span!("deploy", host = field::display(&host.name));332			let hostname = host.name.clone();333			// FIXME: Fix repl concurrency (see build-systems)334			set.spawn_local(335				(async move {336					let built = match build_task(config.clone(), hostname.clone(), "toplevel").await337					{338						Ok(path) => path,339						Err(e) => {340							error!("failed to deploy host: {}", e);341							return;342						}343					};344					if !config.is_local(&hostname) {345						info!("uploading system closure");346						{347							// TODO: Move to remote_derivation method.348							// Alternatively, nix store make-content-addressed can be used,349							// at least for the first deployment, to provide trusted store key.350							//351							// It is much slower, yet doesn't require root on the deployer machine.352							let mut sign = MyCommand::new("nix");353							// Private key for host machine is registered in nix-sign.nix354							sign.arg("store")355								.arg("sign")356								.comparg("--key-file", "/etc/nix/private-key")357								.arg("-r")358								.arg(&built);359							if let Err(e) = sign.sudo().run_nix().await {360								warn!("Failed to sign store paths: {e}");361							};362						}363						let mut tries = 0;364						loop {365							match host.remote_derivation(&built).await {366								Ok(remote) => {367									assert!(remote == built, "CA derivations aren't implemented");368									break;369								}370								Err(e) if tries < 3 => {371									tries += 1;372									warn!("copy failure ({}/3): {}", tries, e);373									sleep(Duration::from_millis(5000)).await;374								}375								Err(e) => {376									error!("upload failed: {e}");377									return;378								}379							}380						}381					}382					if let Err(e) =383						deploy_task(self.action, &host, built, self.disable_rollback).await384					{385						error!("activation failed: {e}");386					}387				})388				.instrument(span),389			);390		}391		set.await;392		Ok(())393	}394}
modifiedcmds/fleet/src/cmds/info.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/info.rs
+++ b/cmds/fleet/src/cmds/info.rs
@@ -1,10 +1,11 @@
 use std::collections::BTreeSet;
 
-use crate::host::Config;
 use anyhow::{ensure, Result};
 use clap::Parser;
 use nix_eval::nix_go_json;
 
+use crate::host::Config;
+
 #[derive(Parser)]
 pub struct Info {
 	#[clap(long)]
modifiedcmds/fleet/src/cmds/mod.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/mod.rs
+++ b/cmds/fleet/src/cmds/mod.rs
@@ -1,4 +1,4 @@
 pub mod build_systems;
+pub mod complete;
 pub mod info;
 pub mod secrets;
-pub mod complete;
modifiedcmds/fleet/src/extra_args.rsdiffbeforeafterboth
--- a/cmds/fleet/src/extra_args.rs
+++ b/cmds/fleet/src/extra_args.rs
@@ -1,7 +1,7 @@
-use anyhow::anyhow;
-use anyhow::Result;
 use std::ffi::{OsStr, OsString};
 
+use anyhow::{anyhow, Result};
+
 pub fn parse_os(os: &OsStr) -> Result<Vec<OsString>> {
 	Ok(shlex::bytes::split(os.as_encoded_bytes())
 		.ok_or_else(|| anyhow!("invalid arguments"))?
modifiedcmds/fleet/src/keys.rsdiffbeforeafterboth
--- a/cmds/fleet/src/keys.rs
+++ b/cmds/fleet/src/keys.rs
@@ -1,12 +1,13 @@
 use std::str::FromStr;
 
-use crate::host::Config;
 use age::Recipient;
 use anyhow::{anyhow, Result};
 use futures::{StreamExt, TryStreamExt};
 use itertools::Itertools;
 use tracing::warn;
 
+use crate::host::Config;
+
 impl Config {
 	pub fn cached_key(&self, host: &str) -> Option<String> {
 		let data = self.data();
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -175,10 +175,20 @@
 	reg.init();
 }
 
-#[tokio::main]
-async fn main() -> ExitCode {
+fn main() -> ExitCode {
+	let opts = RootOpts::parse();
+	if let Opts::Complete(c) = &opts.command {
+		c.run(RootOpts::command());
+		return ExitCode::SUCCESS;
+	}
+
 	setup_logging();
-	if let Err(e) = main_real().await {
+	async_main(opts)
+}
+
+#[tokio::main]
+async fn async_main(opts: RootOpts) -> ExitCode {
+	if let Err(e) = main_real(opts).await {
 		// If I remove this line, the next error!() line gets eaten.
 		// This is a bug in indicatif, it needs to be fixed
 		#[cfg(feature = "indicatif")]
@@ -189,14 +199,13 @@
 	ExitCode::SUCCESS
 }
 
-async fn main_real() -> Result<()> {
+async fn main_real(opts: RootOpts) -> Result<()> {
 	nix_eval::init_tokio();
 
 	let nix_args = std::env::var_os("NIX_ARGS")
 		.map(|a| extra_args::parse_os(&a))
 		.transpose()?
 		.unwrap_or_default();
-	let opts = RootOpts::parse();
 	let config = opts.fleet_opts.build(nix_args).await?;
 
 	match run_command(&config, opts.command).await {
modifiedflake.lockdiffbeforeafterboth
--- a/flake.lock
+++ b/flake.lock
@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717025063,
-        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
+        "lastModified": 1717290123,
+        "narHash": "sha256-K8O2KQEbA+NIAc8BDsWV6QKqU3i9M+YTUi4zzmLRy1s=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
+        "rev": "ae1453ffd0f8f684e863685c317a953317db2b79",
         "type": "github"
       },
       "original": {
@@ -40,11 +40,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1717282945,
-        "narHash": "sha256-Jrn+/CdB/d2hUqduYQdTwGJYDAdaR5cAdlxnq+yEtXI=",
+        "lastModified": 1717336170,
+        "narHash": "sha256-hkD00+n53WNZ4k8hqIbekl5WGDsmb5urhAuDh5XYjyc=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "ab5efd0f3c62dd3b75d21d0de1dd63efc76be5d8",
+        "rev": "73bff846b4e8d0c8156c6fc726bf623fe3f3845c",
         "type": "github"
       },
       "original": {
@@ -56,11 +56,11 @@
     },
     "nixpkgs-stable-for-tests": {
       "locked": {
-        "lastModified": 1716991068,
-        "narHash": "sha256-Av0UWCCiIGJxsZ6TFc+OiKCJNqwoxMNVYDBChmhjNpo=",
+        "lastModified": 1717159533,
+        "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "25cf937a30bf0801447f6bf544fc7486c6309234",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
         "type": "github"
       },
       "original": {
@@ -89,11 +89,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717208326,
-        "narHash": "sha256-4gVhbC+NjSQ4c6cJvJGNCI1oTcD+8jRRNAnOF9faGCE=",
+        "lastModified": 1717294752,
+        "narHash": "sha256-QhlS52cEQyx+iVcgrEoCnEEpWUA6uLdmeLRxk935inI=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "ab69b67fac9a96709fbef0b899db308ca714a120",
+        "rev": "b46857a406d207a1de74e792ef3b83e800c30e08",
         "type": "github"
       },
       "original": {
modifiedpkgs/fleet.nixdiffbeforeafterboth
--- a/pkgs/fleet.nix
+++ b/pkgs/fleet.nix
@@ -1,4 +1,7 @@
-{craneLib}:
+{
+  craneLib,
+  installShellFiles,
+}:
 craneLib.buildPackage rec {
   pname = "fleet";
 
@@ -7,10 +10,12 @@
 
   cargoExtraArgs = "--locked -p ${pname}";
 
+  nativeBuildInputs = [installShellFiles];
+
   postInstall = ''
     for shell in bash fish zsh; do
       installShellCompletion --cmd fleet \
-        --$shell <($out/bin/fleet complete --shell $shell --print)
+        --$shell <($out/bin/fleet complete --shell $shell)
     done
   '';
 }