git.delta.rocks / fleet / refs/commits / 78bc9187eaae

1use std::time::Duration;23use anyhow::{Context as _, Result, anyhow, bail};4use camino::Utf8PathBuf;5use clap::ValueEnum;6use itertools::Itertools;7use remowt_endpoints::fs::FsClient;8use remowt_endpoints::systemd::SystemdClient;9use remowt_fleet::NixClient;10use remowt_link_shared::BifConfig;11use tokio::time::sleep;12use tracing::{Instrument as _, error, info, info_span, warn};1314use crate::host::{Config, ConfigHost, DeployKind, Generation, GenerationStorage};1516#[derive(ValueEnum, Clone, Copy)]17pub enum DeployAction {18	/// Upload derivation, but do not execute the update.19	Upload,20	/// Upload and execute the activation script, old version will be used after reboot.21	Test,22	/// Upload and set as current system profile, but do not execute activation script.23	Boot,24	/// Upload, set current profile, and execute activation script.25	Switch,26}2728impl DeployAction {29	pub(crate) fn name(&self) -> Option<&'static str> {30		match self {31			Self::Upload => None,32			Self::Test => Some("test"),33			Self::Boot => Some("boot"),34			Self::Switch => Some("switch"),35		}36	}37	pub(crate) fn should_switch_profile(&self) -> bool {38		matches!(self, Self::Switch | Self::Boot)39	}40	pub(crate) fn should_activate(&self) -> bool {41		matches!(self, Self::Switch | Self::Test | Self::Boot)42	}43	pub(crate) fn should_create_rollback_marker(&self) -> bool {44		// Upload does nothing on the target machine, other than uploading the closure.45		// In boot case we want to have rollback marker prepared, so that the system may rollback itself on the next boot.46		!matches!(self, Self::Upload)47	}48	pub(crate) fn should_schedule_rollback_run(&self) -> bool {49		matches!(self, Self::Switch | Self::Test)50	}51}5253async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {54	let generations = host.list_generations("system").await?;55	let current = generations56		.into_iter()57		.filter(|g| g.current)58		.at_most_one()59		.map_err(|_e| anyhow!("bad list-generations output"))?60		.ok_or_else(|| anyhow!("failed to find generation"))?;61	Ok(current)62}6364pub async fn deploy_task(65	action: DeployAction,66	host: &ConfigHost,67	built: Utf8PathBuf,68	specialisation: Option<String>,69	disable_rollback: bool,70) -> Result<()> {71	let remowt = host.remowt().await?;72	let deploy_kind = host.deploy_kind().await?;73	if (deploy_kind == DeployKind::NixosInstall || deploy_kind == DeployKind::NixosLustrate)74		&& !matches!(action, DeployAction::Boot | DeployAction::Upload)75	{76		bail!("{deploy_kind:?} deploy kind only supports boot and upload actions");77	}7879	let mut failed = false;8081	// TODO: Lockfile, to prevent concurrent system switch?82	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback83	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to84	// unit name conflict in systemd-run85	// This code is tied to rollback.nix86	if !disable_rollback && action.should_create_rollback_marker() {87		// let _span = info_span!("preparing").entered();88		info!("preparing for rollback");89		let generation = get_current_generation(host).await?;90		info!(91			"rollback target would be {} {}",92			generation.id, generation.datetime93		);94		{95			let mut cmd = remowt.cmd("sh");96			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));97			if let Err(e) = cmd.sudo().run().await {98				error!("failed to set rollback marker: {e}");99				failed = true;100			}101		}102		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.103		// Kicking it on manually will work best.104		//105		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will106		// only allow one instance of it.107108		// TODO: We should also watch how this process is going.109		// After running this command, we have less than 3 minutes to deploy everything,110		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.111		// Anyway, reboot will still help in this case.112		if action.should_schedule_rollback_run() {113			let mut cmd = remowt.cmd("systemd-run");114			cmd.comparg("--on-active", "3min")115				.comparg("--unit", "rollback-watchdog-run")116				.arg("systemctl")117				.arg("start")118				.arg("rollback-watchdog.service");119			if let Err(e) = cmd.sudo().run().await {120				error!("failed to schedule rollback run: {e}");121				failed = true;122			}123		}124	}125126	let remowt = host.remowt().await?;127	let fs = remowt.endpoints::<FsClient<_>>();128	if deploy_kind == DeployKind::NixosLustrate {129		// Fleet could also create this file, but as this operation is potentially disruptive,130		// make user do it themself.131		if !fs132			.file_exists(Utf8PathBuf::from("/etc/NIXOS_LUSTRATE"))133			.await?134		{135			bail!("/etc/NIXOS_LUSTRATE should be created on remote host");136		}137		// Wanted by NixOS to recognize the system as NixOS.138		let mut cmd = remowt.cmd("touch");139		cmd.arg("/etc/NIXOS");140		cmd.sudo().run().await.context("creating /etc/NIXOS")?;141	}142	if deploy_kind == DeployKind::NixosInstall {143		info!(144			"running nixos-install to switch profile, install bootloader, and perform activation"145		);146		let mut cmd = remowt.cmd("nixos-install");147		cmd.arg("--system").arg(&built).args([148			// Channels here aren't fleet host system channels, but channels embedded in installation cd, which might be old.149			// It is possible to copy host channels, but I would prefer non-flake nix just to be unsupported.150			"--no-channel-copy",151			"--root",152			"/mnt",153		]);154		if let Err(e) = cmd.sudo().run().await {155			error!("failed to execute nixos-install: {e}");156			failed = true;157		}158	} else {159		if action.should_switch_profile() && !failed {160			info!("switching system profile generation");161162			match host.ensure_nix_plugin().await {163				Ok(plugin_id) => {164					let nix_elevated = remowt.plugin_endpoints::<NixClient<_>>(plugin_id);165					// To avoid even more problems, using nixos-install for now.166					// // nix build is unable to work with --store argument for some reason, and nix until 2.26 didn't support copy with --profile argument,167					// // falling back to using nix-env command168					// // After stable NixOS starts using 2.26 - use `nix --store /mnt copy --from /mnt --profile ...` here, and instead of nix build below.169					// let mut cmd = host.cmd("nix-env").await?;170					// cmd.args([171					// 	"--store",172					// 	"/mnt",173					// 	"--profile",174					// 	"/mnt/nix/var/nix/profiles/system",175					// 	"--set",176					// ])177					// .arg(&built);178					// if let Err(e) = cmd.sudo().run_nix().await {179					// 	error!("failed to switch system profile generation: {e}");180					// 	failed = true;181					// }182					if let Err(e) = nix_elevated183						.switch_profile("/nix/var/nix/profiles/system".to_owned(), built.clone())184						.await185					{186						error!("failed to switch system profile generation: {e}");187						failed = true;188					}189				}190				Err(e) => {191					failed = true;192					error!("failed to enable nix plugin: {e:?}");193				}194			}195		}196197		// FIXME: Connection might be disconnected after activation run198199		if action.should_activate() && !failed {200			// let _span = info_span!("activating").entered();201			info!("executing activation script");202			let specialised = if let Some(specialisation) = specialisation {203				let mut specialised = built.join("specialisation");204				specialised.push(specialisation);205				specialised206			} else {207				built.clone()208			};209			let switch_script = specialised.join("bin/switch-to-configuration");210			let mut cmd = remowt.cmd("systemd-run");211			if deploy_kind == DeployKind::NixosLustrate {212				cmd.arg("--setenv=NIXOS_INSTALL_BOOTLOADER=1");213			}214			cmd.arg("--setenv=FLEET_ONLINE_ACTIVATION=1")215				.arg("--collect")216				.arg("--no-ask-password")217				.arg("--pipe")218				.arg("--quiet")219				.arg("--service-type=exec")220				.arg("--unit=fleet-switch-to-configuration")221				.arg(switch_script)222				.arg(action.name().expect("upload.should_activate == false"));223			if let Err(e) = cmd.sudo().run().in_current_span().await {224				error!("failed to activate: {e}");225				failed = true;226			}227		}228	}229	if action.should_create_rollback_marker() {230		let elevated_systemd = remowt.run0_endpoints::<SystemdClient<BifConfig>>().await?;231		let elevated_fs = remowt.run0_endpoints::<FsClient<BifConfig>>().await?;232		if !disable_rollback {233			if failed {234				if action.should_schedule_rollback_run() {235					info!("executing rollback");236					if let Err(e) = elevated_systemd237						.start("rollback-watchdog.service".to_owned())238						.instrument(info_span!("rollback"))239						.await240					{241						error!("failed to trigger rollback: {e}")242					}243				}244			} else {245				info!("trying to mark upgrade as successful");246				if let Err(e) = elevated_fs247					.rm_file(Utf8PathBuf::from("/etc/fleet_rollback_marker"))248					.in_current_span()249					.await250				{251					error!(252						"failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}"253					)254				}255			}256			info!("disarming watchdog, just in case");257			if let Err(_e) = elevated_systemd258				.stop("rollback-watchdog.timer".to_owned())259				.await260			{261				// It is ok, if there was no reboot - then timer might not be running.262			}263			if action.should_schedule_rollback_run() {264				if let Err(e) = elevated_systemd265					.stop("rollback-watchdog-run.timer".to_owned())266					.await267				{268					error!("failed to disarm rollback run: {e}");269				}270			}271		} else if let Err(_e) = elevated_fs272			.rm_file(Utf8PathBuf::from("/etc/fleet_rollback_marker"))273			.in_current_span()274			.await275		{276			// Marker might not exist, yet better try to remove it.277		}278	}279	Ok(())280}281282pub async fn upload_task(283	config: &Config,284	host: &ConfigHost,285	location: GenerationStorage,286	generation: Utf8PathBuf,287) -> Result<Utf8PathBuf> {288	if matches!(location, GenerationStorage::Pusher) {289		bail!("pusher is not enabled in this version of fleet");290	}291	if !host.local {292		info!("uploading system closure");293		let mut tries = 0;294		loop {295			match host.remote_derivation(&generation).await {296				Ok(remote) => {297					assert!(remote == generation, "CA derivations aren't implemented");298					return Ok(remote);299				}300				Err(e) if tries < 3 => {301					tries += 1;302					warn!("copy failure ({}/3): {:#}", tries, e);303					sleep(Duration::from_millis(5000)).await;304				}305				Err(e) => {306					bail!("upload failed: {e:#}");307				}308			}309		}310	}311	Ok(generation)312}