git.delta.rocks / fleet / refs/commits / 347078813131

1use std::time::Duration;23use anyhow::{Context as _, Result, anyhow, bail};4use camino::Utf8PathBuf;5use clap::ValueEnum;6use itertools::Itertools;7use remowt_endpoints::fs::FsClient;8use remowt_endpoints::systemd::SystemdClient;9use remowt_fleet::NixClient;10use remowt_link_shared::BifConfig;11use tokio::time::sleep;12use tracing::{Instrument as _, error, info, info_span, warn};1314use crate::host::{ConfigHost, DeployKind};15use crate::pins::{Generation, GenerationStorage};1617#[derive(ValueEnum, Clone, Copy)]18pub enum DeployAction {19	/// Upload derivation, but do not execute the update.20	Upload,21	/// Upload and execute the activation script, old version will be used after reboot.22	Test,23	/// Upload and set as current system profile, but do not execute activation script.24	Boot,25	/// Upload, set current profile, and execute activation script.26	Switch,27}2829impl DeployAction {30	pub(crate) fn name(&self) -> Option<&'static str> {31		match self {32			Self::Upload => None,33			Self::Test => Some("test"),34			Self::Boot => Some("boot"),35			Self::Switch => Some("switch"),36		}37	}38	pub(crate) fn should_switch_profile(&self) -> bool {39		matches!(self, Self::Switch | Self::Boot)40	}41	pub(crate) fn should_activate(&self) -> bool {42		matches!(self, Self::Switch | Self::Test | Self::Boot)43	}44	pub(crate) fn should_create_rollback_marker(&self) -> bool {45		// Upload does nothing on the target machine, other than uploading the closure.46		// In boot case we want to have rollback marker prepared, so that the system may rollback itself on the next boot.47		!matches!(self, Self::Upload)48	}49	pub(crate) fn should_schedule_rollback_run(&self) -> bool {50		matches!(self, Self::Switch | Self::Test)51	}52}5354async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {55	let generations = host.list_generations("system").await?;56	let current = generations57		.into_iter()58		.filter(|g| g.current)59		.at_most_one()60		.map_err(|_e| anyhow!("bad list-generations output"))?61		.ok_or_else(|| anyhow!("failed to find generation"))?;62	Ok(current)63}6465pub async fn deploy_task(66	action: DeployAction,67	host: &ConfigHost,68	built: Utf8PathBuf,69	specialisation: Option<String>,70	disable_rollback: bool,71) -> Result<()> {72	let remowt = host.remowt().await?;73	let deploy_kind = host.deploy_kind().await?;74	if (deploy_kind == DeployKind::NixosInstall || deploy_kind == DeployKind::NixosLustrate)75		&& !matches!(action, DeployAction::Boot | DeployAction::Upload)76	{77		bail!("{deploy_kind:?} deploy kind only supports boot and upload actions");78	}7980	let mut failed = false;8182	// TODO: Lockfile, to prevent concurrent system switch?83	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback84	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to85	// unit name conflict in systemd-run86	// This code is tied to rollback.nix87	if !disable_rollback && action.should_create_rollback_marker() {88		// let _span = info_span!("preparing").entered();89		info!("preparing for rollback");90		let generation = get_current_generation(host).await?;91		info!(92			"rollback target would be {} {}",93			generation.id, generation.datetime94		);95		{96			let mut cmd = remowt.cmd("sh");97			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));98			if let Err(e) = cmd.sudo().run().await {99				error!("failed to set rollback marker: {e}");100				failed = true;101			}102		}103		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.104		// Kicking it on manually will work best.105		//106		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will107		// only allow one instance of it.108109		// TODO: We should also watch how this process is going.110		// After running this command, we have less than 3 minutes to deploy everything,111		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.112		// Anyway, reboot will still help in this case.113		if action.should_schedule_rollback_run() {114			let mut cmd = remowt.cmd("systemd-run");115			cmd.comparg("--on-active", "3min")116				.comparg("--unit", "rollback-watchdog-run")117				.arg("systemctl")118				.arg("start")119				.arg("rollback-watchdog.service");120			if let Err(e) = cmd.sudo().run().await {121				error!("failed to schedule rollback run: {e}");122				failed = true;123			}124		}125	}126127	let remowt = host.remowt().await?;128	let fs = remowt.endpoints::<FsClient<_>>();129	if deploy_kind == DeployKind::NixosLustrate {130		// Fleet could also create this file, but as this operation is potentially disruptive,131		// make user do it themself.132		if !fs133			.file_exists(Utf8PathBuf::from("/etc/NIXOS_LUSTRATE"))134			.await?135		{136			bail!("/etc/NIXOS_LUSTRATE should be created on remote host");137		}138		// Wanted by NixOS to recognize the system as NixOS.139		let mut cmd = remowt.cmd("touch");140		cmd.arg("/etc/NIXOS");141		cmd.sudo().run().await.context("creating /etc/NIXOS")?;142	}143	if deploy_kind == DeployKind::NixosInstall {144		info!(145			"running nixos-install to switch profile, install bootloader, and perform activation"146		);147		let mut cmd = remowt.cmd("nixos-install");148		cmd.arg("--system").arg(&built).args([149			// Channels here aren't fleet host system channels, but channels embedded in installation cd, which might be old.150			// It is possible to copy host channels, but I would prefer non-flake nix just to be unsupported.151			"--no-channel-copy",152			"--root",153			"/mnt",154		]);155		if let Err(e) = cmd.sudo().run().await {156			error!("failed to execute nixos-install: {e}");157			failed = true;158		}159	} else {160		if action.should_switch_profile() && !failed {161			info!("switching system profile generation");162163			match host.ensure_nix_plugin().await {164				Ok(plugin_id) => {165					let nix_elevated = remowt.plugin_endpoints::<NixClient<_>>(plugin_id);166					// To avoid even more problems, using nixos-install for now.167					// // nix build is unable to work with --store argument for some reason, and nix until 2.26 didn't support copy with --profile argument,168					// // falling back to using nix-env command169					// // After stable NixOS starts using 2.26 - use `nix --store /mnt copy --from /mnt --profile ...` here, and instead of nix build below.170					// let mut cmd = host.cmd("nix-env").await?;171					// cmd.args([172					// 	"--store",173					// 	"/mnt",174					// 	"--profile",175					// 	"/mnt/nix/var/nix/profiles/system",176					// 	"--set",177					// ])178					// .arg(&built);179					// if let Err(e) = cmd.sudo().run_nix().await {180					// 	error!("failed to switch system profile generation: {e}");181					// 	failed = true;182					// }183					if let Err(e) = nix_elevated184						.switch_profile("/nix/var/nix/profiles/system".to_owned(), built.clone())185						.await186					{187						error!("failed to switch system profile generation: {e}");188						failed = true;189					}190				}191				Err(e) => {192					failed = true;193					error!("failed to enable nix plugin: {e:?}");194				}195			}196		}197198		// FIXME: Connection might be disconnected after activation run199200		if action.should_activate() && !failed {201			// let _span = info_span!("activating").entered();202			info!("executing activation script");203			let specialised = if let Some(specialisation) = specialisation {204				let mut specialised = built.join("specialisation");205				specialised.push(specialisation);206				specialised207			} else {208				built.clone()209			};210			let switch_script = specialised.join("bin/switch-to-configuration");211			let mut cmd = remowt.cmd("systemd-run");212			if deploy_kind == DeployKind::NixosLustrate {213				cmd.arg("--setenv=NIXOS_INSTALL_BOOTLOADER=1");214			}215			cmd.arg("--setenv=FLEET_ONLINE_ACTIVATION=1")216				.arg("--collect")217				.arg("--no-ask-password")218				.arg("--pipe")219				.arg("--quiet")220				.arg("--service-type=exec")221				.arg("--unit=fleet-switch-to-configuration")222				.arg(switch_script)223				.arg(action.name().expect("upload.should_activate == false"));224			if let Err(e) = cmd.sudo().run().in_current_span().await {225				error!("failed to activate: {e}");226				failed = true;227			}228		}229	}230	if action.should_create_rollback_marker() {231		let elevated_systemd = remowt.run0_endpoints::<SystemdClient<BifConfig>>().await?;232		let elevated_fs = remowt.run0_endpoints::<FsClient<BifConfig>>().await?;233		if !disable_rollback {234			if failed {235				if action.should_schedule_rollback_run() {236					info!("executing rollback");237					if let Err(e) = elevated_systemd238						.start("rollback-watchdog.service".to_owned())239						.instrument(info_span!("rollback"))240						.await241					{242						error!("failed to trigger rollback: {e}")243					}244				}245			} else {246				info!("trying to mark upgrade as successful");247				if let Err(e) = elevated_fs248					.rm_file(Utf8PathBuf::from("/etc/fleet_rollback_marker"))249					.in_current_span()250					.await251				{252					error!(253						"failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}"254					)255				}256			}257			info!("disarming watchdog, just in case");258			if let Err(_e) = elevated_systemd259				.stop("rollback-watchdog.timer".to_owned())260				.await261			{262				// It is ok, if there was no reboot - then timer might not be running.263			}264			if action.should_schedule_rollback_run()265				&& let Err(e) = elevated_systemd266					.stop("rollback-watchdog-run.timer".to_owned())267					.await268			{269				error!("failed to disarm rollback run: {e}");270			}271		} else if let Err(_e) = elevated_fs272			.rm_file(Utf8PathBuf::from("/etc/fleet_rollback_marker"))273			.in_current_span()274			.await275		{276			// Marker might not exist, yet better try to remove it.277		}278	}279	Ok(())280}281282pub async fn upload_task(283	host: &ConfigHost,284	location: GenerationStorage,285	generation: Utf8PathBuf,286) -> Result<Utf8PathBuf> {287	if matches!(location, GenerationStorage::Pusher) {288		bail!("pusher is not enabled in this version of fleet");289	}290	if !host.local {291		info!("uploading system closure");292		let mut tries = 0;293		// TODO: Use gc prefix?..294		loop {295			let name = format!("{}-system", host.gc_root_prefix());296			match host.remote_derivation(&generation, Some(&name)).await {297				Ok(remote) => {298					assert!(remote == generation, "CA derivations aren't implemented");299					return Ok(remote);300				}301				Err(e) if tries < 3 => {302					tries += 1;303					warn!("copy failure ({}/3): {:#}", tries, e);304					sleep(Duration::from_millis(5000)).await;305				}306				Err(e) => {307					bail!("upload failed: {e:#}");308				}309			}310		}311	}312	Ok(generation)313}