git.delta.rocks / jrsonnet / refs/commits / 8fa5c73b5fe4

1use std::{path::PathBuf, time::Duration};23use anyhow::{Context as _, Result, anyhow, bail};4use clap::ValueEnum;5use itertools::Itertools;6use tokio::time::sleep;7use tracing::{Instrument as _, error, info, info_span, warn};89use crate::host::{Config, ConfigHost, DeployKind, Generation, GenerationStorage};1011#[derive(ValueEnum, Clone, Copy)]12pub enum DeployAction {13	/// Upload derivation, but do not execute the update.14	Upload,15	/// Upload and execute the activation script, old version will be used after reboot.16	Test,17	/// Upload and set as current system profile, but do not execute activation script.18	Boot,19	/// Upload, set current profile, and execute activation script.20	Switch,21}2223impl DeployAction {24	pub(crate) fn name(&self) -> Option<&'static str> {25		match self {26			Self::Upload => None,27			Self::Test => Some("test"),28			Self::Boot => Some("boot"),29			Self::Switch => Some("switch"),30		}31	}32	pub(crate) fn should_switch_profile(&self) -> bool {33		matches!(self, Self::Switch | Self::Boot)34	}35	pub(crate) fn should_activate(&self) -> bool {36		matches!(self, Self::Switch | Self::Test | Self::Boot)37	}38	pub(crate) fn should_create_rollback_marker(&self) -> bool {39		// Upload does nothing on the target machine, other than uploading the closure.40		// In boot case we want to have rollback marker prepared, so that the system may rollback itself on the next boot.41		!matches!(self, Self::Upload)42	}43	pub(crate) fn should_schedule_rollback_run(&self) -> bool {44		matches!(self, Self::Switch | Self::Test)45	}46}4748async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {49	let generations = host.list_generations("system").await?;50	let current = generations51		.into_iter()52		.filter(|g| g.current)53		.at_most_one()54		.map_err(|_e| anyhow!("bad list-generations output"))?55		.ok_or_else(|| anyhow!("failed to find generation"))?;56	Ok(current)57}5859pub async fn deploy_task(60	action: DeployAction,61	host: &ConfigHost,62	built: PathBuf,63	specialisation: Option<String>,64	disable_rollback: bool,65) -> Result<()> {66	let deploy_kind = host.deploy_kind().await?;67	if (deploy_kind == DeployKind::NixosInstall || deploy_kind == DeployKind::NixosLustrate)68		&& !matches!(action, DeployAction::Boot | DeployAction::Upload)69	{70		bail!("{deploy_kind:?} deploy kind only supports boot and upload actions");71	}7273	let mut failed = false;7475	// TODO: Lockfile, to prevent concurrent system switch?76	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback77	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to78	// unit name conflict in systemd-run79	// This code is tied to rollback.nix80	if !disable_rollback && action.should_create_rollback_marker() {81		let _span = info_span!("preparing").entered();82		info!("preparing for rollback");83		let generation = get_current_generation(host).await?;84		info!(85			"rollback target would be {} {}",86			generation.id, generation.datetime87		);88		{89			let mut cmd = host.cmd("sh").await?;90			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));91			if let Err(e) = cmd.sudo().run().await {92				error!("failed to set rollback marker: {e}");93				failed = true;94			}95		}96		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.97		// Kicking it on manually will work best.98		//99		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will100		// only allow one instance of it.101102		// TODO: We should also watch how this process is going.103		// After running this command, we have less than 3 minutes to deploy everything,104		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.105		// Anyway, reboot will still help in this case.106		if action.should_schedule_rollback_run() {107			let mut cmd = host.cmd("systemd-run").await?;108			cmd.comparg("--on-active", "3min")109				.comparg("--unit", "rollback-watchdog-run")110				.arg("systemctl")111				.arg("start")112				.arg("rollback-watchdog.service");113			if let Err(e) = cmd.sudo().run().await {114				error!("failed to schedule rollback run: {e}");115				failed = true;116			}117		}118	}119	if deploy_kind == DeployKind::NixosLustrate {120		// Fleet could also create this file, but as this operation is potentially disruptive,121		// make user do it themself.122		if !host.file_exists("/etc/NIXOS_LUSTRATE").await? {123			bail!("/etc/NIXOS_LUSTRATE should be created on remote host");124		}125		// Wanted by NixOS to recognize the system as NixOS.126		let mut cmd = host.cmd("touch").await?;127		cmd.arg("/etc/NIXOS");128		cmd.sudo().run().await.context("creating /etc/NIXOS")?;129	}130	if deploy_kind == DeployKind::NixosInstall {131		info!(132			"running nixos-install to switch profile, install bootloader, and perform activation"133		);134		let mut cmd = host.cmd("nixos-install").await?;135		cmd.arg("--system").arg(&built).args([136			// Channels here aren't fleet host system channels, but channels embedded in installation cd, which might be old.137			// It is possible to copy host channels, but I would prefer non-flake nix just to be unsupported.138			"--no-channel-copy",139			"--root",140			"/mnt",141		]);142		if let Err(e) = cmd.sudo().run().await {143			error!("failed to execute nixos-install: {e}");144			failed = true;145		}146	} else {147		if action.should_switch_profile() && !failed {148			info!("switching system profile generation");149150			// To avoid even more problems, using nixos-install for now.151			// // nix build is unable to work with --store argument for some reason, and nix until 2.26 didn't support copy with --profile argument,152			// // falling back to using nix-env command153			// // After stable NixOS starts using 2.26 - use `nix --store /mnt copy --from /mnt --profile ...` here, and instead of nix build below.154			// let mut cmd = host.cmd("nix-env").await?;155			// cmd.args([156			// 	"--store",157			// 	"/mnt",158			// 	"--profile",159			// 	"/mnt/nix/var/nix/profiles/system",160			// 	"--set",161			// ])162			// .arg(&built);163			// if let Err(e) = cmd.sudo().run_nix().await {164			// 	error!("failed to switch system profile generation: {e}");165			// 	failed = true;166			// }167			// It would also be possible to update profile atomically during copy:168			// https://github.com/NixOS/nix/pull/11657169			let mut cmd = host.nix_cmd().await?;170			cmd.arg("build");171			cmd.comparg("--profile", "/nix/var/nix/profiles/system");172			cmd.arg(&built);173			if let Err(e) = cmd.sudo().run_nix().await {174				error!("failed to switch system profile generation: {e}");175				failed = true;176			}177		}178179		// FIXME: Connection might be disconnected after activation run180181		if action.should_activate() && !failed {182			let _span = info_span!("activating").entered();183			info!("executing activation script");184			let specialised = if let Some(specialisation) = specialisation {185				let mut specialised = built.join("specialisation");186				specialised.push(specialisation);187				specialised188			} else {189				built.clone()190			};191			let switch_script = specialised.join("bin/switch-to-configuration");192			let mut cmd = host.cmd(switch_script).in_current_span().await?;193			if deploy_kind == DeployKind::NixosLustrate {194				cmd.env("NIXOS_INSTALL_BOOTLOADER", "1");195			}196			cmd.env("FLEET_ONLINE_ACTIVATION", "1")197				.arg(action.name().expect("upload.should_activate == false"));198			if let Err(e) = cmd.sudo().run().in_current_span().await {199				error!("failed to activate: {e}");200				failed = true;201			}202		}203	}204	if action.should_create_rollback_marker() {205		if !disable_rollback {206			if failed {207				if action.should_schedule_rollback_run() {208					info!("executing rollback");209					if let Err(e) = host210						.systemctl_start("rollback-watchdog.service")211						.instrument(info_span!("rollback"))212						.await213					{214						error!("failed to trigger rollback: {e}")215					}216				}217			} else {218				info!("trying to mark upgrade as successful");219				if let Err(e) = host220					.rm_file("/etc/fleet_rollback_marker", true)221					.in_current_span()222					.await223				{224					error!(225						"failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}"226					)227				}228			}229			info!("disarming watchdog, just in case");230			if let Err(_e) = host.systemctl_stop("rollback-watchdog.timer").await {231				// It is ok, if there was no reboot - then timer might not be running.232			}233			if action.should_schedule_rollback_run() {234				if let Err(e) = host.systemctl_stop("rollback-watchdog-run.timer").await {235					error!("failed to disarm rollback run: {e}");236				}237			}238		} else if let Err(_e) = host239			.rm_file("/etc/fleet_rollback_marker", true)240			.in_current_span()241			.await242		{243			// Marker might not exist, yet better try to remove it.244		}245	}246	Ok(())247}248249pub async fn upload_task(250	config: &Config,251	host: &ConfigHost,252	location: GenerationStorage,253	generation: PathBuf,254) -> Result<PathBuf> {255	let local_host = config.local_host();256	if matches!(location, GenerationStorage::Pusher) {257		bail!("pusher is not enabled in this version of fleet");258	}259	if !host.local {260		info!("uploading system closure");261		{262			// TODO: Move to remote_derivation method.263			// Alternatively, nix store make-content-addressed can be used,264			// at least for the first deployment, to provide trusted store key.265			//266			// It is much slower, yet doesn't require root on the deployer machine.267			let Ok(mut sign) = local_host.cmd("nix").await else {268				bail!("failed to setup local");269			};270			// Private key for host machine is registered in nix-sign.nix271			sign.arg("store")272				.arg("sign")273				.comparg("--key-file", "/etc/nix/private-key")274				.arg("-r")275				.arg(&generation);276			if let Err(e) = sign.sudo().run_nix().await {277				warn!("failed to sign store paths: {e}");278			};279		}280		let mut tries = 0;281		loop {282			match host.remote_derivation(&generation).await {283				Ok(remote) => {284					assert!(remote == generation, "CA derivations aren't implemented");285					return Ok(remote);286				}287				Err(e) if tries < 3 => {288					tries += 1;289					warn!("copy failure ({}/3): {}", tries, e);290					sleep(Duration::from_millis(5000)).await;291				}292				Err(e) => {293					bail!("upload failed: {e}");294				}295			}296		}297	}298	Ok(generation)299}