git.delta.rocks / jrsonnet / refs/commits / aac7f3854e2a

1use std::{env::current_dir, os::unix::fs::symlink, path::PathBuf, time::Duration};23use anyhow::{anyhow, Result};4use clap::{Parser, ValueEnum};5use itertools::Itertools as _;6use nix_eval::nix_go;7use tokio::{task::LocalSet, time::sleep};8use tracing::{error, field, info, info_span, warn, Instrument};910use crate::{11	command::MyCommand,12	host::{Config, ConfigHost},13};1415#[derive(Parser)]16pub struct Deploy {17	/// Disable automatic rollback18	#[clap(long)]19	disable_rollback: bool,20	/// Action to execute after system is built21	action: DeployAction,22}2324#[derive(ValueEnum, Clone, Copy)]25enum DeployAction {26	/// Upload derivation, but do not execute the update.27	Upload,28	/// Upload and execute the activation script, old version will be used after reboot.29	Test,30	/// Upload and set as current system profile, but do not execute activation script.31	Boot,32	/// Upload, set current profile, and execute activation script.33	Switch,34}3536impl DeployAction {37	pub(crate) fn name(&self) -> Option<&'static str> {38		match self {39			DeployAction::Upload => None,40			DeployAction::Test => Some("test"),41			DeployAction::Boot => Some("boot"),42			DeployAction::Switch => Some("switch"),43		}44	}45	pub(crate) fn should_switch_profile(&self) -> bool {46		matches!(self, Self::Switch | Self::Boot)47	}48	pub(crate) fn should_activate(&self) -> bool {49		matches!(self, Self::Switch | Self::Test)50	}51	pub(crate) fn should_create_rollback_marker(&self) -> bool {52		// Upload does nothing on the target machine, other than uploading the closure.53		// In boot case we want to have rollback marker prepared, so that the system may rollback itself on the next boot.54		!matches!(self, Self::Upload)55	}56	pub(crate) fn should_schedule_rollback_run(&self) -> bool {57		matches!(self, Self::Switch | Self::Test)58	}59}6061#[derive(Parser, Clone)]62pub struct BuildSystems {63	/// Attribute to build. Systems are deployed from "toplevel" attr, well-known used attributes64	/// are "sdImage"/"isoImage", and your configuration may include any other build attributes.65	#[clap(long, default_value = "toplevel")]66	build_attr: String,67}6869struct Generation {70	id: u32,71	current: bool,72	datetime: String,73}74async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {75	let mut cmd = host.cmd("nix-env").await?;76	cmd.comparg("--profile", "/nix/var/nix/profiles/system")77		.arg("--list-generations");78	// Sudo is required due to --list-generations acquiring lock on the profile.79	let data = cmd.sudo().run_string().await?;80	let generations = data81		.split('\n')82		.map(|e| e.trim())83		.filter(|&l| !l.is_empty())84		.filter_map(|g| {85			let gen: Option<Generation> = try {86				let mut parts = g.split_whitespace();87				let id = parts.next()?;88				let id: u32 = id.parse().ok()?;89				let date = parts.next()?;90				let time = parts.next()?;91				let current = if let Some(current) = parts.next() {92					if current == "(current)" {93						Some(true)94					} else {95						None96					}97				} else {98					Some(false)99				};100				let current = current?;101				if parts.next().is_some() {102					warn!("unexpected text after generation: {g}");103				}104				Generation {105					id,106					current,107					datetime: format!("{date} {time}"),108				}109			};110			if gen.is_none() {111				warn!("bad generation: {g}")112			}113			gen114		})115		.collect::<Vec<_>>();116	let current = generations117		.into_iter()118		.filter(|g| g.current)119		.at_most_one()120		.map_err(|_e| anyhow!("bad list-generations output"))?121		.ok_or_else(|| anyhow!("failed to find generation"))?;122	Ok(current)123}124125async fn deploy_task(126	action: DeployAction,127	host: &ConfigHost,128	built: PathBuf,129	disable_rollback: bool,130) -> Result<()> {131	let mut failed = false;132	// TODO: Lockfile, to prevent concurrent system switch?133	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback134	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to135	// unit name conflict in systemd-run136	// This code is tied to rollback.nix137	if !disable_rollback && action.should_create_rollback_marker() {138		let _span = info_span!("preparing").entered();139		info!("preparing for rollback");140		let generation = get_current_generation(host).await?;141		info!(142			"rollback target would be {} {}",143			generation.id, generation.datetime144		);145		{146			let mut cmd = host.cmd("sh").await?;147			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));148			if let Err(e) = cmd.sudo().run().await {149				error!("failed to set rollback marker: {e}");150				failed = true;151			}152		}153		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.154		// Kicking it on manually will work best.155		//156		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will157		// only allow one instance of it.158159		// TODO: We should also watch how this process is going.160		// After running this command, we have less than 3 minutes to deploy everything,161		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.162		// Anyway, reboot will still help in this case.163		if action.should_schedule_rollback_run() {164			let mut cmd = host.cmd("systemd-run").await?;165			cmd.comparg("--on-active", "3min")166				.comparg("--unit", "rollback-watchdog-run")167				.arg("systemctl")168				.arg("start")169				.arg("rollback-watchdog.service");170			if let Err(e) = cmd.sudo().run().await {171				error!("failed to schedule rollback run: {e}");172				failed = true;173			}174		}175	}176177	if action.should_switch_profile() && !failed {178		info!("switching generation");179		let mut cmd = host.cmd("nix-env").await?;180		cmd.comparg("--profile", "/nix/var/nix/profiles/system")181			.comparg("--set", &built);182		if let Err(e) = cmd.sudo().run().await {183			error!("failed to switch generation: {e}");184			failed = true;185		}186	}187188	// FIXME: Connection might be disconnected after activation run189190	if action.should_activate() && !failed {191		let _span = info_span!("activating").entered();192		info!("executing activation script");193		let mut switch_script = built.clone();194		switch_script.push("bin");195		switch_script.push("switch-to-configuration");196		let mut cmd = host.cmd(switch_script).in_current_span().await?;197		cmd.arg(action.name().expect("upload.should_activate == false"));198		if let Err(e) = cmd.sudo().run().in_current_span().await {199			error!("failed to activate: {e}");200			failed = true;201		}202	}203	if action.should_create_rollback_marker() {204		if !disable_rollback {205			if failed {206				if action.should_schedule_rollback_run() {207					info!("executing rollback");208					if let Err(e) = host209						.systemctl_start("rollback-watchdog.service")210						.instrument(info_span!("rollback"))211						.await212					{213						error!("failed to trigger rollback: {e}")214					}215				}216			} else {217				info!("trying to mark upgrade as successful");218				if let Err(e) = host219					.rm_file("/etc/fleet_rollback_marker", true)220					.in_current_span()221					.await222				{223					error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")224				}225			}226			info!("disarming watchdog, just in case");227			if let Err(_e) = host.systemctl_stop("rollback-watchdog.timer").await {228				// It is ok, if there was no reboot - then timer might not be running.229			}230			if action.should_schedule_rollback_run() {231				if let Err(e) = host.systemctl_stop("rollback-watchdog-run.timer").await {232					error!("failed to disarm rollback run: {e}");233				}234			}235		} else if let Err(_e) = host236			.rm_file("/etc/fleet_rollback_marker", true)237			.in_current_span()238			.await239		{240			// Marker might not exist, yet better try to remove it.241		}242	}243	Ok(())244}245246async fn build_task(config: Config, host: String, build_attr: &str) -> Result<PathBuf> {247	info!("building");248	let host = config.host(&host).await?;249	// let action = Action::from(self.subcommand.clone());250	let fleet_config = &config.config_field;251	let drv = nix_go!(252		fleet_config.hosts[{ &host.name }]253			.nixosSystem254			.config255			.system256			.build[{ build_attr }]257	);258	let outputs = drv.build().await.map_err(|e| {259			if build_attr == "sdImage" {260				info!("sd-image build failed");261				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");262			}263			e264		})?;265	let out_output = outputs266		.get("out")267		.ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;268269	Ok(out_output.clone())270}271272impl BuildSystems {273	pub async fn run(self, config: &Config) -> Result<()> {274		let hosts = config.list_hosts().await?;275		let set = LocalSet::new();276		let build_attr = self.build_attr.clone();277		for host in hosts.into_iter() {278			if config.should_skip(&host.name) {279				continue;280			}281			let config = config.clone();282			let span = info_span!("build", host = field::display(&host.name));283			let hostname = host.name;284			let build_attr = build_attr.clone();285			// FIXME: Since the introduction of better-nix-eval,286			// due to single repl used for builds, hosts are waiting for each other to build,287			// instead of building concurrently.288			//289			// Open multiple repls?290			//291			// Create build batcher, which will behave similar to golangs292			// WaitGroup, and start executing once all the build tasks are scheduled?293			// This also allows to cleanup build output, as there will be no longer294			// "waiting for remote machine" messages in the cases when one package is needed for295			// multiple hosts.296			set.spawn_local(297				(async move {298					let built = match build_task(config, hostname.clone(), &build_attr).await {299						Ok(path) => path,300						Err(e) => {301							error!("failed to deploy host: {}", e);302							return;303						}304					};305					// TODO: Handle error306					let mut out = current_dir().expect("cwd exists");307					out.push(format!("built-{}", hostname));308309					info!("linking iso image to {:?}", out);310					if let Err(e) = symlink(built, out) {311						error!("failed to symlink: {e}")312					}313				})314				.instrument(span),315			);316		}317		set.await;318		Ok(())319	}320}321322impl Deploy {323	pub async fn run(self, config: &Config) -> Result<()> {324		let hosts = config.list_hosts().await?;325		let set = LocalSet::new();326		for host in hosts.into_iter() {327			if config.should_skip(&host.name) {328				continue;329			}330			let config = config.clone();331			let span = info_span!("deploy", host = field::display(&host.name));332			let hostname = host.name.clone();333			// FIXME: Fix repl concurrency (see build-systems)334			set.spawn_local(335				(async move {336					let built = match build_task(config.clone(), hostname.clone(), "toplevel").await337					{338						Ok(path) => path,339						Err(e) => {340							error!("failed to deploy host: {}", e);341							return;342						}343					};344					if !config.is_local(&hostname) {345						info!("uploading system closure");346						{347							// TODO: Move to remote_derivation method.348							// Alternatively, nix store make-content-addressed can be used,349							// at least for the first deployment, to provide trusted store key.350							//351							// It is much slower, yet doesn't require root on the deployer machine.352							let mut sign = MyCommand::new("nix");353							// Private key for host machine is registered in nix-sign.nix354							sign.arg("store")355								.arg("sign")356								.comparg("--key-file", "/etc/nix/private-key")357								.arg("-r")358								.arg(&built);359							if let Err(e) = sign.sudo().run_nix().await {360								warn!("Failed to sign store paths: {e}");361							};362						}363						let mut tries = 0;364						loop {365							match host.remote_derivation(&built).await {366								Ok(remote) => {367									assert!(remote == built, "CA derivations aren't implemented");368									break;369								}370								Err(e) if tries < 3 => {371									tries += 1;372									warn!("copy failure ({}/3): {}", tries, e);373									sleep(Duration::from_millis(5000)).await;374								}375								Err(e) => {376									error!("upload failed: {e}");377									return;378								}379							}380						}381					}382					if let Err(e) =383						deploy_task(self.action, &host, built, self.disable_rollback).await384					{385						error!("activation failed: {e}");386					}387				})388				.instrument(span),389			);390		}391		set.await;392		Ok(())393	}394}