git.delta.rocks / fleet / refs/commits / c0c9b96f77be

1use std::os::unix::fs::symlink;2use std::path::PathBuf;3use std::{env::current_dir, time::Duration};45use crate::command::MyCommand;6use crate::host::{Config, ConfigHost};7use crate::nix_go;8use anyhow::{anyhow, Result};9use clap::Parser;10use itertools::Itertools as _;11use tokio::{task::LocalSet, time::sleep};12use tracing::{error, field, info, info_span, warn, Instrument};1314#[derive(Parser, Clone)]15pub struct BuildSystems {16	/// Disable automatic rollback17	#[clap(long)]18	disable_rollback: bool,19	#[clap(subcommand)]20	subcommand: Subcommand,21}2223enum UploadAction {24	Test,25	Boot,26	Switch,27}28impl UploadAction {29	fn name(&self) -> &'static str {30		match self {31			UploadAction::Test => "test",32			UploadAction::Boot => "boot",33			UploadAction::Switch => "switch",34		}35	}3637	pub(crate) fn should_switch_profile(&self) -> bool {38		matches!(self, Self::Switch | Self::Boot)39	}40	pub(crate) fn should_activate(&self) -> bool {41		matches!(self, Self::Switch | Self::Test)42	}43	pub(crate) fn should_schedule_rollback_run(&self) -> bool {44		matches!(self, Self::Switch | Self::Test)45	}46}4748enum PackageAction {49	SdImage,50	InstallationCd,51}52impl PackageAction {53	fn build_attr(&self) -> String {54		match self {55			PackageAction::SdImage => "sdImage".to_owned(),56			PackageAction::InstallationCd => "isoImage".to_owned(),57		}58	}59}6061enum Action {62	Upload { action: Option<UploadAction> },63	Package(PackageAction),64}65impl Action {66	fn build_attr(&self) -> String {67		match self {68			Action::Upload { .. } => "toplevel".to_owned(),69			Action::Package(p) => p.build_attr(),70		}71	}72}7374impl From<Subcommand> for Action {75	fn from(s: Subcommand) -> Self {76		match s {77			Subcommand::Upload => Self::Upload { action: None },78			Subcommand::Test => Self::Upload {79				action: Some(UploadAction::Test),80			},81			Subcommand::Boot => Self::Upload {82				action: Some(UploadAction::Boot),83			},84			Subcommand::Switch => Self::Upload {85				action: Some(UploadAction::Switch),86			},87			Subcommand::SdImage => Self::Package(PackageAction::SdImage),88			Subcommand::InstallationCd => Self::Package(PackageAction::InstallationCd),89		}90	}91}9293#[derive(Parser, Clone)]94enum Subcommand {95	/// Upload, but do not switch96	Upload,97	/// Upload + switch to built system until reboot98	Test,99	/// Upload + switch to built system after reboot100	Boot,101	/// Upload + test + boot102	Switch,103104	/// Build SD .img image105	SdImage,106	/// Build an installation cd ISO image107	InstallationCd,108}109110struct Generation {111	id: u32,112	current: bool,113	datetime: String,114}115async fn get_current_generation(host: &ConfigHost) -> Result<Generation> {116	let mut cmd = host.cmd("nix-env").await?;117	cmd.comparg("--profile", "/nix/var/nix/profiles/system")118		.arg("--list-generations");119	// Sudo is required due to --list-generations acquiring lock on the profile.120	let data = cmd.sudo().run_string().await?;121	let generations = data122		.split('\n')123		.map(|e| e.trim())124		.filter(|&l| !l.is_empty())125		.filter_map(|g| {126			let gen: Option<Generation> = try {127				let mut parts = g.split_whitespace();128				let id = parts.next()?;129				let id: u32 = id.parse().ok()?;130				let date = parts.next()?;131				let time = parts.next()?;132				let current = if let Some(current) = parts.next() {133					if current == "(current)" {134						Some(true)135					} else {136						None137					}138				} else {139					Some(false)140				};141				let current = current?;142				if parts.next().is_some() {143					warn!("unexpected text after generation: {g}");144				}145				Generation {146					id,147					current,148					datetime: format!("{date} {time}"),149				}150			};151			if gen.is_none() {152				warn!("bad generation: {g}")153			}154			gen155		})156		.collect::<Vec<_>>();157	let current = generations158		.into_iter()159		.filter(|g| g.current)160		.at_most_one()161		.map_err(|_e| anyhow!("bad list-generations output"))?162		.ok_or_else(|| anyhow!("failed to find generation"))?;163	Ok(current)164}165166async fn execute_upload(167	build: &BuildSystems,168	action: UploadAction,169	host: &ConfigHost,170	built: PathBuf,171) -> Result<()> {172	let mut failed = false;173	// TODO: Lockfile, to prevent concurrent system switch?174	// TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback175	// is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to176	// unit name conflict in systemd-run177	// This code is tied to rollback.nix178	if !build.disable_rollback {179		let _span = info_span!("preparing").entered();180		info!("preparing for rollback");181		let generation = get_current_generation(host).await?;182		info!(183			"rollback target would be {} {}",184			generation.id, generation.datetime185		);186		{187			let mut cmd = host.cmd("sh").await?;188			cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));189			if let Err(e) = cmd.sudo().run().await {190				error!("failed to set rollback marker: {e}");191				failed = true;192			}193		}194		// Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.195		// Kicking it on manually will work best.196		//197		// There wouldn't be conflict, because here we trigger start of the primary service, and systemd will198		// only allow one instance of it.199200		// TODO: We should also watch how this process is going.201		// After running this command, we have less than 3 minutes to deploy everything,202		// if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.203		// Anyway, reboot will still help in this case.204		if action.should_schedule_rollback_run() {205			let mut cmd = host.cmd("systemd-run").await?;206			cmd.comparg("--on-active", "3min")207				.comparg("--unit", "rollback-watchdog-run")208				.arg("systemctl")209				.arg("start")210				.arg("rollback-watchdog.service");211			if let Err(e) = cmd.sudo().run().await {212				error!("failed to schedule rollback run: {e}");213				failed = true;214			}215		}216	}217218	if action.should_switch_profile() && !failed {219		info!("switching generation");220		let mut cmd = host.cmd("nix-env").await?;221		cmd.comparg("--profile", "/nix/var/nix/profiles/system")222			.comparg("--set", &built);223		if let Err(e) = cmd.sudo().run().await {224			error!("failed to switch generation: {e}");225			failed = true;226		}227	}228229	// FIXME: Connection might be disconnected after activation run230231	if action.should_activate() && !failed {232		let _span = info_span!("activating").entered();233		info!("executing activation script");234		let mut switch_script = built.clone();235		switch_script.push("bin");236		switch_script.push("switch-to-configuration");237		let mut cmd = host.cmd(switch_script).in_current_span().await?;238		cmd.arg(action.name());239		if let Err(e) = cmd.sudo().run().in_current_span().await {240			error!("failed to activate: {e}");241			failed = true;242		}243	}244	if !build.disable_rollback {245		if failed {246			info!("executing rollback");247			if let Err(e) = host248				.systemctl_start("rollback-watchdog.service")249				.instrument(info_span!("rollback"))250				.await251			{252				error!("failed to trigger rollback: {e}")253			}254		} else {255			info!("trying to mark upgrade as successful");256			if let Err(e) = host257				.rm_file("/etc/fleet_rollback_marker", true)258				.in_current_span()259				.await260			{261				error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")262			}263		}264		info!("disarming watchdog, just in case");265		if let Err(_e) = host.systemctl_stop("rollback-watchdog.timer").await {266			// It is ok, if there was no reboot - then timer might not be running.267		}268		if action.should_schedule_rollback_run() {269			if let Err(e) = host.systemctl_stop("rollback-watchdog-run.timer").await {270				error!("failed to disarm rollback run: {e}");271			}272		}273	} else if let Err(_e) = host274		.rm_file("/etc/fleet_rollback_marker", true)275		.in_current_span()276		.await277	{278		// Marker might not exist, yet better try to remove it.279	}280	Ok(())281}282283impl BuildSystems {284	async fn build_task(self, config: Config, host: String) -> Result<()> {285		info!("building");286		let host = config.host(&host).await?;287		let action = Action::from(self.subcommand.clone());288		let fleet_config = &config.config_field;289		let drv = nix_go!(290			fleet_config.hosts[{ &host.name }].nixosSystem.config.system.build[{ action.build_attr() }]291		);292		let outputs = drv.build().await.map_err(|e| {293			if action.build_attr() == "sdImage" {294				info!("sd-image build failed");295				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");296			}297			e298		})?;299		let out_output = outputs300			.get("out")301			.ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;302303		match action {304			Action::Upload { action } => {305				if !config.is_local(&host.name) {306					info!("uploading system closure");307					{308						// TODO: Move to remote_derivation method.309						// Alternatively, nix store make-content-addressed can be used,310						// at least for the first deployment, to provide trusted store key.311						//312						// It is much slower, yet doesn't require root on the deployer machine.313						let mut sign = MyCommand::new("nix");314						// Private key for host machine is registered in nix-sign.nix315						sign.arg("store")316							.arg("sign")317							.comparg("--key-file", "/etc/nix/private-key")318							.arg("-r")319							.arg(out_output);320						if let Err(e) = sign.sudo().run_nix().await {321							warn!("Failed to sign store paths: {e}");322						};323					}324					let mut tries = 0;325					loop {326						match host.remote_derivation(out_output).await {327							Ok(remote) => {328								assert!(&remote == out_output, "CA derivations aren't implemented");329								break;330							}331							Err(e) if tries < 3 => {332								tries += 1;333								warn!("Copy failure ({}/3): {}", tries, e);334								sleep(Duration::from_millis(5000)).await;335							}336							Err(e) => return Err(e),337						}338					}339				}340				if let Some(action) = action {341					execute_upload(&self, action, &host, out_output.clone()).await?342				}343			}344			Action::Package(PackageAction::SdImage) => {345				let mut out = current_dir()?;346				out.push(format!("sd-image-{}", host.name));347348				info!("linking sd image to {:?}", out);349				symlink(out_output, out)?;350			}351			Action::Package(PackageAction::InstallationCd) => {352				let mut out = current_dir()?;353				out.push(format!("installation-cd-{}", host.name));354355				info!("linking iso image to {:?}", out);356				symlink(out_output, out)?;357			}358		};359		Ok(())360	}361362	pub async fn run(self, config: &Config) -> Result<()> {363		let hosts = config.list_hosts().await?;364		let set = LocalSet::new();365		let this = &self;366		for host in hosts.into_iter() {367			if config.should_skip(&host.name) {368				continue;369			}370			let config = config.clone();371			let this = this.clone();372			let span = info_span!("deployment", host = field::display(&host.name));373			let hostname = host.name;374			// FIXME: Since the introduction of better-nix-eval,375			// due to single repl used for builds, hosts are waiting for each other to build,376			// instead of building concurrently.377			//378			// Open multiple repls?379			//380			// Create build batcher, which will behave similar to golangs381			// WaitGroup, and start executing once all the build tasks are scheduled?382			// This also allows to cleanup build output, as there will be no longer383			// "waiting for remote machine" messages in the cases when one package is needed for384			// multiple hosts.385			set.spawn_local(386				(async move {387					match this.build_task(config, hostname).await {388						Ok(_) => {}389						Err(e) => {390							error!("failed to deploy host: {}", e)391						}392					}393				})394				.instrument(span),395			);396		}397		set.await;398		Ok(())399	}400}