1use std::path::PathBuf;2use std::{env::current_dir, time::Duration};34use crate::command::MyCommand;5use crate::host::Config;6use anyhow::{anyhow, Result};7use clap::Parser;8use itertools::Itertools;9use tokio::{task::LocalSet, time::sleep};10use tracing::{error, field, info, info_span, warn, Instrument};1112#[derive(Parser, Clone)]13pub struct BuildSystems {14 15 #[clap(long)]16 fail_fast: bool,17 18 #[clap(long)]19 disable_rollback: bool,20 21 #[clap(long)]22 privileged_build: bool,23 #[clap(subcommand)]24 subcommand: Subcommand,25}2627enum UploadAction {28 Test,29 Boot,30 Switch,31}32impl UploadAction {33 fn name(&self) -> &'static str {34 match self {35 UploadAction::Test => "test",36 UploadAction::Boot => "boot",37 UploadAction::Switch => "switch",38 }39 }4041 pub(crate) fn should_switch_profile(&self) -> bool {42 matches!(self, Self::Switch | Self::Boot)43 }44 pub(crate) fn should_activate(&self) -> bool {45 matches!(self, Self::Switch | Self::Test)46 }47 pub(crate) fn should_schedule_rollback_run(&self) -> bool {48 matches!(self, Self::Switch | Self::Test)49 }50}5152enum PackageAction {53 SdImage,54 InstallationCd,55}56impl PackageAction {57 fn build_attr(&self) -> String {58 match self {59 PackageAction::SdImage => "sdImage".to_owned(),60 PackageAction::InstallationCd => "installationCd".to_owned(),61 }62 }63}6465enum Action {66 Upload { action: Option<UploadAction> },67 Package(PackageAction),68}69impl Action {70 fn build_attr(&self) -> String {71 match self {72 Action::Upload { .. } => "toplevel".to_owned(),73 Action::Package(p) => p.build_attr(),74 }75 }76}7778impl From<Subcommand> for Action {79 fn from(s: Subcommand) -> Self {80 match s {81 Subcommand::Upload => Self::Upload { action: None },82 Subcommand::Test => Self::Upload {83 action: Some(UploadAction::Test),84 },85 Subcommand::Boot => Self::Upload {86 action: Some(UploadAction::Boot),87 },88 Subcommand::Switch => Self::Upload {89 action: Some(UploadAction::Switch),90 },91 Subcommand::SdImage => Self::Package(PackageAction::SdImage),92 Subcommand::InstallationCd => Self::Package(PackageAction::InstallationCd),93 }94 }95}9697#[derive(Parser, Clone)]98enum Subcommand {99 100 Upload,101 102 Test,103 104 Boot,105 106 Switch,107108 109 SdImage,110 111 InstallationCd,112}113114struct Generation {115 id: u32,116 current: bool,117 datetime: String,118}119async fn get_current_generation(config: &Config, host: &str) -> Result<Generation> {120 let mut cmd = MyCommand::new("nix-env");121 cmd.comparg("--profile", "/nix/var/nix/profiles/system")122 .arg("--list-generations");123 124 let data = config.run_string_on(host, cmd, true).await?;125 let generations = data126 .split('\n')127 .map(|e| e.trim())128 .filter(|&l| !l.is_empty())129 .filter_map(|g| {130 let gen: Option<Generation> = try {131 let mut parts = g.split_whitespace();132 let id = parts.next()?;133 let id: u32 = id.parse().ok()?;134 let date = parts.next()?;135 let time = parts.next()?;136 let current = if let Some(current) = parts.next() {137 if current == "(current)" {138 Some(true)139 } else {140 None141 }142 } else {143 Some(false)144 };145 let current = current?;146 if parts.next().is_some() {147 warn!("unexpected text after generation: {g}");148 }149 Generation {150 id,151 current,152 datetime: format!("{date} {time}"),153 }154 };155 if gen.is_none() {156 warn!("bad generation: {g}")157 }158 gen159 })160 .collect::<Vec<_>>();161 let current = generations162 .into_iter()163 .filter(|g| g.current)164 .at_most_one()165 .map_err(|_e| anyhow!("bad list-generations output"))?166 .ok_or_else(|| anyhow!("failed to find generation"))?;167 Ok(current)168}169170async fn systemctl_stop(config: &Config, host: &str, unit: &str) -> Result<()> {171 let mut cmd = MyCommand::new("systemctl");172 cmd.arg("stop").arg(unit);173 config.run_on(host, cmd, true).await174}175176async fn systemctl_start(config: &Config, host: &str, unit: &str) -> Result<()> {177 let mut cmd = MyCommand::new("systemctl");178 cmd.arg("start").arg(unit);179 config.run_on(host, cmd, true).await180}181182async fn execute_upload(183 build: &BuildSystems,184 config: &Config,185 action: UploadAction,186 host: &str,187 built: PathBuf,188) -> Result<()> {189 let mut failed = false;190 191 192 193 194 195 if !build.disable_rollback {196 let _span = info_span!("preparing").entered();197 info!("preparing for rollback");198 let generation = get_current_generation(config, host).await?;199 info!(200 "rollback target would be {} {}",201 generation.id, generation.datetime202 );203 {204 let mut cmd = MyCommand::new("sh");205 cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));206 if let Err(e) = config.run_on(host, cmd, true).await {207 error!("failed to set rollback marker: {e}");208 failed = true;209 }210 }211 212 213 214 215 216217 218 219 220 221 if action.should_schedule_rollback_run() {222 let mut cmd = MyCommand::new("systemd-run");223 cmd.comparg("--on-active", "3min")224 .comparg("--unit", "rollback-watchdog-run")225 .arg("systemctl")226 .arg("start")227 .arg("rollback-watchdog.service");228 if let Err(e) = config.run_on(host, cmd, true).await {229 error!("failed to schedule rollback run: {e}");230 failed = true;231 }232 }233 }234 if action.should_switch_profile() && !failed {235 info!("switching generation");236 let mut cmd = MyCommand::new("nix-env");237 cmd.comparg("--profile", "/nix/var/nix/profiles/system")238 .comparg("--set", &built);239 if let Err(e) = config.run_on(host, cmd, true).await {240 error!("failed to switch generation: {e}");241 failed = true;242 }243 }244 if action.should_activate() && !failed {245 let _span = info_span!("activating").entered();246 info!("executing activation script");247 let mut switch_script = built.clone();248 switch_script.push("bin");249 switch_script.push("switch-to-configuration");250 let mut cmd = MyCommand::new(switch_script);251 cmd.arg(action.name());252 if let Err(e) = config.run_on(host, cmd, true).in_current_span().await {253 error!("failed to activate: {e}");254 failed = true;255 }256 }257 if !build.disable_rollback {258 if failed {259 info!("executing rollback");260 if let Err(e) = systemctl_start(config, host, "rollback-watchdog.service")261 .instrument(info_span!("rollback"))262 .await263 {264 error!("failed to trigger rollback: {e}")265 }266 } else {267 info!("trying to mark upgrade as successful");268 let mut cmd = MyCommand::new("rm");269 cmd.arg("-f").arg("/etc/fleet_rollback_marker");270 if let Err(e) = config.run_on(host, cmd, true).in_current_span().await {271 error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")272 }273 }274 info!("disarming watchdog, just in case");275 if let Err(_e) = systemctl_stop(config, host, "rollback-watchdog.timer").await {276 277 }278 if action.should_schedule_rollback_run() {279 if let Err(e) = systemctl_stop(config, host, "rollback-watchdog-run.timer").await {280 error!("failed to disarm rollback run: {e}");281 }282 }283 } else {284 let mut cmd = MyCommand::new("rm");285 cmd.arg("-f").arg("/etc/fleet_rollback_marker");286 if let Err(_e) = config.run_on(host, cmd, true).in_current_span().await {287 288 }289 }290 Ok(())291}292293impl BuildSystems {294 async fn build_task(self, config: Config, host: String) -> Result<()> {295 info!("building");296 let action = Action::from(self.subcommand.clone());297 let built = {298 let dir = tempfile::tempdir()?;299 dir.path().to_owned()300 };301302 let mut nix_build = MyCommand::new("nix");303 nix_build304 .args([305 "build",306 "--impure",307 "--json",308 309 "--no-link",310 ])311 .comparg("--out-link", &built)312 .arg(313 config.configuration_attr_name(&format!(314 "buildSystems.{}.{host}",315 action.build_attr()316 )),317 )318 .args(&config.nix_args);319320 if self.privileged_build {321 nix_build = nix_build.sudo();322 }323324 nix_build.run_nix().await.map_err(|e| {325 if action.build_attr() == "sdImage" {326 info!("sd-image build failed");327 info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");328 info!("This module was automatically imported before, but was removed for better customization")329 }330 e331 })?;332 let built = std::fs::canonicalize(built)?;333334 match action {335 Action::Upload { action } => {336 if !config.is_local(&host) {337 info!("uploading system closure");338 {339 let mut sign = MyCommand::new("sudo");340 341 sign.arg("nix")342 .arg("store")343 .arg("sign")344 .comparg("--key-file", "/etc/nix/private-key")345 .arg("-r")346 .arg(&built);347 if let Err(e) = sign.run_nix().await {348 warn!("Failed to sign store paths: {e}");349 };350 }351 let mut tries = 0;352 loop {353 let mut nix = MyCommand::new("nix");354 nix.arg("copy")355 .arg("--substitute-on-destination")356 .comparg("--to", format!("ssh-ng://{host}"))357 .arg(&built);358 match nix.run_nix().await {359 Ok(()) => break,360 Err(e) if tries < 3 => {361 tries += 1;362 warn!("Copy failure ({}/3): {}", tries, e);363 sleep(Duration::from_millis(5000)).await;364 }365 Err(e) => return Err(e),366 }367 }368 }369 if let Some(action) = action {370 execute_upload(&self, &config, action, &host, built).await?371 }372 }373 Action::Package(PackageAction::SdImage) => {374 let mut out = current_dir()?;375 out.push(format!("sd-image-{}", host));376377 info!("building sd image to {:?}", out);378 let mut nix_build = MyCommand::new("nix");379 nix_build380 .args(["build", "--impure", "--no-link"])381 .comparg("--out-link", &out)382 .arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))383 .args(&config.nix_args);384 if !self.fail_fast {385 nix_build.arg("--keep-going");386 }387 if self.privileged_build {388 nix_build = nix_build.sudo();389 }390391 nix_build.run_nix().await?;392 }393 Action::Package(PackageAction::InstallationCd) => {394 let mut out = current_dir()?;395 out.push(format!("installation-cd-{}", host));396397 info!("building sd image to {:?}", out);398 let mut nix_build = MyCommand::new("nix");399 nix_build400 .args(["build", "--impure", "--no-link"])401 .comparg("--out-link", &out)402 .arg(403 config.configuration_attr_name(&format!(404 "buildSystems.installationCd.{}",405 host,406 )),407 )408 .args(&config.nix_args);409 if !self.fail_fast {410 nix_build.arg("--keep-going");411 }412 if self.privileged_build {413 nix_build = nix_build.sudo();414 }415416 nix_build.run_nix().await?;417 }418 };419 Ok(())420 }421422 pub async fn run(self, config: &Config) -> Result<()> {423 let hosts = config.list_hosts().await?;424 let set = LocalSet::new();425 let this = &self;426 for host in hosts.into_iter() {427 if config.should_skip(&host.name) {428 continue;429 }430 let config = config.clone();431 let this = this.clone();432 let span = info_span!("deployment", host = field::display(&host.name));433 let hostname = host.name;434 set.spawn_local(435 (async move {436 match this.build_task(config, hostname).await {437 Ok(_) => {}438 Err(e) => {439 error!("failed to deploy host: {}", e)440 }441 }442 })443 .instrument(span),444 );445 }446 set.await;447 Ok(())448 }449}
1use std::path::PathBuf;2use std::{env::current_dir, time::Duration};34use crate::command::MyCommand;5use crate::host::Config;6use anyhow::{anyhow, Result};7use clap::Parser;8use itertools::Itertools;9use tokio::{task::LocalSet, time::sleep};10use tracing::{error, field, info, info_span, warn, Instrument};1112#[derive(Parser, Clone)]13pub struct BuildSystems {14 15 #[clap(long)]16 fail_fast: bool,17 18 #[clap(long)]19 disable_rollback: bool,20 21 #[clap(long)]22 privileged_build: bool,23 #[clap(subcommand)]24 subcommand: Subcommand,25}2627enum UploadAction {28 Test,29 Boot,30 Switch,31}32impl UploadAction {33 fn name(&self) -> &'static str {34 match self {35 UploadAction::Test => "test",36 UploadAction::Boot => "boot",37 UploadAction::Switch => "switch",38 }39 }4041 pub(crate) fn should_switch_profile(&self) -> bool {42 matches!(self, Self::Switch | Self::Boot)43 }44 pub(crate) fn should_activate(&self) -> bool {45 matches!(self, Self::Switch | Self::Test)46 }47 pub(crate) fn should_schedule_rollback_run(&self) -> bool {48 matches!(self, Self::Switch | Self::Test)49 }50}5152enum PackageAction {53 SdImage,54 InstallationCd,55}56impl PackageAction {57 fn build_attr(&self) -> String {58 match self {59 PackageAction::SdImage => "sdImage".to_owned(),60 PackageAction::InstallationCd => "installationCd".to_owned(),61 }62 }63}6465enum Action {66 Upload { action: Option<UploadAction> },67 Package(PackageAction),68}69impl Action {70 fn build_attr(&self) -> String {71 match self {72 Action::Upload { .. } => "toplevel".to_owned(),73 Action::Package(p) => p.build_attr(),74 }75 }76}7778impl From<Subcommand> for Action {79 fn from(s: Subcommand) -> Self {80 match s {81 Subcommand::Upload => Self::Upload { action: None },82 Subcommand::Test => Self::Upload {83 action: Some(UploadAction::Test),84 },85 Subcommand::Boot => Self::Upload {86 action: Some(UploadAction::Boot),87 },88 Subcommand::Switch => Self::Upload {89 action: Some(UploadAction::Switch),90 },91 Subcommand::SdImage => Self::Package(PackageAction::SdImage),92 Subcommand::InstallationCd => Self::Package(PackageAction::InstallationCd),93 }94 }95}9697#[derive(Parser, Clone)]98enum Subcommand {99 100 Upload,101 102 Test,103 104 Boot,105 106 Switch,107108 109 SdImage,110 111 InstallationCd,112}113114struct Generation {115 id: u32,116 current: bool,117 datetime: String,118}119async fn get_current_generation(config: &Config, host: &str) -> Result<Generation> {120 let mut cmd = MyCommand::new("nix-env");121 cmd.comparg("--profile", "/nix/var/nix/profiles/system")122 .arg("--list-generations");123 124 let data = config.run_string_on(host, cmd, true).await?;125 let generations = data126 .split('\n')127 .map(|e| e.trim())128 .filter(|&l| !l.is_empty())129 .filter_map(|g| {130 let gen: Option<Generation> = try {131 let mut parts = g.split_whitespace();132 let id = parts.next()?;133 let id: u32 = id.parse().ok()?;134 let date = parts.next()?;135 let time = parts.next()?;136 let current = if let Some(current) = parts.next() {137 if current == "(current)" {138 Some(true)139 } else {140 None141 }142 } else {143 Some(false)144 };145 let current = current?;146 if parts.next().is_some() {147 warn!("unexpected text after generation: {g}");148 }149 Generation {150 id,151 current,152 datetime: format!("{date} {time}"),153 }154 };155 if gen.is_none() {156 warn!("bad generation: {g}")157 }158 gen159 })160 .collect::<Vec<_>>();161 let current = generations162 .into_iter()163 .filter(|g| g.current)164 .at_most_one()165 .map_err(|_e| anyhow!("bad list-generations output"))?166 .ok_or_else(|| anyhow!("failed to find generation"))?;167 Ok(current)168}169170async fn systemctl_stop(config: &Config, host: &str, unit: &str) -> Result<()> {171 let mut cmd = MyCommand::new("systemctl");172 cmd.arg("stop").arg(unit);173 config.run_on(host, cmd, true).await174}175176async fn systemctl_start(config: &Config, host: &str, unit: &str) -> Result<()> {177 let mut cmd = MyCommand::new("systemctl");178 cmd.arg("start").arg(unit);179 config.run_on(host, cmd, true).await180}181182async fn execute_upload(183 build: &BuildSystems,184 config: &Config,185 action: UploadAction,186 host: &str,187 built: PathBuf,188) -> Result<()> {189 let mut failed = false;190 191 192 193 194 195 if !build.disable_rollback {196 let _span = info_span!("preparing").entered();197 info!("preparing for rollback");198 let generation = get_current_generation(config, host).await?;199 info!(200 "rollback target would be {} {}",201 generation.id, generation.datetime202 );203 {204 let mut cmd = MyCommand::new("sh");205 cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));206 if let Err(e) = config.run_on(host, cmd, true).await {207 error!("failed to set rollback marker: {e}");208 failed = true;209 }210 }211 212 213 214 215 216217 218 219 220 221 if action.should_schedule_rollback_run() {222 let mut cmd = MyCommand::new("systemd-run");223 cmd.comparg("--on-active", "3min")224 .comparg("--unit", "rollback-watchdog-run")225 .arg("systemctl")226 .arg("start")227 .arg("rollback-watchdog.service");228 if let Err(e) = config.run_on(host, cmd, true).await {229 error!("failed to schedule rollback run: {e}");230 failed = true;231 }232 }233 }234 if action.should_switch_profile() && !failed {235 info!("switching generation");236 let mut cmd = MyCommand::new("nix-env");237 cmd.comparg("--profile", "/nix/var/nix/profiles/system")238 .comparg("--set", &built);239 if let Err(e) = config.run_on(host, cmd, true).await {240 error!("failed to switch generation: {e}");241 failed = true;242 }243 }244 if action.should_activate() && !failed {245 let _span = info_span!("activating").entered();246 info!("executing activation script");247 let mut switch_script = built.clone();248 switch_script.push("bin");249 switch_script.push("switch-to-configuration");250 let mut cmd = MyCommand::new(switch_script);251 cmd.arg(action.name());252 if let Err(e) = config.run_on(host, cmd, true).in_current_span().await {253 error!("failed to activate: {e}");254 failed = true;255 }256 }257 if !build.disable_rollback {258 if failed {259 info!("executing rollback");260 if let Err(e) = systemctl_start(config, host, "rollback-watchdog.service")261 .instrument(info_span!("rollback"))262 .await263 {264 error!("failed to trigger rollback: {e}")265 }266 } else {267 info!("trying to mark upgrade as successful");268 let mut cmd = MyCommand::new("rm");269 cmd.arg("-f").arg("/etc/fleet_rollback_marker");270 if let Err(e) = config.run_on(host, cmd, true).in_current_span().await {271 error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")272 }273 }274 info!("disarming watchdog, just in case");275 if let Err(_e) = systemctl_stop(config, host, "rollback-watchdog.timer").await {276 277 }278 if action.should_schedule_rollback_run() {279 if let Err(e) = systemctl_stop(config, host, "rollback-watchdog-run.timer").await {280 error!("failed to disarm rollback run: {e}");281 }282 }283 } else {284 let mut cmd = MyCommand::new("rm");285 cmd.arg("-f").arg("/etc/fleet_rollback_marker");286 if let Err(_e) = config.run_on(host, cmd, true).in_current_span().await {287 288 }289 }290 Ok(())291}292293impl BuildSystems {294 async fn build_task(self, config: Config, host: String) -> Result<()> {295 info!("building");296 let action = Action::from(self.subcommand.clone());297 let built = {298 let dir = tempfile::tempdir()?;299 dir.path().to_owned()300 };301302 let mut nix_build = MyCommand::new("nix");303 nix_build304 .args([305 "build",306 "--impure",307 "--json",308 309 "--no-link",310 ])311 .comparg("--out-link", &built)312 .arg(313 config.configuration_attr_name(&format!(314 "buildSystems.{}.{host}",315 action.build_attr()316 )),317 )318 .args(&config.nix_args);319320 if self.privileged_build {321 nix_build = nix_build.sudo();322 }323324 nix_build.run_nix().await.map_err(|e| {325 if action.build_attr() == "sdImage" {326 info!("sd-image build failed");327 info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");328 info!("This module was automatically imported before, but was removed for better customization")329 }330 e331 })?;332 let built = std::fs::canonicalize(built)?;333334 match action {335 Action::Upload { action } => {336 if !config.is_local(&host) {337 info!("uploading system closure");338 {339 let mut sign = MyCommand::new("nix");340 341 sign.arg("store")342 .arg("sign")343 .comparg("--key-file", "/etc/nix/private-key")344 .arg("-r")345 .arg(&built);346 if let Err(e) = sign.sudo().run_nix().await {347 warn!("Failed to sign store paths: {e}");348 };349 }350 let mut tries = 0;351 loop {352 let mut nix = MyCommand::new("nix");353 nix.arg("copy")354 .arg("--substitute-on-destination")355 .comparg("--to", format!("ssh-ng://{host}"))356 .arg(&built);357 match nix.run_nix().await {358 Ok(()) => break,359 Err(e) if tries < 3 => {360 tries += 1;361 warn!("Copy failure ({}/3): {}", tries, e);362 sleep(Duration::from_millis(5000)).await;363 }364 Err(e) => return Err(e),365 }366 }367 }368 if let Some(action) = action {369 execute_upload(&self, &config, action, &host, built).await?370 }371 }372 Action::Package(PackageAction::SdImage) => {373 let mut out = current_dir()?;374 out.push(format!("sd-image-{}", host));375376 info!("building sd image to {:?}", out);377 let mut nix_build = MyCommand::new("nix");378 nix_build379 .args(["build", "--impure", "--no-link"])380 .comparg("--out-link", &out)381 .arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))382 .args(&config.nix_args);383 if !self.fail_fast {384 nix_build.arg("--keep-going");385 }386 if self.privileged_build {387 nix_build = nix_build.sudo();388 }389390 nix_build.run_nix().await?;391 }392 Action::Package(PackageAction::InstallationCd) => {393 let mut out = current_dir()?;394 out.push(format!("installation-cd-{}", host));395396 info!("building sd image to {:?}", out);397 let mut nix_build = MyCommand::new("nix");398 nix_build399 .args(["build", "--impure", "--no-link"])400 .comparg("--out-link", &out)401 .arg(402 config.configuration_attr_name(&format!(403 "buildSystems.installationCd.{}",404 host,405 )),406 )407 .args(&config.nix_args);408 if !self.fail_fast {409 nix_build.arg("--keep-going");410 }411 if self.privileged_build {412 nix_build = nix_build.sudo();413 }414415 nix_build.run_nix().await?;416 }417 };418 Ok(())419 }420421 pub async fn run(self, config: &Config) -> Result<()> {422 let hosts = config.list_hosts().await?;423 let set = LocalSet::new();424 let this = &self;425 for host in hosts.into_iter() {426 if config.should_skip(&host.name) {427 continue;428 }429 let config = config.clone();430 let this = this.clone();431 let span = info_span!("deployment", host = field::display(&host.name));432 let hostname = host.name;433 set.spawn_local(434 (async move {435 match this.build_task(config, hostname).await {436 Ok(_) => {}437 Err(e) => {438 error!("failed to deploy host: {}", e)439 }440 }441 })442 .instrument(span),443 );444 }445 set.await;446 Ok(())447 }448}