difftreelog
refactor split deployment function
in: trunk
1 file changed
cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth1use std::{env::current_dir, time::Duration};23use crate::command::MyCommand;4use crate::host::Config;5use anyhow::{anyhow, Result};6use clap::Parser;7use itertools::Itertools;8use tokio::{task::LocalSet, time::sleep};9use tracing::{error, field, info, info_span, warn, Instrument};1011#[derive(Parser, Clone)]12pub struct BuildSystems {13 /// Do not continue on error14 #[clap(long)]15 fail_fast: bool,16 /// Disable automatic rollback17 #[clap(long)]18 disable_rollback: bool,19 /// Run builds as sudo20 #[clap(long)]21 privileged_build: bool,22 #[clap(subcommand)]23 subcommand: Subcommand,24}2526enum UploadAction {27 Test,28 Boot,29 Switch,30}31impl UploadAction {32 fn name(&self) -> &'static str {33 match self {34 UploadAction::Test => "test",35 UploadAction::Boot => "boot",36 UploadAction::Switch => "switch",37 }38 }3940 pub(crate) fn should_switch_profile(&self) -> bool {41 matches!(self, Self::Switch | Self::Boot)42 }43 pub(crate) fn should_activate(&self) -> bool {44 matches!(self, Self::Switch | Self::Test)45 }46 pub(crate) fn should_schedule_rollback_run(&self) -> bool {47 matches!(self, Self::Switch | Self::Test)48 }49}5051enum PackageAction {52 SdImage,53 InstallationCd,54}55impl PackageAction {56 fn build_attr(&self) -> String {57 match self {58 PackageAction::SdImage => "sdImage".to_owned(),59 PackageAction::InstallationCd => "installationCd".to_owned(),60 }61 }62}6364enum Action {65 Upload { action: Option<UploadAction> },66 Package(PackageAction),67}68impl Action {69 fn build_attr(&self) -> String {70 match self {71 Action::Upload { .. } => "toplevel".to_owned(),72 Action::Package(p) => p.build_attr(),73 }74 }75}7677impl From<Subcommand> for Action {78 fn from(s: Subcommand) -> Self {79 match s {80 Subcommand::Upload => Self::Upload { action: None },81 Subcommand::Test => Self::Upload {82 action: Some(UploadAction::Test),83 },84 Subcommand::Boot => Self::Upload {85 action: Some(UploadAction::Boot),86 },87 Subcommand::Switch => Self::Upload {88 action: Some(UploadAction::Switch),89 },90 Subcommand::SdImage => Self::Package(PackageAction::SdImage),91 Subcommand::InstallationCd => Self::Package(PackageAction::InstallationCd),92 }93 }94}9596#[derive(Parser, Clone)]97enum Subcommand {98 /// Upload, but do not switch99 Upload,100 /// Upload + switch to built system until reboot101 Test,102 /// Upload + switch to built system after reboot103 Boot,104 /// Upload + test + boot105 Switch,106107 /// Build SD .img image108 SdImage,109 /// Build an installation cd ISO image110 InstallationCd,111}112113struct Generation {114 id: u32,115 current: bool,116 datetime: String,117}118async fn get_current_generation(config: &Config, host: &str) -> Result<Generation> {119 let mut cmd = MyCommand::new("nix-env");120 cmd.comparg("--profile", "/nix/var/nix/profiles/system")121 .arg("--list-generations");122 // Sudo is required due to --list-generations acquiring lock on the profile.123 let data = config.run_string_on(&host, cmd, true).await?;124 let generations = data125 .split('\n')126 .map(|e| e.trim())127 .filter(|&l| l != "")128 .filter_map(|g| {129 let gen: Option<Generation> = try {130 let mut parts = g.split_whitespace();131 let id = parts.next()?;132 let id: u32 = id.parse().ok()?;133 let date = parts.next()?;134 let time = parts.next()?;135 let current = if let Some(current) = parts.next() {136 if current == "(current)" {137 Some(true)138 } else {139 None140 }141 } else {142 Some(false)143 };144 let current = current?;145 if parts.next().is_some() {146 warn!("unexpected text after generation: {g}");147 }148 Generation {149 id,150 current,151 datetime: format!("{date} {time}"),152 }153 };154 if gen.is_none() {155 warn!("bad generation: {g}")156 }157 gen158 })159 .collect::<Vec<_>>();160 let current = generations161 .into_iter()162 .filter(|g| g.current)163 .at_most_one()164 .map_err(|_e| anyhow!("bad list-generations output"))?165 .ok_or_else(|| anyhow!("failed to find generation"))?;166 Ok(current)167}168169impl BuildSystems {170 async fn build_task(self, config: Config, host: String) -> Result<()> {171 info!("building");172 let action = Action::from(self.subcommand.clone());173 let built = {174 let dir = tempfile::tempdir()?;175 dir.path().to_owned()176 };177178 let mut nix_build = MyCommand::new("nix");179 nix_build180 .args([181 "build",182 "--impure",183 "--json",184 // "--show-trace",185 "--no-link",186 "--option",187 "log-lines",188 "200",189 ])190 .comparg("--out-link", &built)191 .arg(192 config.configuration_attr_name(&format!(193 "buildSystems.{}.{host}",194 action.build_attr()195 )),196 )197 .args(&config.nix_args);198199 if self.privileged_build {200 nix_build = nix_build.sudo();201 }202203 nix_build.run_nix().await.map_err(|e| {204 if action.build_attr() == "sdImage" {205 info!("sd-image build failed");206 info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");207 info!("This module was automatically imported before, but was removed for better customization")208 }209 e210 })?;211 let built = std::fs::canonicalize(built)?;212213 match action {214 Action::Upload { action } => {215 if !config.is_local(&host) {216 info!("uploading system closure");217 let mut tries = 0;218 loop {219 let mut nix = MyCommand::new("nix");220 nix.arg("copy")221 .arg("--substitute-on-destination")222 .comparg("--to", format!("ssh://root@{host}"))223 .arg(&built);224 match nix.run_nix().await {225 Ok(()) => break,226 Err(e) if tries < 3 => {227 tries += 1;228 warn!("Copy failure ({}/3): {}", tries, e);229 sleep(Duration::from_millis(5000)).await;230 }231 Err(e) => return Err(e),232 }233 }234 }235 if let Some(action) = action {236 let mut failed = false;237 // TODO: Lockfile, to prevent concurrent system switch?238 // TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback239 // is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to240 // unit name conflict in systemd-run241 if !self.disable_rollback {242 let _span = info_span!("preparing").entered();243 info!("preparing for rollback");244 let generation = get_current_generation(&config, &host).await?;245 info!(246 "rollback target would be {} {}",247 generation.id, generation.datetime248 );249 {250 let mut cmd = MyCommand::new("sh");251 cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));252 if let Err(e) = config.run_on(&host, cmd, true).await {253 error!("failed to set rollback marker: {e}");254 failed = true;255 }256 }257 // Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.258 // Kicking it on manually will work best.259 //260 // There wouldn't be conflict, because here we trigger start of the primary service, and systemd will261 // only allow one instance of it.262 if action.should_schedule_rollback_run() {263 let mut cmd = MyCommand::new("systemd-run");264 cmd.comparg("--on-active", "3min")265 .comparg("--unit", "rollback-watchdog-run")266 .arg("systemctl")267 .arg("start")268 .arg("rollback-watchdog.service");269 if let Err(e) = config.run_on(&host, cmd, true).await {270 error!("failed to schedule rollback run: {e}");271 failed = true;272 }273 }274 }275 if action.should_switch_profile() && !failed {276 info!("switching generation");277 let mut cmd = MyCommand::new("nix-env");278 cmd.comparg("--profile", "/nix/var/nix/profiles/system")279 .comparg("--set", &built);280 if let Err(e) = config.run_on(&host, cmd, true).await {281 error!("failed to switch generation: {e}");282 failed = true;283 }284 }285 if action.should_activate() && !failed {286 let _span = info_span!("activating").entered();287 info!("executing activation script");288 let mut switch_script = built.clone();289 switch_script.push("bin");290 switch_script.push("switch-to-configuration");291 let mut cmd = MyCommand::new(switch_script);292 cmd.arg(action.name());293 if let Err(e) = config.run_on(&host, cmd, true).in_current_span().await {294 error!("failed to activate: {e}");295 failed = true;296 }297 }298 if !self.disable_rollback {299 {300 let _span = info_span!("rollback").entered();301 if failed {302 info!("executing rollback");303 let mut cmd = MyCommand::new("systemctl");304 cmd.arg("start").arg("rollback-watchdog.service");305 if let Err(e) = config.run_on(&host, cmd, true).await {306 error!("failed to rollback: {e}");307 }308 } else {309 info!("marking upgrade as successful");310 let mut cmd = MyCommand::new("rm");311 cmd.arg("-f").arg("/etc/fleet_rollback_marker");312 if let Err(e) =313 config.run_on(&host, cmd, true).in_current_span().await314 {315 error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")316 }317 }318 }319 {320 let _span = info_span!("disarm").entered();321 info!("disarming watchdog, just in case");322 {323 let mut cmd = MyCommand::new("systemctl");324 cmd.arg("stop").arg("rollback-watchdog.timer");325 if let Err(_e) = config.run_on(&host, cmd, true).await {326 // It is ok, if there was no reboot.327 }328 }329 if action.should_schedule_rollback_run() {330 let mut cmd = MyCommand::new("systemctl");331 cmd.arg("stop").arg("rollback-watchdog-run.timer");332 if let Err(e) = config.run_on(&host, cmd, true).await {333 error!("failed to disarm rollback run: {e}");334 }335 }336 }337 }338 }339 }340 Action::Package(PackageAction::SdImage) => {341 let mut out = current_dir()?;342 out.push(format!("sd-image-{}", host));343344 info!("building sd image to {:?}", out);345 let mut nix_build = MyCommand::new("nix");346 nix_build347 .args(["build", "--impure", "--no-link"])348 .comparg("--out-link", &out)349 .arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))350 .args(&config.nix_args);351 if !self.fail_fast {352 nix_build.arg("--keep-going");353 }354 if self.privileged_build {355 nix_build = nix_build.sudo();356 }357358 nix_build.run_nix().await?;359 }360 Action::Package(PackageAction::InstallationCd) => {361 let mut out = current_dir()?;362 out.push(format!("installation-cd-{}", host));363364 info!("building sd image to {:?}", out);365 let mut nix_build = MyCommand::new("nix");366 nix_build367 .args(["build", "--impure", "--no-link"])368 .comparg("--out-link", &out)369 .arg(370 config.configuration_attr_name(&format!(371 "buildSystems.installationCd.{}",372 host,373 )),374 )375 .args(&config.nix_args);376 if !self.fail_fast {377 nix_build.arg("--keep-going");378 }379 if self.privileged_build {380 nix_build = nix_build.sudo();381 }382383 nix_build.run_nix().await?;384 }385 };386 Ok(())387 }388389 pub async fn run(self, config: &Config) -> Result<()> {390 let hosts = config.list_hosts().await?;391 let set = LocalSet::new();392 let this = &self;393 for host in hosts.iter() {394 if config.should_skip(host) {395 continue;396 }397 let config = config.clone();398 let host = host.clone();399 let this = this.clone();400 let span = info_span!("deployment", host = field::display(&host));401 set.spawn_local(402 (async move {403 match this.build_task(config, host).await {404 Ok(_) => {}405 Err(e) => {406 error!("failed to deploy host: {}", e)407 }408 }409 })410 .instrument(span),411 );412 }413 set.await;414 Ok(())415 }416}1use std::path::PathBuf;2use std::{env::current_dir, time::Duration};34use crate::command::MyCommand;5use crate::host::Config;6use anyhow::{anyhow, Result};7use clap::Parser;8use itertools::Itertools;9use tokio::{task::LocalSet, time::sleep};10use tracing::{error, field, info, info_span, warn, Instrument};1112#[derive(Parser, Clone)]13pub struct BuildSystems {14 /// Do not continue on error15 #[clap(long)]16 fail_fast: bool,17 /// Disable automatic rollback18 #[clap(long)]19 disable_rollback: bool,20 /// Run builds as sudo21 #[clap(long)]22 privileged_build: bool,23 #[clap(subcommand)]24 subcommand: Subcommand,25}2627enum UploadAction {28 Test,29 Boot,30 Switch,31}32impl UploadAction {33 fn name(&self) -> &'static str {34 match self {35 UploadAction::Test => "test",36 UploadAction::Boot => "boot",37 UploadAction::Switch => "switch",38 }39 }4041 pub(crate) fn should_switch_profile(&self) -> bool {42 matches!(self, Self::Switch | Self::Boot)43 }44 pub(crate) fn should_activate(&self) -> bool {45 matches!(self, Self::Switch | Self::Test)46 }47 pub(crate) fn should_schedule_rollback_run(&self) -> bool {48 matches!(self, Self::Switch | Self::Test)49 }50}5152enum PackageAction {53 SdImage,54 InstallationCd,55}56impl PackageAction {57 fn build_attr(&self) -> String {58 match self {59 PackageAction::SdImage => "sdImage".to_owned(),60 PackageAction::InstallationCd => "installationCd".to_owned(),61 }62 }63}6465enum Action {66 Upload { action: Option<UploadAction> },67 Package(PackageAction),68}69impl Action {70 fn build_attr(&self) -> String {71 match self {72 Action::Upload { .. } => "toplevel".to_owned(),73 Action::Package(p) => p.build_attr(),74 }75 }76}7778impl From<Subcommand> for Action {79 fn from(s: Subcommand) -> Self {80 match s {81 Subcommand::Upload => Self::Upload { action: None },82 Subcommand::Test => Self::Upload {83 action: Some(UploadAction::Test),84 },85 Subcommand::Boot => Self::Upload {86 action: Some(UploadAction::Boot),87 },88 Subcommand::Switch => Self::Upload {89 action: Some(UploadAction::Switch),90 },91 Subcommand::SdImage => Self::Package(PackageAction::SdImage),92 Subcommand::InstallationCd => Self::Package(PackageAction::InstallationCd),93 }94 }95}9697#[derive(Parser, Clone)]98enum Subcommand {99 /// Upload, but do not switch100 Upload,101 /// Upload + switch to built system until reboot102 Test,103 /// Upload + switch to built system after reboot104 Boot,105 /// Upload + test + boot106 Switch,107108 /// Build SD .img image109 SdImage,110 /// Build an installation cd ISO image111 InstallationCd,112}113114struct Generation {115 id: u32,116 current: bool,117 datetime: String,118}119async fn get_current_generation(config: &Config, host: &str) -> Result<Generation> {120 let mut cmd = MyCommand::new("nix-env");121 cmd.comparg("--profile", "/nix/var/nix/profiles/system")122 .arg("--list-generations");123 // Sudo is required due to --list-generations acquiring lock on the profile.124 let data = config.run_string_on(&host, cmd, true).await?;125 let generations = data126 .split('\n')127 .map(|e| e.trim())128 .filter(|&l| l != "")129 .filter_map(|g| {130 let gen: Option<Generation> = try {131 let mut parts = g.split_whitespace();132 let id = parts.next()?;133 let id: u32 = id.parse().ok()?;134 let date = parts.next()?;135 let time = parts.next()?;136 let current = if let Some(current) = parts.next() {137 if current == "(current)" {138 Some(true)139 } else {140 None141 }142 } else {143 Some(false)144 };145 let current = current?;146 if parts.next().is_some() {147 warn!("unexpected text after generation: {g}");148 }149 Generation {150 id,151 current,152 datetime: format!("{date} {time}"),153 }154 };155 if gen.is_none() {156 warn!("bad generation: {g}")157 }158 gen159 })160 .collect::<Vec<_>>();161 let current = generations162 .into_iter()163 .filter(|g| g.current)164 .at_most_one()165 .map_err(|_e| anyhow!("bad list-generations output"))?166 .ok_or_else(|| anyhow!("failed to find generation"))?;167 Ok(current)168}169170async fn systemctl_stop(config: &Config, host: &str, unit: &str) -> Result<()> {171 let mut cmd = MyCommand::new("systemctl");172 cmd.arg("stop").arg(unit);173 config.run_on(&host, cmd, true).await174}175176async fn systemctl_start(config: &Config, host: &str, unit: &str) -> Result<()> {177 let mut cmd = MyCommand::new("systemctl");178 cmd.arg("start").arg(unit);179 config.run_on(&host, cmd, true).await180}181182async fn execute_upload(183 build: &BuildSystems,184 config: &Config,185 action: UploadAction,186 host: &str,187 built: PathBuf,188) -> Result<()> {189 let mut failed = false;190 // TODO: Lockfile, to prevent concurrent system switch?191 // TODO: If rollback target exists - bail, it should be removed. Lockfile will not work in case if rollback192 // is scheduler on next boot (default behavior). On current boot - rollback activator will fail due to193 // unit name conflict in systemd-run194 if !build.disable_rollback {195 let _span = info_span!("preparing").entered();196 info!("preparing for rollback");197 let generation = get_current_generation(&config, &host).await?;198 info!(199 "rollback target would be {} {}",200 generation.id, generation.datetime201 );202 {203 let mut cmd = MyCommand::new("sh");204 cmd.arg("-c").arg(format!("mark=$(mktemp -p /etc -t fleet_rollback_marker.XXXXX) && echo -n {} > $mark && mv --no-clobber $mark /etc/fleet_rollback_marker", generation.id));205 if let Err(e) = config.run_on(&host, cmd, true).await {206 error!("failed to set rollback marker: {e}");207 failed = true;208 }209 }210 // Activation script also starts rollback-watchdog.timer, however, it is possible that it won't be started.211 // Kicking it on manually will work best.212 //213 // There wouldn't be conflict, because here we trigger start of the primary service, and systemd will214 // only allow one instance of it.215216 // TODO: We should also watch how this process is going.217 // After running this command, we have less than 3 minutes to deploy everything,218 // if we fail to perform generation switch in time, then we will still call the activation script, and this may break something.219 // Anyway, reboot will still help in this case.220 if action.should_schedule_rollback_run() {221 let mut cmd = MyCommand::new("systemd-run");222 cmd.comparg("--on-active", "3min")223 .comparg("--unit", "rollback-watchdog-run")224 .arg("systemctl")225 .arg("start")226 .arg("rollback-watchdog.service");227 if let Err(e) = config.run_on(&host, cmd, true).await {228 error!("failed to schedule rollback run: {e}");229 failed = true;230 }231 }232 }233 if action.should_switch_profile() && !failed {234 info!("switching generation");235 let mut cmd = MyCommand::new("nix-env");236 cmd.comparg("--profile", "/nix/var/nix/profiles/system")237 .comparg("--set", &built);238 if let Err(e) = config.run_on(&host, cmd, true).await {239 error!("failed to switch generation: {e}");240 failed = true;241 }242 }243 if action.should_activate() && !failed {244 let _span = info_span!("activating").entered();245 info!("executing activation script");246 let mut switch_script = built.clone();247 switch_script.push("bin");248 switch_script.push("switch-to-configuration");249 let mut cmd = MyCommand::new(switch_script);250 cmd.arg(action.name());251 if let Err(e) = config.run_on(&host, cmd, true).in_current_span().await {252 error!("failed to activate: {e}");253 failed = true;254 }255 }256 if !build.disable_rollback {257 if failed {258 info!("executing rollback");259 if let Err(e) = systemctl_start(&config, &host, "rollback-watchdog.service")260 .instrument(info_span!("rollback"))261 .await262 {263 error!("failed to trigger rollback: {e}")264 }265 } else {266 info!("trying to mark upgrade as successful");267 let mut cmd = MyCommand::new("rm");268 cmd.arg("-f").arg("/etc/fleet_rollback_marker");269 if let Err(e) = config.run_on(&host, cmd, true).in_current_span().await {270 error!("failed to remove rollback marker. This is bad, as the system will be rolled back by watchdog: {e}")271 }272 }273 info!("disarming watchdog, just in case");274 if let Err(_e) = systemctl_stop(&config, &host, "rollback-watchdog.timer").await {275 // It is ok, if there was no reboot - then timer might not be running.276 }277 if action.should_schedule_rollback_run() {278 if let Err(e) = systemctl_stop(&config, &host, "rollback-watchdog-run.timer").await {279 error!("failed to disarm rollback run: {e}");280 }281 }282 } else {283 let mut cmd = MyCommand::new("rm");284 cmd.arg("-f").arg("/etc/fleet_rollback_marker");285 if let Err(_e) = config.run_on(&host, cmd, true).in_current_span().await {286 // Marker might not exist, yet better try to remove it.287 }288 }289 Ok(())290}291292impl BuildSystems {293 async fn build_task(self, config: Config, host: String) -> Result<()> {294 info!("building");295 let action = Action::from(self.subcommand.clone());296 let built = {297 let dir = tempfile::tempdir()?;298 dir.path().to_owned()299 };300301 let mut nix_build = MyCommand::new("nix");302 nix_build303 .args([304 "build",305 "--impure",306 "--json",307 // "--show-trace",308 "--no-link",309 ])310 .comparg("--out-link", &built)311 .arg(312 config.configuration_attr_name(&format!(313 "buildSystems.{}.{host}",314 action.build_attr()315 )),316 )317 .args(&config.nix_args);318319 if self.privileged_build {320 nix_build = nix_build.sudo();321 }322323 nix_build.run_nix().await.map_err(|e| {324 if action.build_attr() == "sdImage" {325 info!("sd-image build failed");326 info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");327 info!("This module was automatically imported before, but was removed for better customization")328 }329 e330 })?;331 let built = std::fs::canonicalize(built)?;332333 match action {334 Action::Upload { action } => {335 if !config.is_local(&host) {336 info!("uploading system closure");337 let mut tries = 0;338 loop {339 let mut nix = MyCommand::new("nix");340 nix.arg("copy")341 .arg("--substitute-on-destination")342 .comparg("--to", format!("ssh-ng://root@{host}"))343 .arg(&built);344 match nix.run_nix().await {345 Ok(()) => break,346 Err(e) if tries < 3 => {347 tries += 1;348 warn!("Copy failure ({}/3): {}", tries, e);349 sleep(Duration::from_millis(5000)).await;350 }351 Err(e) => return Err(e),352 }353 }354 }355 if let Some(action) = action {356 execute_upload(&self, &config, action, &host, built).await?357 }358 }359 Action::Package(PackageAction::SdImage) => {360 let mut out = current_dir()?;361 out.push(format!("sd-image-{}", host));362363 info!("building sd image to {:?}", out);364 let mut nix_build = MyCommand::new("nix");365 nix_build366 .args(["build", "--impure", "--no-link"])367 .comparg("--out-link", &out)368 .arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))369 .args(&config.nix_args);370 if !self.fail_fast {371 nix_build.arg("--keep-going");372 }373 if self.privileged_build {374 nix_build = nix_build.sudo();375 }376377 nix_build.run_nix().await?;378 }379 Action::Package(PackageAction::InstallationCd) => {380 let mut out = current_dir()?;381 out.push(format!("installation-cd-{}", host));382383 info!("building sd image to {:?}", out);384 let mut nix_build = MyCommand::new("nix");385 nix_build386 .args(["build", "--impure", "--no-link"])387 .comparg("--out-link", &out)388 .arg(389 config.configuration_attr_name(&format!(390 "buildSystems.installationCd.{}",391 host,392 )),393 )394 .args(&config.nix_args);395 if !self.fail_fast {396 nix_build.arg("--keep-going");397 }398 if self.privileged_build {399 nix_build = nix_build.sudo();400 }401402 nix_build.run_nix().await?;403 }404 };405 Ok(())406 }407408 pub async fn run(self, config: &Config) -> Result<()> {409 let hosts = config.list_hosts().await?;410 let set = LocalSet::new();411 let this = &self;412 for host in hosts.iter() {413 if config.should_skip(host) {414 continue;415 }416 let config = config.clone();417 let host = host.clone();418 let this = this.clone();419 let span = info_span!("deployment", host = field::display(&host));420 set.spawn_local(421 (async move {422 match this.build_task(config, host).await {423 Ok(_) => {}424 Err(e) => {425 error!("failed to deploy host: {}", e)426 }427 }428 })429 .instrument(span),430 );431 }432 set.await;433 Ok(())434 }435}