git.delta.rocks / jrsonnet / refs/commits / 81e3c77f96ff

difftreelog

feat parallel host builds

Yaroslav Bolyukin2021-12-26parent: #6a5196a.patch.diff
in: trunk

6 files changed

modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
after · cmds/fleet/src/cmds/build_systems.rs
1use std::{env::current_dir, time::Duration};23use crate::{command::CommandExt, host::Config};4use anyhow::Result;5use structopt::StructOpt;6use tokio::{process::Command, task::LocalSet, time::sleep};7use tracing::{error, field, info, info_span, warn, Instrument};89#[derive(StructOpt, Clone)]10pub struct BuildSystems {11	/// --builders arg for nix12	#[structopt(long)]13	builders: Option<String>,14	/// Jobs to run locally15	#[structopt(long)]16	jobs: Option<usize>,17	/// Do not continue on error18	#[structopt(long)]19	fail_fast: bool,20	#[structopt(long)]21	privileged_build: bool,22	#[structopt(subcommand)]23	subcommand: Subcommand,24	#[structopt(long)]25	show_trace: bool,26}2728enum UploadAction {29	Test,30	Boot,31	Switch,32}33impl UploadAction {34	fn name(&self) -> &'static str {35		match self {36			UploadAction::Test => "test",37			UploadAction::Boot => "boot",38			UploadAction::Switch => "switch",39		}40	}4142	pub(crate) fn should_switch_profile(&self) -> bool {43		matches!(self, Self::Switch | Self::Test)44	}45}4647enum PackageAction {48	SdImage,49}5051enum Action {52	Upload(Option<UploadAction>),53	Package(PackageAction),54}5556impl From<Subcommand> for Action {57	fn from(s: Subcommand) -> Self {58		match s {59			Subcommand::Upload => Self::Upload(None),60			Subcommand::Test => Self::Upload(Some(UploadAction::Test)),61			Subcommand::Boot => Self::Upload(Some(UploadAction::Boot)),62			Subcommand::Switch => Self::Upload(Some(UploadAction::Switch)),63			Subcommand::SdImage => Self::Package(PackageAction::SdImage),64		}65	}66}6768#[derive(StructOpt, Clone)]69enum Subcommand {70	/// Upload, but do not switch71	Upload,72	/// Upload + switch to built system until reboot73	Test,74	/// Upload + switch to built system after reboot75	Boot,76	/// Upload + test + boot77	Switch,7879	/// Build sd image80	SdImage,81}8283impl BuildSystems {84	async fn build_task(self, config: Config, host: String) -> Result<()> {85		info!("building");86		let built = {87			let dir = tempfile::tempdir()?;88			dir.path().to_owned()89		};9091		let mut nix_build = if self.privileged_build {92			let mut out = Command::new("sudo");93			out.arg("nix");94			out95		} else {96			Command::new("nix")97		};98		nix_build99			.args(&[100				"build",101				"--impure",102				"--json",103				// "--show-trace",104				"--no-link",105				"--out-link",106			])107			.arg(&built)108			.arg(config.configuration_attr_name(&format!(109				"configuredSystems.{}.config.system.build.toplevel",110				host111			)));112113		if self.show_trace {114			nix_build.arg("--show-trace");115		}116		if let Some(builders) = &self.builders {117			nix_build.arg("--builders").arg(builders);118		}119		if let Some(jobs) = &self.jobs {120			nix_build.arg("--max-jobs");121			nix_build.arg(format!("{}", jobs));122		}123		if !self.fail_fast {124			nix_build.arg("--keep-going");125		}126127		nix_build.run_nix().await?;128		let built = std::fs::canonicalize(built)?;129130		let action = Action::from(self.subcommand.clone());131132		match action {133			Action::Upload(action) => {134				if !config.is_local(&host) {135					info!("uploading system closure");136					let mut tries = 0;137					loop {138						match Command::new("nix")139							.args(&["copy", "--to"])140							.arg(format!("ssh://root@{}", host))141							.arg(&built)142							.inherit_stdio()143							.run_nix()144							.await145						{146							Ok(()) => break,147							Err(e) if tries < 3 => {148								tries += 1;149								warn!("Copy failure ({}/3): {}", tries, e);150								sleep(Duration::from_millis(5000)).await;151							}152							Err(e) => return Err(e),153						}154					}155				}156				if let Some(action) = action {157					if action.should_switch_profile() {158						info!("switching generation");159						config160							.command_on(&host, "nix-env", true)161							.args(&["-p", "/nix/var/nix/profiles/system", "--set"])162							.arg(&built)163							.inherit_stdio()164							.run()165							.await?;166					}167					info!("executing activation script");168					let mut switch_script = built.clone();169					switch_script.push("bin");170					switch_script.push("switch-to-configuration");171					config172						.command_on(&host, switch_script, true)173						.arg(action.name())174						.inherit_stdio()175						.run()176						.await?;177				}178			}179			Action::Package(PackageAction::SdImage) => {180				let mut out = current_dir()?;181				out.push(format!("sd-image-{}", host));182183				info!("building sd image to {:?}", out);184				let mut nix_build = if self.privileged_build {185					let mut out = Command::new("sudo");186					out.arg("nix");187					out188				} else {189					Command::new("nix")190				};191				nix_build192					.args(&["build", "--impure", "--no-link", "--out-link"])193					.arg(&out)194					.arg(config.configuration_attr_name(&format!(195						"configuredSystems.{}.config.system.build.sdImage",196						host,197					)));198				if let Some(builders) = &self.builders {199					nix_build.arg("--builders").arg(builders);200				}201				if let Some(jobs) = &self.jobs {202					nix_build.arg("--max-jobs");203					nix_build.arg(format!("{}", jobs));204				}205				if !self.fail_fast {206					nix_build.arg("--keep-going");207				}208209				nix_build.inherit_stdio().run_nix().await?;210			}211		};212		Ok(())213	}214215	pub async fn run(self, config: &Config) -> Result<()> {216		let hosts = config.list_hosts().await?;217		let set = LocalSet::new();218		let this = &self;219		for host in hosts.iter() {220			if config.should_skip(host) {221				continue;222			}223			let config = config.clone();224			let host = host.clone();225			let this = this.clone();226			let span = info_span!("deployment", host = field::display(&host));227			set.spawn_local(228				(async move {229					match this.build_task(config, host).await {230						Ok(_) => {}231						Err(e) => {232							error!("failed to deploy host: {}", e)233						}234					}235				})236				.instrument(span),237			);238		}239		set.await;240		Ok(())241	}242}
modifiedcmds/fleet/src/command.rsdiffbeforeafterboth
--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -150,6 +150,12 @@
 										info!(target: "nix", "{}", text)
 									}
 								},
+								NixLog::Start { text, level: 0, typ: 108, .. } if text == "" => {
+									// Cache lookup? Coupled with copy log
+								},
+								NixLog::Start { text, level: 4, typ: 101, .. } if text.starts_with("downloading ") => {
+									// NAR downloading, coupled with copy log
+								}
 								NixLog::Stop { .. } => {},
 								NixLog::Result { .. } => {},
 								_ => warn!("unknown log: {:?}", log)
modifiedcmds/fleet/src/host.rsdiffbeforeafterboth
--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -16,6 +16,7 @@
 use crate::{command::CommandExt, fleetdata::FleetData};
 
 pub struct FleetConfigInternals {
+	pub local_system: String,
 	pub directory: PathBuf,
 	pub opts: FleetOpts,
 	pub data: RefCell<FleetData>,
@@ -66,17 +67,21 @@
 		}
 	}
 
-	pub fn full_attr_name(&self, attr_name: &str) -> OsString {
+	pub fn configuration_attr_name(&self, name: &str) -> OsString {
 		let mut str = self.directory.as_os_str().to_owned();
 		str.push("#");
-		str.push(attr_name);
+		str.push(&format!(
+			"fleetConfigurations.default.{}.{}",
+			self.local_system,
+			name
+		));
 		str
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<String>> {
 		Command::new("nix")
 			.arg("eval")
-			.arg(self.full_attr_name("fleetConfigurations.default.configuredHosts"))
+			.arg(self.configuration_attr_name("configuredHosts"))
 			.args(&["--apply", "builtins.attrNames", "--json", "--show-trace"])
 			.run_nix_json()
 			.await
@@ -84,10 +89,7 @@
 	pub async fn config_attr<T: DeserializeOwned>(&self, host: &str, attr: &str) -> Result<T> {
 		Command::new("nix")
 			.arg("eval")
-			.arg(self.full_attr_name(&format!(
-				"fleetConfigurations.default.configuredSystems.{}.config.{}",
-				host, attr
-			)))
+			.arg(self.configuration_attr_name(&format!("configuredSystems.{}.config.{}", host, attr)))
 			.args(&["--json", "--show-trace"])
 			.run_nix_json()
 			.await
@@ -129,10 +131,14 @@
 	/// Host, which should be threaten as current machine
 	#[structopt(long)]
 	pub localhost: Option<String>,
+
+	#[structopt(long, default_value = "x86_64-linux")]
+	pub local_system: String,
 }
 
 impl FleetOpts {
 	pub fn build(mut self) -> Result<Config> {
+		let local_system = self.local_system.clone();
 		if self.localhost.is_none() {
 			self.localhost
 				.replace(hostname::get().unwrap().to_str().unwrap().to_owned());
@@ -148,6 +154,7 @@
 			opts: self,
 			directory,
 			data,
+			local_system,
 		})))
 	}
 }
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -1,11 +1,7 @@
-#![feature(try_blocks)]
-
 pub mod command;
 pub mod host;
 pub mod keys;
-
 pub mod cmds;
-pub mod nix;
 
 mod fleetdata;
 
deletedcmds/fleet/src/nix.rsdiffbeforeafterboth
--- a/cmds/fleet/src/nix.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-pub const HOSTS_ATTRIBUTE: &str = ".#fleetConfigurations.default.configuredHosts";
-pub const SECRETS_ATTRIBUTE: &str = ".#fleetConfigurations.default.configuredSecrets";
-pub const SYSTEMS_ATTRIBUTE: &str = ".#fleetConfigurations.default.configuredSystems";
modifiedlib/default.nixdiffbeforeafterboth
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -2,15 +2,16 @@
   fleetConfiguration = { data, nixpkgs, hosts, ... }@allConfig:
     let
       config = builtins.removeAttrs allConfig [ "nixpkgs" "data" ];
+      fleetLib = import ./fleetLib.nix {
+        inherit nixpkgs hosts;
+      };
     in
-    flake-utils.lib.eachDefaultSystem (system: rec {
+    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: rec {
       root = nixpkgs.lib.evalModules {
         modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];
         specialArgs = {
           inherit nixpkgs;
-          fleet = import ./fleetLib.nix {
-            inherit nixpkgs hosts;
-          };
+          fleet = fleetLib;
         };
       };
       configuredHosts = root.config.hosts;
@@ -27,12 +28,16 @@
                   else [ ]
                 ) ++ [
                   ({ ... }: {
+                    nixpkgs.system = system;
                     nixpkgs.localSystem.system = system;
                     nixpkgs.crossSystem = if system == configuredHosts.${name}.system then null else {
                       system = configuredHosts.${name}.system;
                     };
                   })
                 ];
+                specialArgs = {
+                  fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
+                };
               };
             }
           )