git.delta.rocks / jrsonnet / refs/commits / 81e3c77f96ff

difftreelog

feat parallel host builds

Yaroslav Bolyukin2021-12-26parent: #6a5196a.patch.diff
in: trunk

6 files changed

modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
before · cmds/fleet/src/cmds/build_systems.rs
1use std::{env::current_dir, time::Duration};23use crate::{command::CommandExt, host::Config, nix::SYSTEMS_ATTRIBUTE};4use anyhow::Result;5use structopt::StructOpt;6use tokio::{process::Command, task::LocalSet, time::sleep};7use tracing::{error, field, info, info_span, warn, Instrument};89#[derive(StructOpt, Clone)]10pub struct BuildSystems {11	/// --builders arg for nix12	#[structopt(long)]13	builders: Option<String>,14	/// Jobs to run locally15	#[structopt(long)]16	jobs: Option<usize>,17	/// Do not continue on error18	#[structopt(long)]19	fail_fast: bool,20	#[structopt(long)]21	privileged_build: bool,22	#[structopt(subcommand)]23	subcommand: Subcommand,24}2526enum UploadAction {27	Test,28	Boot,29	Switch,30}31impl UploadAction {32	fn name(&self) -> &'static str {33		match self {34			UploadAction::Test => "test",35			UploadAction::Boot => "boot",36			UploadAction::Switch => "switch",37		}38	}3940	pub(crate) fn should_switch_profile(&self) -> bool {41		matches!(self, Self::Switch | Self::Test)42	}43}4445enum PackageAction {46	SdImage,47}4849enum Action {50	Upload(Option<UploadAction>),51	Package(PackageAction),52}5354impl From<Subcommand> for Action {55	fn from(s: Subcommand) -> Self {56		match s {57			Subcommand::Upload => Self::Upload(None),58			Subcommand::Test => Self::Upload(Some(UploadAction::Test)),59			Subcommand::Boot => Self::Upload(Some(UploadAction::Boot)),60			Subcommand::Switch => Self::Upload(Some(UploadAction::Switch)),61			Subcommand::SdImage => Self::Package(PackageAction::SdImage),62		}63	}64}6566#[derive(StructOpt, Clone)]67enum Subcommand {68	/// Upload, but do not switch69	Upload,70	/// Upload + switch to built system until reboot71	Test,72	/// Upload + switch to built system after reboot73	Boot,74	/// Upload + test + boot75	Switch,7677	/// Build sd image78	SdImage,79}8081impl BuildSystems {82	pub async fn run(self, config: &Config) -> Result<()> {83		let hosts = config.list_hosts().await?;84		let set = LocalSet::new();85		let this = &self;86		for host in hosts.iter() {87			if config.should_skip(host) {88				continue;89			}90			let config = config.clone();91			let host = host.clone();92			let this = this.clone();93			let span = info_span!("deployment", host = field::display(&host));94			set.spawn_local(95				(async move {96					let res: Result<()> = try {97						info!("building");98						let built = {99							let dir = tempfile::tempdir()?;100							dir.path().to_owned()101						};102103						let mut nix_build = if this.privileged_build {104							let mut out = Command::new("sudo");105							out.arg("nix");106							out107						} else {108							Command::new("nix")109						};110						nix_build111							.args(&[112								"build",113								"--impure",114								"--json",115								// "--show-trace",116								"--no-link",117								"--out-link",118							])119							.arg(&built)120							.arg(format!(121								"{}.{}.config.system.build.toplevel",122								SYSTEMS_ATTRIBUTE, host,123							));124125						if let Some(builders) = &this.builders {126							nix_build.arg("--builders").arg(builders);127						}128						if let Some(jobs) = &this.jobs {129							nix_build.arg("--max-jobs");130							nix_build.arg(format!("{}", jobs));131						}132						if !this.fail_fast {133							nix_build.arg("--keep-going");134						}135136						nix_build.run_nix().await?;137						let built = std::fs::canonicalize(built)?;138139						let action = Action::from(this.subcommand.clone());140141						match action {142							Action::Upload(action) => {143								if !config.is_local(&host) {144									info!("uploading system closure");145									let mut tries = 0;146									loop {147										match Command::new("nix")148											.args(&["copy", "--to"])149											.arg(format!("ssh://root@{}", host))150											.arg(&built)151											.inherit_stdio()152											.run_nix()153											.await154										{155											Ok(()) => break,156											Err(e) if tries < 3 => {157												tries += 1;158												warn!("Copy failure ({}/3): {}", tries, e);159												sleep(Duration::from_millis(5000)).await;160											}161											Err(e) => return Err(e),162										}163									}164								}165								if let Some(action) = action {166									if action.should_switch_profile() {167										info!("switching generation");168										config169											.command_on(&host, "nix-env", true)170											.args(&["-p", "/nix/var/nix/profiles/system", "--set"])171											.arg(&built)172											.inherit_stdio()173											.run()174											.await?;175									}176									info!("executing activation script");177									let mut switch_script = built.clone();178									switch_script.push("bin");179									switch_script.push("switch-to-configuration");180									config181										.command_on(&host, switch_script, true)182										.arg(action.name())183										.inherit_stdio()184										.run()185										.await?;186								}187							}188							Action::Package(PackageAction::SdImage) => {189								let mut out = current_dir()?;190								out.push(format!("sd-image-{}", host));191192								info!("building sd image to {:?}", out);193								let mut nix_build = if this.privileged_build {194									let mut out = Command::new("sudo");195									out.arg("nix");196									out197								} else {198									Command::new("nix")199								};200								nix_build201									.args(&["build", "--impure", "--no-link", "--out-link"])202									.arg(&out)203									.arg(format!(204										"{}.{}.config.system.build.sdImage",205										SYSTEMS_ATTRIBUTE, host,206									));207								if let Some(builders) = &this.builders {208									nix_build.arg("--builders").arg(builders);209								}210								if let Some(jobs) = &this.jobs {211									nix_build.arg("--max-jobs");212									nix_build.arg(format!("{}", jobs));213								}214								if !this.fail_fast {215									nix_build.arg("--keep-going");216								}217218								nix_build.inherit_stdio().run_nix().await?;219							}220						};221					};222					match res {223						Ok(_) => {}224						Err(e) => {225							error!("failed to deploy host: {}", e)226						}227					}228					Ok(())229				})230				.instrument(span),231			);232		}233		set.await;234		Ok(())235	}236}
modifiedcmds/fleet/src/command.rsdiffbeforeafterboth
--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -150,6 +150,12 @@
 										info!(target: "nix", "{}", text)
 									}
 								},
+								NixLog::Start { text, level: 0, typ: 108, .. } if text == "" => {
+									// Cache lookup? Coupled with copy log
+								},
+								NixLog::Start { text, level: 4, typ: 101, .. } if text.starts_with("downloading ") => {
+									// NAR downloading, coupled with copy log
+								}
 								NixLog::Stop { .. } => {},
 								NixLog::Result { .. } => {},
 								_ => warn!("unknown log: {:?}", log)
modifiedcmds/fleet/src/host.rsdiffbeforeafterboth
--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -16,6 +16,7 @@
 use crate::{command::CommandExt, fleetdata::FleetData};
 
 pub struct FleetConfigInternals {
+	pub local_system: String,
 	pub directory: PathBuf,
 	pub opts: FleetOpts,
 	pub data: RefCell<FleetData>,
@@ -66,17 +67,21 @@
 		}
 	}
 
-	pub fn full_attr_name(&self, attr_name: &str) -> OsString {
+	pub fn configuration_attr_name(&self, name: &str) -> OsString {
 		let mut str = self.directory.as_os_str().to_owned();
 		str.push("#");
-		str.push(attr_name);
+		str.push(&format!(
+			"fleetConfigurations.default.{}.{}",
+			self.local_system,
+			name
+		));
 		str
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<String>> {
 		Command::new("nix")
 			.arg("eval")
-			.arg(self.full_attr_name("fleetConfigurations.default.configuredHosts"))
+			.arg(self.configuration_attr_name("configuredHosts"))
 			.args(&["--apply", "builtins.attrNames", "--json", "--show-trace"])
 			.run_nix_json()
 			.await
@@ -84,10 +89,7 @@
 	pub async fn config_attr<T: DeserializeOwned>(&self, host: &str, attr: &str) -> Result<T> {
 		Command::new("nix")
 			.arg("eval")
-			.arg(self.full_attr_name(&format!(
-				"fleetConfigurations.default.configuredSystems.{}.config.{}",
-				host, attr
-			)))
+			.arg(self.configuration_attr_name(&format!("configuredSystems.{}.config.{}", host, attr)))
 			.args(&["--json", "--show-trace"])
 			.run_nix_json()
 			.await
@@ -129,10 +131,14 @@
 	/// Host, which should be threaten as current machine
 	#[structopt(long)]
 	pub localhost: Option<String>,
+
+	#[structopt(long, default_value = "x86_64-linux")]
+	pub local_system: String,
 }
 
 impl FleetOpts {
 	pub fn build(mut self) -> Result<Config> {
+		let local_system = self.local_system.clone();
 		if self.localhost.is_none() {
 			self.localhost
 				.replace(hostname::get().unwrap().to_str().unwrap().to_owned());
@@ -148,6 +154,7 @@
 			opts: self,
 			directory,
 			data,
+			local_system,
 		})))
 	}
 }
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -1,11 +1,7 @@
-#![feature(try_blocks)]
-
 pub mod command;
 pub mod host;
 pub mod keys;
-
 pub mod cmds;
-pub mod nix;
 
 mod fleetdata;
 
deletedcmds/fleet/src/nix.rsdiffbeforeafterboth
--- a/cmds/fleet/src/nix.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-pub const HOSTS_ATTRIBUTE: &str = ".#fleetConfigurations.default.configuredHosts";
-pub const SECRETS_ATTRIBUTE: &str = ".#fleetConfigurations.default.configuredSecrets";
-pub const SYSTEMS_ATTRIBUTE: &str = ".#fleetConfigurations.default.configuredSystems";
modifiedlib/default.nixdiffbeforeafterboth
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -2,15 +2,16 @@
   fleetConfiguration = { data, nixpkgs, hosts, ... }@allConfig:
     let
       config = builtins.removeAttrs allConfig [ "nixpkgs" "data" ];
+      fleetLib = import ./fleetLib.nix {
+        inherit nixpkgs hosts;
+      };
     in
-    flake-utils.lib.eachDefaultSystem (system: rec {
+    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: rec {
       root = nixpkgs.lib.evalModules {
         modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];
         specialArgs = {
           inherit nixpkgs;
-          fleet = import ./fleetLib.nix {
-            inherit nixpkgs hosts;
-          };
+          fleet = fleetLib;
         };
       };
       configuredHosts = root.config.hosts;
@@ -27,12 +28,16 @@
                   else [ ]
                 ) ++ [
                   ({ ... }: {
+                    nixpkgs.system = system;
                     nixpkgs.localSystem.system = system;
                     nixpkgs.crossSystem = if system == configuredHosts.${name}.system then null else {
                       system = configuredHosts.${name}.system;
                     };
                   })
                 ];
+                specialArgs = {
+                  fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
+                };
               };
             }
           )