git.delta.rocks / jrsonnet / refs/commits / 89d35672dcfd

1use std::collections::HashMap;
1use std::ffi::{OsStr, OsString};2use std::ffi::{OsStr, OsString};
2use std::fmt::Display;3use std::fmt::{self, Display};
4use std::path::PathBuf;
3use std::process::Stdio;5use std::process::Stdio;
4use std::sync::{Arc, OnceLock};6use std::sync::{Arc, OnceLock};
57
8use itertools::Itertools;10use itertools::Itertools;
9use r2d2::{Pool, PooledConnection};11use r2d2::{Pool, PooledConnection};
10use serde::de::DeserializeOwned;12use serde::de::DeserializeOwned;
11use serde::Deserialize;13use serde::{Deserialize, Serialize};
12use tokio::io::AsyncWriteExt;14use tokio::io::AsyncWriteExt;
13use tokio::process::{ChildStderr, ChildStdin, ChildStdout, Command};15use tokio::process::{ChildStderr, ChildStdin, ChildStdout, Command};
14use tokio::select;16use tokio::select;
150	}158	}
151}159}
160
161struct WarnHandler;
162impl Handler for WarnHandler {
163	fn handle_line(&mut self, e: &str) {
164		warn!(target: "nix", "{e}")
165	}
166}
152167
153impl NixSessionInner {168impl NixSessionInner {
154	async fn new(flake: &OsStr, extra_args: impl IntoIterator<Item = &OsStr>) -> Result<Self> {169	async fn new(flake: &OsStr, extra_args: impl IntoIterator<Item = &OsStr>) -> Result<Self> {
174		stdin.flush().await?;189		stdin.flush().await?;
175		let nix_handler = NixHandler::default();190		let nix_handler = NixHandler::default();
176		let mut full_delimiter = None;191		let mut full_delimiter = None;
192		let mut errors = vec![];
177		while let Some(line) = out.next().await {193		while let Some(line) = out.next().await {
178			let line = match line {194			let line = match line {
179				OutputLine::Out(o) => o,195				OutputLine::Out(o) => o,
180				OutputLine::Err(_e) => {196				OutputLine::Err(_e) => {
181					// Handle startup errors, but skip repl hello?197					// Handle startup errors, but skip repl hello?
182					//nix_handler.handle_line(&e);198					errors.push(_e);
183					continue;199					continue;
184				}200				}
185			};201			};
190			}206			}
191		}207		}
192		let Some(full_delimiter) = full_delimiter else {208		let Some(full_delimiter) = full_delimiter else {
209			for e in errors {
210				error!("{e}");
211			}
193			bail!("failed to discover delimiter");212			bail!("failed to discover delimiter");
194		};213		};
195		let mut res = Self {214		let mut res = Self {
342#[derive(Clone)]361#[derive(Clone)]
343pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);362pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);
363
364#[macro_export]
365macro_rules! nix_path {
366	(@o($o:ident) $var:ident $($tt:tt)*) => {{
367		$o.push(Index::var(stringify!($var)));
368		nix_path!(@o($o) $($tt)*);
369	}};
370	(@o($o:ident) . $var:ident $($tt:tt)*) => {{
371		$o.push(Index::attr(stringify!($var)));
372		nix_path!(@o($o) $($tt)*);
373	}};
374	(@o($o:ident) . $var:literal $($tt:tt)*) => {{
375		$o.push(Index::attr($var));
376		nix_path!(@o($o) $($tt)*);
377	}};
378	(@o($o:ident) . { $var:expr } $($tt:tt)*) => {{
379		$o.push(Index::attr($var));
380		nix_path!(@o($o) $($tt)*);
381	}};
382	(@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{
383		$o.push(Index::idx($var));
384		nix_path!(@o($o) $($tt)*);
385	}};
386	(@o($o:ident) ($e:expr) $($tt:tt)*) => {
387		$o.push(Index::apply($e));
388		nix_path!(@o($o) $($tt)*);
389	};
390	(@o($o:ident)) => {};
391	($($tt:tt)+) => {{
392		use $crate::{nix_path, better_nix_eval::Index};
393		let mut out = vec![];
394		nix_path!(@o(out) $($tt)*);
395		out
396	}}
397}
344398
345#[derive(Clone)]399#[derive(Clone)]
346enum Index {400pub enum Index {
401	Var(String),
347	String(String),402	String(String),
348	// Idx(u32),403	Apply(String),
404	Idx(u32),
349}405}
406impl Index {
407	pub fn var(v: impl AsRef<str>) -> Self {
408		let v = v.as_ref();
409		assert!(
410			!(v.contains('.') | v.contains(' ')),
411			"bad variable name: {v}"
412		);
413		Self::Var(v.to_owned())
414	}
415	pub fn attr(v: impl AsRef<str>) -> Self {
416		Self::String(v.as_ref().to_owned())
417	}
418	pub fn idx(v: u32) -> Self {
419		Self::Idx(v)
420	}
421	pub fn apply(v: impl Serialize) -> Self {
422		let serialized = nixlike::serialize(v).expect("invalid value for apply");
423		Self::Apply(serialized)
424	}
425}
350impl Display for Index {426impl Display for Index {
351	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {427	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
352		match self {428		match self {
429			Index::Var(v) => {
430				write!(f, "{v}")
431			}
353			Index::String(k) => {432			Index::String(k) => {
354				let v = nixlike::format_identifier(k.as_str());433				let v = nixlike::format_identifier(k.as_str());
355				write!(f, ".{v}")434				write!(f, ".{v}")
356			}435			}
436			Index::Apply(o) => {
437				let v = nixlike::serialize(o).map_err(|_| fmt::Error)?;
438				write!(f, "<apply>({v})")
439			}
440			Index::Idx(i) => {
441				write!(f, "[{i}]")
442			}
357		}443		}
358	}444	}
359}445}
446impl fmt::Debug for Index {
447	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
448		write!(f, "{self}")
449	}
450}
360struct PathDisplay<'i>(&'i [Index]);451struct PathDisplay<'i>(&'i [Index]);
361impl Display for PathDisplay<'_> {452impl Display for PathDisplay<'_> {
362	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {453	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
381		}472		}
382	}473	}
383	pub async fn field(session: NixSession, field: &str) -> Result<Self> {474	pub async fn field(session: NixSession, field: &str) -> Result<Self> {
384		Self::root(session).get_field_deep([field]).await475		Self::root(session)
476			.select([Index::var(field)])
477			.await
385	}478	}
386	pub async fn get_json_deep<'a, V: DeserializeOwned>(479	pub async fn get_json_deep<'a, V: DeserializeOwned>(
387		&self,480		&self,
388		name: impl IntoIterator<Item = &'a str>,481		name: impl IntoIterator<Item = Index>,
389	) -> Result<V> {482	) -> Result<V> {
390		let field = self.get_field_deep(name).await?;483		let field = self.select(name).await?;
391		field.as_json().await484		field.as_json().await
392	}485	}
393	pub async fn get_field(&self, name: &str) -> Result<Self> {
394		self.get_field_deep([name]).await
395	}
396	pub async fn get_field_deep<'a>(486	pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {
397		&self,
398		name: impl IntoIterator<Item = &'a str>,
399	) -> Result<Self> {
400		let mut iter = name.into_iter();487		let mut name = name.into_iter();
401488
402		let mut full_path = self.full_path.clone();489		let mut full_path = self.full_path.clone();
403		let mut query = if let Some(id) = self.value {490		let mut query = if let Some(id) = self.value {
404			format!("sess_field_{id}")491			format!("sess_field_{id}")
405		} else {492		} else {
406			let first = iter.next().expect("name not empty");493			let first = name.next();
407			ensure!(494			if let Some(Index::Var(i)) = first {
408				!(first.contains('.') | first.contains(' ')),
409				"bad name for root query: {first}"
410			);
411			full_path.push(Index::String(first.to_string()));495				full_path.push(Index::Var(i.clone()));
412			first.to_string()496				i.clone()
497			} else {
498				panic!("first path item should be variable, got {first:?}")
499			}
413		};500		};
414		for v in iter {501		for v in name {
415			full_path.push(Index::String(v.to_string()));502			full_path.push(v.clone());
416			// Escape503			match v {
504				Index::Var(_) => panic!("var item may only be first"),
505				Index::String(s) => {
417			let escaped = nixlike::serialize(v)?;506					let escaped = nixlike::serialize(s)?;
418			let escaped = escaped.trim();507					query.push('.');
508					query.push_str(escaped.trim());
509				}
510				Index::Apply(a) => {
419			query.push('.');511					query.push(' ');
420			query.push_str(escaped);512					query.push_str(&a);
513				}
514				Index::Idx(idx) => {
515					query = format!("builtins.elemAt ({query}) {idx}");
516				}
517			}
421		}518		}
422519
423		let vid = self520		let vid = self
454			.await551			.await
455			.with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))552			.with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))
456	}553	}
554	pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {
555		let id = self.value.expect("can't use build on not-value");
556		let vid = self
557			.session
558			.0
559			.lock()
560			.await
561			.execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default())
562			.await?;
563		ensure!(!vid.is_empty(), "build failed");
564		let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")
565		else {
566			panic!("unexpected build output: {vid:?}");
567		};
568		let outputs = vid
569			.split('\n')
570			.filter(|v| !v.is_empty())
571			.map(|v| v.split_once(" -> ").expect("unexpected build output"))
572			.map(|(a, b)| (a.trim_start().to_owned(), PathBuf::from(b)))
573			.collect();
574		Ok(outputs)
575	}
457}576}
458impl Drop for Field {577impl Drop for Field {
459	fn drop(&mut self) {578	fn drop(&mut self) {

--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -1,8 +1,10 @@
+use std::os::unix::fs::symlink;
 use std::path::PathBuf;
 use std::{env::current_dir, time::Duration};
 
 use crate::command::MyCommand;
 use crate::host::Config;
+use crate::nix_path;
 use anyhow::{anyhow, Result};
 use clap::Parser;
 use itertools::Itertools;
@@ -11,15 +13,9 @@
 
 #[derive(Parser, Clone)]
 pub struct BuildSystems {
-	/// Do not continue on error
-	#[clap(long)]
-	fail_fast: bool,
 	/// Disable automatic rollback
 	#[clap(long)]
 	disable_rollback: bool,
-	/// Run builds as sudo
-	#[clap(long)]
-	privileged_build: bool,
 	#[clap(subcommand)]
 	subcommand: Subcommand,
 }
@@ -294,34 +290,11 @@
 	async fn build_task(self, config: Config, host: String) -> Result<()> {
 		info!("building");
 		let action = Action::from(self.subcommand.clone());
-		let built = {
-			let dir = tempfile::tempdir()?;
-			dir.path().to_owned()
-		};
-
-		let mut nix_build = MyCommand::new("nix");
-		nix_build
-			.args([
-				"build",
-				"--impure",
-				"--json",
-				// "--show-trace",
-				"--no-link",
-			])
-			.comparg("--out-link", &built)
-			.arg(
-				config.configuration_attr_name(&format!(
-					"buildSystems.{}.{host}",
-					action.build_attr()
-				)),
-			)
-			.args(&config.nix_args);
-
-		if self.privileged_build {
-			nix_build = nix_build.sudo();
-		}
-
-		nix_build.run_nix().await.map_err(|e| {
+		let drv = config
+			.fleet_field
+			.select(nix_path!(.buildSystems.{action.build_attr()}.{&host}))
+			.await?;
+		let outputs = drv.build().await.map_err(|e| {
 			if action.build_attr() == "sdImage" {
 				info!("sd-image build failed");
 				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");
@@ -329,7 +302,9 @@
 			}
 			e
 		})?;
-		let built = std::fs::canonicalize(built)?;
+		let out_output = outputs
+			.get("out")
+			.ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;
 
 		match action {
 			Action::Upload { action } => {
@@ -342,7 +317,7 @@
 							.arg("sign")
 							.comparg("--key-file", "/etc/nix/private-key")
 							.arg("-r")
-							.arg(&built);
+							.arg(out_output);
 						if let Err(e) = sign.sudo().run_nix().await {
 							warn!("Failed to sign store paths: {e}");
 						};
@@ -353,7 +328,7 @@
 						nix.arg("copy")
 							.arg("--substitute-on-destination")
 							.comparg("--to", format!("ssh-ng://{host}"))
-							.arg(&built);
+							.arg(out_output);
 						match nix.run_nix().await {
 							Ok(()) => break,
 							Err(e) if tries < 3 => {
@@ -366,53 +341,22 @@
 					}
 				}
 				if let Some(action) = action {
-					execute_upload(&self, &config, action, &host, built).await?
+					execute_upload(&self, &config, action, &host, out_output.clone()).await?
 				}
 			}
 			Action::Package(PackageAction::SdImage) => {
 				let mut out = current_dir()?;
 				out.push(format!("sd-image-{}", host));
 
-				info!("building sd image to {:?}", out);
-				let mut nix_build = MyCommand::new("nix");
-				nix_build
-					.args(["build", "--impure", "--no-link"])
-					.comparg("--out-link", &out)
-					.arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))
-					.args(&config.nix_args);
-				if !self.fail_fast {
-					nix_build.arg("--keep-going");
-				}
-				if self.privileged_build {
-					nix_build = nix_build.sudo();
-				}
-
-				nix_build.run_nix().await?;
+				info!("linking sd image to {:?}", out);
+				symlink(out_output, out)?;
 			}
 			Action::Package(PackageAction::InstallationCd) => {
 				let mut out = current_dir()?;
 				out.push(format!("installation-cd-{}", host));
 
-				info!("building sd image to {:?}", out);
-				let mut nix_build = MyCommand::new("nix");
-				nix_build
-					.args(["build", "--impure", "--no-link"])
-					.comparg("--out-link", &out)
-					.arg(
-						config.configuration_attr_name(&format!(
-							"buildSystems.installationCd.{}",
-							host,
-						)),
-					)
-					.args(&config.nix_args);
-				if !self.fail_fast {
-					nix_build.arg("--keep-going");
-				}
-				if self.privileged_build {
-					nix_build = nix_build.sudo();
-				}
-
-				nix_build.run_nix().await?;
+				info!("linking iso image to {:?}", out);
+				symlink(out_output, out)?;
 			}
 		};
 		Ok(())

--- a/cmds/fleet/src/cmds/info.rs
+++ b/cmds/fleet/src/cmds/info.rs
@@ -1,6 +1,7 @@
 use std::collections::BTreeSet;
 
 use crate::host::Config;
+use crate::nix_path;
 use anyhow::{ensure, Result};
 use clap::Parser;
 
@@ -38,7 +39,7 @@
 					if !tagged.is_empty() {
 						let tags: Vec<String> = config
 							.fleet_field
-							.get_field_deep(["configuredSystems", &host.name, "config", "tags"])
+							.select(nix_path!(.configuredSystems.{&host.name}.config.tags))
 							.await?
 							.as_json()
 							.await?;
@@ -64,7 +65,7 @@
 				let host = config.system_config(&host).await?;
 				if external {
 					out.extend(
-						host.get_field_deep(["network", "externalIps"])
+						host.select(nix_path!(.network.externalIps))
 							.await?
 							.as_json::<Vec<String>>()
 							.await?,
@@ -72,7 +73,7 @@
 				}
 				if internal {
 					out.extend(
-						host.get_field_deep(["network", "internalIps"])
+						host.select(nix_path!(.network.internalIps))
 							.await?
 							.as_json::<Vec<String>>()
 							.await?,

--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -1,6 +1,6 @@
 use crate::{
 	fleetdata::{FleetSecret, FleetSharedSecret},
-	host::Config,
+	host::Config, nix_path,
 };
 use anyhow::{bail, ensure, Context, Result};
 use chrono::Utc;
@@ -339,7 +339,7 @@
 					let mut data = config.shared_secret(name)?;
 					let expected_owners: Vec<String> = config
 						.config_field
-						.get_json_deep(["sharedSecrets", name, "expectedOwners"])
+						.get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners))
 						.await?;
 					if expected_owners.is_empty() {
 						warn!("secret was removed from fleet config: {name}, removing from data");
@@ -352,7 +352,7 @@
 					if set != expected_set {
 						let owner_dependent: bool = config
 							.config_field
-							.get_json_deep(["sharedSecrets", name, "ownerDependent"])
+							.get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent))
 							.await?;
 						if !owner_dependent {
 							warn!("reencrypting secret '{name}' for new owner set");

--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -1,5 +1,4 @@
 use std::{
-	borrow::Cow,
 	collections::HashMap,
 	ffi::OsStr,
 	process::Stdio,
@@ -247,10 +246,14 @@
 pub struct NixHandler {
 	spans: HashMap<u64, Span>,
 }
-fn process_message(m: &str) -> Cow<'_, str> {
+fn process_message(m: &str) -> String {
 	static OSC_CLEANER: Lazy<Regex> =
 		Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());
-	OSC_CLEANER.replace_all(m, "")
+	static DETABBER: Lazy<Regex> = Lazy::new(|| Regex::new(r"\t").unwrap());
+	let m = OSC_CLEANER.replace_all(m, "");
+	// Indicatif can't format tabs. This is not the correct tab formatting, as correct one should be aligned,
+	// and not just be replaced with the constant number of spaces, but it's ok for now, as statuses are single-line.
+	DETABBER.replace_all(m.as_ref(), "  ").to_string()
 }
 impl Handler for NixHandler {
 	fn handle_line(&mut self, e: &str) {

--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -13,9 +13,10 @@
 use tempfile::NamedTempFile;
 
 use crate::{
-	better_nix_eval::{Field, NixSessionPool},
+	better_nix_eval::{Field, Index, NixSessionPool},
 	command::MyCommand,
 	fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
+	nix_path,
 };
 
 pub struct FleetConfigInternals {
@@ -24,9 +25,9 @@
 	pub opts: FleetOpts,
 	pub data: Mutex<FleetData>,
 	pub nix_args: Vec<OsString>,
-	// fleetConfigurations.<name>
+	/// fleetConfigurations.<name>.<localSystem>
 	pub fleet_field: Field,
-	// fleet_config.configUnchecked
+	/// fleet_config.configUnchecked
 	pub config_field: Field,
 }
 
@@ -91,22 +92,12 @@
 			command = command.ssh(host);
 		}
 		command.run_string().await
-	}
-
-	pub fn configuration_attr_name(&self, name: &str) -> OsString {
-		let mut str = self.directory.as_os_str().to_owned();
-		str.push("#");
-		str.push(&format!(
-			"fleetConfigurations.default.{}.{}",
-			self.local_system, name
-		));
-		str
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
 		let names = self
 			.fleet_field
-			.get_field_deep(["configuredHosts"])
+			.select(nix_path!(.configuredHosts))
 			.await?
 			.list_fields()
 			.await?;
@@ -118,7 +109,7 @@
 	}
 	pub async fn system_config(&self, host: &str) -> Result<Field> {
 		self.fleet_field
-			.get_field_deep(["configuredSystems", host, "config"])
+			.select(nix_path!(.configuredSystems.{host}.config))
 			.await
 	}
 
@@ -131,7 +122,7 @@
 	/// Shared secrets configured in fleet.nix or in flake
 	pub async fn list_configured_shared(&self) -> Result<Vec<String>> {
 		self.config_field
-			.get_field("sharedSecrets")
+			.select(nix_path!(.sharedSecrets))
 			.await?
 			.list_fields()
 			.await
@@ -221,7 +212,7 @@
 	}
 	pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
 		self.config_field
-			.get_field_deep(["sharedSecrets", secret, "expectedOwners"])
+			.select(nix_path!(.sharedSecrets.{secret}.expectedOwners))
 			.await?
 			.as_json()
 			.await
@@ -279,7 +270,9 @@
 
 		if self.local_system == "detect" {
 			let builtins_field = Field::field(root_field.clone(), "builtins").await?;
-			let system = builtins_field.get_field("currentSystem").await?;
+			let system = builtins_field
+				.select(nix_path!(.currentSystem))
+				.await?;
 			self.local_system = system.as_json().await?;
 		}
 		let local_system = self.local_system.clone();
@@ -287,9 +280,11 @@
 		let fleet_root = Field::field(root_field, "fleetConfigurations").await?;
 
 		let fleet_field = fleet_root
-			.get_field_deep(["default", &local_system])
+			.select(nix_path!(.default.{&local_system}))
+			.await?;
+		let config_field = fleet_field
+			.select(nix_path!(.configUnchecked))
 			.await?;
-		let config_field = fleet_field.get_field("configUnchecked").await?;
 
 		let mut fleet_data_path = directory.clone();
 		fleet_data_path.push("fleet.nix");

--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -1,3 +1,4 @@
+#![recursion_limit = "512"]
 #![feature(try_blocks)]
 
 pub(crate) mod cmds;

--- a/flake.nix
+++ b/flake.nix
@@ -19,6 +19,7 @@
       rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; };
     in
     {
+		packages = (import ./pkgs) pkgs pkgs;
       devShell = (pkgs.mkShell.override { stdenv = llvmPkgs.stdenv; }) {
         nativeBuildInputs = with pkgs; [
           rust