git.delta.rocks / jrsonnet / refs/commits / 89d35672dcfd

--- a/cmds/fleet/src/better_nix_eval.rs
+++ b/cmds/fleet/src/better_nix_eval.rs
@@ -1,5 +1,7 @@
+use std::collections::HashMap;
 use std::ffi::{OsStr, OsString};
-use std::fmt::Display;
+use std::fmt::{self, Display};
+use std::path::PathBuf;
 use std::process::Stdio;
 use std::sync::{Arc, OnceLock};
 
@@ -8,7 +10,7 @@
 use itertools::Itertools;
 use r2d2::{Pool, PooledConnection};
 use serde::de::DeserializeOwned;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
 use tokio::process::{ChildStderr, ChildStdin, ChildStdout, Command};
 use tokio::select;
@@ -72,14 +74,20 @@
 		// 	s.split('\n').filter(|s| !s.trim().is_empty()).map(|v| v.)
 		// }
 		if !self.collected.is_empty() {
-			bail!("{}", self.collected.iter().map(|v| {
-				if let Some(f) = v.strip_prefix("\u{1b}[31;1merror:\u{1b}[0m ") {
-					let v = unindent::unindent(f.trim_start());
-					v.trim().to_owned()
-				} else {
-					v.to_owned()
-				}
-			}).join("\n"));
+			bail!(
+				"{}",
+				self.collected
+					.iter()
+					.map(|v| {
+						if let Some(f) = v.strip_prefix("\u{1b}[31;1merror:\u{1b}[0m ") {
+							let v = unindent::unindent(f.trim_start());
+							v.trim().to_owned()
+						} else {
+							v.to_owned()
+						}
+					})
+					.join("\n")
+			);
 		}
 		Ok(())
 	}
@@ -150,6 +158,13 @@
 	}
 }
 
+struct WarnHandler;
+impl Handler for WarnHandler {
+	fn handle_line(&mut self, e: &str) {
+		warn!(target: "nix", "{e}")
+	}
+}
+
 impl NixSessionInner {
 	async fn new(flake: &OsStr, extra_args: impl IntoIterator<Item = &OsStr>) -> Result<Self> {
 		let mut cmd = Command::new("nix");
@@ -174,12 +189,13 @@
 		stdin.flush().await?;
 		let nix_handler = NixHandler::default();
 		let mut full_delimiter = None;
+		let mut errors = vec![];
 		while let Some(line) = out.next().await {
 			let line = match line {
 				OutputLine::Out(o) => o,
 				OutputLine::Err(_e) => {
 					// Handle startup errors, but skip repl hello?
-					//nix_handler.handle_line(&e);
+					errors.push(_e);
 					continue;
 				}
 			};
@@ -190,6 +206,9 @@
 			}
 		}
 		let Some(full_delimiter) = full_delimiter else {
+			for e in errors {
+				error!("{e}");
+			}
 			bail!("failed to discover delimiter");
 		};
 		let mut res = Self {
@@ -342,21 +361,93 @@
 #[derive(Clone)]
 pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);
 
+#[macro_export]
+macro_rules! nix_path {
+	(@o($o:ident) $var:ident $($tt:tt)*) => {{
+		$o.push(Index::var(stringify!($var)));
+		nix_path!(@o($o) $($tt)*);
+	}};
+	(@o($o:ident) . $var:ident $($tt:tt)*) => {{
+		$o.push(Index::attr(stringify!($var)));
+		nix_path!(@o($o) $($tt)*);
+	}};
+	(@o($o:ident) . $var:literal $($tt:tt)*) => {{
+		$o.push(Index::attr($var));
+		nix_path!(@o($o) $($tt)*);
+	}};
+	(@o($o:ident) . { $var:expr } $($tt:tt)*) => {{
+		$o.push(Index::attr($var));
+		nix_path!(@o($o) $($tt)*);
+	}};
+	(@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{
+		$o.push(Index::idx($var));
+		nix_path!(@o($o) $($tt)*);
+	}};
+	(@o($o:ident) ($e:expr) $($tt:tt)*) => {
+		$o.push(Index::apply($e));
+		nix_path!(@o($o) $($tt)*);
+	};
+	(@o($o:ident)) => {};
+	($($tt:tt)+) => {{
+		use $crate::{nix_path, better_nix_eval::Index};
+		let mut out = vec![];
+		nix_path!(@o(out) $($tt)*);
+		out
+	}}
+}
+
 #[derive(Clone)]
-enum Index {
+pub enum Index {
+	Var(String),
 	String(String),
-	// Idx(u32),
+	Apply(String),
+	Idx(u32),
 }
+impl Index {
+	pub fn var(v: impl AsRef<str>) -> Self {
+		let v = v.as_ref();
+		assert!(
+			!(v.contains('.') | v.contains(' ')),
+			"bad variable name: {v}"
+		);
+		Self::Var(v.to_owned())
+	}
+	pub fn attr(v: impl AsRef<str>) -> Self {
+		Self::String(v.as_ref().to_owned())
+	}
+	pub fn idx(v: u32) -> Self {
+		Self::Idx(v)
+	}
+	pub fn apply(v: impl Serialize) -> Self {
+		let serialized = nixlike::serialize(v).expect("invalid value for apply");
+		Self::Apply(serialized)
+	}
+}
 impl Display for Index {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		match self {
+			Index::Var(v) => {
+				write!(f, "{v}")
+			}
 			Index::String(k) => {
 				let v = nixlike::format_identifier(k.as_str());
 				write!(f, ".{v}")
 			}
+			Index::Apply(o) => {
+				let v = nixlike::serialize(o).map_err(|_| fmt::Error)?;
+				write!(f, "<apply>({v})")
+			}
+			Index::Idx(i) => {
+				write!(f, "[{i}]")
+			}
 		}
 	}
 }
+impl fmt::Debug for Index {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "{self}")
+	}
+}
 struct PathDisplay<'i>(&'i [Index]);
 impl Display for PathDisplay<'_> {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -381,43 +472,49 @@
 		}
 	}
 	pub async fn field(session: NixSession, field: &str) -> Result<Self> {
-		Self::root(session).get_field_deep([field]).await
+		Self::root(session)
+			.select([Index::var(field)])
+			.await
 	}
 	pub async fn get_json_deep<'a, V: DeserializeOwned>(
 		&self,
-		name: impl IntoIterator<Item = &'a str>,
+		name: impl IntoIterator<Item = Index>,
 	) -> Result<V> {
-		let field = self.get_field_deep(name).await?;
+		let field = self.select(name).await?;
 		field.as_json().await
 	}
-	pub async fn get_field(&self, name: &str) -> Result<Self> {
-		self.get_field_deep([name]).await
-	}
-	pub async fn get_field_deep<'a>(
-		&self,
-		name: impl IntoIterator<Item = &'a str>,
-	) -> Result<Self> {
-		let mut iter = name.into_iter();
+	pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {
+		let mut name = name.into_iter();
 
 		let mut full_path = self.full_path.clone();
 		let mut query = if let Some(id) = self.value {
 			format!("sess_field_{id}")
 		} else {
-			let first = iter.next().expect("name not empty");
-			ensure!(
-				!(first.contains('.') | first.contains(' ')),
-				"bad name for root query: {first}"
-			);
-			full_path.push(Index::String(first.to_string()));
-			first.to_string()
+			let first = name.next();
+			if let Some(Index::Var(i)) = first {
+				full_path.push(Index::Var(i.clone()));
+				i.clone()
+			} else {
+				panic!("first path item should be variable, got {first:?}")
+			}
 		};
-		for v in iter {
-			full_path.push(Index::String(v.to_string()));
-			// Escape
-			let escaped = nixlike::serialize(v)?;
-			let escaped = escaped.trim();
-			query.push('.');
-			query.push_str(escaped);
+		for v in name {
+			full_path.push(v.clone());
+			match v {
+				Index::Var(_) => panic!("var item may only be first"),
+				Index::String(s) => {
+					let escaped = nixlike::serialize(s)?;
+					query.push('.');
+					query.push_str(escaped.trim());
+				}
+				Index::Apply(a) => {
+					query.push(' ');
+					query.push_str(&a);
+				}
+				Index::Idx(idx) => {
+					query = format!("builtins.elemAt ({query}) {idx}");
+				}
+			}
 		}
 
 		let vid = self
@@ -454,6 +551,28 @@
 			.await
 			.with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))
 	}
+	pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {
+		let id = self.value.expect("can't use build on not-value");
+		let vid = self
+			.session
+			.0
+			.lock()
+			.await
+			.execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default())
+			.await?;
+		ensure!(!vid.is_empty(), "build failed");
+		let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")
+		else {
+			panic!("unexpected build output: {vid:?}");
+		};
+		let outputs = vid
+			.split('\n')
+			.filter(|v| !v.is_empty())
+			.map(|v| v.split_once(" -> ").expect("unexpected build output"))
+			.map(|(a, b)| (a.trim_start().to_owned(), PathBuf::from(b)))
+			.collect();
+		Ok(outputs)
+	}
 }
 impl Drop for Field {
 	fn drop(&mut self) {

--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -1,8 +1,10 @@
+use std::os::unix::fs::symlink;
 use std::path::PathBuf;
 use std::{env::current_dir, time::Duration};
 
 use crate::command::MyCommand;
 use crate::host::Config;
+use crate::nix_path;
 use anyhow::{anyhow, Result};
 use clap::Parser;
 use itertools::Itertools;
@@ -11,15 +13,9 @@
 
 #[derive(Parser, Clone)]
 pub struct BuildSystems {
-	/// Do not continue on error
-	#[clap(long)]
-	fail_fast: bool,
 	/// Disable automatic rollback
 	#[clap(long)]
 	disable_rollback: bool,
-	/// Run builds as sudo
-	#[clap(long)]
-	privileged_build: bool,
 	#[clap(subcommand)]
 	subcommand: Subcommand,
 }
@@ -294,34 +290,11 @@
 	async fn build_task(self, config: Config, host: String) -> Result<()> {
 		info!("building");
 		let action = Action::from(self.subcommand.clone());
-		let built = {
-			let dir = tempfile::tempdir()?;
-			dir.path().to_owned()
-		};
-
-		let mut nix_build = MyCommand::new("nix");
-		nix_build
-			.args([
-				"build",
-				"--impure",
-				"--json",
-				// "--show-trace",
-				"--no-link",
-			])
-			.comparg("--out-link", &built)
-			.arg(
-				config.configuration_attr_name(&format!(
-					"buildSystems.{}.{host}",
-					action.build_attr()
-				)),
-			)
-			.args(&config.nix_args);
-
-		if self.privileged_build {
-			nix_build = nix_build.sudo();
-		}
-
-		nix_build.run_nix().await.map_err(|e| {
+		let drv = config
+			.fleet_field
+			.select(nix_path!(.buildSystems.{action.build_attr()}.{&host}))
+			.await?;
+		let outputs = drv.build().await.map_err(|e| {
 			if action.build_attr() == "sdImage" {
 				info!("sd-image build failed");
 				info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");
@@ -329,7 +302,9 @@
 			}
 			e
 		})?;
-		let built = std::fs::canonicalize(built)?;
+		let out_output = outputs
+			.get("out")
+			.ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;
 
 		match action {
 			Action::Upload { action } => {
@@ -342,7 +317,7 @@
 							.arg("sign")
 							.comparg("--key-file", "/etc/nix/private-key")
 							.arg("-r")
-							.arg(&built);
+							.arg(out_output);
 						if let Err(e) = sign.sudo().run_nix().await {
 							warn!("Failed to sign store paths: {e}");
 						};
@@ -353,7 +328,7 @@
 						nix.arg("copy")
 							.arg("--substitute-on-destination")
 							.comparg("--to", format!("ssh-ng://{host}"))
-							.arg(&built);
+							.arg(out_output);
 						match nix.run_nix().await {
 							Ok(()) => break,
 							Err(e) if tries < 3 => {
@@ -366,53 +341,22 @@
 					}
 				}
 				if let Some(action) = action {
-					execute_upload(&self, &config, action, &host, built).await?
+					execute_upload(&self, &config, action, &host, out_output.clone()).await?
 				}
 			}
 			Action::Package(PackageAction::SdImage) => {
 				let mut out = current_dir()?;
 				out.push(format!("sd-image-{}", host));
 
-				info!("building sd image to {:?}", out);
-				let mut nix_build = MyCommand::new("nix");
-				nix_build
-					.args(["build", "--impure", "--no-link"])
-					.comparg("--out-link", &out)
-					.arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))
-					.args(&config.nix_args);
-				if !self.fail_fast {
-					nix_build.arg("--keep-going");
-				}
-				if self.privileged_build {
-					nix_build = nix_build.sudo();
-				}
-
-				nix_build.run_nix().await?;
+				info!("linking sd image to {:?}", out);
+				symlink(out_output, out)?;
 			}
 			Action::Package(PackageAction::InstallationCd) => {
 				let mut out = current_dir()?;
 				out.push(format!("installation-cd-{}", host));
 
-				info!("building sd image to {:?}", out);
-				let mut nix_build = MyCommand::new("nix");
-				nix_build
-					.args(["build", "--impure", "--no-link"])
-					.comparg("--out-link", &out)
-					.arg(
-						config.configuration_attr_name(&format!(
-							"buildSystems.installationCd.{}",
-							host,
-						)),
-					)
-					.args(&config.nix_args);
-				if !self.fail_fast {
-					nix_build.arg("--keep-going");
-				}
-				if self.privileged_build {
-					nix_build = nix_build.sudo();
-				}
-
-				nix_build.run_nix().await?;
+				info!("linking iso image to {:?}", out);
+				symlink(out_output, out)?;
 			}
 		};
 		Ok(())

--- a/cmds/fleet/src/cmds/info.rs
+++ b/cmds/fleet/src/cmds/info.rs
@@ -1,6 +1,7 @@
 use std::collections::BTreeSet;
 
 use crate::host::Config;
+use crate::nix_path;
 use anyhow::{ensure, Result};
 use clap::Parser;
 
@@ -38,7 +39,7 @@
 					if !tagged.is_empty() {
 						let tags: Vec<String> = config
 							.fleet_field
-							.get_field_deep(["configuredSystems", &host.name, "config", "tags"])
+							.select(nix_path!(.configuredSystems.{&host.name}.config.tags))
 							.await?
 							.as_json()
 							.await?;
@@ -64,7 +65,7 @@
 				let host = config.system_config(&host).await?;
 				if external {
 					out.extend(
-						host.get_field_deep(["network", "externalIps"])
+						host.select(nix_path!(.network.externalIps))
 							.await?
 							.as_json::<Vec<String>>()
 							.await?,
@@ -72,7 +73,7 @@
 				}
 				if internal {
 					out.extend(
-						host.get_field_deep(["network", "internalIps"])
+						host.select(nix_path!(.network.internalIps))
 							.await?
 							.as_json::<Vec<String>>()
 							.await?,

--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -1,6 +1,6 @@
 use crate::{
 	fleetdata::{FleetSecret, FleetSharedSecret},
-	host::Config,
+	host::Config, nix_path,
 };
 use anyhow::{bail, ensure, Context, Result};
 use chrono::Utc;
@@ -339,7 +339,7 @@
 					let mut data = config.shared_secret(name)?;
 					let expected_owners: Vec<String> = config
 						.config_field
-						.get_json_deep(["sharedSecrets", name, "expectedOwners"])
+						.get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners))
 						.await?;
 					if expected_owners.is_empty() {
 						warn!("secret was removed from fleet config: {name}, removing from data");
@@ -352,7 +352,7 @@
 					if set != expected_set {
 						let owner_dependent: bool = config
 							.config_field
-							.get_json_deep(["sharedSecrets", name, "ownerDependent"])
+							.get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent))
 							.await?;
 						if !owner_dependent {
 							warn!("reencrypting secret '{name}' for new owner set");

--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -13,9 +13,10 @@
 use tempfile::NamedTempFile;
 
 use crate::{
-	better_nix_eval::{Field, NixSessionPool},
+	better_nix_eval::{Field, Index, NixSessionPool},
 	command::MyCommand,
 	fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
+	nix_path,
 };
 
 pub struct FleetConfigInternals {
@@ -24,9 +25,9 @@
 	pub opts: FleetOpts,
 	pub data: Mutex<FleetData>,
 	pub nix_args: Vec<OsString>,
-	// fleetConfigurations.<name>
+	/// fleetConfigurations.<name>.<localSystem>
 	pub fleet_field: Field,
-	// fleet_config.configUnchecked
+	/// fleet_config.configUnchecked
 	pub config_field: Field,
 }
 
@@ -91,22 +92,12 @@
 			command = command.ssh(host);
 		}
 		command.run_string().await
-	}
-
-	pub fn configuration_attr_name(&self, name: &str) -> OsString {
-		let mut str = self.directory.as_os_str().to_owned();
-		str.push("#");
-		str.push(&format!(
-			"fleetConfigurations.default.{}.{}",
-			self.local_system, name
-		));
-		str
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
 		let names = self
 			.fleet_field
-			.get_field_deep(["configuredHosts"])
+			.select(nix_path!(.configuredHosts))
 			.await?
 			.list_fields()
 			.await?;
@@ -118,7 +109,7 @@
 	}
 	pub async fn system_config(&self, host: &str) -> Result<Field> {
 		self.fleet_field
-			.get_field_deep(["configuredSystems", host, "config"])
+			.select(nix_path!(.configuredSystems.{host}.config))
 			.await
 	}
 
@@ -131,7 +122,7 @@
 	/// Shared secrets configured in fleet.nix or in flake
 	pub async fn list_configured_shared(&self) -> Result<Vec<String>> {
 		self.config_field
-			.get_field("sharedSecrets")
+			.select(nix_path!(.sharedSecrets))
 			.await?
 			.list_fields()
 			.await
@@ -221,7 +212,7 @@
 	}
 	pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
 		self.config_field
-			.get_field_deep(["sharedSecrets", secret, "expectedOwners"])
+			.select(nix_path!(.sharedSecrets.{secret}.expectedOwners))
 			.await?
 			.as_json()
 			.await
@@ -279,7 +270,9 @@
 
 		if self.local_system == "detect" {
 			let builtins_field = Field::field(root_field.clone(), "builtins").await?;
-			let system = builtins_field.get_field("currentSystem").await?;
+			let system = builtins_field
+				.select(nix_path!(.currentSystem))
+				.await?;
 			self.local_system = system.as_json().await?;
 		}
 		let local_system = self.local_system.clone();
@@ -287,9 +280,11 @@
 		let fleet_root = Field::field(root_field, "fleetConfigurations").await?;
 
 		let fleet_field = fleet_root
-			.get_field_deep(["default", &local_system])
+			.select(nix_path!(.default.{&local_system}))
+			.await?;
+		let config_field = fleet_field
+			.select(nix_path!(.configUnchecked))
 			.await?;
-		let config_field = fleet_field.get_field("configUnchecked").await?;
 
 		let mut fleet_data_path = directory.clone();
 		fleet_data_path.push("fleet.nix");

--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -1,3 +1,4 @@
+#![recursion_limit = "512"]
 #![feature(try_blocks)]
 
 pub(crate) mod cmds;

--- a/flake.nix
+++ b/flake.nix
@@ -19,6 +19,7 @@
       rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; };
     in
     {
+		packages = (import ./pkgs) pkgs pkgs;
       devShell = (pkgs.mkShell.override { stdenv = llvmPkgs.stdenv; }) {
         nativeBuildInputs = with pkgs; [
           rust