difftreelog
refactor perform build using nix repl
in: trunk
8 files changed
cmds/fleet/src/better_nix_eval.rsdiffbeforeafterboth--- a/cmds/fleet/src/better_nix_eval.rs
+++ b/cmds/fleet/src/better_nix_eval.rs
@@ -1,5 +1,7 @@
+use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
-use std::fmt::Display;
+use std::fmt::{self, Display};
+use std::path::PathBuf;
use std::process::Stdio;
use std::sync::{Arc, OnceLock};
@@ -8,7 +10,7 @@
use itertools::Itertools;
use r2d2::{Pool, PooledConnection};
use serde::de::DeserializeOwned;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tokio::process::{ChildStderr, ChildStdin, ChildStdout, Command};
use tokio::select;
@@ -72,14 +74,20 @@
// s.split('\n').filter(|s| !s.trim().is_empty()).map(|v| v.)
// }
if !self.collected.is_empty() {
- bail!("{}", self.collected.iter().map(|v| {
- if let Some(f) = v.strip_prefix("\u{1b}[31;1merror:\u{1b}[0m ") {
- let v = unindent::unindent(f.trim_start());
- v.trim().to_owned()
- } else {
- v.to_owned()
- }
- }).join("\n"));
+ bail!(
+ "{}",
+ self.collected
+ .iter()
+ .map(|v| {
+ if let Some(f) = v.strip_prefix("\u{1b}[31;1merror:\u{1b}[0m ") {
+ let v = unindent::unindent(f.trim_start());
+ v.trim().to_owned()
+ } else {
+ v.to_owned()
+ }
+ })
+ .join("\n")
+ );
}
Ok(())
}
@@ -150,6 +158,13 @@
}
}
+struct WarnHandler;
+impl Handler for WarnHandler {
+ fn handle_line(&mut self, e: &str) {
+ warn!(target: "nix", "{e}")
+ }
+}
+
impl NixSessionInner {
async fn new(flake: &OsStr, extra_args: impl IntoIterator<Item = &OsStr>) -> Result<Self> {
let mut cmd = Command::new("nix");
@@ -174,12 +189,13 @@
stdin.flush().await?;
let nix_handler = NixHandler::default();
let mut full_delimiter = None;
+ let mut errors = vec![];
while let Some(line) = out.next().await {
let line = match line {
OutputLine::Out(o) => o,
OutputLine::Err(_e) => {
// Handle startup errors, but skip repl hello?
- //nix_handler.handle_line(&e);
+ errors.push(_e);
continue;
}
};
@@ -190,6 +206,9 @@
}
}
let Some(full_delimiter) = full_delimiter else {
+ for e in errors {
+ error!("{e}");
+ }
bail!("failed to discover delimiter");
};
let mut res = Self {
@@ -342,21 +361,93 @@
#[derive(Clone)]
pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);
+#[macro_export]
+macro_rules! nix_path {
+ (@o($o:ident) $var:ident $($tt:tt)*) => {{
+ $o.push(Index::var(stringify!($var)));
+ nix_path!(@o($o) $($tt)*);
+ }};
+ (@o($o:ident) . $var:ident $($tt:tt)*) => {{
+ $o.push(Index::attr(stringify!($var)));
+ nix_path!(@o($o) $($tt)*);
+ }};
+ (@o($o:ident) . $var:literal $($tt:tt)*) => {{
+ $o.push(Index::attr($var));
+ nix_path!(@o($o) $($tt)*);
+ }};
+ (@o($o:ident) . { $var:expr } $($tt:tt)*) => {{
+ $o.push(Index::attr($var));
+ nix_path!(@o($o) $($tt)*);
+ }};
+ (@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{
+ $o.push(Index::idx($var));
+ nix_path!(@o($o) $($tt)*);
+ }};
+ (@o($o:ident) ($e:expr) $($tt:tt)*) => {
+ $o.push(Index::apply($e));
+ nix_path!(@o($o) $($tt)*);
+ };
+ (@o($o:ident)) => {};
+ ($($tt:tt)+) => {{
+ use $crate::{nix_path, better_nix_eval::Index};
+ let mut out = vec![];
+ nix_path!(@o(out) $($tt)*);
+ out
+ }}
+}
+
#[derive(Clone)]
-enum Index {
+pub enum Index {
+ Var(String),
String(String),
- // Idx(u32),
+ Apply(String),
+ Idx(u32),
}
+impl Index {
+ pub fn var(v: impl AsRef<str>) -> Self {
+ let v = v.as_ref();
+ assert!(
+ !(v.contains('.') | v.contains(' ')),
+ "bad variable name: {v}"
+ );
+ Self::Var(v.to_owned())
+ }
+ pub fn attr(v: impl AsRef<str>) -> Self {
+ Self::String(v.as_ref().to_owned())
+ }
+ pub fn idx(v: u32) -> Self {
+ Self::Idx(v)
+ }
+ pub fn apply(v: impl Serialize) -> Self {
+ let serialized = nixlike::serialize(v).expect("invalid value for apply");
+ Self::Apply(serialized)
+ }
+}
impl Display for Index {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
+ Index::Var(v) => {
+ write!(f, "{v}")
+ }
Index::String(k) => {
let v = nixlike::format_identifier(k.as_str());
write!(f, ".{v}")
}
+ Index::Apply(o) => {
+ let v = nixlike::serialize(o).map_err(|_| fmt::Error)?;
+ write!(f, "<apply>({v})")
+ }
+ Index::Idx(i) => {
+ write!(f, "[{i}]")
+ }
}
}
}
+impl fmt::Debug for Index {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "{self}")
+ }
+}
struct PathDisplay<'i>(&'i [Index]);
impl Display for PathDisplay<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -381,43 +472,49 @@
}
}
pub async fn field(session: NixSession, field: &str) -> Result<Self> {
- Self::root(session).get_field_deep([field]).await
+ Self::root(session)
+ .select([Index::var(field)])
+ .await
}
pub async fn get_json_deep<'a, V: DeserializeOwned>(
&self,
- name: impl IntoIterator<Item = &'a str>,
+ name: impl IntoIterator<Item = Index>,
) -> Result<V> {
- let field = self.get_field_deep(name).await?;
+ let field = self.select(name).await?;
field.as_json().await
}
- pub async fn get_field(&self, name: &str) -> Result<Self> {
- self.get_field_deep([name]).await
- }
- pub async fn get_field_deep<'a>(
- &self,
- name: impl IntoIterator<Item = &'a str>,
- ) -> Result<Self> {
- let mut iter = name.into_iter();
+ pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {
+ let mut name = name.into_iter();
let mut full_path = self.full_path.clone();
let mut query = if let Some(id) = self.value {
format!("sess_field_{id}")
} else {
- let first = iter.next().expect("name not empty");
- ensure!(
- !(first.contains('.') | first.contains(' ')),
- "bad name for root query: {first}"
- );
- full_path.push(Index::String(first.to_string()));
- first.to_string()
+ let first = name.next();
+ if let Some(Index::Var(i)) = first {
+ full_path.push(Index::Var(i.clone()));
+ i.clone()
+ } else {
+ panic!("first path item should be variable, got {first:?}")
+ }
};
- for v in iter {
- full_path.push(Index::String(v.to_string()));
- // Escape
- let escaped = nixlike::serialize(v)?;
- let escaped = escaped.trim();
- query.push('.');
- query.push_str(escaped);
+ for v in name {
+ full_path.push(v.clone());
+ match v {
+ Index::Var(_) => panic!("var item may only be first"),
+ Index::String(s) => {
+ let escaped = nixlike::serialize(s)?;
+ query.push('.');
+ query.push_str(escaped.trim());
+ }
+ Index::Apply(a) => {
+ query.push(' ');
+ query.push_str(&a);
+ }
+ Index::Idx(idx) => {
+ query = format!("builtins.elemAt ({query}) {idx}");
+ }
+ }
}
let vid = self
@@ -454,6 +551,28 @@
.await
.with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))
}
+ pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {
+ let id = self.value.expect("can't use build on not-value");
+ let vid = self
+ .session
+ .0
+ .lock()
+ .await
+ .execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default())
+ .await?;
+ ensure!(!vid.is_empty(), "build failed");
+ let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")
+ else {
+ panic!("unexpected build output: {vid:?}");
+ };
+ let outputs = vid
+ .split('\n')
+ .filter(|v| !v.is_empty())
+ .map(|v| v.split_once(" -> ").expect("unexpected build output"))
+ .map(|(a, b)| (a.trim_start().to_owned(), PathBuf::from(b)))
+ .collect();
+ Ok(outputs)
+ }
}
impl Drop for Field {
fn drop(&mut self) {
cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth1use std::os::unix::fs::symlink;1use std::path::PathBuf;2use std::path::PathBuf;2use std::{env::current_dir, time::Duration};3use std::{env::current_dir, time::Duration};344use crate::command::MyCommand;5use crate::command::MyCommand;5use crate::host::Config;6use crate::host::Config;7use crate::nix_path;6use anyhow::{anyhow, Result};8use anyhow::{anyhow, Result};7use clap::Parser;9use clap::Parser;8use itertools::Itertools;10use itertools::Itertools;111312#[derive(Parser, Clone)]14#[derive(Parser, Clone)]13pub struct BuildSystems {15pub struct BuildSystems {14 /// Do not continue on error15 #[clap(long)]16 fail_fast: bool,17 /// Disable automatic rollback16 /// Disable automatic rollback18 #[clap(long)]17 #[clap(long)]19 disable_rollback: bool,18 disable_rollback: bool,20 /// Run builds as sudo21 #[clap(long)]22 privileged_build: bool,23 #[clap(subcommand)]19 #[clap(subcommand)]24 subcommand: Subcommand,20 subcommand: Subcommand,25}21}294 async fn build_task(self, config: Config, host: String) -> Result<()> {290 async fn build_task(self, config: Config, host: String) -> Result<()> {295 info!("building");291 info!("building");296 let action = Action::from(self.subcommand.clone());292 let action = Action::from(self.subcommand.clone());297 let built = {298 let dir = tempfile::tempdir()?;299 dir.path().to_owned()300 };301302 let mut nix_build = MyCommand::new("nix");293 let drv = config303 nix_build304 .args([305 "build",306 "--impure",307 "--json",308 // "--show-trace",309 "--no-link",310 ])311 .comparg("--out-link", &built)294 .fleet_field312 .arg(295 .select(nix_path!(.buildSystems.{action.build_attr()}.{&host}))313 config.configuration_attr_name(&format!(314 "buildSystems.{}.{host}",315 action.build_attr()316 )),317 )318 .args(&config.nix_args);296 .await?;319320 if self.privileged_build {321 nix_build = nix_build.sudo();322 }323324 nix_build.run_nix().await.map_err(|e| {297 let outputs = drv.build().await.map_err(|e| {325 if action.build_attr() == "sdImage" {298 if action.build_attr() == "sdImage" {326 info!("sd-image build failed");299 info!("sd-image build failed");327 info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");300 info!("Make sure you have imported modulesPath/installer/sd-card/sd-image-<arch>[-installer].nix (For installer, you may want to check config)");328 info!("This module was automatically imported before, but was removed for better customization")301 info!("This module was automatically imported before, but was removed for better customization")329 }302 }330 e303 e331 })?;304 })?;332 let built = std::fs::canonicalize(built)?;305 let out_output = outputs306 .get("out")307 .ok_or_else(|| anyhow!("system build should produce \"out\" output"))?;333308334 match action {309 match action {335 Action::Upload { action } => {310 Action::Upload { action } => {342 .arg("sign")317 .arg("sign")343 .comparg("--key-file", "/etc/nix/private-key")318 .comparg("--key-file", "/etc/nix/private-key")344 .arg("-r")319 .arg("-r")345 .arg(&built);320 .arg(out_output);346 if let Err(e) = sign.sudo().run_nix().await {321 if let Err(e) = sign.sudo().run_nix().await {347 warn!("Failed to sign store paths: {e}");322 warn!("Failed to sign store paths: {e}");348 };323 };353 nix.arg("copy")328 nix.arg("copy")354 .arg("--substitute-on-destination")329 .arg("--substitute-on-destination")355 .comparg("--to", format!("ssh-ng://{host}"))330 .comparg("--to", format!("ssh-ng://{host}"))356 .arg(&built);331 .arg(out_output);357 match nix.run_nix().await {332 match nix.run_nix().await {358 Ok(()) => break,333 Ok(()) => break,359 Err(e) if tries < 3 => {334 Err(e) if tries < 3 => {366 }341 }367 }342 }368 if let Some(action) = action {343 if let Some(action) = action {369 execute_upload(&self, &config, action, &host, built).await?344 execute_upload(&self, &config, action, &host, out_output.clone()).await?370 }345 }371 }346 }372 Action::Package(PackageAction::SdImage) => {347 Action::Package(PackageAction::SdImage) => {373 let mut out = current_dir()?;348 let mut out = current_dir()?;374 out.push(format!("sd-image-{}", host));349 out.push(format!("sd-image-{}", host));375350376 info!("building sd image to {:?}", out);351 info!("linking sd image to {:?}", out);377 let mut nix_build = MyCommand::new("nix");378 nix_build379 .args(["build", "--impure", "--no-link"])380 .comparg("--out-link", &out)352 symlink(out_output, out)?;381 .arg(config.configuration_attr_name(&format!("buildSystems.sdImage.{}", host,)))382 .args(&config.nix_args);383 if !self.fail_fast {384 nix_build.arg("--keep-going");385 }386 if self.privileged_build {387 nix_build = nix_build.sudo();388 }389390 nix_build.run_nix().await?;391 }353 }392 Action::Package(PackageAction::InstallationCd) => {354 Action::Package(PackageAction::InstallationCd) => {393 let mut out = current_dir()?;355 let mut out = current_dir()?;394 out.push(format!("installation-cd-{}", host));356 out.push(format!("installation-cd-{}", host));395357396 info!("building sd image to {:?}", out);358 info!("linking iso image to {:?}", out);397 let mut nix_build = MyCommand::new("nix");398 nix_build399 .args(["build", "--impure", "--no-link"])400 .comparg("--out-link", &out)359 symlink(out_output, out)?;401 .arg(402 config.configuration_attr_name(&format!(403 "buildSystems.installationCd.{}",404 host,405 )),406 )407 .args(&config.nix_args);408 if !self.fail_fast {409 nix_build.arg("--keep-going");410 }411 if self.privileged_build {412 nix_build = nix_build.sudo();413 }414415 nix_build.run_nix().await?;416 }360 }417 };361 };418 Ok(())362 Ok(())cmds/fleet/src/cmds/info.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/info.rs
+++ b/cmds/fleet/src/cmds/info.rs
@@ -1,6 +1,7 @@
use std::collections::BTreeSet;
use crate::host::Config;
+use crate::nix_path;
use anyhow::{ensure, Result};
use clap::Parser;
@@ -38,7 +39,7 @@
if !tagged.is_empty() {
let tags: Vec<String> = config
.fleet_field
- .get_field_deep(["configuredSystems", &host.name, "config", "tags"])
+ .select(nix_path!(.configuredSystems.{&host.name}.config.tags))
.await?
.as_json()
.await?;
@@ -64,7 +65,7 @@
let host = config.system_config(&host).await?;
if external {
out.extend(
- host.get_field_deep(["network", "externalIps"])
+ host.select(nix_path!(.network.externalIps))
.await?
.as_json::<Vec<String>>()
.await?,
@@ -72,7 +73,7 @@
}
if internal {
out.extend(
- host.get_field_deep(["network", "internalIps"])
+ host.select(nix_path!(.network.internalIps))
.await?
.as_json::<Vec<String>>()
.await?,
cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -1,6 +1,6 @@
use crate::{
fleetdata::{FleetSecret, FleetSharedSecret},
- host::Config,
+ host::Config, nix_path,
};
use anyhow::{bail, ensure, Context, Result};
use chrono::Utc;
@@ -339,7 +339,7 @@
let mut data = config.shared_secret(name)?;
let expected_owners: Vec<String> = config
.config_field
- .get_json_deep(["sharedSecrets", name, "expectedOwners"])
+ .get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners))
.await?;
if expected_owners.is_empty() {
warn!("secret was removed from fleet config: {name}, removing from data");
@@ -352,7 +352,7 @@
if set != expected_set {
let owner_dependent: bool = config
.config_field
- .get_json_deep(["sharedSecrets", name, "ownerDependent"])
+ .get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent))
.await?;
if !owner_dependent {
warn!("reencrypting secret '{name}' for new owner set");
cmds/fleet/src/command.rsdiffbeforeafterboth--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -1,5 +1,4 @@
use std::{
- borrow::Cow,
collections::HashMap,
ffi::OsStr,
process::Stdio,
@@ -247,10 +246,14 @@
pub struct NixHandler {
spans: HashMap<u64, Span>,
}
-fn process_message(m: &str) -> Cow<'_, str> {
+fn process_message(m: &str) -> String {
static OSC_CLEANER: Lazy<Regex> =
Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());
- OSC_CLEANER.replace_all(m, "")
+ static DETABBER: Lazy<Regex> = Lazy::new(|| Regex::new(r"\t").unwrap());
+ let m = OSC_CLEANER.replace_all(m, "");
+ // Indicatif can't format tabs. This is not the correct tab formatting, as correct one should be aligned,
+ // and not just be replaced with the constant number of spaces, but it's ok for now, as statuses are single-line.
+ DETABBER.replace_all(m.as_ref(), " ").to_string()
}
impl Handler for NixHandler {
fn handle_line(&mut self, e: &str) {
cmds/fleet/src/host.rsdiffbeforeafterboth--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -13,9 +13,10 @@
use tempfile::NamedTempFile;
use crate::{
- better_nix_eval::{Field, NixSessionPool},
+ better_nix_eval::{Field, Index, NixSessionPool},
command::MyCommand,
fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
+ nix_path,
};
pub struct FleetConfigInternals {
@@ -24,9 +25,9 @@
pub opts: FleetOpts,
pub data: Mutex<FleetData>,
pub nix_args: Vec<OsString>,
- // fleetConfigurations.<name>
+ /// fleetConfigurations.<name>.<localSystem>
pub fleet_field: Field,
- // fleet_config.configUnchecked
+ /// fleet_config.configUnchecked
pub config_field: Field,
}
@@ -91,22 +92,12 @@
command = command.ssh(host);
}
command.run_string().await
- }
-
- pub fn configuration_attr_name(&self, name: &str) -> OsString {
- let mut str = self.directory.as_os_str().to_owned();
- str.push("#");
- str.push(&format!(
- "fleetConfigurations.default.{}.{}",
- self.local_system, name
- ));
- str
}
pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
let names = self
.fleet_field
- .get_field_deep(["configuredHosts"])
+ .select(nix_path!(.configuredHosts))
.await?
.list_fields()
.await?;
@@ -118,7 +109,7 @@
}
pub async fn system_config(&self, host: &str) -> Result<Field> {
self.fleet_field
- .get_field_deep(["configuredSystems", host, "config"])
+ .select(nix_path!(.configuredSystems.{host}.config))
.await
}
@@ -131,7 +122,7 @@
/// Shared secrets configured in fleet.nix or in flake
pub async fn list_configured_shared(&self) -> Result<Vec<String>> {
self.config_field
- .get_field("sharedSecrets")
+ .select(nix_path!(.sharedSecrets))
.await?
.list_fields()
.await
@@ -221,7 +212,7 @@
}
pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
self.config_field
- .get_field_deep(["sharedSecrets", secret, "expectedOwners"])
+ .select(nix_path!(.sharedSecrets.{secret}.expectedOwners))
.await?
.as_json()
.await
@@ -279,7 +270,9 @@
if self.local_system == "detect" {
let builtins_field = Field::field(root_field.clone(), "builtins").await?;
- let system = builtins_field.get_field("currentSystem").await?;
+ let system = builtins_field
+ .select(nix_path!(.currentSystem))
+ .await?;
self.local_system = system.as_json().await?;
}
let local_system = self.local_system.clone();
@@ -287,9 +280,11 @@
let fleet_root = Field::field(root_field, "fleetConfigurations").await?;
let fleet_field = fleet_root
- .get_field_deep(["default", &local_system])
+ .select(nix_path!(.default.{&local_system}))
+ .await?;
+ let config_field = fleet_field
+ .select(nix_path!(.configUnchecked))
.await?;
- let config_field = fleet_field.get_field("configUnchecked").await?;
let mut fleet_data_path = directory.clone();
fleet_data_path.push("fleet.nix");
cmds/fleet/src/main.rsdiffbeforeafterboth--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -1,3 +1,4 @@
+#![recursion_limit = "512"]
#![feature(try_blocks)]
pub(crate) mod cmds;
flake.nixdiffbeforeafterboth--- a/flake.nix
+++ b/flake.nix
@@ -19,6 +19,7 @@
rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; };
in
{
+ packages = (import ./pkgs) pkgs pkgs;
devShell = (pkgs.mkShell.override { stdenv = llvmPkgs.stdenv; }) {
nativeBuildInputs = with pkgs; [
rust