difftreelog
refactor nix-eval is now fully sync, parallelism should be explicit at the callsite
in: trunk
6 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2070,8 +2070,6 @@
"serde_json",
"test-log",
"thiserror 2.0.16",
- "tokio",
- "tokio-util",
"tracing",
"tracing-indicatif",
"vte 0.15.0",
cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -8,7 +8,7 @@
opts::FleetOpts,
};
use nix_eval::nix_go;
-use tokio::task::LocalSet;
+use tokio::task::{LocalSet, spawn_blocking};
use tracing::{Instrument, error, field, info, info_span, warn};
#[derive(Parser)]
@@ -34,7 +34,9 @@
// let action = Action::from(self.subcommand.clone());
let nixos = host.nixos_config().await?;
let drv = nix_go!(nixos.system.build[{ build_attr }]);
- let out_output = drv.build("out").await?;
+ let out_output = spawn_blocking(move || drv.build("out"))
+ .await
+ .expect("system derivation build should not panic")?;
// We already have system profiles for backups.
if !host.local {
cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -19,7 +19,7 @@
use owo_colors::OwoColorize;
use serde::Deserialize;
use tabled::{Table, Tabled};
-use tokio::fs::read;
+use tokio::{fs::read, task::spawn_blocking};
use tracing::{Instrument, error, info, info_span, warn};
#[derive(Parser)]
@@ -288,7 +288,9 @@
let generator = nix_go!(call_package(generator)(Obj {}));
- let generator = generator.build("out").await?;
+ let generator = spawn_blocking(move || generator.build("out"))
+ .await
+ .expect("nix build shouldn't fail")?;
let generator = host.remote_derivation(&generator).await?;
let out_parent = host.mktemp_dir().await?;
cmds/fleet/src/cmds/tf.rsdiffbeforeafterboth1use std::{collections::BTreeMap, ffi::OsString, path::PathBuf};23use anyhow::{Context, Result};4use clap::Parser;5use fleet_base::host::Config;6use nix_eval::nix_go;7use serde::Deserialize;8use serde_json::Value;9use tempfile::NamedTempFile;10use tokio::{11 fs::{self, create_dir_all},12 process::Command,13};14use tracing::debug;1516#[derive(Deserialize, Debug)]17pub struct TfData {18 // Dummy19 #[allow(dead_code)]20 managed: bool,21 // Host => Data22 #[serde(default)]23 #[serde(skip_serializing_if = "BTreeMap::is_empty")]24 pub hosts: BTreeMap<String, Value>,25}2627#[derive(Parser)]28pub struct Tf {29 args: Vec<OsString>,30}31impl Tf {32 pub async fn run(&self, config: &Config) -> Result<()> {33 let dir = config.directory.join(".fleet/tf/default");34 // TODO: consider postponing fleet init until this step, as it might be35 // highly preferred to extract terraform configuration using multithreaded nix or36 // lazy-trees nix. lazy-trees nix is very fast and perfect for this task.37 {38 debug!("generating terraform configs");39 let system = &config.local_system;40 let config = &config.config_field;41 let data: PathBuf = nix_go!(config.tf({ system })).build("out").await?;42 let data = fs::read(&data).await?;4344 create_dir_all(&dir).await?;4546 let tmp = NamedTempFile::new_in(&dir)?;47 fs::write(tmp.path(), data).await?;48 tmp.persist(dir.join("fleet.tf.json"))?;49 }5051 {52 debug!("running terraform command");53 Command::new("terraform")54 .current_dir(&dir)55 .args(&self.args)56 .status()57 .await?;58 }59 {60 debug!("syncing terraform data");61 let data = Command::new("terraform")62 .current_dir(dir)63 .arg("output")64 .arg("-json")65 .arg("fleet")66 .output()67 .await?;68 let tf_data: TfData = serde_json::from_slice(&data.stdout)69 .context("failed to parse terraform fleet output")?;7071 let mut data = config.data();72 debug!("synchronized done = {tf_data:?}");73 data.extra.insert(74 "terraformHosts".to_owned(),75 serde_json::to_value(tf_data.hosts).expect("should be valid extra"),76 );77 }7879 Ok(())80 }81}1use std::{collections::BTreeMap, ffi::OsString, path::PathBuf};23use anyhow::{Context, Result};4use clap::Parser;5use fleet_base::host::Config;6use nix_eval::nix_go;7use serde::Deserialize;8use serde_json::Value;9use tempfile::NamedTempFile;10use tokio::{11 fs::{self, create_dir_all},12 process::Command,13 task::spawn_blocking,14};15use tracing::debug;1617#[derive(Deserialize, Debug)]18pub struct TfData {19 // Dummy20 #[allow(dead_code)]21 managed: bool,22 // Host => Data23 #[serde(default)]24 #[serde(skip_serializing_if = "BTreeMap::is_empty")]25 pub hosts: BTreeMap<String, Value>,26}2728#[derive(Parser)]29pub struct Tf {30 args: Vec<OsString>,31}32impl Tf {33 pub async fn run(&self, config: &Config) -> Result<()> {34 let dir = config.directory.join(".fleet/tf/default");35 // TODO: consider postponing fleet init until this step, as it might be36 // highly preferred to extract terraform configuration using multithreaded nix or37 // lazy-trees nix. lazy-trees nix is very fast and perfect for this task.38 {39 debug!("generating terraform configs");40 let system = &config.local_system;41 let config = &config.config_field;42 let data = nix_go!(config.tf({ system }));43 let data: PathBuf = spawn_blocking(move || data.build("out"))44 .await45 .expect("tf.json derivation should not fail")?;46 let data = fs::read(&data).await?;4748 create_dir_all(&dir).await?;4950 let tmp = NamedTempFile::new_in(&dir)?;51 fs::write(tmp.path(), data).await?;52 tmp.persist(dir.join("fleet.tf.json"))?;53 }5455 {56 debug!("running terraform command");57 Command::new("terraform")58 .current_dir(&dir)59 .args(&self.args)60 .status()61 .await?;62 }63 {64 debug!("syncing terraform data");65 let data = Command::new("terraform")66 .current_dir(dir)67 .arg("output")68 .arg("-json")69 .arg("fleet")70 .output()71 .await?;72 let tf_data: TfData = serde_json::from_slice(&data.stdout)73 .context("failed to parse terraform fleet output")?;7475 let mut data = config.data();76 debug!("synchronized done = {tf_data:?}");77 data.extra.insert(78 "terraformHosts".to_owned(),79 serde_json::to_value(tf_data.hosts).expect("should be valid extra"),80 );81 }8283 Ok(())84 }85}crates/nix-eval/Cargo.tomldiffbeforeafterboth--- a/crates/nix-eval/Cargo.toml
+++ b/crates/nix-eval/Cargo.toml
@@ -11,8 +11,6 @@
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
thiserror.workspace = true
-tokio = { workspace = true }
-tokio-util.workspace = true
tracing.workspace = true
cxx = "1.0.168"
crates/nix-eval/src/lib.rsdiffbeforeafterboth--- a/crates/nix-eval/src/lib.rs
+++ b/crates/nix-eval/src/lib.rs
@@ -750,7 +750,7 @@
})?;
Ok(out)
}
- pub async fn build(&self, output: &str) -> Result<PathBuf> {
+ pub fn build(&self, output: &str) -> Result<PathBuf> {
if !self.is_derivation() {
bail!("expected derivation to build")
}