--- a/cmds/fleet/src/better_nix_eval.rs +++ b/cmds/fleet/src/better_nix_eval.rs @@ -365,47 +365,176 @@ #[derive(Clone)] pub struct NixSession(Arc>>); +#[derive(Clone)] +pub struct NixExprBuilder { + out: String, + used_fields: Vec, +} +impl NixExprBuilder { + pub fn object() -> Self { + NixExprBuilder { + out: "{ ".to_owned(), + used_fields: Vec::new(), + } + } + pub fn string(s: &str) -> Self { + NixExprBuilder { + out: nixlike::serialize(s) + .expect("no problems with serializing_string") + .trim_end() + .to_owned(), + used_fields: Vec::new(), + } + } + pub fn serialized(v: impl Serialize) -> Self { + let serialized = nixlike::serialize(v).expect("invalid value for apply"); + Self { + out: serialized.trim_end().to_owned(), + used_fields: Vec::new(), + } + } + pub fn field(f: Field) -> Self { + Self { + out: format!("sess_field_{}", f.0.value.expect("no value")), + used_fields: vec![f], + } + } + pub fn end_obj(&mut self) { + self.out.push('}'); + } + pub fn obj_key(&mut self, name: Self, value: Self) { + self.out.push_str(r#""${"#); + self.extend(name); + self.out.push_str(r#"}" = "#); + self.extend(value); + self.out.push_str("; "); + } + + pub fn extend(&mut self, e: Self) { + self.out.push_str(&e.out); + self.used_fields.extend(e.used_fields); + } + + pub fn session(&self) -> NixSession { + let mut session = None; + for ele in &self.used_fields { + if session.is_none() { + session = Some(ele.0.session.clone()); + continue; + } + let session = &session.as_ref().expect("checked").0; + let ele_sess = &ele.0.session.0; + assert!( + Arc::ptr_eq(session, ele_sess), + "can't mix fields from different session" + ); + } + session.expect("expr without fields used") + } + pub fn index_attr(&mut self, s: &str) { + let escaped = nixlike::serialize(s).expect("string"); + self.out.push('.'); + self.out.push_str(escaped.trim_end()); + } +} + +#[macro_export] +macro_rules! nix_expr_inner { + (Obj { $($ident:ident: $($val:tt)+),* $(,)? }) => {{ + use $crate::better_nix_eval::NixExprBuilder; + let mut out = NixExprBuilder::object(); + $( + out.obj_key( + NixExprBuilder::string(stringify!($ident)), + $crate::nix_expr_inner!($($val)+), + ); + )* + out.end_obj(); + out + }}; + (@field($o:ident) . $var:ident $($tt:tt)*) => {{ + $o.index_attr(stringify!($var)); + nix_expr_inner!(@field($o) $($tt)*); + }}; + (@field($o:ident) [{ $v:expr }] $($tt:tt)*) => {{ + $o.push(Index::attr(&$v)); + nix_expr_inner!(@o($o) $($tt)*); + }}; + (@field($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{ + $o.push(Index::Expr($crate::nix_expr_inner!($($var)+))); + nix_expr_inner!(@o($o) $($tt)*); + }}; + (@field($o:ident) ($($var:tt)*) $($tt:tt)*) => { + $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+))); + nix_expr_inner!(@o($o) $($tt)*); + }; + (@field($o:ident)) => {}; + ($field:ident $($tt:tt)*) => {{ + use $crate::{better_nix_eval::NixExprBuilder, nix_expr_inner}; + #[allow(unused_mut, reason = "might be used if indexed")] + let mut out = NixExprBuilder::field($field); + nix_expr_inner!(@field(out) $($tt)*); + out + }}; + ($v:literal) => {{ + use $crate::better_nix_eval::NixExprBuilder; + NixExprBuilder::string($v) + }}; + ({$v:expr}) => {{ + use $crate::better_nix_eval::NixExprBuilder; + NixExprBuilder::serialized(&$v) + }} +} #[macro_export] -macro_rules! nix_path { - (@o($o:ident) $var:ident $($tt:tt)*) => {{ - $o.push(Index::var(stringify!($var))); - nix_path!(@o($o) $($tt)*); +macro_rules! nix_expr { + ($($tt:tt)+) => {{ + use $crate::{better_nix_eval::{NixExprBuilder, Field}, nix_expr_inner}; + let expr = nix_expr_inner!($($tt)+); + Field::new(expr.session(), expr.out) }}; +} + +#[macro_export] +macro_rules! nix_go { (@o($o:ident) . $var:ident $($tt:tt)*) => {{ $o.push(Index::attr(stringify!($var))); - nix_path!(@o($o) $($tt)*); + nix_go!(@o($o) $($tt)*); }}; - (@o($o:ident) . $var:literal $($tt:tt)*) => {{ - $o.push(Index::attr($var)); - nix_path!(@o($o) $($tt)*); + (@o($o:ident) [{ $v:expr }] $($tt:tt)*) => {{ + $o.push(Index::attr(&$v)); + nix_go!(@o($o) $($tt)*); }}; - (@o($o:ident) . { $var:expr } $($tt:tt)*) => {{ - $o.push(Index::attr($var)); - nix_path!(@o($o) $($tt)*); + (@o($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{ + $o.push(Index::Expr($crate::nix_expr_inner!($($var)+))); + nix_go!(@o($o) $($tt)*); }}; - (@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{ - $o.push(Index::idx($var)); - nix_path!(@o($o) $($tt)*); - }}; - (@o($o:ident) ($e:expr) $($tt:tt)*) => { - $o.push(Index::apply($e)); - nix_path!(@o($o) $($tt)*); + (@o($o:ident) ($($var:tt)*) $($tt:tt)*) => { + $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+))); + nix_go!(@o($o) $($tt)*); }; (@o($o:ident)) => {}; - ($($tt:tt)+) => {{ - use $crate::{nix_path, better_nix_eval::Index}; + ($field:ident $($tt:tt)+) => {{ + use $crate::{nix_go, better_nix_eval::Index}; + let field = $field.clone(); let mut out = vec![]; - nix_path!(@o(out) $($tt)*); - out + nix_go!(@o(out) $($tt)*); + field.select(out).await? }} } +#[macro_export] +macro_rules! nix_go_json { + ($($tt:tt)*) => {{ + $crate::nix_go!($($tt)*).as_json().await? + }}; +} #[derive(Clone)] pub enum Index { Var(String), String(String), Apply(String), - Idx(u32), + Expr(NixExprBuilder), + ExprApply(NixExprBuilder), } impl Index { pub fn var(v: impl AsRef) -> Self { @@ -419,9 +548,6 @@ pub fn attr(v: impl AsRef) -> Self { Self::String(v.as_ref().to_owned()) } - pub fn idx(v: u32) -> Self { - Self::Idx(v) - } pub fn apply(v: impl Serialize) -> Self { let serialized = nixlike::serialize(v).expect("invalid value for apply"); Self::Apply(serialized.trim_end().to_owned()) @@ -440,9 +566,12 @@ Index::Apply(o) => { write!(f, "({o})") } - Index::Idx(i) => { - write!(f, "[{i}]") + Index::Expr(e) => { + write!(f, "[{}]", e.out) } + Index::ExprApply(e) => { + write!(f, "({})", e.out) + } } } } @@ -460,24 +589,45 @@ Ok(()) } } -pub struct Field { - full_path: Vec, +struct FieldInner { + full_path: Option>, session: NixSession, value: Option, } +fn context(full_path: Option<&[Index]>, query: &str) -> String { + if let Some(full_path) = &full_path { + format!("full path: {}", PathDisplay(full_path)) + } else { + format!("query: {query:?}") + } +} +#[derive(Clone)] +pub struct Field(Arc); impl Field { fn root(session: NixSession) -> Self { - Self { - full_path: vec![], + Self(Arc::new(FieldInner { + full_path: Some(vec![]), session, value: None, - } + })) } - pub async fn field(session: NixSession, field: &str) -> Result { - Self::root(session) - .select([Index::var(field)]) + async fn new(session: NixSession, query: &str) -> Result { + let vid = session + .0 + .lock() .await + .execute_assign(query) + .await + .with_context(|| context(None, query))?; + Ok(Self(Arc::new(FieldInner { + full_path: None, + session, + value: Some(vid), + }))) } + pub async fn field(session: NixSession, field: &str) -> Result { + Self::root(session).select([Index::var(field)]).await + } pub async fn get_json_deep<'a, V: DeserializeOwned>( &self, name: impl IntoIterator, @@ -486,22 +636,27 @@ field.as_json().await } pub async fn select<'a>(&self, name: impl IntoIterator) -> Result { + let mut used_fields = Vec::new(); let mut name = name.into_iter(); - let mut full_path = self.full_path.clone(); - let mut query = if let Some(id) = self.value { + let mut full_path = self.0.full_path.clone(); + let mut query = if let Some(id) = self.0.value { format!("sess_field_{id}") } else { let first = name.next(); if let Some(Index::Var(i)) = first { - full_path.push(Index::Var(i.clone())); + if let Some(full_path) = &mut full_path { + full_path.push(Index::Var(i.clone())); + } i.clone() } else { panic!("first path item should be variable, got {first:?}") } }; for v in name { - full_path.push(v.clone()); + if let Some(full_path) = &mut full_path { + full_path.push(v.clone()); + } match v { Index::Var(_) => panic!("var item may only be first"), Index::String(s) => { @@ -513,56 +668,85 @@ // In cases like `a {}.b` first `{}.b` will be evaluated, so `a {}` should be encased in `()` query = format!("({query} {a})"); } - Index::Idx(idx) => { - query = format!("builtins.elemAt ({query}) {idx}"); + Index::Expr(e) => { + let index = Field::new(self.0.session.clone(), &e.out).await?; + used_fields.push(index.clone()); + query.push('.'); + let index = format!("${{sess_field_{}}}", index.0.value.expect("value")); + query.push_str(&index); + } + Index::ExprApply(e) => { + let index = Field::new(self.0.session.clone(), &e.out).await?; + used_fields.push(index.clone()); + query.push(' '); + let index = format!("sess_field_{}", index.0.value.expect("value")); + query.push_str(&index); + query = format!("({query})"); } } } let vid = self + .0 .session .0 .lock() .await .execute_assign(&query) .await - .with_context(|| format!("full path: {}", PathDisplay(&full_path)))?; - Ok(Self { + .with_context(|| { + if let Some(full_path) = &full_path { + format!("full path: {}", PathDisplay(full_path)) + } else { + format!("query: {query:?}") + } + })?; + Ok(Self(Arc::new(FieldInner { full_path, - session: self.session.clone(), + session: self.0.session.clone(), value: Some(vid), - }) + }))) } pub async fn as_json(&self) -> Result { - let id = self.value.expect("can't serialize root field"); - self.session + let id = self.0.value.expect("can't serialize root field"); + let query = format!("sess_field_{id}"); + self.0 + .session .0 .lock() .await - .execute_expression_to_json(&format!("sess_field_{id}")) + .execute_expression_to_json(&query) .await - .with_context(|| format!("full path: {}", PathDisplay(&self.full_path))) + .with_context(|| context(self.0.full_path.as_deref(), &query)) } pub async fn list_fields(&self) -> Result> { - let id = self.value.expect("can't list root fields"); - self.session + let id = self.0.value.expect("can't list root fields"); + let query = format!("builtins.attrNames sess_field_{id}"); + self.0 + .session .0 .lock() .await - .execute_expression_to_json(&format!("builtins.attrNames sess_field_{id}")) + .execute_expression_to_json(&query) .await - .with_context(|| format!("full path: {}", PathDisplay(&self.full_path))) + .with_context(|| context(self.0.full_path.as_deref(), &query)) } pub async fn build(&self) -> Result> { - let id = self.value.expect("can't use build on not-value"); + let id = self.0.value.expect("can't use build on not-value"); + let query = format!(":b sess_field_{id}"); let vid = self + .0 .session .0 .lock() .await - .execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default()) + .execute_expression_raw(&query, &mut NixHandler::default()) .await?; - ensure!(!vid.is_empty(), "build failed: {}", PathDisplay(&self.full_path)); + ensure!( + !vid.is_empty(), + "build failed: {}", + context(self.0.full_path.as_deref(), &query), + ); let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n") else { panic!("unexpected build output: {vid:?}"); @@ -576,7 +760,7 @@ Ok(outputs) } } -impl Drop for Field { +impl Drop for FieldInner { fn drop(&mut self) { if let Some(id) = self.value { if let Ok(mut lock) = self.session.0.try_lock() { --- a/cmds/fleet/src/cmds/build_systems.rs +++ b/cmds/fleet/src/cmds/build_systems.rs @@ -4,8 +4,8 @@ use crate::command::MyCommand; use crate::host::Config; -use crate::nix_path; -use anyhow::{anyhow, Result, Context}; +use crate::nix_go; +use anyhow::{anyhow, Result}; use clap::Parser; use itertools::Itertools; use tokio::{task::LocalSet, time::sleep}; @@ -290,12 +290,10 @@ async fn build_task(self, config: Config, host: String) -> Result<()> { info!("building"); let action = Action::from(self.subcommand.clone()); - let drv = config - .fleet_field - .select(nix_path!(.buildSystems((serde_json::json!({ - "localSystem": config.local_system.clone(), - }))).{action.build_attr()}.{&host})) - .await.context("system attribute")?; + let fleet_field = &config.fleet_field; + let drv = nix_go!(fleet_field.buildSystems(Obj { + localSystem: { config.local_system.clone() } + })); let outputs = drv.build().await.map_err(|e| { if action.build_attr() == "sdImage" { info!("sd-image build failed"); --- a/cmds/fleet/src/cmds/info.rs +++ b/cmds/fleet/src/cmds/info.rs @@ -1,7 +1,7 @@ use std::collections::BTreeSet; use crate::host::Config; -use crate::nix_path; +use crate::nix_go_json; use anyhow::{ensure, Result}; use clap::Parser; @@ -37,12 +37,9 @@ InfoCmd::ListHosts { ref tagged } => { 'host: for host in config.list_hosts().await? { if !tagged.is_empty() { - let tags: Vec = config - .fleet_field - .select(nix_path!(.configuredSystems.{&host.name}.config.tags)) - .await? - .as_json() - .await?; + let fleet_field = &config.fleet_field; + let tags: Vec = + nix_go_json!(fleet_field.configuredSystems[{ host.name }].config.tags); for tag in tagged { if !tags.contains(tag) { continue 'host; @@ -64,20 +61,12 @@ let mut out = >::new(); let host = config.system_config(&host).await?; if external { - out.extend( - host.select(nix_path!(.network.externalIps)) - .await? - .as_json::>() - .await?, - ); + let data: Vec = nix_go_json!(host.network.externalIps); + out.extend(data); } if internal { - out.extend( - host.select(nix_path!(.network.internalIps)) - .await? - .as_json::>() - .await?, - ); + let data: Vec = nix_go_json!(host.network.internalIps); + out.extend(data); } for ip in out { data.push(ip); --- a/cmds/fleet/src/cmds/secrets/mod.rs +++ b/cmds/fleet/src/cmds/secrets/mod.rs @@ -1,9 +1,10 @@ use crate::{ fleetdata::{FleetSecret, FleetSharedSecret}, - host::Config, nix_path, + host::Config, + nix_go, nix_go_json, }; -use anyhow::{bail, ensure, Context, Result}; -use chrono::Utc; +use anyhow::{anyhow, bail, ensure, Context, Result}; +use chrono::{DateTime, Utc}; use clap::Parser; use futures::{StreamExt, TryStreamExt}; use owo_colors::OwoColorize; @@ -17,8 +18,8 @@ use tracing::{error, info, info_span, warn}; #[derive(Parser)] -pub enum Secrets { - /// Force load keys for all defined hosts +pub enum Secret { + /// Force load host keys for all defined hosts ForceKeys, /// Add secret, data should be provided in stdin AddShared { @@ -29,14 +30,20 @@ /// Override secret if already present #[clap(long)] force: bool, + /// Secret public part #[clap(long)] public: Option, + /// Load public part from specified file #[clap(long)] public_file: Option, + /// Create a notification on secret expiration + #[clap(long)] + expires_at: Option>, + /// Secret with this name already exists, override its value while keeping the same owners. #[clap(long)] - readd: bool, + re_add: bool, }, /// Add secret, data should be provided in stdin Add { @@ -81,12 +88,33 @@ prefer_identities: Vec, }, List {}, + InvokeGenerator, } -impl Secrets { +impl Secret { pub async fn run(self, config: &Config) -> Result<()> { match self { - Secrets::ForceKeys => { + Secret::InvokeGenerator => { + let config_field = &config.config_unchecked_field; + + let generate_impure = + nix_go!(config_field.sharedSecrets["kube-apiserver.pem"].generateImpure); + let on = nix_go!(generate_impure.on); + let call_package = nix_go!( + config_field.buildableSystems(Obj { + localSystem: { config.local_system.clone() } + })[on] + .config + .nixpkgs + .pkgs + .callPackage + ); + let generator = nix_go!(call_package(generate_impure.generator)); + let built = generator.build().await?; + // .as_json().await?; + dbg!(&built); + } + Secret::ForceKeys => { for host in config.list_hosts().await? { if config.should_skip(&host.name) { continue; @@ -94,19 +122,20 @@ config.key(&host.name).await?; } } - Secrets::AddShared { + Secret::AddShared { mut machines, name, force, public, public_file, - readd, + expires_at, + re_add, } => { let exists = config.has_shared(&name); - if exists && !force && !readd { + if exists && !force && !re_add { bail!("secret already defined"); } - if readd { + if re_add { // Fixme: use clap to limit this usage ensure!(!force, "--force and --readd are not compatible"); ensure!(exists, "secret doesn't exists"); @@ -137,7 +166,7 @@ .map(|r| Box::new(r) as Box) .collect(); let mut encryptor = age::Encryptor::with_recipients(recipients) - .expect("recipients provided") + .ok_or_else(|| anyhow!("no recipients provided"))? .wrap_output(&mut encrypted)?; io::copy(&mut Cursor::new(input), &mut encryptor)?; encryptor.finish()?; @@ -150,7 +179,7 @@ owners: machines, secret: FleetSecret { created_at: Utc::now(), - expires_at: None, + expires_at, secret, public: match (public, public_file) { (Some(v), None) => Some(v), @@ -164,7 +193,7 @@ }, ); } - Secrets::Add { + Secret::Add { machine, name, force, @@ -211,7 +240,7 @@ } // TODO: Instead of using sudo, decode secret on remote machine #[allow(clippy::await_holding_refcell_ref)] - Secrets::Read { + Secret::Read { name, machine, plaintext, @@ -228,7 +257,7 @@ println!("{}", z85::encode(&data)); } } - Secrets::UpdateShared { + Secret::UpdateShared { name, machines, mut add_machines, @@ -321,7 +350,7 @@ secret.secret.secret = encrypted; config.replace_shared(name, secret); } - Secrets::Regenerate { prefer_identities } => { + Secret::Regenerate { prefer_identities } => { { let expected_shared_set = config .list_configured_shared() @@ -337,10 +366,9 @@ for name in &config.list_shared() { info!("updating secret: {name}"); let mut data = config.shared_secret(name)?; - let expected_owners: Vec = config - .config_field - .get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners)) - .await?; + let config_field = &config.config_field; + let expected_owners: Vec = + nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners); if expected_owners.is_empty() { warn!("secret was removed from fleet config: {name}, removing from data"); to_remove.push(name.to_string()); @@ -350,10 +378,8 @@ let expected_set = expected_owners.iter().collect::>(); let should_remove = set.difference(&expected_set).next().is_some(); if set != expected_set { - let owner_dependent: bool = config - .config_field - .get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent)) - .await?; + let owner_dependent: bool = + nix_go_json!(config_field.sharedSecrets[{ name }].ownerDependent); if !owner_dependent { warn!("reencrypting secret '{name}' for new owner set"); // TODO: force regeneration @@ -401,7 +427,7 @@ config.remove_shared(&k); } } - Secrets::List {} => { + Secret::List {} => { let _span = info_span!("loading secrets").entered(); let configured = config.list_configured_shared().await?; #[derive(Tabled)] --- a/cmds/fleet/src/command.rs +++ b/cmds/fleet/src/command.rs @@ -337,6 +337,8 @@ if !text.is_empty() && text != "querying info about missing paths" && text != "copying 0 paths" + // Too much spam on lazy-trees branch + && !(text.starts_with("copying '") && text.ends_with("' to the store")) { let span = info_span!("job"); span.pb_start(); --- a/cmds/fleet/src/host.rs +++ b/cmds/fleet/src/host.rs @@ -16,7 +16,7 @@ better_nix_eval::{Field, NixSessionPool}, command::MyCommand, fleetdata::{FleetData, FleetSecret, FleetSharedSecret}, - nix_path, + nix_go, nix_go_json, }; pub struct FleetConfigInternals { @@ -29,6 +29,8 @@ pub fleet_field: Field, /// fleet_config.configUnchecked pub config_field: Field, + /// fleet_config.unchecked + pub config_unchecked_field: Field, } #[derive(Clone)] @@ -95,12 +97,8 @@ } pub async fn list_hosts(&self) -> Result> { - let names = self - .fleet_field - .select(nix_path!(.configuredHosts)) - .await? - .list_fields() - .await?; + let fleet_field = &self.fleet_field; + let names = nix_go!(fleet_field.configuredHosts).list_fields().await?; let mut out = vec![]; for name in names { out.push(ConfigHost { name }) @@ -108,9 +106,8 @@ Ok(out) } pub async fn system_config(&self, host: &str) -> Result { - self.fleet_field - .select(nix_path!(.configuredSystems.{host}.config)) - .await + let fleet_field = &self.fleet_field; + Ok(nix_go!(fleet_field.configuredSystems[{ host }].config)) } pub(super) fn data(&self) -> MutexGuard { @@ -121,11 +118,8 @@ } /// Shared secrets configured in fleet.nix or in flake pub async fn list_configured_shared(&self) -> Result> { - self.config_field - .select(nix_path!(.sharedSecrets)) - .await? - .list_fields() - .await + let config_field = &self.config_field; + nix_go!(config_field.sharedSecrets).list_fields().await } /// Shared secrets configured in fleet.nix pub fn list_shared(&self) -> Vec { @@ -211,11 +205,10 @@ Ok(secret.clone()) } pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result> { - self.config_field - .select(nix_path!(.sharedSecrets.{secret}.expectedOwners)) - .await? - .as_json() - .await + let config_field = &self.config_field; + Ok(nix_go_json!( + config_field.sharedSecrets[{ secret }].expectedOwners + )) } pub fn save(&self) -> Result<()> { @@ -269,21 +262,15 @@ if self.local_system == "detect" { let builtins_field = Field::field(root_field.clone(), "builtins").await?; - let system = builtins_field - .select(nix_path!(.currentSystem)) - .await?; - self.local_system = system.as_json().await?; + self.local_system = nix_go_json!(builtins_field.currentSystem); } let local_system = self.local_system.clone(); let fleet_root = Field::field(root_field, "fleetConfigurations").await?; - let fleet_field = fleet_root - .select(nix_path!(.default)) - .await?; - let config_field = fleet_field - .select(nix_path!(.configUnchecked)) - .await?; + let fleet_field = nix_go!(fleet_root.default); + let config_field = nix_go!(fleet_field.configUnchecked); + let config_unchecked_field = nix_go!(fleet_field.unchecked); let mut fleet_data_path = directory.clone(); fleet_data_path.push("fleet.nix"); @@ -298,6 +285,7 @@ nix_args, fleet_field, config_field, + config_unchecked_field, }))) } } --- a/cmds/fleet/src/main.rs +++ b/cmds/fleet/src/main.rs @@ -1,5 +1,5 @@ #![recursion_limit = "512"] -#![feature(try_blocks)] +#![feature(try_blocks, lint_reasons)] pub(crate) mod cmds; pub(crate) mod command; @@ -17,7 +17,7 @@ use anyhow::{bail, Result}; use clap::Parser; -use cmds::{build_systems::BuildSystems, info::Info, secrets::Secrets}; +use cmds::{build_systems::BuildSystems, info::Info, secrets::Secret}; use futures::future::LocalBoxFuture; use futures::stream::FuturesUnordered; use futures::TryStreamExt; @@ -73,7 +73,7 @@ BuildSystems(BuildSystems), /// Secret management #[clap(subcommand)] - Secrets(Secrets), + Secret(Secret), /// Upload prefetch directory to the nix store Prefetch(Prefetch), /// Config parsing @@ -92,7 +92,7 @@ async fn run_command(config: &Config, command: Opts) -> Result<()> { match command { Opts::BuildSystems(c) => c.run(config).await?, - Opts::Secrets(s) => s.run(config).await?, + Opts::Secret(s) => s.run(config).await?, Opts::Info(i) => i.run(config).await?, Opts::Prefetch(p) => p.run(config).await?, }; --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1694529238, - "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", "owner": "numtide", "repo": "flake-utils", - "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", "type": "github" }, "original": { @@ -38,11 +38,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1698350982, - "narHash": "sha256-zoEV8Ad3bOAejp0ys/mOpaHSWrzK+GupZwGGYfuWuEY=", + "lastModified": 1703705939, + "narHash": "sha256-9s2Ep3NyRDj9HUgfv2TQUwQEanRUAmeXkvKIr/o1XbY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "dd83f9de26ff7c0326468b659ea4729fa5cf6262", + "rev": "1ada32da4ba24d7310653c9ac54888bee463f455", "type": "github" }, "original": { @@ -67,11 +67,11 @@ ] }, "locked": { - "lastModified": 1698199907, - "narHash": "sha256-n8RtHBIb0rLuYs4RDehW6mj6r6Yam/ODY1af/VCcurw=", + "lastModified": 1703643208, + "narHash": "sha256-UL4KO8JxnD5rOycwHqBAf84lExF1/VnYMDC7b/wpPDU=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "22b8d29fd22cfaa2c311e0d6fd8a0ed9c2a1152b", + "rev": "ce117f3e0de8262be8cd324ee6357775228687cf", "type": "github" }, "original": { --- a/flake.nix +++ b/flake.nix @@ -3,35 +3,52 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/master"; - rust-overlay = { url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; }; - flake-utils = { url = "github:numtide/flake-utils"; }; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-utils = {url = "github:numtide/flake-utils";}; }; - outputs = { self, rust-overlay, flake-utils, nixpkgs }: with nixpkgs.lib; rec { - lib = import ./lib { inherit flake-utils; }; - } // flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs - { - inherit system; overlays = [ (import rust-overlay) ]; - }; - llvmPkgs = pkgs.buildPackages.llvmPackages_11; - rust = (pkgs.rustChannelOf { date = "2023-10-20"; channel = "nightly"; }).default.override { extensions = [ "rust-src" "rust-analyzer" ]; }; - rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; }; - in - { - packages = (import ./pkgs) pkgs pkgs; - devShell = (pkgs.mkShell.override { stdenv = llvmPkgs.stdenv; }) { - nativeBuildInputs = with pkgs; [ - rust - lld - cargo-edit - cargo-udeps - cargo-fuzz + outputs = { + self, + rust-overlay, + flake-utils, + nixpkgs, + }: + with nixpkgs.lib; + { + lib = import ./lib {inherit flake-utils;}; + } + // flake-utils.lib.eachDefaultSystem (system: let + pkgs = + import nixpkgs + { + inherit system; + overlays = [(import rust-overlay)]; + }; + llvmPkgs = pkgs.buildPackages.llvmPackages_11; + rust = + (pkgs.rustChannelOf { + date = "2023-12-26"; + channel = "nightly"; + }) + .default + .override {extensions = ["rust-src" "rust-analyzer"];}; + in { + packages = (import ./pkgs) pkgs pkgs; + devShell = (pkgs.mkShell.override {stdenv = llvmPkgs.stdenv;}) { + nativeBuildInputs = with pkgs; [ + rust + lld + cargo-edit + cargo-udeps + cargo-fuzz + cargo-watch - pkg-config - openssl - bacon - ]; - }; - }); + pkg-config + openssl + bacon + ]; + }; + }); } --- a/lib/default.nix +++ b/lib/default.nix @@ -10,80 +10,99 @@ fleetLib = import ./fleetLib.nix { inherit nixpkgs hostNames; }; - in - let - withData = data: rec { - root = nixpkgs.lib.evalModules { - modules = (import ../modules/fleet/_modules.nix) ++ [config data]; - specialArgs = { - inherit nixpkgs fleetLib; - }; - }; - failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions); - rootAssertWarn = - if failedAssertions != [] - then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}" - else nixpkgs.lib.showWarnings root.config.warnings root; - configuredHosts = rootAssertWarn.config.hosts; - configuredSecrets = rootAssertWarn.config.secrets; - configuredSystems = configuredSystemsWithExtraModules []; - configuredSystemsWithExtraModules = extraModules: - nixpkgs.lib.listToAttrs ( - map - ( - name: { - inherit name; - value = nixpkgs.lib.nixosSystem { - system = configuredHosts.${name}.system; - modules = configuredHosts.${name}.modules ++ extraModules; - specialArgs = { - inherit fleetLib; - fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config); - }; + in let + root = nixpkgs.lib.evalModules { + modules = (import ../modules/fleet/_modules.nix) ++ [config data]; + specialArgs = { + inherit nixpkgs fleetLib; + }; + }; + failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions); + checkedRoot = + if failedAssertions != [] + then throw "Fleet failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}" + else nixpkgs.lib.showWarnings root.config.warnings root; + withData = { + root, + data, + }: rec { + configuredHosts = root.config.hosts; + configuredUncheckedHosts = root.config.hosts; + configuredSystems = configuredSystemsWithExtraModules []; + configuredSystemsWithExtraModules = extraModules: + nixpkgs.lib.listToAttrs ( + map + ( + name: { + inherit name; + value = nixpkgs.lib.nixosSystem { + system = configuredHosts.${name}.system; + modules = configuredHosts.${name}.modules ++ extraModules; + specialArgs = { + inherit fleetLib; + fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config); }; - } - ) - (builtins.attrNames rootAssertWarn.config.hosts) - ); - buildSystems = {localSystem}: let - buildConfigurationModule = {config, ...}: { - # Equivalent to nixpkgs.localSystem - # nixpkgs.system = localSystem; - nixpkgs.buildPlatform.system = localSystem; - }; - in { - toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [ - buildConfigurationModule - ({...}: { - buildTarget = "toplevel"; - }) - ]); - sdImage = builtins.mapAttrs (_name: value: value.config.system.build.sdImage) (configuredSystemsWithExtraModules [ - buildConfigurationModule - #(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64-installer.nix") - ({...}: { - buildTarget = "sd-image"; - }) - ]); - installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [ - buildConfigurationModule - (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix") - ({lib, ...}: { - buildTarget = "installation-cd"; - # Needed for https://github.com/NixOS/nixpkgs/issues/58959 - boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"]; - }) - ]); + }; + } + ) + (builtins.attrNames root.config.hosts) + ); + buildableSystems = {localSystem}: let + buildConfigurationModule = {config, ...}: { + # Equivalent to nixpkgs.localSystem + # nixpkgs.system = localSystem; + nixpkgs.buildPlatform.system = localSystem; + }; + in + configuredSystemsWithExtraModules [ + buildConfigurationModule + ]; + buildSystems = {localSystem}: let + buildConfigurationModule = {config, ...}: { + # Equivalent to nixpkgs.localSystem + # nixpkgs.system = localSystem; + nixpkgs.buildPlatform.system = localSystem; }; - configUnchecked = root.config; - }; - defaultData = withData data; - in rec { - inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked; - injectData = data: let - injectedData = withData data; in { - inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked; + toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [ + buildConfigurationModule + ({...}: { + buildTarget = "toplevel"; + }) + ]); + sdImage = builtins.mapAttrs (_name: value: value.config.system.build.sdImage) (configuredSystemsWithExtraModules [ + buildConfigurationModule + #(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64-installer.nix") + ({...}: { + buildTarget = "sd-image"; + }) + ]); + installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [ + buildConfigurationModule + (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix") + ({lib, ...}: { + buildTarget = "installation-cd"; + # Needed for https://github.com/NixOS/nixpkgs/issues/58959 + boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"]; + }) + ]); }; + configUnchecked = root.config; + }; + defaultData = withData { + inherit data; + root = checkedRoot; + }; + uncheckedData = withData {inherit data root;}; + in rec { + inherit (defaultData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems; + unchecked = { + inherit (uncheckedData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems; + }; + injectData = data: let + injectedData = withData data; + in { + inherit (injectedData) configuredHosts configuredSystems buildSystems configUnchecked; }; + }; } --- a/modules/fleet/secrets.nix +++ b/modules/fleet/secrets.nix @@ -15,6 +15,9 @@ type = bool; description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted"; }; + generateImpure = mkOption { + type = unspecified; + }; generator = mkOption { type = nullOr (submodule { packages = mkOption {