difftreelog
refactor more repl abstractions
in: trunk
11 files changed
cmds/fleet/src/better_nix_eval.rsdiffbeforeafterboth--- a/cmds/fleet/src/better_nix_eval.rs
+++ b/cmds/fleet/src/better_nix_eval.rs
@@ -365,47 +365,176 @@
#[derive(Clone)]
pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);
+#[derive(Clone)]
+pub struct NixExprBuilder {
+ out: String,
+ used_fields: Vec<Field>,
+}
+impl NixExprBuilder {
+ pub fn object() -> Self {
+ NixExprBuilder {
+ out: "{ ".to_owned(),
+ used_fields: Vec::new(),
+ }
+ }
+ pub fn string(s: &str) -> Self {
+ NixExprBuilder {
+ out: nixlike::serialize(s)
+ .expect("no problems with serializing_string")
+ .trim_end()
+ .to_owned(),
+ used_fields: Vec::new(),
+ }
+ }
+ pub fn serialized(v: impl Serialize) -> Self {
+ let serialized = nixlike::serialize(v).expect("invalid value for apply");
+ Self {
+ out: serialized.trim_end().to_owned(),
+ used_fields: Vec::new(),
+ }
+ }
+ pub fn field(f: Field) -> Self {
+ Self {
+ out: format!("sess_field_{}", f.0.value.expect("no value")),
+ used_fields: vec![f],
+ }
+ }
+ pub fn end_obj(&mut self) {
+ self.out.push('}');
+ }
+ pub fn obj_key(&mut self, name: Self, value: Self) {
+ self.out.push_str(r#""${"#);
+ self.extend(name);
+ self.out.push_str(r#"}" = "#);
+ self.extend(value);
+ self.out.push_str("; ");
+ }
+
+ pub fn extend(&mut self, e: Self) {
+ self.out.push_str(&e.out);
+ self.used_fields.extend(e.used_fields);
+ }
+
+ pub fn session(&self) -> NixSession {
+ let mut session = None;
+ for ele in &self.used_fields {
+ if session.is_none() {
+ session = Some(ele.0.session.clone());
+ continue;
+ }
+ let session = &session.as_ref().expect("checked").0;
+ let ele_sess = &ele.0.session.0;
+ assert!(
+ Arc::ptr_eq(session, ele_sess),
+ "can't mix fields from different session"
+ );
+ }
+ session.expect("expr without fields used")
+ }
+ pub fn index_attr(&mut self, s: &str) {
+ let escaped = nixlike::serialize(s).expect("string");
+ self.out.push('.');
+ self.out.push_str(escaped.trim_end());
+ }
+}
+
+#[macro_export]
+macro_rules! nix_expr_inner {
+ (Obj { $($ident:ident: $($val:tt)+),* $(,)? }) => {{
+ use $crate::better_nix_eval::NixExprBuilder;
+ let mut out = NixExprBuilder::object();
+ $(
+ out.obj_key(
+ NixExprBuilder::string(stringify!($ident)),
+ $crate::nix_expr_inner!($($val)+),
+ );
+ )*
+ out.end_obj();
+ out
+ }};
+ (@field($o:ident) . $var:ident $($tt:tt)*) => {{
+ $o.index_attr(stringify!($var));
+ nix_expr_inner!(@field($o) $($tt)*);
+ }};
+ (@field($o:ident) [{ $v:expr }] $($tt:tt)*) => {{
+ $o.push(Index::attr(&$v));
+ nix_expr_inner!(@o($o) $($tt)*);
+ }};
+ (@field($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{
+ $o.push(Index::Expr($crate::nix_expr_inner!($($var)+)));
+ nix_expr_inner!(@o($o) $($tt)*);
+ }};
+ (@field($o:ident) ($($var:tt)*) $($tt:tt)*) => {
+ $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+)));
+ nix_expr_inner!(@o($o) $($tt)*);
+ };
+ (@field($o:ident)) => {};
+ ($field:ident $($tt:tt)*) => {{
+ use $crate::{better_nix_eval::NixExprBuilder, nix_expr_inner};
+ #[allow(unused_mut, reason = "might be used if indexed")]
+ let mut out = NixExprBuilder::field($field);
+ nix_expr_inner!(@field(out) $($tt)*);
+ out
+ }};
+ ($v:literal) => {{
+ use $crate::better_nix_eval::NixExprBuilder;
+ NixExprBuilder::string($v)
+ }};
+ ({$v:expr}) => {{
+ use $crate::better_nix_eval::NixExprBuilder;
+ NixExprBuilder::serialized(&$v)
+ }}
+}
#[macro_export]
-macro_rules! nix_path {
- (@o($o:ident) $var:ident $($tt:tt)*) => {{
- $o.push(Index::var(stringify!($var)));
- nix_path!(@o($o) $($tt)*);
+macro_rules! nix_expr {
+ ($($tt:tt)+) => {{
+ use $crate::{better_nix_eval::{NixExprBuilder, Field}, nix_expr_inner};
+ let expr = nix_expr_inner!($($tt)+);
+ Field::new(expr.session(), expr.out)
}};
+}
+
+#[macro_export]
+macro_rules! nix_go {
(@o($o:ident) . $var:ident $($tt:tt)*) => {{
$o.push(Index::attr(stringify!($var)));
- nix_path!(@o($o) $($tt)*);
+ nix_go!(@o($o) $($tt)*);
}};
- (@o($o:ident) . $var:literal $($tt:tt)*) => {{
- $o.push(Index::attr($var));
- nix_path!(@o($o) $($tt)*);
+ (@o($o:ident) [{ $v:expr }] $($tt:tt)*) => {{
+ $o.push(Index::attr(&$v));
+ nix_go!(@o($o) $($tt)*);
}};
- (@o($o:ident) . { $var:expr } $($tt:tt)*) => {{
- $o.push(Index::attr($var));
- nix_path!(@o($o) $($tt)*);
+ (@o($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{
+ $o.push(Index::Expr($crate::nix_expr_inner!($($var)+)));
+ nix_go!(@o($o) $($tt)*);
}};
- (@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{
- $o.push(Index::idx($var));
- nix_path!(@o($o) $($tt)*);
- }};
- (@o($o:ident) ($e:expr) $($tt:tt)*) => {
- $o.push(Index::apply($e));
- nix_path!(@o($o) $($tt)*);
+ (@o($o:ident) ($($var:tt)*) $($tt:tt)*) => {
+ $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+)));
+ nix_go!(@o($o) $($tt)*);
};
(@o($o:ident)) => {};
- ($($tt:tt)+) => {{
- use $crate::{nix_path, better_nix_eval::Index};
+ ($field:ident $($tt:tt)+) => {{
+ use $crate::{nix_go, better_nix_eval::Index};
+ let field = $field.clone();
let mut out = vec![];
- nix_path!(@o(out) $($tt)*);
- out
+ nix_go!(@o(out) $($tt)*);
+ field.select(out).await?
}}
}
+#[macro_export]
+macro_rules! nix_go_json {
+ ($($tt:tt)*) => {{
+ $crate::nix_go!($($tt)*).as_json().await?
+ }};
+}
#[derive(Clone)]
pub enum Index {
Var(String),
String(String),
Apply(String),
- Idx(u32),
+ Expr(NixExprBuilder),
+ ExprApply(NixExprBuilder),
}
impl Index {
pub fn var(v: impl AsRef<str>) -> Self {
@@ -419,9 +548,6 @@
pub fn attr(v: impl AsRef<str>) -> Self {
Self::String(v.as_ref().to_owned())
}
- pub fn idx(v: u32) -> Self {
- Self::Idx(v)
- }
pub fn apply(v: impl Serialize) -> Self {
let serialized = nixlike::serialize(v).expect("invalid value for apply");
Self::Apply(serialized.trim_end().to_owned())
@@ -440,9 +566,12 @@
Index::Apply(o) => {
write!(f, "<apply>({o})")
}
- Index::Idx(i) => {
- write!(f, "[{i}]")
+ Index::Expr(e) => {
+ write!(f, "[{}]", e.out)
}
+ Index::ExprApply(e) => {
+ write!(f, "<apply>({})", e.out)
+ }
}
}
}
@@ -460,24 +589,45 @@
Ok(())
}
}
-pub struct Field {
- full_path: Vec<Index>,
+struct FieldInner {
+ full_path: Option<Vec<Index>>,
session: NixSession,
value: Option<u32>,
}
+fn context(full_path: Option<&[Index]>, query: &str) -> String {
+ if let Some(full_path) = &full_path {
+ format!("full path: {}", PathDisplay(full_path))
+ } else {
+ format!("query: {query:?}")
+ }
+}
+#[derive(Clone)]
+pub struct Field(Arc<FieldInner>);
impl Field {
fn root(session: NixSession) -> Self {
- Self {
- full_path: vec![],
+ Self(Arc::new(FieldInner {
+ full_path: Some(vec![]),
session,
value: None,
- }
+ }))
}
- pub async fn field(session: NixSession, field: &str) -> Result<Self> {
- Self::root(session)
- .select([Index::var(field)])
+ async fn new(session: NixSession, query: &str) -> Result<Self> {
+ let vid = session
+ .0
+ .lock()
.await
+ .execute_assign(query)
+ .await
+ .with_context(|| context(None, query))?;
+ Ok(Self(Arc::new(FieldInner {
+ full_path: None,
+ session,
+ value: Some(vid),
+ })))
}
+ pub async fn field(session: NixSession, field: &str) -> Result<Self> {
+ Self::root(session).select([Index::var(field)]).await
+ }
pub async fn get_json_deep<'a, V: DeserializeOwned>(
&self,
name: impl IntoIterator<Item = Index>,
@@ -486,22 +636,27 @@
field.as_json().await
}
pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {
+ let mut used_fields = Vec::new();
let mut name = name.into_iter();
- let mut full_path = self.full_path.clone();
- let mut query = if let Some(id) = self.value {
+ let mut full_path = self.0.full_path.clone();
+ let mut query = if let Some(id) = self.0.value {
format!("sess_field_{id}")
} else {
let first = name.next();
if let Some(Index::Var(i)) = first {
- full_path.push(Index::Var(i.clone()));
+ if let Some(full_path) = &mut full_path {
+ full_path.push(Index::Var(i.clone()));
+ }
i.clone()
} else {
panic!("first path item should be variable, got {first:?}")
}
};
for v in name {
- full_path.push(v.clone());
+ if let Some(full_path) = &mut full_path {
+ full_path.push(v.clone());
+ }
match v {
Index::Var(_) => panic!("var item may only be first"),
Index::String(s) => {
@@ -513,56 +668,85 @@
// In cases like `a {}.b` first `{}.b` will be evaluated, so `a {}` should be encased in `()`
query = format!("({query} {a})");
}
- Index::Idx(idx) => {
- query = format!("builtins.elemAt ({query}) {idx}");
+ Index::Expr(e) => {
+ let index = Field::new(self.0.session.clone(), &e.out).await?;
+ used_fields.push(index.clone());
+ query.push('.');
+ let index = format!("${{sess_field_{}}}", index.0.value.expect("value"));
+ query.push_str(&index);
+ }
+ Index::ExprApply(e) => {
+ let index = Field::new(self.0.session.clone(), &e.out).await?;
+ used_fields.push(index.clone());
+ query.push(' ');
+ let index = format!("sess_field_{}", index.0.value.expect("value"));
+ query.push_str(&index);
+ query = format!("({query})");
}
}
}
let vid = self
+ .0
.session
.0
.lock()
.await
.execute_assign(&query)
.await
- .with_context(|| format!("full path: {}", PathDisplay(&full_path)))?;
- Ok(Self {
+ .with_context(|| {
+ if let Some(full_path) = &full_path {
+ format!("full path: {}", PathDisplay(full_path))
+ } else {
+ format!("query: {query:?}")
+ }
+ })?;
+ Ok(Self(Arc::new(FieldInner {
full_path,
- session: self.session.clone(),
+ session: self.0.session.clone(),
value: Some(vid),
- })
+ })))
}
pub async fn as_json<V: DeserializeOwned>(&self) -> Result<V> {
- let id = self.value.expect("can't serialize root field");
- self.session
+ let id = self.0.value.expect("can't serialize root field");
+ let query = format!("sess_field_{id}");
+ self.0
+ .session
.0
.lock()
.await
- .execute_expression_to_json(&format!("sess_field_{id}"))
+ .execute_expression_to_json(&query)
.await
- .with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))
+ .with_context(|| context(self.0.full_path.as_deref(), &query))
}
pub async fn list_fields(&self) -> Result<Vec<String>> {
- let id = self.value.expect("can't list root fields");
- self.session
+ let id = self.0.value.expect("can't list root fields");
+ let query = format!("builtins.attrNames sess_field_{id}");
+ self.0
+ .session
.0
.lock()
.await
- .execute_expression_to_json(&format!("builtins.attrNames sess_field_{id}"))
+ .execute_expression_to_json(&query)
.await
- .with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))
+ .with_context(|| context(self.0.full_path.as_deref(), &query))
}
pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {
- let id = self.value.expect("can't use build on not-value");
+ let id = self.0.value.expect("can't use build on not-value");
+ let query = format!(":b sess_field_{id}");
let vid = self
+ .0
.session
.0
.lock()
.await
- .execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default())
+ .execute_expression_raw(&query, &mut NixHandler::default())
.await?;
- ensure!(!vid.is_empty(), "build failed: {}", PathDisplay(&self.full_path));
+ ensure!(
+ !vid.is_empty(),
+ "build failed: {}",
+ context(self.0.full_path.as_deref(), &query),
+ );
let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")
else {
panic!("unexpected build output: {vid:?}");
@@ -576,7 +760,7 @@
Ok(outputs)
}
}
-impl Drop for Field {
+impl Drop for FieldInner {
fn drop(&mut self) {
if let Some(id) = self.value {
if let Ok(mut lock) = self.session.0.try_lock() {
cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -4,8 +4,8 @@
use crate::command::MyCommand;
use crate::host::Config;
-use crate::nix_path;
-use anyhow::{anyhow, Result, Context};
+use crate::nix_go;
+use anyhow::{anyhow, Result};
use clap::Parser;
use itertools::Itertools;
use tokio::{task::LocalSet, time::sleep};
@@ -290,12 +290,10 @@
async fn build_task(self, config: Config, host: String) -> Result<()> {
info!("building");
let action = Action::from(self.subcommand.clone());
- let drv = config
- .fleet_field
- .select(nix_path!(.buildSystems((serde_json::json!({
- "localSystem": config.local_system.clone(),
- }))).{action.build_attr()}.{&host}))
- .await.context("system attribute")?;
+ let fleet_field = &config.fleet_field;
+ let drv = nix_go!(fleet_field.buildSystems(Obj {
+ localSystem: { config.local_system.clone() }
+ }));
let outputs = drv.build().await.map_err(|e| {
if action.build_attr() == "sdImage" {
info!("sd-image build failed");
cmds/fleet/src/cmds/info.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/info.rs
+++ b/cmds/fleet/src/cmds/info.rs
@@ -1,7 +1,7 @@
use std::collections::BTreeSet;
use crate::host::Config;
-use crate::nix_path;
+use crate::nix_go_json;
use anyhow::{ensure, Result};
use clap::Parser;
@@ -37,12 +37,9 @@
InfoCmd::ListHosts { ref tagged } => {
'host: for host in config.list_hosts().await? {
if !tagged.is_empty() {
- let tags: Vec<String> = config
- .fleet_field
- .select(nix_path!(.configuredSystems.{&host.name}.config.tags))
- .await?
- .as_json()
- .await?;
+ let fleet_field = &config.fleet_field;
+ let tags: Vec<String> =
+ nix_go_json!(fleet_field.configuredSystems[{ host.name }].config.tags);
for tag in tagged {
if !tags.contains(tag) {
continue 'host;
@@ -64,20 +61,12 @@
let mut out = <BTreeSet<String>>::new();
let host = config.system_config(&host).await?;
if external {
- out.extend(
- host.select(nix_path!(.network.externalIps))
- .await?
- .as_json::<Vec<String>>()
- .await?,
- );
+ let data: Vec<String> = nix_go_json!(host.network.externalIps);
+ out.extend(data);
}
if internal {
- out.extend(
- host.select(nix_path!(.network.internalIps))
- .await?
- .as_json::<Vec<String>>()
- .await?,
- );
+ let data: Vec<String> = nix_go_json!(host.network.internalIps);
+ out.extend(data);
}
for ip in out {
data.push(ip);
cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth1use crate::{1use crate::{2 fleetdata::{FleetSecret, FleetSharedSecret},2 fleetdata::{FleetSecret, FleetSharedSecret},3 host::Config, nix_path,3 host::Config,4 nix_go, nix_go_json,4};5};5use anyhow::{bail, ensure, Context, Result};6use anyhow::{anyhow, bail, ensure, Context, Result};6use chrono::Utc;7use chrono::{DateTime, Utc};7use clap::Parser;8use clap::Parser;8use futures::{StreamExt, TryStreamExt};9use futures::{StreamExt, TryStreamExt};9use owo_colors::OwoColorize;10use owo_colors::OwoColorize;17use tracing::{error, info, info_span, warn};18use tracing::{error, info, info_span, warn};181919#[derive(Parser)]20#[derive(Parser)]20pub enum Secrets {21pub enum Secret {21 /// Force load keys for all defined hosts22 /// Force load host keys for all defined hosts22 ForceKeys,23 ForceKeys,23 /// Add secret, data should be provided in stdin24 /// Add secret, data should be provided in stdin24 AddShared {25 AddShared {29 /// Override secret if already present30 /// Override secret if already present30 #[clap(long)]31 #[clap(long)]31 force: bool,32 force: bool,33 /// Secret public part32 #[clap(long)]34 #[clap(long)]33 public: Option<String>,35 public: Option<String>,36 /// Load public part from specified file34 #[clap(long)]37 #[clap(long)]35 public_file: Option<PathBuf>,38 public_file: Option<PathBuf>,3940 /// Create a notification on secret expiration41 #[clap(long)]42 expires_at: Option<DateTime<Utc>>,364337 /// Secret with this name already exists, override its value while keeping the same owners.44 /// Secret with this name already exists, override its value while keeping the same owners.38 #[clap(long)]45 #[clap(long)]39 readd: bool,46 re_add: bool,40 },47 },41 /// Add secret, data should be provided in stdin48 /// Add secret, data should be provided in stdin42 Add {49 Add {81 prefer_identities: Vec<String>,88 prefer_identities: Vec<String>,82 },89 },83 List {},90 List {},91 InvokeGenerator,84}92}859386impl Secrets {94impl Secret {87 pub async fn run(self, config: &Config) -> Result<()> {95 pub async fn run(self, config: &Config) -> Result<()> {88 match self {96 match self {97 Secret::InvokeGenerator => {98 let config_field = &config.config_unchecked_field;99100 let generate_impure =101 nix_go!(config_field.sharedSecrets["kube-apiserver.pem"].generateImpure);102 let on = nix_go!(generate_impure.on);103 let call_package = nix_go!(104 config_field.buildableSystems(Obj {105 localSystem: { config.local_system.clone() }106 })[on]107 .config108 .nixpkgs109 .pkgs110 .callPackage111 );112 let generator = nix_go!(call_package(generate_impure.generator));113 let built = generator.build().await?;114 // .as_json().await?;115 dbg!(&built);116 }89 Secrets::ForceKeys => {117 Secret::ForceKeys => {90 for host in config.list_hosts().await? {118 for host in config.list_hosts().await? {91 if config.should_skip(&host.name) {119 if config.should_skip(&host.name) {92 continue;120 continue;93 }121 }94 config.key(&host.name).await?;122 config.key(&host.name).await?;95 }123 }96 }124 }97 Secrets::AddShared {125 Secret::AddShared {98 mut machines,126 mut machines,99 name,127 name,100 force,128 force,101 public,129 public,102 public_file,130 public_file,103 readd,131 expires_at,132 re_add,104 } => {133 } => {105 let exists = config.has_shared(&name);134 let exists = config.has_shared(&name);106 if exists && !force && !readd {135 if exists && !force && !re_add {107 bail!("secret already defined");136 bail!("secret already defined");108 }137 }109 if readd {138 if re_add {110 // Fixme: use clap to limit this usage139 // Fixme: use clap to limit this usage111 ensure!(!force, "--force and --readd are not compatible");140 ensure!(!force, "--force and --readd are not compatible");112 ensure!(exists, "secret doesn't exists");141 ensure!(exists, "secret doesn't exists");137 .map(|r| Box::new(r) as Box<dyn age::Recipient + Send>)166 .map(|r| Box::new(r) as Box<dyn age::Recipient + Send>)138 .collect();167 .collect();139 let mut encryptor = age::Encryptor::with_recipients(recipients)168 let mut encryptor = age::Encryptor::with_recipients(recipients)140 .expect("recipients provided")169 .ok_or_else(|| anyhow!("no recipients provided"))?141 .wrap_output(&mut encrypted)?;170 .wrap_output(&mut encrypted)?;142 io::copy(&mut Cursor::new(input), &mut encryptor)?;171 io::copy(&mut Cursor::new(input), &mut encryptor)?;143 encryptor.finish()?;172 encryptor.finish()?;150 owners: machines,179 owners: machines,151 secret: FleetSecret {180 secret: FleetSecret {152 created_at: Utc::now(),181 created_at: Utc::now(),153 expires_at: None,182 expires_at,154 secret,183 secret,155 public: match (public, public_file) {184 public: match (public, public_file) {156 (Some(v), None) => Some(v),185 (Some(v), None) => Some(v),164 },193 },165 );194 );166 }195 }167 Secrets::Add {196 Secret::Add {168 machine,197 machine,169 name,198 name,170 force,199 force,211 }240 }212 // TODO: Instead of using sudo, decode secret on remote machine241 // TODO: Instead of using sudo, decode secret on remote machine213 #[allow(clippy::await_holding_refcell_ref)]242 #[allow(clippy::await_holding_refcell_ref)]214 Secrets::Read {243 Secret::Read {215 name,244 name,216 machine,245 machine,217 plaintext,246 plaintext,228 println!("{}", z85::encode(&data));257 println!("{}", z85::encode(&data));229 }258 }230 }259 }231 Secrets::UpdateShared {260 Secret::UpdateShared {232 name,261 name,233 machines,262 machines,234 mut add_machines,263 mut add_machines,321 secret.secret.secret = encrypted;350 secret.secret.secret = encrypted;322 config.replace_shared(name, secret);351 config.replace_shared(name, secret);323 }352 }324 Secrets::Regenerate { prefer_identities } => {353 Secret::Regenerate { prefer_identities } => {325 {354 {326 let expected_shared_set = config355 let expected_shared_set = config327 .list_configured_shared()356 .list_configured_shared()337 for name in &config.list_shared() {366 for name in &config.list_shared() {338 info!("updating secret: {name}");367 info!("updating secret: {name}");339 let mut data = config.shared_secret(name)?;368 let mut data = config.shared_secret(name)?;369 let config_field = &config.config_field;340 let expected_owners: Vec<String> = config370 let expected_owners: Vec<String> =341 .config_field342 .get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners))371 nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners);343 .await?;344 if expected_owners.is_empty() {372 if expected_owners.is_empty() {345 warn!("secret was removed from fleet config: {name}, removing from data");373 warn!("secret was removed from fleet config: {name}, removing from data");346 to_remove.push(name.to_string());374 to_remove.push(name.to_string());350 let expected_set = expected_owners.iter().collect::<HashSet<_>>();378 let expected_set = expected_owners.iter().collect::<HashSet<_>>();351 let should_remove = set.difference(&expected_set).next().is_some();379 let should_remove = set.difference(&expected_set).next().is_some();352 if set != expected_set {380 if set != expected_set {353 let owner_dependent: bool = config381 let owner_dependent: bool =354 .config_field355 .get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent))382 nix_go_json!(config_field.sharedSecrets[{ name }].ownerDependent);356 .await?;357 if !owner_dependent {383 if !owner_dependent {358 warn!("reencrypting secret '{name}' for new owner set");384 warn!("reencrypting secret '{name}' for new owner set");359 // TODO: force regeneration385 // TODO: force regeneration401 config.remove_shared(&k);427 config.remove_shared(&k);402 }428 }403 }429 }404 Secrets::List {} => {430 Secret::List {} => {405 let _span = info_span!("loading secrets").entered();431 let _span = info_span!("loading secrets").entered();406 let configured = config.list_configured_shared().await?;432 let configured = config.list_configured_shared().await?;407 #[derive(Tabled)]433 #[derive(Tabled)]cmds/fleet/src/command.rsdiffbeforeafterboth--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -337,6 +337,8 @@
if !text.is_empty()
&& text != "querying info about missing paths"
&& text != "copying 0 paths"
+ // Too much spam on lazy-trees branch
+ && !(text.starts_with("copying '") && text.ends_with("' to the store"))
{
let span = info_span!("job");
span.pb_start();
cmds/fleet/src/host.rsdiffbeforeafterboth--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -16,7 +16,7 @@
better_nix_eval::{Field, NixSessionPool},
command::MyCommand,
fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
- nix_path,
+ nix_go, nix_go_json,
};
pub struct FleetConfigInternals {
@@ -29,6 +29,8 @@
pub fleet_field: Field,
/// fleet_config.configUnchecked
pub config_field: Field,
+ /// fleet_config.unchecked
+ pub config_unchecked_field: Field,
}
#[derive(Clone)]
@@ -95,12 +97,8 @@
}
pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
- let names = self
- .fleet_field
- .select(nix_path!(.configuredHosts))
- .await?
- .list_fields()
- .await?;
+ let fleet_field = &self.fleet_field;
+ let names = nix_go!(fleet_field.configuredHosts).list_fields().await?;
let mut out = vec![];
for name in names {
out.push(ConfigHost { name })
@@ -108,9 +106,8 @@
Ok(out)
}
pub async fn system_config(&self, host: &str) -> Result<Field> {
- self.fleet_field
- .select(nix_path!(.configuredSystems.{host}.config))
- .await
+ let fleet_field = &self.fleet_field;
+ Ok(nix_go!(fleet_field.configuredSystems[{ host }].config))
}
pub(super) fn data(&self) -> MutexGuard<FleetData> {
@@ -121,11 +118,8 @@
}
/// Shared secrets configured in fleet.nix or in flake
pub async fn list_configured_shared(&self) -> Result<Vec<String>> {
- self.config_field
- .select(nix_path!(.sharedSecrets))
- .await?
- .list_fields()
- .await
+ let config_field = &self.config_field;
+ nix_go!(config_field.sharedSecrets).list_fields().await
}
/// Shared secrets configured in fleet.nix
pub fn list_shared(&self) -> Vec<String> {
@@ -211,11 +205,10 @@
Ok(secret.clone())
}
pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
- self.config_field
- .select(nix_path!(.sharedSecrets.{secret}.expectedOwners))
- .await?
- .as_json()
- .await
+ let config_field = &self.config_field;
+ Ok(nix_go_json!(
+ config_field.sharedSecrets[{ secret }].expectedOwners
+ ))
}
pub fn save(&self) -> Result<()> {
@@ -269,21 +262,15 @@
if self.local_system == "detect" {
let builtins_field = Field::field(root_field.clone(), "builtins").await?;
- let system = builtins_field
- .select(nix_path!(.currentSystem))
- .await?;
- self.local_system = system.as_json().await?;
+ self.local_system = nix_go_json!(builtins_field.currentSystem);
}
let local_system = self.local_system.clone();
let fleet_root = Field::field(root_field, "fleetConfigurations").await?;
- let fleet_field = fleet_root
- .select(nix_path!(.default))
- .await?;
- let config_field = fleet_field
- .select(nix_path!(.configUnchecked))
- .await?;
+ let fleet_field = nix_go!(fleet_root.default);
+ let config_field = nix_go!(fleet_field.configUnchecked);
+ let config_unchecked_field = nix_go!(fleet_field.unchecked);
let mut fleet_data_path = directory.clone();
fleet_data_path.push("fleet.nix");
@@ -298,6 +285,7 @@
nix_args,
fleet_field,
config_field,
+ config_unchecked_field,
})))
}
}
cmds/fleet/src/main.rsdiffbeforeafterboth--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -1,5 +1,5 @@
#![recursion_limit = "512"]
-#![feature(try_blocks)]
+#![feature(try_blocks, lint_reasons)]
pub(crate) mod cmds;
pub(crate) mod command;
@@ -17,7 +17,7 @@
use anyhow::{bail, Result};
use clap::Parser;
-use cmds::{build_systems::BuildSystems, info::Info, secrets::Secrets};
+use cmds::{build_systems::BuildSystems, info::Info, secrets::Secret};
use futures::future::LocalBoxFuture;
use futures::stream::FuturesUnordered;
use futures::TryStreamExt;
@@ -73,7 +73,7 @@
BuildSystems(BuildSystems),
/// Secret management
#[clap(subcommand)]
- Secrets(Secrets),
+ Secret(Secret),
/// Upload prefetch directory to the nix store
Prefetch(Prefetch),
/// Config parsing
@@ -92,7 +92,7 @@
async fn run_command(config: &Config, command: Opts) -> Result<()> {
match command {
Opts::BuildSystems(c) => c.run(config).await?,
- Opts::Secrets(s) => s.run(config).await?,
+ Opts::Secret(s) => s.run(config).await?,
Opts::Info(i) => i.run(config).await?,
Opts::Prefetch(p) => p.run(config).await?,
};
flake.lockdiffbeforeafterboth--- a/flake.lock
+++ b/flake.lock
@@ -5,11 +5,11 @@
"systems": "systems"
},
"locked": {
- "lastModified": 1694529238,
- "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+ "lastModified": 1701680307,
+ "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
"owner": "numtide",
"repo": "flake-utils",
- "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+ "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
"type": "github"
},
"original": {
@@ -38,11 +38,11 @@
},
"nixpkgs": {
"locked": {
- "lastModified": 1698350982,
- "narHash": "sha256-zoEV8Ad3bOAejp0ys/mOpaHSWrzK+GupZwGGYfuWuEY=",
+ "lastModified": 1703705939,
+ "narHash": "sha256-9s2Ep3NyRDj9HUgfv2TQUwQEanRUAmeXkvKIr/o1XbY=",
"owner": "nixos",
"repo": "nixpkgs",
- "rev": "dd83f9de26ff7c0326468b659ea4729fa5cf6262",
+ "rev": "1ada32da4ba24d7310653c9ac54888bee463f455",
"type": "github"
},
"original": {
@@ -67,11 +67,11 @@
]
},
"locked": {
- "lastModified": 1698199907,
- "narHash": "sha256-n8RtHBIb0rLuYs4RDehW6mj6r6Yam/ODY1af/VCcurw=",
+ "lastModified": 1703643208,
+ "narHash": "sha256-UL4KO8JxnD5rOycwHqBAf84lExF1/VnYMDC7b/wpPDU=",
"owner": "oxalica",
"repo": "rust-overlay",
- "rev": "22b8d29fd22cfaa2c311e0d6fd8a0ed9c2a1152b",
+ "rev": "ce117f3e0de8262be8cd324ee6357775228687cf",
"type": "github"
},
"original": {
flake.nixdiffbeforeafterboth--- a/flake.nix
+++ b/flake.nix
@@ -3,35 +3,52 @@
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/master";
- rust-overlay = { url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; };
- flake-utils = { url = "github:numtide/flake-utils"; };
+ rust-overlay = {
+ url = "github:oxalica/rust-overlay";
+ inputs.nixpkgs.follows = "nixpkgs";
+ };
+ flake-utils = {url = "github:numtide/flake-utils";};
};
- outputs = { self, rust-overlay, flake-utils, nixpkgs }: with nixpkgs.lib; rec {
- lib = import ./lib { inherit flake-utils; };
- } // flake-utils.lib.eachDefaultSystem (system:
- let
- pkgs = import nixpkgs
- {
- inherit system; overlays = [ (import rust-overlay) ];
- };
- llvmPkgs = pkgs.buildPackages.llvmPackages_11;
- rust = (pkgs.rustChannelOf { date = "2023-10-20"; channel = "nightly"; }).default.override { extensions = [ "rust-src" "rust-analyzer" ]; };
- rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; };
- in
- {
- packages = (import ./pkgs) pkgs pkgs;
- devShell = (pkgs.mkShell.override { stdenv = llvmPkgs.stdenv; }) {
- nativeBuildInputs = with pkgs; [
- rust
- lld
- cargo-edit
- cargo-udeps
- cargo-fuzz
+ outputs = {
+ self,
+ rust-overlay,
+ flake-utils,
+ nixpkgs,
+ }:
+ with nixpkgs.lib;
+ {
+ lib = import ./lib {inherit flake-utils;};
+ }
+ // flake-utils.lib.eachDefaultSystem (system: let
+ pkgs =
+ import nixpkgs
+ {
+ inherit system;
+ overlays = [(import rust-overlay)];
+ };
+ llvmPkgs = pkgs.buildPackages.llvmPackages_11;
+ rust =
+ (pkgs.rustChannelOf {
+ date = "2023-12-26";
+ channel = "nightly";
+ })
+ .default
+ .override {extensions = ["rust-src" "rust-analyzer"];};
+ in {
+ packages = (import ./pkgs) pkgs pkgs;
+ devShell = (pkgs.mkShell.override {stdenv = llvmPkgs.stdenv;}) {
+ nativeBuildInputs = with pkgs; [
+ rust
+ lld
+ cargo-edit
+ cargo-udeps
+ cargo-fuzz
+ cargo-watch
- pkg-config
- openssl
- bacon
- ];
- };
- });
+ pkg-config
+ openssl
+ bacon
+ ];
+ };
+ });
}
lib/default.nixdiffbeforeafterboth--- a/lib/default.nix
+++ b/lib/default.nix
@@ -10,80 +10,99 @@
fleetLib = import ./fleetLib.nix {
inherit nixpkgs hostNames;
};
- in
- let
- withData = data: rec {
- root = nixpkgs.lib.evalModules {
- modules = (import ../modules/fleet/_modules.nix) ++ [config data];
- specialArgs = {
- inherit nixpkgs fleetLib;
- };
- };
- failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
- rootAssertWarn =
- if failedAssertions != []
- then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
- else nixpkgs.lib.showWarnings root.config.warnings root;
- configuredHosts = rootAssertWarn.config.hosts;
- configuredSecrets = rootAssertWarn.config.secrets;
- configuredSystems = configuredSystemsWithExtraModules [];
- configuredSystemsWithExtraModules = extraModules:
- nixpkgs.lib.listToAttrs (
- map
- (
- name: {
- inherit name;
- value = nixpkgs.lib.nixosSystem {
- system = configuredHosts.${name}.system;
- modules = configuredHosts.${name}.modules ++ extraModules;
- specialArgs = {
- inherit fleetLib;
- fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
- };
+ in let
+ root = nixpkgs.lib.evalModules {
+ modules = (import ../modules/fleet/_modules.nix) ++ [config data];
+ specialArgs = {
+ inherit nixpkgs fleetLib;
+ };
+ };
+ failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
+ checkedRoot =
+ if failedAssertions != []
+ then throw "Fleet failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
+ else nixpkgs.lib.showWarnings root.config.warnings root;
+ withData = {
+ root,
+ data,
+ }: rec {
+ configuredHosts = root.config.hosts;
+ configuredUncheckedHosts = root.config.hosts;
+ configuredSystems = configuredSystemsWithExtraModules [];
+ configuredSystemsWithExtraModules = extraModules:
+ nixpkgs.lib.listToAttrs (
+ map
+ (
+ name: {
+ inherit name;
+ value = nixpkgs.lib.nixosSystem {
+ system = configuredHosts.${name}.system;
+ modules = configuredHosts.${name}.modules ++ extraModules;
+ specialArgs = {
+ inherit fleetLib;
+ fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
};
- }
- )
- (builtins.attrNames rootAssertWarn.config.hosts)
- );
- buildSystems = {localSystem}: let
- buildConfigurationModule = {config, ...}: {
- # Equivalent to nixpkgs.localSystem
- # nixpkgs.system = localSystem;
- nixpkgs.buildPlatform.system = localSystem;
- };
- in {
- toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
- buildConfigurationModule
- ({...}: {
- buildTarget = "toplevel";
- })
- ]);
- sdImage = builtins.mapAttrs (_name: value: value.config.system.build.sdImage) (configuredSystemsWithExtraModules [
- buildConfigurationModule
- #(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64-installer.nix")
- ({...}: {
- buildTarget = "sd-image";
- })
- ]);
- installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
- buildConfigurationModule
- (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
- ({lib, ...}: {
- buildTarget = "installation-cd";
- # Needed for https://github.com/NixOS/nixpkgs/issues/58959
- boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
- })
- ]);
+ };
+ }
+ )
+ (builtins.attrNames root.config.hosts)
+ );
+ buildableSystems = {localSystem}: let
+ buildConfigurationModule = {config, ...}: {
+ # Equivalent to nixpkgs.localSystem
+ # nixpkgs.system = localSystem;
+ nixpkgs.buildPlatform.system = localSystem;
+ };
+ in
+ configuredSystemsWithExtraModules [
+ buildConfigurationModule
+ ];
+ buildSystems = {localSystem}: let
+ buildConfigurationModule = {config, ...}: {
+ # Equivalent to nixpkgs.localSystem
+ # nixpkgs.system = localSystem;
+ nixpkgs.buildPlatform.system = localSystem;
};
- configUnchecked = root.config;
- };
- defaultData = withData data;
- in rec {
- inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
- injectData = data: let
- injectedData = withData data;
in {
- inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+ toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
+ buildConfigurationModule
+ ({...}: {
+ buildTarget = "toplevel";
+ })
+ ]);
+ sdImage = builtins.mapAttrs (_name: value: value.config.system.build.sdImage) (configuredSystemsWithExtraModules [
+ buildConfigurationModule
+ #(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64-installer.nix")
+ ({...}: {
+ buildTarget = "sd-image";
+ })
+ ]);
+ installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
+ buildConfigurationModule
+ (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
+ ({lib, ...}: {
+ buildTarget = "installation-cd";
+ # Needed for https://github.com/NixOS/nixpkgs/issues/58959
+ boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
+ })
+ ]);
};
+ configUnchecked = root.config;
+ };
+ defaultData = withData {
+ inherit data;
+ root = checkedRoot;
+ };
+ uncheckedData = withData {inherit data root;};
+ in rec {
+ inherit (defaultData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems;
+ unchecked = {
+ inherit (uncheckedData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems;
+ };
+ injectData = data: let
+ injectedData = withData data;
+ in {
+ inherit (injectedData) configuredHosts configuredSystems buildSystems configUnchecked;
};
+ };
}
modules/fleet/secrets.nixdiffbeforeafterboth--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -15,6 +15,9 @@
type = bool;
description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted";
};
+ generateImpure = mkOption {
+ type = unspecified;
+ };
generator = mkOption {
type = nullOr (submodule {
packages = mkOption {