difftreelog
refactor more repl abstractions
in: trunk
11 files changed
cmds/fleet/src/better_nix_eval.rsdiffbeforeafterboth365#[derive(Clone)]365#[derive(Clone)]366pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);366pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);367368#[derive(Clone)]369pub struct NixExprBuilder {370 out: String,371 used_fields: Vec<Field>,372}373impl NixExprBuilder {374 pub fn object() -> Self {375 NixExprBuilder {376 out: "{ ".to_owned(),377 used_fields: Vec::new(),378 }379 }380 pub fn string(s: &str) -> Self {381 NixExprBuilder {382 out: nixlike::serialize(s)383 .expect("no problems with serializing_string")384 .trim_end()385 .to_owned(),386 used_fields: Vec::new(),387 }388 }389 pub fn serialized(v: impl Serialize) -> Self {390 let serialized = nixlike::serialize(v).expect("invalid value for apply");391 Self {392 out: serialized.trim_end().to_owned(),393 used_fields: Vec::new(),394 }395 }396 pub fn field(f: Field) -> Self {397 Self {398 out: format!("sess_field_{}", f.0.value.expect("no value")),399 used_fields: vec![f],400 }401 }402 pub fn end_obj(&mut self) {403 self.out.push('}');404 }405 pub fn obj_key(&mut self, name: Self, value: Self) {406 self.out.push_str(r#""${"#);407 self.extend(name);408 self.out.push_str(r#"}" = "#);409 self.extend(value);410 self.out.push_str("; ");411 }412413 pub fn extend(&mut self, e: Self) {414 self.out.push_str(&e.out);415 self.used_fields.extend(e.used_fields);416 }417418 pub fn session(&self) -> NixSession {419 let mut session = None;420 for ele in &self.used_fields {421 if session.is_none() {422 session = Some(ele.0.session.clone());423 continue;424 }425 let session = &session.as_ref().expect("checked").0;426 let ele_sess = &ele.0.session.0;427 assert!(428 Arc::ptr_eq(session, ele_sess),429 "can't mix fields from different session"430 );431 }432 session.expect("expr without fields used")433 }434 pub fn index_attr(&mut self, s: &str) {435 let escaped = nixlike::serialize(s).expect("string");436 self.out.push('.');437 self.out.push_str(escaped.trim_end());438 }439}367440368#[macro_export]441#[macro_export]369macro_rules! nix_path {442macro_rules! nix_expr_inner {370 (@o($o:ident) $var:ident $($tt:tt)*) => {{443 (Obj { $($ident:ident: $($val:tt)+),* $(,)? }) => {{371 $o.push(Index::var(stringify!($var)));444 use $crate::better_nix_eval::NixExprBuilder;445 let mut out = NixExprBuilder::object();446 $(447 out.obj_key(448 NixExprBuilder::string(stringify!($ident)),449 $crate::nix_expr_inner!($($val)+),450 );451 )*452 out.end_obj();453 out454 }};455 (@field($o:ident) . $var:ident $($tt:tt)*) => {{456 $o.index_attr(stringify!($var));457 nix_expr_inner!(@field($o) $($tt)*);458 }};459 (@field($o:ident) [{ $v:expr }] $($tt:tt)*) => {{460 $o.push(Index::attr(&$v));461 nix_expr_inner!(@o($o) $($tt)*);462 }};463 (@field($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{464 $o.push(Index::Expr($crate::nix_expr_inner!($($var)+)));465 nix_expr_inner!(@o($o) $($tt)*);466 }};467 (@field($o:ident) ($($var:tt)*) $($tt:tt)*) => {468 $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+)));469 nix_expr_inner!(@o($o) $($tt)*);470 };471 (@field($o:ident)) => {};472 ($field:ident $($tt:tt)*) => {{473 use $crate::{better_nix_eval::NixExprBuilder, nix_expr_inner};474 #[allow(unused_mut, reason = "might be used if indexed")]475 let mut out = NixExprBuilder::field($field);476 nix_expr_inner!(@field(out) $($tt)*);477 out478 }};479 ($v:literal) => {{480 use $crate::better_nix_eval::NixExprBuilder;481 NixExprBuilder::string($v)482 }};483 ({$v:expr}) => {{484 use $crate::better_nix_eval::NixExprBuilder;485 NixExprBuilder::serialized(&$v)486 }}487}488#[macro_export]372 nix_path!(@o($o) $($tt)*);489macro_rules! nix_expr {373 }};374 (@o($o:ident) . $var:ident $($tt:tt)*) => {{490 ($($tt:tt)+) => {{375 $o.push(Index::attr(stringify!($var)));491 use $crate::{better_nix_eval::{NixExprBuilder, Field}, nix_expr_inner};376 nix_path!(@o($o) $($tt)*);492 let expr = nix_expr_inner!($($tt)+);493 Field::new(expr.session(), expr.out)377 }};494 }};495}496497#[macro_export]498macro_rules! nix_go {378 (@o($o:ident) . $var:literal $($tt:tt)*) => {{499 (@o($o:ident) . $var:ident $($tt:tt)*) => {{379 $o.push(Index::attr($var));500 $o.push(Index::attr(stringify!($var)));380 nix_path!(@o($o) $($tt)*);501 nix_go!(@o($o) $($tt)*);381 }};502 }};382 (@o($o:ident) . { $var:expr } $($tt:tt)*) => {{503 (@o($o:ident) [{ $v:expr }] $($tt:tt)*) => {{383 $o.push(Index::attr($var));504 $o.push(Index::attr(&$v));384 nix_path!(@o($o) $($tt)*);505 nix_go!(@o($o) $($tt)*);385 }};506 }};386 (@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{507 (@o($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{387 $o.push(Index::idx($var));508 $o.push(Index::Expr($crate::nix_expr_inner!($($var)+)));388 nix_path!(@o($o) $($tt)*);509 nix_go!(@o($o) $($tt)*);389 }};510 }};390 (@o($o:ident) ($e:expr) $($tt:tt)*) => {511 (@o($o:ident) ($($var:tt)*) $($tt:tt)*) => {391 $o.push(Index::apply($e));512 $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+)));392 nix_path!(@o($o) $($tt)*);513 nix_go!(@o($o) $($tt)*);393 };514 };394 (@o($o:ident)) => {};515 (@o($o:ident)) => {};395 ($($tt:tt)+) => {{516 ($field:ident $($tt:tt)+) => {{396 use $crate::{nix_path, better_nix_eval::Index};517 use $crate::{nix_go, better_nix_eval::Index};518 let field = $field.clone();397 let mut out = vec![];519 let mut out = vec![];398 nix_path!(@o(out) $($tt)*);520 nix_go!(@o(out) $($tt)*);399 out521 field.select(out).await?400 }}522 }}401}523}524#[macro_export]525macro_rules! nix_go_json {526 ($($tt:tt)*) => {{527 $crate::nix_go!($($tt)*).as_json().await?528 }};529}402530403#[derive(Clone)]531#[derive(Clone)]404pub enum Index {532pub enum Index {405 Var(String),533 Var(String),406 String(String),534 String(String),407 Apply(String),535 Apply(String),408 Idx(u32),536 Expr(NixExprBuilder),537 ExprApply(NixExprBuilder),409}538}410impl Index {539impl Index {411 pub fn var(v: impl AsRef<str>) -> Self {540 pub fn var(v: impl AsRef<str>) -> Self {419 pub fn attr(v: impl AsRef<str>) -> Self {548 pub fn attr(v: impl AsRef<str>) -> Self {420 Self::String(v.as_ref().to_owned())549 Self::String(v.as_ref().to_owned())421 }550 }422 pub fn idx(v: u32) -> Self {423 Self::Idx(v)424 }425 pub fn apply(v: impl Serialize) -> Self {551 pub fn apply(v: impl Serialize) -> Self {426 let serialized = nixlike::serialize(v).expect("invalid value for apply");552 let serialized = nixlike::serialize(v).expect("invalid value for apply");427 Self::Apply(serialized.trim_end().to_owned())553 Self::Apply(serialized.trim_end().to_owned())440 Index::Apply(o) => {566 Index::Apply(o) => {441 write!(f, "<apply>({o})")567 write!(f, "<apply>({o})")442 }568 }443 Index::Idx(i) => {569 Index::Expr(e) => {444 write!(f, "[{i}]")570 write!(f, "[{}]", e.out)445 }571 }572 Index::ExprApply(e) => {573 write!(f, "<apply>({})", e.out)574 }446 }575 }447 }576 }448}577}460 Ok(())589 Ok(())461 }590 }462}591}463pub struct Field {592struct FieldInner {464 full_path: Vec<Index>,593 full_path: Option<Vec<Index>>,465 session: NixSession,594 session: NixSession,466 value: Option<u32>,595 value: Option<u32>,467}596}597fn context(full_path: Option<&[Index]>, query: &str) -> String {598 if let Some(full_path) = &full_path {599 format!("full path: {}", PathDisplay(full_path))600 } else {601 format!("query: {query:?}")602 }603}604#[derive(Clone)]605pub struct Field(Arc<FieldInner>);468impl Field {606impl Field {469 fn root(session: NixSession) -> Self {607 fn root(session: NixSession) -> Self {470 Self {608 Self(Arc::new(FieldInner {471 full_path: vec![],609 full_path: Some(vec![]),472 session,610 session,473 value: None,611 value: None,474 }612 }))475 }613 }614 async fn new(session: NixSession, query: &str) -> Result<Self> {615 let vid = session616 .0617 .lock()618 .await619 .execute_assign(query)620 .await621 .with_context(|| context(None, query))?;622 Ok(Self(Arc::new(FieldInner {623 full_path: None,624 session,625 value: Some(vid),626 })))627 }476 pub async fn field(session: NixSession, field: &str) -> Result<Self> {628 pub async fn field(session: NixSession, field: &str) -> Result<Self> {477 Self::root(session)629 Self::root(session).select([Index::var(field)]).await478 .select([Index::var(field)])486 field.as_json().await636 field.as_json().await487 }637 }488 pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {638 pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {639 let mut used_fields = Vec::new();489 let mut name = name.into_iter();640 let mut name = name.into_iter();490641491 let mut full_path = self.full_path.clone();642 let mut full_path = self.0.full_path.clone();492 let mut query = if let Some(id) = self.value {643 let mut query = if let Some(id) = self.0.value {493 format!("sess_field_{id}")644 format!("sess_field_{id}")494 } else {645 } else {495 let first = name.next();646 let first = name.next();496 if let Some(Index::Var(i)) = first {647 if let Some(Index::Var(i)) = first {648 if let Some(full_path) = &mut full_path {497 full_path.push(Index::Var(i.clone()));649 full_path.push(Index::Var(i.clone()));650 }498 i.clone()651 i.clone()499 } else {652 } else {500 panic!("first path item should be variable, got {first:?}")653 panic!("first path item should be variable, got {first:?}")501 }654 }502 };655 };503 for v in name {656 for v in name {657 if let Some(full_path) = &mut full_path {504 full_path.push(v.clone());658 full_path.push(v.clone());659 }505 match v {660 match v {506 Index::Var(_) => panic!("var item may only be first"),661 Index::Var(_) => panic!("var item may only be first"),507 Index::String(s) => {662 Index::String(s) => {513 // In cases like `a {}.b` first `{}.b` will be evaluated, so `a {}` should be encased in `()`668 // In cases like `a {}.b` first `{}.b` will be evaluated, so `a {}` should be encased in `()`514 query = format!("({query} {a})");669 query = format!("({query} {a})");515 }670 }671 Index::Expr(e) => {672 let index = Field::new(self.0.session.clone(), &e.out).await?;673 used_fields.push(index.clone());674 query.push('.');675 let index = format!("${{sess_field_{}}}", index.0.value.expect("value"));676 query.push_str(&index);677 }516 Index::Idx(idx) => {678 Index::ExprApply(e) => {679 let index = Field::new(self.0.session.clone(), &e.out).await?;680 used_fields.push(index.clone());681 query.push(' ');682 let index = format!("sess_field_{}", index.0.value.expect("value"));683 query.push_str(&index);517 query = format!("builtins.elemAt ({query}) {idx}");684 query = format!("({query})");518 }685 }519 }686 }520 }687 }521688522 let vid = self689 let vid = self690 .0523 .session691 .session524 .0692 .0525 .lock()693 .lock()526 .await694 .await527 .execute_assign(&query)695 .execute_assign(&query)528 .await696 .await529 .with_context(|| format!("full path: {}", PathDisplay(&full_path)))?;697 .with_context(|| {698 if let Some(full_path) = &full_path {699 format!("full path: {}", PathDisplay(full_path))700 } else {701 format!("query: {query:?}")702 }703 })?;530 Ok(Self {704 Ok(Self(Arc::new(FieldInner {531 full_path,705 full_path,532 session: self.session.clone(),706 session: self.0.session.clone(),533 value: Some(vid),707 value: Some(vid),534 })708 })))535 }709 }536 pub async fn as_json<V: DeserializeOwned>(&self) -> Result<V> {710 pub async fn as_json<V: DeserializeOwned>(&self) -> Result<V> {537 let id = self.value.expect("can't serialize root field");711 let id = self.0.value.expect("can't serialize root field");712 let query = format!("sess_field_{id}");538 self.session713 self.0714 .session539 .0715 .0540 .lock()716 .lock()541 .await717 .await542 .execute_expression_to_json(&format!("sess_field_{id}"))718 .execute_expression_to_json(&query)543 .await719 .await544 .with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))720 .with_context(|| context(self.0.full_path.as_deref(), &query))545 }721 }546 pub async fn list_fields(&self) -> Result<Vec<String>> {722 pub async fn list_fields(&self) -> Result<Vec<String>> {547 let id = self.value.expect("can't list root fields");723 let id = self.0.value.expect("can't list root fields");724 let query = format!("builtins.attrNames sess_field_{id}");548 self.session725 self.0726 .session549 .0727 .0550 .lock()728 .lock()551 .await729 .await552 .execute_expression_to_json(&format!("builtins.attrNames sess_field_{id}"))730 .execute_expression_to_json(&query)553 .await731 .await554 .with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))732 .with_context(|| context(self.0.full_path.as_deref(), &query))555 }733 }556 pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {734 pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {557 let id = self.value.expect("can't use build on not-value");735 let id = self.0.value.expect("can't use build on not-value");736 let query = format!(":b sess_field_{id}");558 let vid = self737 let vid = self738 .0559 .session739 .session560 .0740 .0561 .lock()741 .lock()562 .await742 .await563 .execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default())743 .execute_expression_raw(&query, &mut NixHandler::default())564 .await?;744 .await?;565 ensure!(!vid.is_empty(), "build failed: {}", PathDisplay(&self.full_path));745 ensure!(746 !vid.is_empty(),747 "build failed: {}",748 context(self.0.full_path.as_deref(), &query),749 );566 let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")750 let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")567 else {751 else {576 Ok(outputs)760 Ok(outputs)577 }761 }578}762}579impl Drop for Field {763impl Drop for FieldInner {580 fn drop(&mut self) {764 fn drop(&mut self) {581 if let Some(id) = self.value {765 if let Some(id) = self.value {582 if let Ok(mut lock) = self.session.0.try_lock() {766 if let Ok(mut lock) = self.session.0.try_lock() {cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth445use crate::command::MyCommand;5use crate::command::MyCommand;6use crate::host::Config;6use crate::host::Config;7use crate::nix_path;7use crate::nix_go;8use anyhow::{anyhow, Result, Context};8use anyhow::{anyhow, Result};9use clap::Parser;9use clap::Parser;10use itertools::Itertools;10use itertools::Itertools;11use tokio::{task::LocalSet, time::sleep};11use tokio::{task::LocalSet, time::sleep};290 async fn build_task(self, config: Config, host: String) -> Result<()> {290 async fn build_task(self, config: Config, host: String) -> Result<()> {291 info!("building");291 info!("building");292 let action = Action::from(self.subcommand.clone());292 let action = Action::from(self.subcommand.clone());293 let drv = config293 let fleet_field = &config.fleet_field;294 .fleet_field295 .select(nix_path!(.buildSystems((serde_json::json!({294 let drv = nix_go!(fleet_field.buildSystems(Obj {296 "localSystem": config.local_system.clone(),295 localSystem: { config.local_system.clone() }297 }))).{action.build_attr()}.{&host}))296 }));298 .await.context("system attribute")?;299 let outputs = drv.build().await.map_err(|e| {297 let outputs = drv.build().await.map_err(|e| {300 if action.build_attr() == "sdImage" {298 if action.build_attr() == "sdImage" {301 info!("sd-image build failed");299 info!("sd-image build failed");cmds/fleet/src/cmds/info.rsdiffbeforeafterboth1use std::collections::BTreeSet;1use std::collections::BTreeSet;223use crate::host::Config;3use crate::host::Config;4use crate::nix_path;4use crate::nix_go_json;5use anyhow::{ensure, Result};5use anyhow::{ensure, Result};6use clap::Parser;6use clap::Parser;7737 InfoCmd::ListHosts { ref tagged } => {37 InfoCmd::ListHosts { ref tagged } => {38 'host: for host in config.list_hosts().await? {38 'host: for host in config.list_hosts().await? {39 if !tagged.is_empty() {39 if !tagged.is_empty() {40 let fleet_field = &config.fleet_field;40 let tags: Vec<String> = config41 let tags: Vec<String> =41 .fleet_field42 .select(nix_path!(.configuredSystems.{&host.name}.config.tags))42 nix_go_json!(fleet_field.configuredSystems[{ host.name }].config.tags);43 .await?44 .as_json()45 .await?;46 for tag in tagged {43 for tag in tagged {47 if !tags.contains(tag) {44 if !tags.contains(tag) {48 continue 'host;45 continue 'host;64 let mut out = <BTreeSet<String>>::new();61 let mut out = <BTreeSet<String>>::new();65 let host = config.system_config(&host).await?;62 let host = config.system_config(&host).await?;66 if external {63 if external {67 out.extend(68 host.select(nix_path!(.network.externalIps))64 let data: Vec<String> = nix_go_json!(host.network.externalIps);69 .await?65 out.extend(data);70 .as_json::<Vec<String>>()71 .await?,72 );73 }66 }74 if internal {67 if internal {75 out.extend(76 host.select(nix_path!(.network.internalIps))68 let data: Vec<String> = nix_go_json!(host.network.internalIps);77 .await?69 out.extend(data);78 .as_json::<Vec<String>>()79 .await?,80 );81 }70 }82 for ip in out {71 for ip in out {83 data.push(ip);72 data.push(ip);cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth1use crate::{1use crate::{2 fleetdata::{FleetSecret, FleetSharedSecret},2 fleetdata::{FleetSecret, FleetSharedSecret},3 host::Config, nix_path,3 host::Config,4 nix_go, nix_go_json,4};5};5use anyhow::{bail, ensure, Context, Result};6use anyhow::{anyhow, bail, ensure, Context, Result};6use chrono::Utc;7use chrono::{DateTime, Utc};7use clap::Parser;8use clap::Parser;8use futures::{StreamExt, TryStreamExt};9use futures::{StreamExt, TryStreamExt};9use owo_colors::OwoColorize;10use owo_colors::OwoColorize;17use tracing::{error, info, info_span, warn};18use tracing::{error, info, info_span, warn};181919#[derive(Parser)]20#[derive(Parser)]20pub enum Secrets {21pub enum Secret {21 /// Force load keys for all defined hosts22 /// Force load host keys for all defined hosts22 ForceKeys,23 ForceKeys,23 /// Add secret, data should be provided in stdin24 /// Add secret, data should be provided in stdin24 AddShared {25 AddShared {29 /// Override secret if already present30 /// Override secret if already present30 #[clap(long)]31 #[clap(long)]31 force: bool,32 force: bool,33 /// Secret public part32 #[clap(long)]34 #[clap(long)]33 public: Option<String>,35 public: Option<String>,36 /// Load public part from specified file34 #[clap(long)]37 #[clap(long)]35 public_file: Option<PathBuf>,38 public_file: Option<PathBuf>,3940 /// Create a notification on secret expiration41 #[clap(long)]42 expires_at: Option<DateTime<Utc>>,364337 /// Secret with this name already exists, override its value while keeping the same owners.44 /// Secret with this name already exists, override its value while keeping the same owners.38 #[clap(long)]45 #[clap(long)]39 readd: bool,46 re_add: bool,40 },47 },41 /// Add secret, data should be provided in stdin48 /// Add secret, data should be provided in stdin42 Add {49 Add {81 prefer_identities: Vec<String>,88 prefer_identities: Vec<String>,82 },89 },83 List {},90 List {},91 InvokeGenerator,84}92}859386impl Secrets {94impl Secret {87 pub async fn run(self, config: &Config) -> Result<()> {95 pub async fn run(self, config: &Config) -> Result<()> {88 match self {96 match self {97 Secret::InvokeGenerator => {98 let config_field = &config.config_unchecked_field;99100 let generate_impure =101 nix_go!(config_field.sharedSecrets["kube-apiserver.pem"].generateImpure);102 let on = nix_go!(generate_impure.on);103 let call_package = nix_go!(104 config_field.buildableSystems(Obj {105 localSystem: { config.local_system.clone() }106 })[on]107 .config108 .nixpkgs109 .pkgs110 .callPackage111 );112 let generator = nix_go!(call_package(generate_impure.generator));113 let built = generator.build().await?;114 // .as_json().await?;115 dbg!(&built);116 }89 Secrets::ForceKeys => {117 Secret::ForceKeys => {90 for host in config.list_hosts().await? {118 for host in config.list_hosts().await? {91 if config.should_skip(&host.name) {119 if config.should_skip(&host.name) {92 continue;120 continue;93 }121 }94 config.key(&host.name).await?;122 config.key(&host.name).await?;95 }123 }96 }124 }97 Secrets::AddShared {125 Secret::AddShared {98 mut machines,126 mut machines,99 name,127 name,100 force,128 force,101 public,129 public,102 public_file,130 public_file,103 readd,131 expires_at,132 re_add,104 } => {133 } => {105 let exists = config.has_shared(&name);134 let exists = config.has_shared(&name);106 if exists && !force && !readd {135 if exists && !force && !re_add {107 bail!("secret already defined");136 bail!("secret already defined");108 }137 }109 if readd {138 if re_add {110 // Fixme: use clap to limit this usage139 // Fixme: use clap to limit this usage111 ensure!(!force, "--force and --readd are not compatible");140 ensure!(!force, "--force and --readd are not compatible");112 ensure!(exists, "secret doesn't exists");141 ensure!(exists, "secret doesn't exists");137 .map(|r| Box::new(r) as Box<dyn age::Recipient + Send>)166 .map(|r| Box::new(r) as Box<dyn age::Recipient + Send>)138 .collect();167 .collect();139 let mut encryptor = age::Encryptor::with_recipients(recipients)168 let mut encryptor = age::Encryptor::with_recipients(recipients)140 .expect("recipients provided")169 .ok_or_else(|| anyhow!("no recipients provided"))?141 .wrap_output(&mut encrypted)?;170 .wrap_output(&mut encrypted)?;142 io::copy(&mut Cursor::new(input), &mut encryptor)?;171 io::copy(&mut Cursor::new(input), &mut encryptor)?;143 encryptor.finish()?;172 encryptor.finish()?;150 owners: machines,179 owners: machines,151 secret: FleetSecret {180 secret: FleetSecret {152 created_at: Utc::now(),181 created_at: Utc::now(),153 expires_at: None,182 expires_at,154 secret,183 secret,155 public: match (public, public_file) {184 public: match (public, public_file) {156 (Some(v), None) => Some(v),185 (Some(v), None) => Some(v),164 },193 },165 );194 );166 }195 }167 Secrets::Add {196 Secret::Add {168 machine,197 machine,169 name,198 name,170 force,199 force,211 }240 }212 // TODO: Instead of using sudo, decode secret on remote machine241 // TODO: Instead of using sudo, decode secret on remote machine213 #[allow(clippy::await_holding_refcell_ref)]242 #[allow(clippy::await_holding_refcell_ref)]214 Secrets::Read {243 Secret::Read {215 name,244 name,216 machine,245 machine,217 plaintext,246 plaintext,228 println!("{}", z85::encode(&data));257 println!("{}", z85::encode(&data));229 }258 }230 }259 }231 Secrets::UpdateShared {260 Secret::UpdateShared {232 name,261 name,233 machines,262 machines,234 mut add_machines,263 mut add_machines,321 secret.secret.secret = encrypted;350 secret.secret.secret = encrypted;322 config.replace_shared(name, secret);351 config.replace_shared(name, secret);323 }352 }324 Secrets::Regenerate { prefer_identities } => {353 Secret::Regenerate { prefer_identities } => {325 {354 {326 let expected_shared_set = config355 let expected_shared_set = config327 .list_configured_shared()356 .list_configured_shared()337 for name in &config.list_shared() {366 for name in &config.list_shared() {338 info!("updating secret: {name}");367 info!("updating secret: {name}");339 let mut data = config.shared_secret(name)?;368 let mut data = config.shared_secret(name)?;369 let config_field = &config.config_field;340 let expected_owners: Vec<String> = config370 let expected_owners: Vec<String> =341 .config_field342 .get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners))371 nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners);343 .await?;344 if expected_owners.is_empty() {372 if expected_owners.is_empty() {345 warn!("secret was removed from fleet config: {name}, removing from data");373 warn!("secret was removed from fleet config: {name}, removing from data");346 to_remove.push(name.to_string());374 to_remove.push(name.to_string());350 let expected_set = expected_owners.iter().collect::<HashSet<_>>();378 let expected_set = expected_owners.iter().collect::<HashSet<_>>();351 let should_remove = set.difference(&expected_set).next().is_some();379 let should_remove = set.difference(&expected_set).next().is_some();352 if set != expected_set {380 if set != expected_set {353 let owner_dependent: bool = config381 let owner_dependent: bool =354 .config_field355 .get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent))382 nix_go_json!(config_field.sharedSecrets[{ name }].ownerDependent);356 .await?;357 if !owner_dependent {383 if !owner_dependent {358 warn!("reencrypting secret '{name}' for new owner set");384 warn!("reencrypting secret '{name}' for new owner set");359 // TODO: force regeneration385 // TODO: force regeneration401 config.remove_shared(&k);427 config.remove_shared(&k);402 }428 }403 }429 }404 Secrets::List {} => {430 Secret::List {} => {405 let _span = info_span!("loading secrets").entered();431 let _span = info_span!("loading secrets").entered();406 let configured = config.list_configured_shared().await?;432 let configured = config.list_configured_shared().await?;407 #[derive(Tabled)]433 #[derive(Tabled)]cmds/fleet/src/command.rsdiffbeforeafterboth337 if !text.is_empty()337 if !text.is_empty()338 && text != "querying info about missing paths"338 && text != "querying info about missing paths"339 && text != "copying 0 paths"339 && text != "copying 0 paths"340 // Too much spam on lazy-trees branch341 && !(text.starts_with("copying '") && text.ends_with("' to the store"))340 {342 {341 let span = info_span!("job");343 let span = info_span!("job");342 span.pb_start();344 span.pb_start();cmds/fleet/src/host.rsdiffbeforeafterboth16 better_nix_eval::{Field, NixSessionPool},16 better_nix_eval::{Field, NixSessionPool},17 command::MyCommand,17 command::MyCommand,18 fleetdata::{FleetData, FleetSecret, FleetSharedSecret},18 fleetdata::{FleetData, FleetSecret, FleetSharedSecret},19 nix_path,19 nix_go, nix_go_json,20};20};212122pub struct FleetConfigInternals {22pub struct FleetConfigInternals {29 pub fleet_field: Field,29 pub fleet_field: Field,30 /// fleet_config.configUnchecked30 /// fleet_config.configUnchecked31 pub config_field: Field,31 pub config_field: Field,32 /// fleet_config.unchecked33 pub config_unchecked_field: Field,32}34}333534#[derive(Clone)]36#[derive(Clone)]95 }97 }969897 pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {99 pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {100 let fleet_field = &self.fleet_field;98 let names = self101 let names = nix_go!(fleet_field.configuredHosts).list_fields().await?;99 .fleet_field100 .select(nix_path!(.configuredHosts))101 .await?108 Ok(out)106 Ok(out)109 }107 }110 pub async fn system_config(&self, host: &str) -> Result<Field> {108 pub async fn system_config(&self, host: &str) -> Result<Field> {111 self.fleet_field109 let fleet_field = &self.fleet_field;112 .select(nix_path!(.configuredSystems.{host}.config))110 Ok(nix_go!(fleet_field.configuredSystems[{ host }].config))113 .await114 }111 }115112116 pub(super) fn data(&self) -> MutexGuard<FleetData> {113 pub(super) fn data(&self) -> MutexGuard<FleetData> {121 }118 }122 /// Shared secrets configured in fleet.nix or in flake119 /// Shared secrets configured in fleet.nix or in flake123 pub async fn list_configured_shared(&self) -> Result<Vec<String>> {120 pub async fn list_configured_shared(&self) -> Result<Vec<String>> {124 self.config_field121 let config_field = &self.config_field;125 .select(nix_path!(.sharedSecrets))122 nix_go!(config_field.sharedSecrets).list_fields().await126 .await?127 .list_fields()128 .await129 }123 }211 Ok(secret.clone())205 Ok(secret.clone())212 }206 }213 pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {207 pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {214 self.config_field208 let config_field = &self.config_field;215 .select(nix_path!(.sharedSecrets.{secret}.expectedOwners))209 Ok(nix_go_json!(216 .await?210 config_field.sharedSecrets[{ secret }].expectedOwners217 .as_json()211 ))218 .await219 }212 }220213221 pub fn save(&self) -> Result<()> {214 pub fn save(&self) -> Result<()> {269262270 if self.local_system == "detect" {263 if self.local_system == "detect" {271 let builtins_field = Field::field(root_field.clone(), "builtins").await?;264 let builtins_field = Field::field(root_field.clone(), "builtins").await?;272 let system = builtins_field273 .select(nix_path!(.currentSystem))274 .await?;275 self.local_system = system.as_json().await?;265 self.local_system = nix_go_json!(builtins_field.currentSystem);276 }266 }277 let local_system = self.local_system.clone();267 let local_system = self.local_system.clone();278268279 let fleet_root = Field::field(root_field, "fleetConfigurations").await?;269 let fleet_root = Field::field(root_field, "fleetConfigurations").await?;280270281 let fleet_field = fleet_root271 let fleet_field = nix_go!(fleet_root.default);282 .select(nix_path!(.default))283 .await?;284 let config_field = fleet_field272 let config_field = nix_go!(fleet_field.configUnchecked);285 .select(nix_path!(.configUnchecked))273 let config_unchecked_field = nix_go!(fleet_field.unchecked);286 .await?;287274288 let mut fleet_data_path = directory.clone();275 let mut fleet_data_path = directory.clone();289 fleet_data_path.push("fleet.nix");276 fleet_data_path.push("fleet.nix");298 nix_args,285 nix_args,299 fleet_field,286 fleet_field,300 config_field,287 config_field,288 config_unchecked_field,301 })))289 })))302 }290 }303}291}cmds/fleet/src/main.rsdiffbeforeafterboth1#![recursion_limit = "512"]1#![recursion_limit = "512"]2#![feature(try_blocks)]2#![feature(try_blocks, lint_reasons)]334pub(crate) mod cmds;4pub(crate) mod cmds;5pub(crate) mod command;5pub(crate) mod command;17use anyhow::{bail, Result};17use anyhow::{bail, Result};18use clap::Parser;18use clap::Parser;191920use cmds::{build_systems::BuildSystems, info::Info, secrets::Secrets};20use cmds::{build_systems::BuildSystems, info::Info, secrets::Secret};21use futures::future::LocalBoxFuture;21use futures::future::LocalBoxFuture;22use futures::stream::FuturesUnordered;22use futures::stream::FuturesUnordered;23use futures::TryStreamExt;23use futures::TryStreamExt;73 BuildSystems(BuildSystems),73 BuildSystems(BuildSystems),74 /// Secret management74 /// Secret management75 #[clap(subcommand)]75 #[clap(subcommand)]76 Secrets(Secrets),76 Secret(Secret),77 /// Upload prefetch directory to the nix store77 /// Upload prefetch directory to the nix store78 Prefetch(Prefetch),78 Prefetch(Prefetch),79 /// Config parsing79 /// Config parsing92async fn run_command(config: &Config, command: Opts) -> Result<()> {92async fn run_command(config: &Config, command: Opts) -> Result<()> {93 match command {93 match command {94 Opts::BuildSystems(c) => c.run(config).await?,94 Opts::BuildSystems(c) => c.run(config).await?,95 Opts::Secrets(s) => s.run(config).await?,95 Opts::Secret(s) => s.run(config).await?,96 Opts::Info(i) => i.run(config).await?,96 Opts::Info(i) => i.run(config).await?,97 Opts::Prefetch(p) => p.run(config).await?,97 Opts::Prefetch(p) => p.run(config).await?,98 };98 };flake.lockdiffbeforeafterboth5 "systems": "systems"5 "systems": "systems"6 },6 },7 "locked": {7 "locked": {8 "lastModified": 1694529238,8 "lastModified": 1701680307,9 "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",9 "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",10 "owner": "numtide",10 "owner": "numtide",11 "repo": "flake-utils",11 "repo": "flake-utils",12 "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",12 "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",13 "type": "github"13 "type": "github"14 },14 },15 "original": {15 "original": {38 },38 },39 "nixpkgs": {39 "nixpkgs": {40 "locked": {40 "locked": {41 "lastModified": 1698350982,41 "lastModified": 1703705939,42 "narHash": "sha256-zoEV8Ad3bOAejp0ys/mOpaHSWrzK+GupZwGGYfuWuEY=",42 "narHash": "sha256-9s2Ep3NyRDj9HUgfv2TQUwQEanRUAmeXkvKIr/o1XbY=",43 "owner": "nixos",43 "owner": "nixos",44 "repo": "nixpkgs",44 "repo": "nixpkgs",45 "rev": "dd83f9de26ff7c0326468b659ea4729fa5cf6262",45 "rev": "1ada32da4ba24d7310653c9ac54888bee463f455",46 "type": "github"46 "type": "github"47 },47 },48 "original": {48 "original": {67 ]67 ]68 },68 },69 "locked": {69 "locked": {70 "lastModified": 1698199907,70 "lastModified": 1703643208,71 "narHash": "sha256-n8RtHBIb0rLuYs4RDehW6mj6r6Yam/ODY1af/VCcurw=",71 "narHash": "sha256-UL4KO8JxnD5rOycwHqBAf84lExF1/VnYMDC7b/wpPDU=",72 "owner": "oxalica",72 "owner": "oxalica",73 "repo": "rust-overlay",73 "repo": "rust-overlay",74 "rev": "22b8d29fd22cfaa2c311e0d6fd8a0ed9c2a1152b",74 "rev": "ce117f3e0de8262be8cd324ee6357775228687cf",75 "type": "github"75 "type": "github"76 },76 },77 "original": {77 "original": {flake.nixdiffbeforeafterboth9 };7 flake-utils = { url = "github:numtide/flake-utils"; };10 flake-utils = {url = "github:numtide/flake-utils";};8 };11 };9 outputs = { self, rust-overlay, flake-utils, nixpkgs }: with nixpkgs.lib; rec {12 outputs = {13 self,14 rust-overlay,15 flake-utils,16 nixpkgs,17 }:18 with nixpkgs.lib;19 {27 overlays = [(import rust-overlay)];16 };28 };17 llvmPkgs = pkgs.buildPackages.llvmPackages_11;29 llvmPkgs = pkgs.buildPackages.llvmPackages_11;18 rust = (pkgs.rustChannelOf { date = "2023-10-20"; channel = "nightly"; }).default.override { extensions = [ "rust-src" "rust-analyzer" ]; };30 rust =19 rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; };31 (pkgs.rustChannelOf {32 date = "2023-12-26";33 channel = "nightly";34 })35 .default27 cargo-edit43 cargo-edit28 cargo-udeps44 cargo-udeps29 cargo-fuzz45 cargo-fuzz46 cargo-watch304731 pkg-config48 pkg-config32 openssl49 openssllib/default.nixdiffbeforeafterboth12 };12 };13 in13 in let14 let15 withData = data: rec {16 root = nixpkgs.lib.evalModules {14 root = nixpkgs.lib.evalModules {17 modules = (import ../modules/fleet/_modules.nix) ++ [config data];15 modules = (import ../modules/fleet/_modules.nix) ++ [config data];18 specialArgs = {16 specialArgs = {19 inherit nixpkgs fleetLib;17 inherit nixpkgs fleetLib;20 };18 };21 };19 };22 failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);20 failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);23 rootAssertWarn =21 checkedRoot =24 if failedAssertions != []22 if failedAssertions != []25 then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"23 then throw "Fleet failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"26 else nixpkgs.lib.showWarnings root.config.warnings root;24 else nixpkgs.lib.showWarnings root.config.warnings root;25 withData = {26 root,27 data,28 }: rec {27 configuredHosts = rootAssertWarn.config.hosts;29 configuredHosts = root.config.hosts;28 configuredSecrets = rootAssertWarn.config.secrets;30 configuredUncheckedHosts = root.config.hosts;29 configuredSystems = configuredSystemsWithExtraModules [];31 configuredSystems = configuredSystemsWithExtraModules [];30 configuredSystemsWithExtraModules = extraModules:32 configuredSystemsWithExtraModules = extraModules:31 nixpkgs.lib.listToAttrs (33 nixpkgs.lib.listToAttrs (43 };45 };44 }46 }45 )47 )46 (builtins.attrNames rootAssertWarn.config.hosts)48 (builtins.attrNames root.config.hosts)47 );49 );50 buildableSystems = {localSystem}: let51 buildConfigurationModule = {config, ...}: {52 # Equivalent to nixpkgs.localSystem53 # nixpkgs.system = localSystem;54 nixpkgs.buildPlatform.system = localSystem;55 };56 in57 configuredSystemsWithExtraModules [58 buildConfigurationModule59 ];48 buildSystems = {localSystem}: let60 buildSystems = {localSystem}: let49 buildConfigurationModule = {config, ...}: {61 buildConfigurationModule = {config, ...}: {50 # Equivalent to nixpkgs.localSystem62 # Equivalent to nixpkgs.localSystem76 ]);88 ]);77 };89 };78 configUnchecked = root.config;90 configUnchecked = root.config;79 };91 };80 defaultData = withData data;92 defaultData = withData {93 inherit data;94 root = checkedRoot;95 };96 uncheckedData = withData {inherit data root;};81 in rec {97 in rec {82 inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;98 inherit (defaultData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems;99 unchecked = {100 inherit (uncheckedData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems;101 };83 injectData = data: let102 injectData = data: let84 injectedData = withData data;103 injectedData = withData data;85 in {104 in {86 inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;105 inherit (injectedData) configuredHosts configuredSystems buildSystems configUnchecked;87 };106 };88 };107 };89}108}modules/fleet/secrets.nixdiffbeforeafterboth15 type = bool;15 type = bool;16 description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted";16 description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted";17 };17 };18 generateImpure = mkOption {19 type = unspecified;20 };18 generator = mkOption {21 generator = mkOption {19 type = nullOr (submodule {22 type = nullOr (submodule {20 packages = mkOption {23 packages = mkOption {