git.delta.rocks / jrsonnet / refs/commits / 7e2e5c591e04

difftreelog

refactor more repl abstractions

Yaroslav Bolyukin2023-12-27parent: #624fe7e.patch.diff
in: trunk

11 files changed

modifiedcmds/fleet/src/better_nix_eval.rsdiffbeforeafterboth
365#[derive(Clone)]365#[derive(Clone)]
366pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);366pub struct NixSession(Arc<tokio::sync::Mutex<PooledConnection<NixSessionPoolInner>>>);
367
368#[derive(Clone)]
369pub struct NixExprBuilder {
370 out: String,
371 used_fields: Vec<Field>,
372}
373impl NixExprBuilder {
374 pub fn object() -> Self {
375 NixExprBuilder {
376 out: "{ ".to_owned(),
377 used_fields: Vec::new(),
378 }
379 }
380 pub fn string(s: &str) -> Self {
381 NixExprBuilder {
382 out: nixlike::serialize(s)
383 .expect("no problems with serializing_string")
384 .trim_end()
385 .to_owned(),
386 used_fields: Vec::new(),
387 }
388 }
389 pub fn serialized(v: impl Serialize) -> Self {
390 let serialized = nixlike::serialize(v).expect("invalid value for apply");
391 Self {
392 out: serialized.trim_end().to_owned(),
393 used_fields: Vec::new(),
394 }
395 }
396 pub fn field(f: Field) -> Self {
397 Self {
398 out: format!("sess_field_{}", f.0.value.expect("no value")),
399 used_fields: vec![f],
400 }
401 }
402 pub fn end_obj(&mut self) {
403 self.out.push('}');
404 }
405 pub fn obj_key(&mut self, name: Self, value: Self) {
406 self.out.push_str(r#""${"#);
407 self.extend(name);
408 self.out.push_str(r#"}" = "#);
409 self.extend(value);
410 self.out.push_str("; ");
411 }
412
413 pub fn extend(&mut self, e: Self) {
414 self.out.push_str(&e.out);
415 self.used_fields.extend(e.used_fields);
416 }
417
418 pub fn session(&self) -> NixSession {
419 let mut session = None;
420 for ele in &self.used_fields {
421 if session.is_none() {
422 session = Some(ele.0.session.clone());
423 continue;
424 }
425 let session = &session.as_ref().expect("checked").0;
426 let ele_sess = &ele.0.session.0;
427 assert!(
428 Arc::ptr_eq(session, ele_sess),
429 "can't mix fields from different session"
430 );
431 }
432 session.expect("expr without fields used")
433 }
434 pub fn index_attr(&mut self, s: &str) {
435 let escaped = nixlike::serialize(s).expect("string");
436 self.out.push('.');
437 self.out.push_str(escaped.trim_end());
438 }
439}
367440
368#[macro_export]441#[macro_export]
369macro_rules! nix_path {442macro_rules! nix_expr_inner {
370 (@o($o:ident) $var:ident $($tt:tt)*) => {{443 (Obj { $($ident:ident: $($val:tt)+),* $(,)? }) => {{
371 $o.push(Index::var(stringify!($var)));444 use $crate::better_nix_eval::NixExprBuilder;
445 let mut out = NixExprBuilder::object();
446 $(
447 out.obj_key(
448 NixExprBuilder::string(stringify!($ident)),
449 $crate::nix_expr_inner!($($val)+),
450 );
451 )*
452 out.end_obj();
453 out
454 }};
455 (@field($o:ident) . $var:ident $($tt:tt)*) => {{
456 $o.index_attr(stringify!($var));
457 nix_expr_inner!(@field($o) $($tt)*);
458 }};
459 (@field($o:ident) [{ $v:expr }] $($tt:tt)*) => {{
460 $o.push(Index::attr(&$v));
461 nix_expr_inner!(@o($o) $($tt)*);
462 }};
463 (@field($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{
464 $o.push(Index::Expr($crate::nix_expr_inner!($($var)+)));
465 nix_expr_inner!(@o($o) $($tt)*);
466 }};
467 (@field($o:ident) ($($var:tt)*) $($tt:tt)*) => {
468 $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+)));
469 nix_expr_inner!(@o($o) $($tt)*);
470 };
471 (@field($o:ident)) => {};
472 ($field:ident $($tt:tt)*) => {{
473 use $crate::{better_nix_eval::NixExprBuilder, nix_expr_inner};
474 #[allow(unused_mut, reason = "might be used if indexed")]
475 let mut out = NixExprBuilder::field($field);
476 nix_expr_inner!(@field(out) $($tt)*);
477 out
478 }};
479 ($v:literal) => {{
480 use $crate::better_nix_eval::NixExprBuilder;
481 NixExprBuilder::string($v)
482 }};
483 ({$v:expr}) => {{
484 use $crate::better_nix_eval::NixExprBuilder;
485 NixExprBuilder::serialized(&$v)
486 }}
487}
488#[macro_export]
372 nix_path!(@o($o) $($tt)*);489macro_rules! nix_expr {
373 }};
374 (@o($o:ident) . $var:ident $($tt:tt)*) => {{490 ($($tt:tt)+) => {{
375 $o.push(Index::attr(stringify!($var)));491 use $crate::{better_nix_eval::{NixExprBuilder, Field}, nix_expr_inner};
376 nix_path!(@o($o) $($tt)*);492 let expr = nix_expr_inner!($($tt)+);
493 Field::new(expr.session(), expr.out)
377 }};494 }};
495}
496
497#[macro_export]
498macro_rules! nix_go {
378 (@o($o:ident) . $var:literal $($tt:tt)*) => {{499 (@o($o:ident) . $var:ident $($tt:tt)*) => {{
379 $o.push(Index::attr($var));500 $o.push(Index::attr(stringify!($var)));
380 nix_path!(@o($o) $($tt)*);501 nix_go!(@o($o) $($tt)*);
381 }};502 }};
382 (@o($o:ident) . { $var:expr } $($tt:tt)*) => {{503 (@o($o:ident) [{ $v:expr }] $($tt:tt)*) => {{
383 $o.push(Index::attr($var));504 $o.push(Index::attr(&$v));
384 nix_path!(@o($o) $($tt)*);505 nix_go!(@o($o) $($tt)*);
385 }};506 }};
386 (@o($o:ident) [ $var:literal ] $($tt:tt)*) => {{507 (@o($o:ident) [ $($var:tt)+ ] $($tt:tt)*) => {{
387 $o.push(Index::idx($var));508 $o.push(Index::Expr($crate::nix_expr_inner!($($var)+)));
388 nix_path!(@o($o) $($tt)*);509 nix_go!(@o($o) $($tt)*);
389 }};510 }};
390 (@o($o:ident) ($e:expr) $($tt:tt)*) => {511 (@o($o:ident) ($($var:tt)*) $($tt:tt)*) => {
391 $o.push(Index::apply($e));512 $o.push(Index::ExprApply($crate::nix_expr_inner!($($var)+)));
392 nix_path!(@o($o) $($tt)*);513 nix_go!(@o($o) $($tt)*);
393 };514 };
394 (@o($o:ident)) => {};515 (@o($o:ident)) => {};
395 ($($tt:tt)+) => {{516 ($field:ident $($tt:tt)+) => {{
396 use $crate::{nix_path, better_nix_eval::Index};517 use $crate::{nix_go, better_nix_eval::Index};
518 let field = $field.clone();
397 let mut out = vec![];519 let mut out = vec![];
398 nix_path!(@o(out) $($tt)*);520 nix_go!(@o(out) $($tt)*);
399 out521 field.select(out).await?
400 }}522 }}
401}523}
524#[macro_export]
525macro_rules! nix_go_json {
526 ($($tt:tt)*) => {{
527 $crate::nix_go!($($tt)*).as_json().await?
528 }};
529}
402530
403#[derive(Clone)]531#[derive(Clone)]
404pub enum Index {532pub enum Index {
405 Var(String),533 Var(String),
406 String(String),534 String(String),
407 Apply(String),535 Apply(String),
408 Idx(u32),536 Expr(NixExprBuilder),
537 ExprApply(NixExprBuilder),
409}538}
410impl Index {539impl Index {
411 pub fn var(v: impl AsRef<str>) -> Self {540 pub fn var(v: impl AsRef<str>) -> Self {
419 pub fn attr(v: impl AsRef<str>) -> Self {548 pub fn attr(v: impl AsRef<str>) -> Self {
420 Self::String(v.as_ref().to_owned())549 Self::String(v.as_ref().to_owned())
421 }550 }
422 pub fn idx(v: u32) -> Self {
423 Self::Idx(v)
424 }
425 pub fn apply(v: impl Serialize) -> Self {551 pub fn apply(v: impl Serialize) -> Self {
426 let serialized = nixlike::serialize(v).expect("invalid value for apply");552 let serialized = nixlike::serialize(v).expect("invalid value for apply");
427 Self::Apply(serialized.trim_end().to_owned())553 Self::Apply(serialized.trim_end().to_owned())
440 Index::Apply(o) => {566 Index::Apply(o) => {
441 write!(f, "<apply>({o})")567 write!(f, "<apply>({o})")
442 }568 }
443 Index::Idx(i) => {569 Index::Expr(e) => {
444 write!(f, "[{i}]")570 write!(f, "[{}]", e.out)
445 }571 }
572 Index::ExprApply(e) => {
573 write!(f, "<apply>({})", e.out)
574 }
446 }575 }
447 }576 }
448}577}
460 Ok(())589 Ok(())
461 }590 }
462}591}
463pub struct Field {592struct FieldInner {
464 full_path: Vec<Index>,593 full_path: Option<Vec<Index>>,
465 session: NixSession,594 session: NixSession,
466 value: Option<u32>,595 value: Option<u32>,
467}596}
597fn context(full_path: Option<&[Index]>, query: &str) -> String {
598 if let Some(full_path) = &full_path {
599 format!("full path: {}", PathDisplay(full_path))
600 } else {
601 format!("query: {query:?}")
602 }
603}
604#[derive(Clone)]
605pub struct Field(Arc<FieldInner>);
468impl Field {606impl Field {
469 fn root(session: NixSession) -> Self {607 fn root(session: NixSession) -> Self {
470 Self {608 Self(Arc::new(FieldInner {
471 full_path: vec![],609 full_path: Some(vec![]),
472 session,610 session,
473 value: None,611 value: None,
474 }612 }))
475 }613 }
614 async fn new(session: NixSession, query: &str) -> Result<Self> {
615 let vid = session
616 .0
617 .lock()
618 .await
619 .execute_assign(query)
620 .await
621 .with_context(|| context(None, query))?;
622 Ok(Self(Arc::new(FieldInner {
623 full_path: None,
624 session,
625 value: Some(vid),
626 })))
627 }
476 pub async fn field(session: NixSession, field: &str) -> Result<Self> {628 pub async fn field(session: NixSession, field: &str) -> Result<Self> {
477 Self::root(session)629 Self::root(session).select([Index::var(field)]).await
478 .select([Index::var(field)])
486 field.as_json().await636 field.as_json().await
487 }637 }
488 pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {638 pub async fn select<'a>(&self, name: impl IntoIterator<Item = Index>) -> Result<Self> {
639 let mut used_fields = Vec::new();
489 let mut name = name.into_iter();640 let mut name = name.into_iter();
490641
491 let mut full_path = self.full_path.clone();642 let mut full_path = self.0.full_path.clone();
492 let mut query = if let Some(id) = self.value {643 let mut query = if let Some(id) = self.0.value {
493 format!("sess_field_{id}")644 format!("sess_field_{id}")
494 } else {645 } else {
495 let first = name.next();646 let first = name.next();
496 if let Some(Index::Var(i)) = first {647 if let Some(Index::Var(i)) = first {
648 if let Some(full_path) = &mut full_path {
497 full_path.push(Index::Var(i.clone()));649 full_path.push(Index::Var(i.clone()));
650 }
498 i.clone()651 i.clone()
499 } else {652 } else {
500 panic!("first path item should be variable, got {first:?}")653 panic!("first path item should be variable, got {first:?}")
501 }654 }
502 };655 };
503 for v in name {656 for v in name {
657 if let Some(full_path) = &mut full_path {
504 full_path.push(v.clone());658 full_path.push(v.clone());
659 }
505 match v {660 match v {
506 Index::Var(_) => panic!("var item may only be first"),661 Index::Var(_) => panic!("var item may only be first"),
507 Index::String(s) => {662 Index::String(s) => {
513 // In cases like `a {}.b` first `{}.b` will be evaluated, so `a {}` should be encased in `()`668 // In cases like `a {}.b` first `{}.b` will be evaluated, so `a {}` should be encased in `()`
514 query = format!("({query} {a})");669 query = format!("({query} {a})");
515 }670 }
671 Index::Expr(e) => {
672 let index = Field::new(self.0.session.clone(), &e.out).await?;
673 used_fields.push(index.clone());
674 query.push('.');
675 let index = format!("${{sess_field_{}}}", index.0.value.expect("value"));
676 query.push_str(&index);
677 }
516 Index::Idx(idx) => {678 Index::ExprApply(e) => {
679 let index = Field::new(self.0.session.clone(), &e.out).await?;
680 used_fields.push(index.clone());
681 query.push(' ');
682 let index = format!("sess_field_{}", index.0.value.expect("value"));
683 query.push_str(&index);
517 query = format!("builtins.elemAt ({query}) {idx}");684 query = format!("({query})");
518 }685 }
519 }686 }
520 }687 }
521688
522 let vid = self689 let vid = self
690 .0
523 .session691 .session
524 .0692 .0
525 .lock()693 .lock()
526 .await694 .await
527 .execute_assign(&query)695 .execute_assign(&query)
528 .await696 .await
529 .with_context(|| format!("full path: {}", PathDisplay(&full_path)))?;697 .with_context(|| {
698 if let Some(full_path) = &full_path {
699 format!("full path: {}", PathDisplay(full_path))
700 } else {
701 format!("query: {query:?}")
702 }
703 })?;
530 Ok(Self {704 Ok(Self(Arc::new(FieldInner {
531 full_path,705 full_path,
532 session: self.session.clone(),706 session: self.0.session.clone(),
533 value: Some(vid),707 value: Some(vid),
534 })708 })))
535 }709 }
536 pub async fn as_json<V: DeserializeOwned>(&self) -> Result<V> {710 pub async fn as_json<V: DeserializeOwned>(&self) -> Result<V> {
537 let id = self.value.expect("can't serialize root field");711 let id = self.0.value.expect("can't serialize root field");
712 let query = format!("sess_field_{id}");
538 self.session713 self.0
714 .session
539 .0715 .0
540 .lock()716 .lock()
541 .await717 .await
542 .execute_expression_to_json(&format!("sess_field_{id}"))718 .execute_expression_to_json(&query)
543 .await719 .await
544 .with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))720 .with_context(|| context(self.0.full_path.as_deref(), &query))
545 }721 }
546 pub async fn list_fields(&self) -> Result<Vec<String>> {722 pub async fn list_fields(&self) -> Result<Vec<String>> {
547 let id = self.value.expect("can't list root fields");723 let id = self.0.value.expect("can't list root fields");
724 let query = format!("builtins.attrNames sess_field_{id}");
548 self.session725 self.0
726 .session
549 .0727 .0
550 .lock()728 .lock()
551 .await729 .await
552 .execute_expression_to_json(&format!("builtins.attrNames sess_field_{id}"))730 .execute_expression_to_json(&query)
553 .await731 .await
554 .with_context(|| format!("full path: {}", PathDisplay(&self.full_path)))732 .with_context(|| context(self.0.full_path.as_deref(), &query))
555 }733 }
556 pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {734 pub async fn build(&self) -> Result<HashMap<String, PathBuf>> {
557 let id = self.value.expect("can't use build on not-value");735 let id = self.0.value.expect("can't use build on not-value");
736 let query = format!(":b sess_field_{id}");
558 let vid = self737 let vid = self
738 .0
559 .session739 .session
560 .0740 .0
561 .lock()741 .lock()
562 .await742 .await
563 .execute_expression_raw(&format!(":b sess_field_{id}"), &mut NixHandler::default())743 .execute_expression_raw(&query, &mut NixHandler::default())
564 .await?;744 .await?;
565 ensure!(!vid.is_empty(), "build failed: {}", PathDisplay(&self.full_path));745 ensure!(
746 !vid.is_empty(),
747 "build failed: {}",
748 context(self.0.full_path.as_deref(), &query),
749 );
566 let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")750 let Some(vid) = vid.strip_prefix("This derivation produced the following outputs:\n")
567 else {751 else {
576 Ok(outputs)760 Ok(outputs)
577 }761 }
578}762}
579impl Drop for Field {763impl Drop for FieldInner {
580 fn drop(&mut self) {764 fn drop(&mut self) {
581 if let Some(id) = self.value {765 if let Some(id) = self.value {
582 if let Ok(mut lock) = self.session.0.try_lock() {766 if let Ok(mut lock) = self.session.0.try_lock() {
modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
44
5use crate::command::MyCommand;5use crate::command::MyCommand;
6use crate::host::Config;6use crate::host::Config;
7use crate::nix_path;7use crate::nix_go;
8use anyhow::{anyhow, Result, Context};8use anyhow::{anyhow, Result};
9use clap::Parser;9use clap::Parser;
10use itertools::Itertools;10use itertools::Itertools;
11use tokio::{task::LocalSet, time::sleep};11use tokio::{task::LocalSet, time::sleep};
290 async fn build_task(self, config: Config, host: String) -> Result<()> {290 async fn build_task(self, config: Config, host: String) -> Result<()> {
291 info!("building");291 info!("building");
292 let action = Action::from(self.subcommand.clone());292 let action = Action::from(self.subcommand.clone());
293 let drv = config293 let fleet_field = &config.fleet_field;
294 .fleet_field
295 .select(nix_path!(.buildSystems((serde_json::json!({294 let drv = nix_go!(fleet_field.buildSystems(Obj {
296 "localSystem": config.local_system.clone(),295 localSystem: { config.local_system.clone() }
297 }))).{action.build_attr()}.{&host}))296 }));
298 .await.context("system attribute")?;
299 let outputs = drv.build().await.map_err(|e| {297 let outputs = drv.build().await.map_err(|e| {
300 if action.build_attr() == "sdImage" {298 if action.build_attr() == "sdImage" {
301 info!("sd-image build failed");299 info!("sd-image build failed");
modifiedcmds/fleet/src/cmds/info.rsdiffbeforeafterboth
1use std::collections::BTreeSet;1use std::collections::BTreeSet;
22
3use crate::host::Config;3use crate::host::Config;
4use crate::nix_path;4use crate::nix_go_json;
5use anyhow::{ensure, Result};5use anyhow::{ensure, Result};
6use clap::Parser;6use clap::Parser;
77
37 InfoCmd::ListHosts { ref tagged } => {37 InfoCmd::ListHosts { ref tagged } => {
38 'host: for host in config.list_hosts().await? {38 'host: for host in config.list_hosts().await? {
39 if !tagged.is_empty() {39 if !tagged.is_empty() {
40 let fleet_field = &config.fleet_field;
40 let tags: Vec<String> = config41 let tags: Vec<String> =
41 .fleet_field
42 .select(nix_path!(.configuredSystems.{&host.name}.config.tags))42 nix_go_json!(fleet_field.configuredSystems[{ host.name }].config.tags);
43 .await?
44 .as_json()
45 .await?;
46 for tag in tagged {43 for tag in tagged {
47 if !tags.contains(tag) {44 if !tags.contains(tag) {
48 continue 'host;45 continue 'host;
64 let mut out = <BTreeSet<String>>::new();61 let mut out = <BTreeSet<String>>::new();
65 let host = config.system_config(&host).await?;62 let host = config.system_config(&host).await?;
66 if external {63 if external {
67 out.extend(
68 host.select(nix_path!(.network.externalIps))64 let data: Vec<String> = nix_go_json!(host.network.externalIps);
69 .await?65 out.extend(data);
70 .as_json::<Vec<String>>()
71 .await?,
72 );
73 }66 }
74 if internal {67 if internal {
75 out.extend(
76 host.select(nix_path!(.network.internalIps))68 let data: Vec<String> = nix_go_json!(host.network.internalIps);
77 .await?69 out.extend(data);
78 .as_json::<Vec<String>>()
79 .await?,
80 );
81 }70 }
82 for ip in out {71 for ip in out {
83 data.push(ip);72 data.push(ip);
modifiedcmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth
1use crate::{1use crate::{
2 fleetdata::{FleetSecret, FleetSharedSecret},2 fleetdata::{FleetSecret, FleetSharedSecret},
3 host::Config, nix_path,3 host::Config,
4 nix_go, nix_go_json,
4};5};
5use anyhow::{bail, ensure, Context, Result};6use anyhow::{anyhow, bail, ensure, Context, Result};
6use chrono::Utc;7use chrono::{DateTime, Utc};
7use clap::Parser;8use clap::Parser;
8use futures::{StreamExt, TryStreamExt};9use futures::{StreamExt, TryStreamExt};
9use owo_colors::OwoColorize;10use owo_colors::OwoColorize;
17use tracing::{error, info, info_span, warn};18use tracing::{error, info, info_span, warn};
1819
19#[derive(Parser)]20#[derive(Parser)]
20pub enum Secrets {21pub enum Secret {
21 /// Force load keys for all defined hosts22 /// Force load host keys for all defined hosts
22 ForceKeys,23 ForceKeys,
23 /// Add secret, data should be provided in stdin24 /// Add secret, data should be provided in stdin
24 AddShared {25 AddShared {
29 /// Override secret if already present30 /// Override secret if already present
30 #[clap(long)]31 #[clap(long)]
31 force: bool,32 force: bool,
33 /// Secret public part
32 #[clap(long)]34 #[clap(long)]
33 public: Option<String>,35 public: Option<String>,
36 /// Load public part from specified file
34 #[clap(long)]37 #[clap(long)]
35 public_file: Option<PathBuf>,38 public_file: Option<PathBuf>,
39
40 /// Create a notification on secret expiration
41 #[clap(long)]
42 expires_at: Option<DateTime<Utc>>,
3643
37 /// Secret with this name already exists, override its value while keeping the same owners.44 /// Secret with this name already exists, override its value while keeping the same owners.
38 #[clap(long)]45 #[clap(long)]
39 readd: bool,46 re_add: bool,
40 },47 },
41 /// Add secret, data should be provided in stdin48 /// Add secret, data should be provided in stdin
42 Add {49 Add {
81 prefer_identities: Vec<String>,88 prefer_identities: Vec<String>,
82 },89 },
83 List {},90 List {},
91 InvokeGenerator,
84}92}
8593
86impl Secrets {94impl Secret {
87 pub async fn run(self, config: &Config) -> Result<()> {95 pub async fn run(self, config: &Config) -> Result<()> {
88 match self {96 match self {
97 Secret::InvokeGenerator => {
98 let config_field = &config.config_unchecked_field;
99
100 let generate_impure =
101 nix_go!(config_field.sharedSecrets["kube-apiserver.pem"].generateImpure);
102 let on = nix_go!(generate_impure.on);
103 let call_package = nix_go!(
104 config_field.buildableSystems(Obj {
105 localSystem: { config.local_system.clone() }
106 })[on]
107 .config
108 .nixpkgs
109 .pkgs
110 .callPackage
111 );
112 let generator = nix_go!(call_package(generate_impure.generator));
113 let built = generator.build().await?;
114 // .as_json().await?;
115 dbg!(&built);
116 }
89 Secrets::ForceKeys => {117 Secret::ForceKeys => {
90 for host in config.list_hosts().await? {118 for host in config.list_hosts().await? {
91 if config.should_skip(&host.name) {119 if config.should_skip(&host.name) {
92 continue;120 continue;
93 }121 }
94 config.key(&host.name).await?;122 config.key(&host.name).await?;
95 }123 }
96 }124 }
97 Secrets::AddShared {125 Secret::AddShared {
98 mut machines,126 mut machines,
99 name,127 name,
100 force,128 force,
101 public,129 public,
102 public_file,130 public_file,
103 readd,131 expires_at,
132 re_add,
104 } => {133 } => {
105 let exists = config.has_shared(&name);134 let exists = config.has_shared(&name);
106 if exists && !force && !readd {135 if exists && !force && !re_add {
107 bail!("secret already defined");136 bail!("secret already defined");
108 }137 }
109 if readd {138 if re_add {
110 // Fixme: use clap to limit this usage139 // Fixme: use clap to limit this usage
111 ensure!(!force, "--force and --readd are not compatible");140 ensure!(!force, "--force and --readd are not compatible");
112 ensure!(exists, "secret doesn't exists");141 ensure!(exists, "secret doesn't exists");
137 .map(|r| Box::new(r) as Box<dyn age::Recipient + Send>)166 .map(|r| Box::new(r) as Box<dyn age::Recipient + Send>)
138 .collect();167 .collect();
139 let mut encryptor = age::Encryptor::with_recipients(recipients)168 let mut encryptor = age::Encryptor::with_recipients(recipients)
140 .expect("recipients provided")169 .ok_or_else(|| anyhow!("no recipients provided"))?
141 .wrap_output(&mut encrypted)?;170 .wrap_output(&mut encrypted)?;
142 io::copy(&mut Cursor::new(input), &mut encryptor)?;171 io::copy(&mut Cursor::new(input), &mut encryptor)?;
143 encryptor.finish()?;172 encryptor.finish()?;
150 owners: machines,179 owners: machines,
151 secret: FleetSecret {180 secret: FleetSecret {
152 created_at: Utc::now(),181 created_at: Utc::now(),
153 expires_at: None,182 expires_at,
154 secret,183 secret,
155 public: match (public, public_file) {184 public: match (public, public_file) {
156 (Some(v), None) => Some(v),185 (Some(v), None) => Some(v),
164 },193 },
165 );194 );
166 }195 }
167 Secrets::Add {196 Secret::Add {
168 machine,197 machine,
169 name,198 name,
170 force,199 force,
211 }240 }
212 // TODO: Instead of using sudo, decode secret on remote machine241 // TODO: Instead of using sudo, decode secret on remote machine
213 #[allow(clippy::await_holding_refcell_ref)]242 #[allow(clippy::await_holding_refcell_ref)]
214 Secrets::Read {243 Secret::Read {
215 name,244 name,
216 machine,245 machine,
217 plaintext,246 plaintext,
228 println!("{}", z85::encode(&data));257 println!("{}", z85::encode(&data));
229 }258 }
230 }259 }
231 Secrets::UpdateShared {260 Secret::UpdateShared {
232 name,261 name,
233 machines,262 machines,
234 mut add_machines,263 mut add_machines,
321 secret.secret.secret = encrypted;350 secret.secret.secret = encrypted;
322 config.replace_shared(name, secret);351 config.replace_shared(name, secret);
323 }352 }
324 Secrets::Regenerate { prefer_identities } => {353 Secret::Regenerate { prefer_identities } => {
325 {354 {
326 let expected_shared_set = config355 let expected_shared_set = config
327 .list_configured_shared()356 .list_configured_shared()
337 for name in &config.list_shared() {366 for name in &config.list_shared() {
338 info!("updating secret: {name}");367 info!("updating secret: {name}");
339 let mut data = config.shared_secret(name)?;368 let mut data = config.shared_secret(name)?;
369 let config_field = &config.config_field;
340 let expected_owners: Vec<String> = config370 let expected_owners: Vec<String> =
341 .config_field
342 .get_json_deep(nix_path!(sharedSecrets.{name}.expectedOwners))371 nix_go_json!(config_field.sharedSecrets[{ name }].expectedOwners);
343 .await?;
344 if expected_owners.is_empty() {372 if expected_owners.is_empty() {
345 warn!("secret was removed from fleet config: {name}, removing from data");373 warn!("secret was removed from fleet config: {name}, removing from data");
346 to_remove.push(name.to_string());374 to_remove.push(name.to_string());
350 let expected_set = expected_owners.iter().collect::<HashSet<_>>();378 let expected_set = expected_owners.iter().collect::<HashSet<_>>();
351 let should_remove = set.difference(&expected_set).next().is_some();379 let should_remove = set.difference(&expected_set).next().is_some();
352 if set != expected_set {380 if set != expected_set {
353 let owner_dependent: bool = config381 let owner_dependent: bool =
354 .config_field
355 .get_json_deep(nix_path!(.sharedSecrets.{name}.ownerDependent))382 nix_go_json!(config_field.sharedSecrets[{ name }].ownerDependent);
356 .await?;
357 if !owner_dependent {383 if !owner_dependent {
358 warn!("reencrypting secret '{name}' for new owner set");384 warn!("reencrypting secret '{name}' for new owner set");
359 // TODO: force regeneration385 // TODO: force regeneration
401 config.remove_shared(&k);427 config.remove_shared(&k);
402 }428 }
403 }429 }
404 Secrets::List {} => {430 Secret::List {} => {
405 let _span = info_span!("loading secrets").entered();431 let _span = info_span!("loading secrets").entered();
406 let configured = config.list_configured_shared().await?;432 let configured = config.list_configured_shared().await?;
407 #[derive(Tabled)]433 #[derive(Tabled)]
modifiedcmds/fleet/src/command.rsdiffbeforeafterboth
337 if !text.is_empty()337 if !text.is_empty()
338 && text != "querying info about missing paths"338 && text != "querying info about missing paths"
339 && text != "copying 0 paths"339 && text != "copying 0 paths"
340 // Too much spam on lazy-trees branch
341 && !(text.starts_with("copying '") && text.ends_with("' to the store"))
340 {342 {
341 let span = info_span!("job");343 let span = info_span!("job");
342 span.pb_start();344 span.pb_start();
modifiedcmds/fleet/src/host.rsdiffbeforeafterboth
16 better_nix_eval::{Field, NixSessionPool},16 better_nix_eval::{Field, NixSessionPool},
17 command::MyCommand,17 command::MyCommand,
18 fleetdata::{FleetData, FleetSecret, FleetSharedSecret},18 fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
19 nix_path,19 nix_go, nix_go_json,
20};20};
2121
22pub struct FleetConfigInternals {22pub struct FleetConfigInternals {
29 pub fleet_field: Field,29 pub fleet_field: Field,
30 /// fleet_config.configUnchecked30 /// fleet_config.configUnchecked
31 pub config_field: Field,31 pub config_field: Field,
32 /// fleet_config.unchecked
33 pub config_unchecked_field: Field,
32}34}
3335
34#[derive(Clone)]36#[derive(Clone)]
95 }97 }
9698
97 pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {99 pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
100 let fleet_field = &self.fleet_field;
98 let names = self101 let names = nix_go!(fleet_field.configuredHosts).list_fields().await?;
99 .fleet_field
100 .select(nix_path!(.configuredHosts))
101 .await?
108 Ok(out)106 Ok(out)
109 }107 }
110 pub async fn system_config(&self, host: &str) -> Result<Field> {108 pub async fn system_config(&self, host: &str) -> Result<Field> {
111 self.fleet_field109 let fleet_field = &self.fleet_field;
112 .select(nix_path!(.configuredSystems.{host}.config))110 Ok(nix_go!(fleet_field.configuredSystems[{ host }].config))
113 .await
114 }111 }
115112
116 pub(super) fn data(&self) -> MutexGuard<FleetData> {113 pub(super) fn data(&self) -> MutexGuard<FleetData> {
121 }118 }
122 /// Shared secrets configured in fleet.nix or in flake119 /// Shared secrets configured in fleet.nix or in flake
123 pub async fn list_configured_shared(&self) -> Result<Vec<String>> {120 pub async fn list_configured_shared(&self) -> Result<Vec<String>> {
124 self.config_field121 let config_field = &self.config_field;
125 .select(nix_path!(.sharedSecrets))122 nix_go!(config_field.sharedSecrets).list_fields().await
126 .await?
127 .list_fields()
128 .await
129 }123 }
211 Ok(secret.clone())205 Ok(secret.clone())
212 }206 }
213 pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {207 pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {
214 self.config_field208 let config_field = &self.config_field;
215 .select(nix_path!(.sharedSecrets.{secret}.expectedOwners))209 Ok(nix_go_json!(
216 .await?210 config_field.sharedSecrets[{ secret }].expectedOwners
217 .as_json()211 ))
218 .await
219 }212 }
220213
221 pub fn save(&self) -> Result<()> {214 pub fn save(&self) -> Result<()> {
269262
270 if self.local_system == "detect" {263 if self.local_system == "detect" {
271 let builtins_field = Field::field(root_field.clone(), "builtins").await?;264 let builtins_field = Field::field(root_field.clone(), "builtins").await?;
272 let system = builtins_field
273 .select(nix_path!(.currentSystem))
274 .await?;
275 self.local_system = system.as_json().await?;265 self.local_system = nix_go_json!(builtins_field.currentSystem);
276 }266 }
277 let local_system = self.local_system.clone();267 let local_system = self.local_system.clone();
278268
279 let fleet_root = Field::field(root_field, "fleetConfigurations").await?;269 let fleet_root = Field::field(root_field, "fleetConfigurations").await?;
280270
281 let fleet_field = fleet_root271 let fleet_field = nix_go!(fleet_root.default);
282 .select(nix_path!(.default))
283 .await?;
284 let config_field = fleet_field272 let config_field = nix_go!(fleet_field.configUnchecked);
285 .select(nix_path!(.configUnchecked))273 let config_unchecked_field = nix_go!(fleet_field.unchecked);
286 .await?;
287274
288 let mut fleet_data_path = directory.clone();275 let mut fleet_data_path = directory.clone();
289 fleet_data_path.push("fleet.nix");276 fleet_data_path.push("fleet.nix");
298 nix_args,285 nix_args,
299 fleet_field,286 fleet_field,
300 config_field,287 config_field,
288 config_unchecked_field,
301 })))289 })))
302 }290 }
303}291}
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
1#![recursion_limit = "512"]1#![recursion_limit = "512"]
2#![feature(try_blocks)]2#![feature(try_blocks, lint_reasons)]
33
4pub(crate) mod cmds;4pub(crate) mod cmds;
5pub(crate) mod command;5pub(crate) mod command;
17use anyhow::{bail, Result};17use anyhow::{bail, Result};
18use clap::Parser;18use clap::Parser;
1919
20use cmds::{build_systems::BuildSystems, info::Info, secrets::Secrets};20use cmds::{build_systems::BuildSystems, info::Info, secrets::Secret};
21use futures::future::LocalBoxFuture;21use futures::future::LocalBoxFuture;
22use futures::stream::FuturesUnordered;22use futures::stream::FuturesUnordered;
23use futures::TryStreamExt;23use futures::TryStreamExt;
73 BuildSystems(BuildSystems),73 BuildSystems(BuildSystems),
74 /// Secret management74 /// Secret management
75 #[clap(subcommand)]75 #[clap(subcommand)]
76 Secrets(Secrets),76 Secret(Secret),
77 /// Upload prefetch directory to the nix store77 /// Upload prefetch directory to the nix store
78 Prefetch(Prefetch),78 Prefetch(Prefetch),
79 /// Config parsing79 /// Config parsing
92async fn run_command(config: &Config, command: Opts) -> Result<()> {92async fn run_command(config: &Config, command: Opts) -> Result<()> {
93 match command {93 match command {
94 Opts::BuildSystems(c) => c.run(config).await?,94 Opts::BuildSystems(c) => c.run(config).await?,
95 Opts::Secrets(s) => s.run(config).await?,95 Opts::Secret(s) => s.run(config).await?,
96 Opts::Info(i) => i.run(config).await?,96 Opts::Info(i) => i.run(config).await?,
97 Opts::Prefetch(p) => p.run(config).await?,97 Opts::Prefetch(p) => p.run(config).await?,
98 };98 };
modifiedflake.lockdiffbeforeafterboth
5 "systems": "systems"5 "systems": "systems"
6 },6 },
7 "locked": {7 "locked": {
8 "lastModified": 1694529238,8 "lastModified": 1701680307,
9 "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",9 "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
10 "owner": "numtide",10 "owner": "numtide",
11 "repo": "flake-utils",11 "repo": "flake-utils",
12 "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",12 "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
13 "type": "github"13 "type": "github"
14 },14 },
15 "original": {15 "original": {
38 },38 },
39 "nixpkgs": {39 "nixpkgs": {
40 "locked": {40 "locked": {
41 "lastModified": 1698350982,41 "lastModified": 1703705939,
42 "narHash": "sha256-zoEV8Ad3bOAejp0ys/mOpaHSWrzK+GupZwGGYfuWuEY=",42 "narHash": "sha256-9s2Ep3NyRDj9HUgfv2TQUwQEanRUAmeXkvKIr/o1XbY=",
43 "owner": "nixos",43 "owner": "nixos",
44 "repo": "nixpkgs",44 "repo": "nixpkgs",
45 "rev": "dd83f9de26ff7c0326468b659ea4729fa5cf6262",45 "rev": "1ada32da4ba24d7310653c9ac54888bee463f455",
46 "type": "github"46 "type": "github"
47 },47 },
48 "original": {48 "original": {
67 ]67 ]
68 },68 },
69 "locked": {69 "locked": {
70 "lastModified": 1698199907,70 "lastModified": 1703643208,
71 "narHash": "sha256-n8RtHBIb0rLuYs4RDehW6mj6r6Yam/ODY1af/VCcurw=",71 "narHash": "sha256-UL4KO8JxnD5rOycwHqBAf84lExF1/VnYMDC7b/wpPDU=",
72 "owner": "oxalica",72 "owner": "oxalica",
73 "repo": "rust-overlay",73 "repo": "rust-overlay",
74 "rev": "22b8d29fd22cfaa2c311e0d6fd8a0ed9c2a1152b",74 "rev": "ce117f3e0de8262be8cd324ee6357775228687cf",
75 "type": "github"75 "type": "github"
76 },76 },
77 "original": {77 "original": {
modifiedflake.nixdiffbeforeafterboth
9 };
7 flake-utils = { url = "github:numtide/flake-utils"; };10 flake-utils = {url = "github:numtide/flake-utils";};
8 };11 };
9 outputs = { self, rust-overlay, flake-utils, nixpkgs }: with nixpkgs.lib; rec {12 outputs = {
13 self,
14 rust-overlay,
15 flake-utils,
16 nixpkgs,
17 }:
18 with nixpkgs.lib;
19 {
27 overlays = [(import rust-overlay)];
16 };28 };
17 llvmPkgs = pkgs.buildPackages.llvmPackages_11;29 llvmPkgs = pkgs.buildPackages.llvmPackages_11;
18 rust = (pkgs.rustChannelOf { date = "2023-10-20"; channel = "nightly"; }).default.override { extensions = [ "rust-src" "rust-analyzer" ]; };30 rust =
19 rustPlatform = pkgs.makeRustPlatform { cargo = rust; rustc = rust; };31 (pkgs.rustChannelOf {
32 date = "2023-12-26";
33 channel = "nightly";
34 })
35 .default
27 cargo-edit43 cargo-edit
28 cargo-udeps44 cargo-udeps
29 cargo-fuzz45 cargo-fuzz
46 cargo-watch
3047
31 pkg-config48 pkg-config
32 openssl49 openssl
modifiedlib/default.nixdiffbeforeafterboth
12 };12 };
13 in13 in let
14 let
15 withData = data: rec {
16 root = nixpkgs.lib.evalModules {14 root = nixpkgs.lib.evalModules {
17 modules = (import ../modules/fleet/_modules.nix) ++ [config data];15 modules = (import ../modules/fleet/_modules.nix) ++ [config data];
18 specialArgs = {16 specialArgs = {
19 inherit nixpkgs fleetLib;17 inherit nixpkgs fleetLib;
20 };18 };
21 };19 };
22 failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);20 failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
23 rootAssertWarn =21 checkedRoot =
24 if failedAssertions != []22 if failedAssertions != []
25 then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"23 then throw "Fleet failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
26 else nixpkgs.lib.showWarnings root.config.warnings root;24 else nixpkgs.lib.showWarnings root.config.warnings root;
25 withData = {
26 root,
27 data,
28 }: rec {
27 configuredHosts = rootAssertWarn.config.hosts;29 configuredHosts = root.config.hosts;
28 configuredSecrets = rootAssertWarn.config.secrets;30 configuredUncheckedHosts = root.config.hosts;
29 configuredSystems = configuredSystemsWithExtraModules [];31 configuredSystems = configuredSystemsWithExtraModules [];
30 configuredSystemsWithExtraModules = extraModules:32 configuredSystemsWithExtraModules = extraModules:
31 nixpkgs.lib.listToAttrs (33 nixpkgs.lib.listToAttrs (
43 };45 };
44 }46 }
45 )47 )
46 (builtins.attrNames rootAssertWarn.config.hosts)48 (builtins.attrNames root.config.hosts)
47 );49 );
50 buildableSystems = {localSystem}: let
51 buildConfigurationModule = {config, ...}: {
52 # Equivalent to nixpkgs.localSystem
53 # nixpkgs.system = localSystem;
54 nixpkgs.buildPlatform.system = localSystem;
55 };
56 in
57 configuredSystemsWithExtraModules [
58 buildConfigurationModule
59 ];
48 buildSystems = {localSystem}: let60 buildSystems = {localSystem}: let
49 buildConfigurationModule = {config, ...}: {61 buildConfigurationModule = {config, ...}: {
50 # Equivalent to nixpkgs.localSystem62 # Equivalent to nixpkgs.localSystem
76 ]);88 ]);
77 };89 };
78 configUnchecked = root.config;90 configUnchecked = root.config;
79 };91 };
80 defaultData = withData data;92 defaultData = withData {
93 inherit data;
94 root = checkedRoot;
95 };
96 uncheckedData = withData {inherit data root;};
81 in rec {97 in rec {
82 inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;98 inherit (defaultData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems;
99 unchecked = {
100 inherit (uncheckedData) configuredHosts configuredSystems buildSystems configUnchecked buildableSystems;
101 };
83 injectData = data: let102 injectData = data: let
84 injectedData = withData data;103 injectedData = withData data;
85 in {104 in {
86 inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;105 inherit (injectedData) configuredHosts configuredSystems buildSystems configUnchecked;
87 };106 };
88 };107 };
89}108}
modifiedmodules/fleet/secrets.nixdiffbeforeafterboth
15 type = bool;15 type = bool;
16 description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted";16 description = "Is this secret owner-dependent, and needs to be regenerated on ownership set change, or it may be just reencrypted";
17 };17 };
18 generateImpure = mkOption {
19 type = unspecified;
20 };
18 generator = mkOption {21 generator = mkOption {
19 type = nullOr (submodule {22 type = nullOr (submodule {
20 packages = mkOption {23 packages = mkOption {