git.delta.rocks / jrsonnet / refs/commits / d8c8e6d33ef5

--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,7 @@
 [workspace]
 members = ["crates/*", "cmds/*"]
 resolver = "2"
+package.version = "0.1.0"
 
 [workspace.dependencies]
 nixlike = { path = "./crates/nixlike" }

--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -1,6 +1,5 @@
 use crate::{
 	better_nix_eval::Field,
-	command::MyCommand,
 	fleetdata::{FleetSecret, FleetSharedSecret, SecretData},
 	host::Config,
 	nix_go, nix_go_json,
@@ -9,16 +8,14 @@
 use chrono::{DateTime, Utc};
 use clap::Parser;
 use owo_colors::OwoColorize;
-use serde::{de::DeserializeOwned, Deserialize};
+use serde::Deserialize;
 use std::{
 	collections::{BTreeSet, HashSet},
 	io::{self, Cursor, Read},
-	path::{Path, PathBuf},
-	str::FromStr,
+	path::PathBuf,
 };
 use tabled::{Table, Tabled};
-use tempfile::tempdir;
-use tokio::fs::{self, read_to_string};
+use tokio::fs::read_to_string;
 use tracing::{error, info, info_span, warn, Instrument};
 
 #[derive(Parser)]
@@ -162,84 +159,13 @@
 }
 
 async fn generate_pure(
-	config: &Config,
+	_config: &Config,
 	_display_name: &str,
-	secret: Field,
-	default_generator: Field,
-	owners: &[String],
+	_secret: Field,
+	_default_generator: Field,
+	_owners: &[String],
 ) -> Result<FleetSecret> {
-	// TODO: pure secrets are supposed to be generated by nix daemon itself,
-	// inside of a sandbox... But we aren't here yet.
-	let config_field = &config.config_unchecked_field;
-	let generator = nix_go!(secret.generator);
-	let default_pkgs = &config.default_pkgs;
-
-	let call_package = nix_go!(default_pkgs.callPackage);
-
-	let generator = nix_go!(call_package(generator)(Obj {}));
-	let generator = generator.build().await?;
-	let generator = generator
-		.get("out")
-		.ok_or_else(|| anyhow!("missing generate out"))?;
-
-	let mut recipients = String::new();
-	for owner in owners {
-		let key = config.key(owner).await?;
-		recipients.push_str(&format!("-r \"{key}\" "));
-	}
-	recipients.push_str("-e");
-
-	let out = tempdir()?;
-
-	let mut gen = MyCommand::new(generator);
-	gen.env("rageArgs", recipients);
-	gen.env(
-		"out",
-		out.path().to_str().expect("sane tempdir should be utf-8"),
-	);
-	gen.run().await.context("impure generator")?;
-
-	{
-		let mut marker_path = out.path().to_owned();
-		marker_path.push("marker");
-		let marker = fs::read_to_string(&marker_path).await?;
-		ensure!(marker == "SUCCESS", "generation not succeeded");
-	}
-
-	let mut public_path = out.path().to_owned();
-	public_path.push("public");
-	let mut secret_path = out.path().to_owned();
-	secret_path.push("secret");
-	let public = fs::read_to_string(&public_path).await.ok();
-	let secret = fs::read(&secret_path).await.ok();
-	if let Some(secret) = &secret {
-		ensure!(
-			age::Decryptor::new(Cursor::new(&secret)).is_ok(),
-			"builder produced non-encrypted value as secret, this is highly insecure, and not allowed."
-		);
-	}
-
-	let mut created_at_path = out.path().to_owned();
-	created_at_path.push("created_at");
-	let mut expires_at_path = out.path().to_owned();
-	expires_at_path.push("expires_at");
-
-	async fn read_value<T: FromStr>(path: &Path) -> Result<T> {
-		dbg!(path);
-		let raw = fs::read(path).await?;
-		let raw = String::from_utf8(raw)?;
-		raw.parse().map_err(|_| anyhow!("fromStr failed"))
-	}
-
-	let created_at = read_value(&created_at_path).await?;
-	let expires_at = read_value(&expires_at_path).await.ok();
-
-	Ok(FleetSecret {
-		created_at,
-		expires_at,
-		public,
-		secret: secret.map(SecretData),
-	})
+	bail!("pure generators are broken for now")
 }
 async fn generate_impure(
 	config: &Config,
@@ -248,39 +174,53 @@
 	default_generator: Field,
 	owners: &[String],
 ) -> Result<FleetSecret> {
-	let config_field = &config.config_unchecked_field;
 	let generator = nix_go!(secret.generator);
+	let on: Option<String> = nix_go_json!(default_generator.impureOn);
 
-	let on: String = nix_go_json!(default_generator.impureOn);
-	let call_package = nix_go!(
-		config_field.hosts[{ on }]
-			.nixosSystem
-			.config
-			.nixpkgs
-			.resolvedPkgs
-			.callPackage
-	);
+	let host = if let Some(on) = &on {
+		config.host(on).await?
+	} else {
+		config.local_host()
+	};
+	let on_pkgs = host.pkgs().await?;
+	let call_package = nix_go!(on_pkgs.callPackage);
+	let mk_encrypt_secret = nix_go!(on_pkgs.mkEncryptSecret);
 
-	let host = config.host(&on).await?;
+	let mut recipients = Vec::new();
+	for owner in owners {
+		let key = config.key(owner).await?;
+		recipients.push(key);
+	}
+	let encrypt = nix_go!(mk_encrypt_secret(Obj {
+		recipients: { recipients },
+	}));
+
+	let generator = nix_go!(call_package(generator)(Obj {
+		encrypt,
+		rustfmt_please_newline: { true },
+	}));
 
-	let generator = nix_go!(call_package(generator)(Obj {}));
 	let generator = generator.build().await?;
 	let generator = generator
 		.get("out")
 		.ok_or_else(|| anyhow!("missing generateImpure out"))?;
 	let generator = host.remote_derivation(generator).await?;
 
-	let mut recipients = String::new();
-	for owner in owners {
-		let key = config.key(owner).await?;
-		recipients.push_str(&format!("-r \"{key}\" "));
-	}
-	recipients.push_str("-e");
-
-	let out = host.mktemp_dir().await?;
+	let out_parent = host.mktemp_dir().await?;
+	let out = format!("{out_parent}/out");
 
 	let mut gen = host.cmd(generator).await?;
-	gen.env("rageArgs", recipients).env("out", &out);
+	gen.env("out", &out);
+	if on.is_none() {
+		// This path is local, thus we can feed `OsString` directly to env var... But I don't think that's necessary to handle.
+		let project_path: String = config
+			.directory
+			.clone()
+			.into_os_string()
+			.into_string()
+			.map_err(|s| anyhow!("fleet project path is not utf-8: {s:?}"))?;
+		gen.env("FLEET_PROJECT", project_path);
+	}
 	gen.run().await.context("impure generator")?;
 
 	{
@@ -549,10 +489,7 @@
 					println!("{}", z85::encode(&data));
 				}
 			}
-			Secret::ReadPublic {
-				name,
-				machine,
-			} => {
+			Secret::ReadPublic { name, machine } => {
 				let secret = config.host_secret(&machine, &name)?;
 				let Some(public) = secret.public else {
 					bail!("no secret {name}");

--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -54,7 +54,7 @@
 	pub local: bool,
 	pub session: OnceLock<Arc<openssh::Session>>,
 
-	pub nixos_config: Field,
+	pub nixos_config: Option<Field>,
 }
 impl ConfigHost {
 	async fn open_session(&self) -> Result<Arc<openssh::Session>> {
@@ -169,7 +169,9 @@
 	}
 
 	pub async fn list_configured_secrets(&self) -> Result<Vec<String>> {
-		let nixos = &self.nixos_config;
+		let Some(nixos) = &self.nixos_config else {
+			return Ok(vec![]);
+		};
 		let secrets = nix_go!(nixos.secrets);
 		let mut out = Vec::new();
 		for name in secrets.list_fields().await? {
@@ -183,9 +185,19 @@
 		Ok(out)
 	}
 	pub async fn secret_field(&self, name: &str) -> Result<Field> {
-		let nixos = &self.nixos_config;
+		let Some(nixos) = &self.nixos_config else {
+			bail!("host is virtual and has no secrets");
+		};
 		Ok(nix_go!(nixos.secrets[{ name }]))
 	}
+
+	/// Packages for this host, resolved with nixpkgs overlays
+	pub async fn pkgs(&self) -> Result<Field> {
+		let Some(nixos) = &self.nixos_config else {
+			return Ok(self.config.default_pkgs.clone());
+		};
+		Ok(nix_go!(nixos.nixpkgs.resolvedPkgs))
+	}
 }
 
 impl Config {
@@ -202,6 +214,16 @@
 		self.opts.localhost.as_ref().map(|s| s as &str) == Some(host)
 	}
 
+	pub fn local_host(&self) -> ConfigHost {
+		ConfigHost {
+			config: self.clone(),
+			name: "<virtual localhost>".to_owned(),
+			local: true,
+			session: OnceLock::new(),
+			nixos_config: None,
+		}
+	}
+
 	pub async fn host(&self, name: &str) -> Result<ConfigHost> {
 		let config = &self.config_unchecked_field;
 		let nixos_config = nix_go!(config.hosts[{ name }].nixosSystem.config);
@@ -210,7 +232,7 @@
 			name: name.to_owned(),
 			local: self.is_local(name),
 			session: OnceLock::new(),
-			nixos_config,
+			nixos_config: Some(nixos_config),
 		})
 	}
 	pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {

--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -11,7 +11,6 @@
 
 mod fleetdata;
 
-use std::time::Duration;
 use std::{ffi::OsString, process::ExitCode};
 
 use anyhow::{bail, Result};
@@ -158,7 +157,7 @@
 	let reg = tracing_subscriber::registry().with({
 		let sub = tracing_subscriber::fmt::layer()
 			.without_time()
-			.with_target(true);
+			.with_target(false);
 		#[cfg(feature = "indicatif")]
 		let sub = sub.with_writer(indicatif_layer.get_stdout_writer());
 		sub.with_filter(filter) // .withou,

--- a/cmds/install-secrets/src/main.rs
+++ b/cmds/install-secrets/src/main.rs
@@ -13,7 +13,7 @@
 use std::path::Path;
 use std::str::{from_utf8, FromStr};
 use std::{collections::HashMap, path::PathBuf};
-use tracing::{error, info, warn};
+use tracing::{error, info, info_span, warn};
 use tracing_subscriber::filter::LevelFilter;
 use tracing_subscriber::EnvFilter;
 
@@ -213,12 +213,9 @@
 
 	let mut failed = false;
 	for (name, value) in data {
-		info!("initializing secret {name}");
+		let _span = info_span!("init", name = name);
 		if let Err(e) = init_secret(&identity, value) {
-			error!(
-				"{:?}",
-				e.context(format!("failed to initialize secret {}", name))
-			);
+			error!("{e}");
 			failed = true;
 		}
 	}
@@ -237,6 +234,7 @@
 				.from_env_lossy(),
 		)
 		.without_time()
+		.with_target(false)
 		.init();
 
 	let opts = Opts::parse();

--- a/crates/better-command/src/handler.rs
+++ b/crates/better-command/src/handler.rs
@@ -274,7 +274,10 @@
 							#[cfg(feature = "indicatif")]
 							span.pb_set_message(&process_message(s.trim()));
 							#[cfg(not(feature = "indicatif"))]
-							info!("{}", process_message(s));
+							{
+								let _span = span.enter();
+								info!("{}", process_message(s));
+							}
 						} else {
 							warn!("bad fields: {fields:?}");
 						}

--- a/crates/better-command/src/lib.rs
+++ b/crates/better-command/src/lib.rs
@@ -1,5 +1,5 @@
 mod handler;
-pub use handler::{Handler, PlainHandler, NoopHandler, NixHandler, ClonableHandler};
+pub use handler::{ClonableHandler, Handler, NixHandler, NoopHandler, PlainHandler};
 
 pub fn add(left: usize, right: usize) -> usize {
 	left + right

--- a/flake.nix
+++ b/flake.nix
@@ -5,15 +5,23 @@
     nixpkgs.url = "github:nixos/nixpkgs/master";
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        flake-utils.follows = "flake-utils";
+      };
+    };
+    flake-utils.url = "github:numtide/flake-utils";
+    crane = {
+      url = "github:ipetkov/crane";
       inputs.nixpkgs.follows = "nixpkgs";
     };
-    flake-utils = {url = "github:numtide/flake-utils";};
   };
   outputs = {
     self,
     rust-overlay,
     flake-utils,
     nixpkgs,
+    crane,
   }:
     with nixpkgs.lib;
       {
@@ -26,20 +34,16 @@
             inherit system;
             overlays = [(import rust-overlay)];
           };
-        llvmPkgs = pkgs.buildPackages.llvmPackages_11;
-        rust =
-          (pkgs.rustChannelOf {
-            date = "2024-02-10";
-            channel = "nightly";
-          })
-          .default
-          .override {extensions = ["rust-src" "rust-analyzer"];};
+        rust = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
+        craneLib = (crane.mkLib pkgs).overrideToolchain rust;
       in {
-        packages = (import ./pkgs) pkgs pkgs;
-        devShell = (pkgs.mkShell.override {stdenv = llvmPkgs.stdenv;}) {
+        packages = import ./pkgs {
+          inherit (pkgs) callPackage;
+          inherit craneLib;
+        };
+        devShell = craneLib.devShell {
           nativeBuildInputs = with pkgs; [
             alejandra
-            rust
             lld
             cargo-edit
             cargo-udeps

--- a/lib/default.nix
+++ b/lib/default.nix
@@ -10,11 +10,14 @@
     hosts,
     modules,
     globalModules ? [],
+    extraFleetLib ? {},
   }: let
     hostNames = nixpkgs.lib.attrNames hosts;
-    fleetLib = import ./fleetLib.nix {
-      inherit nixpkgs hostNames;
-    };
+    fleetLib =
+      (import ./fleetLib.nix {
+        inherit nixpkgs hostNames;
+      })
+      // extraFleetLib;
   in let
     root = nixpkgs.lib.evalModules {
       modules =

--- a/lib/fleetLib.nix
+++ b/lib/fleetLib.nix
@@ -39,4 +39,32 @@
   mkFleetDefault = mkOverride 999;
   # Some generators use mkDefault, but optionDefault is set by nixpkgs.
   mkFleetGeneratorDefault = mkOverride 1001;
+
+  mkPassword = {size ? 32}: {
+    coreutils,
+    encrypt,
+    mkSecretGenerator,
+  }:
+    mkSecretGenerator {
+      script = ''
+        ${coreutils}/bin/tr -dc 'A-Za-z0-9!?%=' < /dev/random \
+          | ${coreutils}/bin/head -c ${toString size} \
+          | ${encrypt} > $out/secret
+      '';
+    };
+
+  mkRsa = {size ? 4096}: {
+    openssl,
+    encrypt,
+    mkSecretGenerator,
+  }:
+    mkSecretGenerator {
+      script = ''
+        ${openssl}/bin/openssl genrsa -out rsa_private.key ${toString size}
+        ${openssl}/bin/openssl rsa -in rsa_private.key -pubout -out rsa_public.key
+
+        sudo cat rsa_private.key | ${encrypt} > $out/secret
+        sudo cat rsa_public.key > $out/public
+      '';
+    };
 }

--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -8,6 +8,13 @@
 with fleetLib; let
   sharedSecret = with types; ({config, ...}: {
     options = {
+      managed = mkOption {
+        type = bool;
+        description = ''
+          Is this secret managed by configuration (I.e will work with reencrypt/etc), or it is configured by user
+        '';
+      };
+
       expectedOwners = mkOption {
         type = nullOr (listOf str);
         description = ''
@@ -146,77 +153,81 @@
     overlays = [
       (final: prev: let
         lib = final.lib;
+        inherit (lib) strings;
+        inherit (strings) escapeShellArgs;
       in {
-        mkPassword = {size ? 32}:
-          final.mkSecretGenerator ''
-            ${final.coreutils}/bin/tr -dc 'A-Za-z0-9!?%=' < /dev/random \
-              | ${final.coreutils}/bin/head -c ${toString size} \
-              | encrypt > $out/secret
-          '';
-        mkRsa = {size ? 4096}:
-          final.mkSecretGenerator ''
-            ${final.openssl}/bin/openssl genrsa -out rsa_private.key ${toString size}
-            ${final.openssl}/bin/openssl rsa -in rsa_private.key -pubout -out rsa_public.key
-
-            sudo cat rsa_private.key | encrypt > $out/secret
-            sudo cat rsa_public.key > $out/public
+        mkEncryptSecret = {
+          rage ? prev.rage,
+          recipients,
+        }:
+          prev.writeShellScript "encryptor" ''
+            #!/bin/sh
+            exec ${rage}/bin/rage ${escapeShellArgs recipients} -e "$@"
           '';
         # TODO: Move to fleet
         # TODO: Merge both generators to one with consistent options syntax?
         # Impure generator is built on local machine, then built closure is copied to remote machine,
         # and then it is ran in inpure context, so that this generator may access HSMs and other things.
-        mkImpureSecretGenerator = generatorText: machine:
+        mkImpureSecretGenerator = {
+          script,
+          # If set - script will be run on remote machine, otherwise it will be run with fleet project in CWD
+          # (Some secrets-encryption-in-git/managed PKI solution is expected)
+          impureOn ? null,
+        }:
           (prev.writeShellScript "impureGenerator.sh" ''
             #!/bin/sh
             set -eu
-
-            # TODO: Provide encryption function as script passed to `callPackage generator {encrypt = ...;}`
-            function encrypt() {
-              eval ${final.rage}/bin/rage $rageArgs
-            }
+            cd /var/empty
 
             created_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ")
-            echo -n $created_at > $out/created_at
 
-            ${generatorText}
+            ${script}
 
+            if ! test -d $out; then
+              echo "impure generator script did not produce expected \$out output"
+              exit 1
+            fi
+
+            echo -n $created_at > $out/created_at
             echo -n SUCCESS > $out/marker
           '')
           .overrideAttrs (old: {
             passthru = {
+              inherit impureOn;
               generatorKind = "impure";
-              impureOn = machine;
             };
           });
+        # Pure generators are disabled for now
+        mkSecretGenerator = {script}: final.mkImpureSecretGenerator {inherit script;};
+
         # TODO: Implement consistent naming
         # Pure secret generator is supposed to be run entirely by nix, using `__impure` derivation type...
         # But for now, it is ran the same way as `impureSecretGenerator`, but on the local machine.
-        mkSecretGenerator = generatorText:
-          (prev.writeShellScript "generator.sh" ''
-            #!/bin/sh
-            set -eu
-            # TODO: User should create output directory by themselves.
-            cd $out
-
-            # TODO: Provide encryption function as script passed to `callPackage generator {encrypt = ...;}`
-            function encrypt() {
-              eval ${final.rage}/bin/rage $rageArgs
-            }
-
-            created_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ")
-            echo -n $created_at > $out/created_at
-
-            ${generatorText}
-
-            echo -n SUCCESS > $out/marker
-          '')
-          .overrideAttrs (old: {
-            passthru = {
-              generatorKind = "pure";
-            };
-            # TODO: make nix daemon build secret, not just the script.
-            # __impure = true;
-          });
+        # mkSecretGenerator = {script}:
+        #   (prev.writeShellScript "generator.sh" ''
+        #     #!/bin/sh
+        #     set -eu
+        #     # TODO: make nix daemon build secret, not just the script.
+        #     cd /var/empty
+        #
+        #     created_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ")
+        #
+        #     ${script}
+        #     if ! test -d $out; then
+        #       echo "impure generator script did not produce expected \$out output"
+        #       exit 1
+        #     fi
+        #
+        #     echo -n $created_at > $out/created_at
+        #     echo -n SUCCESS > $out/marker
+        #   '')
+        #   .overrideAttrs (old: {
+        #     passthru = {
+        #       generatorKind = "pure";
+        #     };
+        #     # TODO: make nix daemon build secret, not just the script.
+        #     # __impure = true;
+        #   });
       })
     ];
   };

--- a/nixos/fleetPkgs.nix
+++ b/nixos/fleetPkgs.nix
@@ -1,3 +1,24 @@
-{ ... }: {
-  nixpkgs.overlays = [ (import ../pkgs) ];
+{...}: {
+  nixpkgs.overlays = [
+    # Not using craneLib here, because we don't want to have two different rust versions for some platforms.
+    (final: prev: {
+      fleet-install-secrets = prev.callPackage ({rustPlatform}:
+        rustPlatform.buildRustPackage rec {
+          pname = "fleet-install-secrets";
+          name = "${pname}";
+
+          src = ../.;
+          strictDeps = true;
+
+          buildAndTestSubdir = "cmds/install-secrets";
+
+          cargoLock = {
+            lockFile = ../Cargo.lock;
+            outputHashes = {
+              "alejandra-3.0.0" = "sha256-lStDIPizbJipd1JpNKX1olBKzyIosyC2U/mVFwJPcZE=";
+            };
+          };
+        }) {};
+    })
+  ];
 }

--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -1,6 +1,9 @@
-pkgs: super:
-with pkgs;
 {
-  fleet-install-secrets = callPackage ./fleet-install-secrets.nix { };
-  fleet = callPackage ./fleet.nix { };
+  callPackage,
+  craneLib,
+}: rec {
+  default = fleet;
+
+  fleet-install-secrets = callPackage ./fleet-install-secrets.nix {inherit craneLib;};
+  fleet = callPackage ./fleet.nix {inherit craneLib;};
 }

--- a/pkgs/fleet-install-secrets.nix
+++ b/pkgs/fleet-install-secrets.nix
@@ -1,16 +1,9 @@
-{ rustPlatform, lib }:
-
-rustPlatform.buildRustPackage rec {
+{craneLib}:
+craneLib.buildPackage rec {
   pname = "fleet-install-secrets";
-  version = "0.0.1";
-  name = "${pname}-${version}";
 
-  src = ../.;
-  buildAndTestSubdir = "cmds/install-secrets";
-  cargoLock = {
-    lockFile = ../Cargo.lock;
-    outputHashes = {
-      "alejandra-3.0.0" = "sha256-lStDIPizbJipd1JpNKX1olBKzyIosyC2U/mVFwJPcZE=";
-    };
-  };
+  src = craneLib.cleanCargoSource (craneLib.path ../.);
+  strictDeps = true;
+
+  cargoExtraArgs = "--locked -p ${pname}";
 }

--- a/pkgs/fleet.nix
+++ b/pkgs/fleet.nix
@@ -1,16 +1,9 @@
-{ rustPlatform }:
-
-rustPlatform.buildRustPackage rec {
+{craneLib}:
+craneLib.buildPackage rec {
   pname = "fleet";
-  version = "0.0.1";
-  name = "${pname}-${version}";
 
-  src = ../.;
-  cargoBuildFlags = "-p ${pname}";
-  cargoLock = {
-    lockFile = ../Cargo.lock;
-    outputHashes = {
-      "alejandra-3.0.0" = "sha256-YSdHsJ73G7TEFzbmpZ2peuMefIa9/vNB2g+xdiyma3U=";
-    };
-  };
+  src = craneLib.cleanCargoSource (craneLib.path ../.);
+  strictDeps = true;
+
+  cargoExtraArgs = "--locked -p ${pname}";
 }