difftreelog
secret management
in: trunk
11 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1092,6 +1092,7 @@
"time",
"tokio",
"tokio-util",
+ "toml_edit",
"tracing",
]
@@ -2744,6 +2745,13 @@
checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298"
[[package]]
+name = "repl-plugin-unstable"
+version = "0.1.0"
+dependencies = [
+ "fleet-base",
+]
+
+[[package]]
name = "reqwest"
version = "0.12.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3618,6 +3626,43 @@
]
[[package]]
+name = "toml_datetime"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.23.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
+dependencies = [
+ "indexmap 2.11.4",
+ "toml_datetime",
+ "toml_parser",
+ "toml_writer",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e"
+dependencies = [
+ "winnow",
+]
+
+[[package]]
+name = "toml_writer"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2"
+
+[[package]]
name = "tonic"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4469,6 +4514,15 @@
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
+name = "winnow"
+version = "0.7.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
name = "wit-bindgen"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,7 +3,7 @@
resolver = "2"
package.version = "0.1.0"
package.edition = "2024"
-package.rust-version = "1.86.0"
+package.rust-version = "1.89.0"
[workspace.dependencies]
better-command = { path = "./crates/better-command" }
cmds/repl-plugin-unstable/Cargo.tomldiffbeforeafterboth--- /dev/null
+++ b/cmds/repl-plugin-unstable/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "repl-plugin-unstable"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+fleet-base = { version = "0.1.0", path = "../../crates/fleet-base" }
cmds/repl-plugin-unstable/src/lib.rsdiffbeforeafterboth--- /dev/null
+++ b/cmds/repl-plugin-unstable/src/lib.rs
@@ -0,0 +1,6 @@
+use fleet_base::primops::init_primops;
+
+#[unsafe(no_mangle)]
+fn nix_plugin_entry() {
+ init_primops();
+}
crates/fleet-base/Cargo.tomldiffbeforeafterboth--- a/crates/fleet-base/Cargo.toml
+++ b/crates/fleet-base/Cargo.toml
@@ -8,7 +8,7 @@
age.workspace = true
anyhow.workspace = true
better-command.workspace = true
-chrono = "0.4.41"
+chrono = { version = "0.4.41", features = ["serde"] }
clap = { workspace = true, features = ["derive"] }
fleet-shared.workspace = true
futures = "0.3.31"
@@ -27,5 +27,6 @@
thiserror.workspace = true
time = { version = "0.3.41", features = ["parsing"] }
tokio.workspace = true
-tokio-util = "0.7.15"
+tokio-util = { version = "0.7.15", features = ["codec"] }
+toml_edit = "0.23.7"
tracing.workspace = true
crates/fleet-base/src/lib.rsdiffbeforeafterboth--- a/crates/fleet-base/src/lib.rs
+++ b/crates/fleet-base/src/lib.rs
@@ -4,4 +4,6 @@
pub mod host;
mod keys;
pub mod opts;
+pub mod primops;
pub mod secret;
+pub mod secret_storage;
crates/fleet-base/src/primops.rsdiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-base/src/primops.rs
@@ -0,0 +1,45 @@
+use nix_eval::NativeFn;
+
+#[derive(thiserror::Error, Debug)]
+enum Error {}
+
+struct Parts {
+ encrypted: Vec<String>,
+ public: Vec<String>,
+}
+
+trait SecretsBackend {
+ fn has_shared(&self, name: &str);
+ fn has_host(&self, host: &str, name: &str);
+ fn shared_parts(&self, name: &str) -> Parts;
+ fn host_parts(&self, host: &str, name: &str) -> Parts;
+}
+
+struct FsSecretsBackend {
+
+}
+
+pub fn init_primops() {
+ NativeFn::new(
+ c"fleet_ensure_secret",
+ c"Ensure secret existence for a host, regenerating it in case of some mismatch",
+ [
+ c"host",
+ c"secret",
+ c"expected_parts",
+ c"expected_encrypted_parts",
+ c"generator",
+ ],
+ |[
+ host,
+ secret,
+ expected_parts,
+ expected_encrypted_parts,
+ generator,
+ ]| {
+
+ todo!()
+ },
+ )
+ .register();
+}
crates/fleet-base/src/secret_storage.rsdiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-base/src/secret_storage.rs
@@ -0,0 +1,278 @@
+use anyhow::{Result, bail, ensure};
+use itertools::Itertools;
+use std::fs::{File, metadata};
+use std::io::{self, ErrorKind, Read, Write};
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::{env, fs};
+
+use tempfile::{TempPath, tempfile_in};
+use toml_edit::{Document, DocumentMut, Formatted, Item, Value};
+
+struct Name(String);
+
+fn encode_name(name: &str) -> Name {
+ assert!(
+ !name.starts_with(['_', '.']),
+ "groups should not start with _ or ."
+ );
+ assert!(
+ !name.chars().any(|c| c == '/'),
+ "group name should not contain internal slash"
+ );
+ Name(name.to_owned())
+}
+
+enum RewriteError {
+ ConcurrentCreate,
+ ConcurrentDelete,
+ ConcurrentModify,
+ ConcurrentWrite,
+ Io(io::Error),
+ Persist(tempfile::PersistError),
+}
+
+fn safe_rewrite(
+ path: &PathBuf,
+ old_content: Option<Vec<u8>>,
+ new_content: Option<Vec<u8>>,
+) -> Result<(), RewriteError> {
+ let mut f = match (old_content.is_some(), new_content.is_some()) {
+ (false, true) => match File::create_new(path) {
+ Ok(v) => v,
+ Err(e) if e.kind() == ErrorKind::AlreadyExists => {
+ return Err(RewriteError::ConcurrentCreate);
+ }
+ Err(e) => return Err(RewriteError::Io(e)),
+ },
+ (true, _) => match File::open(&path) {
+ Ok(v) => v,
+ Err(_e) => return Err(RewriteError::ConcurrentDelete),
+ },
+ (false, false) => match metadata(&path) {
+ Err(e) if e.kind() == ErrorKind::NotFound => {
+ return Ok(());
+ }
+ Ok(_) => return Err(RewriteError::ConcurrentCreate),
+ Err(e) => return Err(RewriteError::Io(e)),
+ },
+ };
+ f.lock().map_err(RewriteError::Io)?;
+ let mut check_content = vec![];
+ f.read_to_end(&mut check_content)
+ .map_err(RewriteError::Io)?;
+ match &old_content {
+ Some(old) => {
+ if old != &check_content {
+ return Err(RewriteError::ConcurrentModify);
+ }
+ }
+ None => {
+ if !check_content.is_empty() {
+ return Err(RewriteError::ConcurrentDelete);
+ }
+ }
+ }
+ if let Some(new_content) = new_content {
+ if Some(&new_content) == old_content.as_ref() {
+ return Ok(());
+ }
+ let dir = path.parent().expect("file is in directory, thus not root");
+ let mut tempfile = tempfile::Builder::new()
+ .prefix(".rewrite-")
+ .tempfile_in(dir)
+ .map_err(RewriteError::Io)?;
+ tempfile.write_all(&new_content).map_err(RewriteError::Io)?;
+ tempfile.flush().map_err(RewriteError::Io)?;
+ tempfile.persist(path).map_err(RewriteError::Persist)?;
+ } else {
+ fs::remove_file(path).map_err(RewriteError::Io)?;
+ }
+ let _ = f.unlock();
+ Ok(())
+}
+fn update_string(path: PathBuf, modify: impl Fn(&mut Option<String>) -> Result<()>) -> Result<()> {
+ loop {
+ let orig = match fs::read_to_string(&path) {
+ Ok(v) => Some(v),
+ Err(e) if e.kind() == ErrorKind::NotFound => None,
+ Err(e) => return Err(e.into()),
+ };
+ let mut edit = orig.clone();
+ modify(&mut edit);
+
+ match safe_rewrite(&path, orig.map(String::into), edit.map(String::into)) {
+ Ok(()) => return Ok(()),
+ Err(
+ RewriteError::ConcurrentCreate
+ | RewriteError::ConcurrentModify
+ | RewriteError::ConcurrentWrite
+ | RewriteError::ConcurrentDelete,
+ ) => {
+ continue;
+ }
+ Err(RewriteError::Io(io)) => return Err(io.into()),
+ Err(RewriteError::Persist(io)) => return Err(io.into()),
+ }
+ }
+}
+fn update_toml(path: PathBuf, modify: impl Fn(&mut DocumentMut) -> Result<()>) -> Result<()> {
+ update_string(path, |str| {
+ let mut doc = match str {
+ None => DocumentMut::new(),
+ Some(v) => DocumentMut::from_str(v)?,
+ };
+ modify(&mut doc)?;
+ if doc.is_empty() {
+ *str = None
+ } else {
+ *str = Some(doc.to_string())
+ }
+ Ok(())
+ })
+}
+fn update_lines(path: PathBuf, modify: impl Fn(&mut Vec<String>) -> Result<()>) -> Result<()> {
+ update_string(path, |str| {
+ let mut list = if let Some(str) = str {
+ str.split('\n').map(|s| s.to_owned()).collect_vec()
+ } else {
+ vec![]
+ };
+ let had_end_newline = if list.last().map(|v| v.as_str()) == Some("") {
+ list.pop();
+ true
+ } else {
+ false
+ };
+ modify(&mut list)?;
+ if list.is_empty() {
+ *str = None
+ } else {
+ if had_end_newline {
+ list.push("".to_owned())
+ }
+ *str = Some(list.join("\n"));
+ }
+ Ok(())
+ })
+}
+fn update_section(
+ data: &mut Vec<String>,
+ start: &str,
+ end: &str,
+ modify: impl Fn(&mut Vec<String>) -> Result<()>,
+) -> Result<()> {
+ let first = data
+ .iter()
+ .enumerate()
+ .filter(|(_, v)| *v == start)
+ .at_most_one()
+ .map_err(|_| anyhow::anyhow!("there should be at most one section start"))?
+ .map(|(v, _)| v);
+ let last = data
+ .iter()
+ .enumerate()
+ .filter(|(_, v)| *v == end)
+ .at_most_one()
+ .map_err(|_| anyhow::anyhow!("there should be at most one section end"))?
+ .map(|(v, _)| v);
+
+ match (first, last) {
+ (None, None) => {
+ let mut out = Vec::new();
+ modify(&mut out)?;
+ if out.is_empty() {
+ return Ok(());
+ }
+ data.push(start.to_owned());
+ data.extend(out);
+ data.push(end.to_owned());
+ Ok(())
+ }
+ (None, Some(_)) | (Some(_), None) => {
+ bail!("mismatched section start/end")
+ }
+ (Some(first), Some(last)) => {
+ ensure!(first < last, "section end should come after start");
+ let mut out = data[first + 1..last]
+ .iter()
+ .map(|v| v.to_owned())
+ .collect_vec();
+ modify(&mut out)?;
+ if out.is_empty() {
+ data.drain(first..=last);
+ } else {
+ data.splice(first + 1..last, out);
+ }
+ Ok(())
+ }
+ }
+}
+
+struct Group {
+ path: PathBuf,
+}
+impl Group {
+ fn new(path: PathBuf) -> Self {
+ Self { path }
+ }
+ fn manage(&self, manager: &str) {}
+ fn ensure_managing(&self, manager: &str) {
+ if !self.has_stored() {
+ return;
+ }
+ let managed = match fs::read_to_string(self.path.join(".managed_by")) {
+ Ok(found_manager) => found_manager.lines().any(|line| line == manager),
+ Err(e) if e.kind() == ErrorKind::NotFound => true,
+ Err(e) => panic!("{e}"),
+ };
+ assert!(managed);
+ }
+ fn has_stored(&self) -> bool {
+ match fs::metadata(&self.path) {
+ Ok(d) => d.is_dir(),
+ Err(e) if e.kind() == ErrorKind::NotFound => false,
+ Err(e) => panic!("{e}"),
+ }
+ }
+}
+
+struct Root {
+ path: PathBuf,
+}
+impl Root {
+ fn new(path: PathBuf) -> Self {
+ Self { path }
+ }
+ fn subgroup(&self, name: &str) -> Group {
+ Group::new(self.path.join(name))
+ }
+}
+
+#[test]
+fn test() {
+ let mut data = vec![
+ "a".to_owned(),
+ "b".to_owned(),
+ "start".to_owned(),
+ "c".to_owned(),
+ "d".to_owned(),
+ "end".to_owned(),
+ "e".to_owned(),
+ "f".to_owned(),
+ ];
+ update_section(&mut data, "start", "end", |a| {
+ a.push("vv".to_owned());
+ Ok(())
+ })
+ .unwrap();
+ dbg!(&data);
+ // for v in 0..1000 {
+ // update_toml(PathBuf::from("./test.toml"), |e| {
+ // e.as_table_mut()
+ // .insert("hello", Item::Value(Value::Integer(Formatted::new(v))));
+ // })
+ // .expect("update")
+ // }
+ // v.subgroup(name)
+}
crates/fleet-base/test.tomldiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-base/test.toml
@@ -0,0 +1 @@
+hello = 999
crates/nix-eval/src/lib.rsdiffbeforeafterboth--- a/crates/nix-eval/src/lib.rs
+++ b/crates/nix-eval/src/lib.rs
@@ -952,7 +952,7 @@
type UserClosure<const N: usize> = Box<dyn Fn([&Value; N]) -> Result<Value>>;
-struct NativeFn(*mut PrimOp);
+pub struct NativeFn(*mut PrimOp);
impl NativeFn {
pub fn new<const N: usize>(
name: &'static CStr,
flake.nixdiffbeforeafterboth1{2 description = "NixOS cluster configuration management";34 inputs = {5 nixpkgs.url = "github:nixos/nixpkgs/release-25.05";6 rust-overlay = {7 url = "github:oxalica/rust-overlay";8 inputs.nixpkgs.follows = "nixpkgs";9 };10 flake-parts = {11 url = "github:hercules-ci/flake-parts";12 inputs.nixpkgs-lib.follows = "nixpkgs";13 };14 crane.url = "github:ipetkov/crane";15 shelly.url = "github:CertainLach/shelly";16 fleet-tf = {17 url = "github:CertainLach/fleet-tf";18 inputs.nixpkgs.follows = "nixpkgs";19 inputs.shelly.follows = "shelly";20 inputs.flake-parts.follows = "flake-parts";21 };22 treefmt-nix = {23 url = "github:numtide/treefmt-nix";24 inputs.nixpkgs.follows = "nixpkgs";25 };26 # DeterminateSystem's nix fork is controversial, but I don't mind it,27 # and it has lazy-trees support which is useful for fleet.28 nix = {29 url = "github:deltarocks/nix/fleet";30 inputs.nixpkgs.follows = "nixpkgs";31 inputs.flake-parts.follows = "flake-parts";32 };33 };34 outputs =35 inputs:36 inputs.flake-parts.lib.mkFlake37 {38 inherit inputs;39 }40 {41 imports = [ inputs.shelly.flakeModule ];42 flake = rec {43 lib =44 (import ./lib {45 inherit (inputs.nixpkgs) lib;46 })47 // {48 fleetConfiguration = throw "function-based interface is deprecated, use flake-parts syntax instead";49 };50 flakeModules.default = import ./lib/flakePart.nix {51 inherit (inputs) crane;52 };53 flakeModule = flakeModules.default;5455 flakeModules.fleet-tf = ./modules/extras/tf.nix;5657 # Used to test nix-eval bindings58 testData = {59 testObj = {60 v = "Hello";61 };62 testString = "hello";63 testPrimop = op: "PREFIX_" + (op "body" "_SUFFIX");64 };6566 # To be used with https://github.com/NixOS/nix/pull/889267 schemas =68 let69 inherit (inputs.nixpkgs.lib) mapAttrs;70 in71 {72 fleetConfigurations = {73 version = 1;74 doc = ''75 The `fleetConfigurations` flake output defines fleet cluster configurations.76 '';77 inventory = output: {78 children = mapAttrs (configName: cluster: {79 what = "fleet cluster configuration";8081 children = mapAttrs (hostName: host: {82 what = "host [${host.system}]";83 }) cluster.config.hosts;84 # It is possible to implement this inventory right now, but I want to85 # get rid of `fleet.nix` file in the future.86 # children.secrets = { };87 }) output;88 };89 };90 };91 };92 # Supported and tested list of deployment targets.93 systems = [94 "x86_64-linux"95 "aarch64-linux"96 "armv7l-linux"97 "armv6l-linux"98 ];99 perSystem =100 {101 config,102 system,103 pkgs,104 self,105 inputs',106 ...107 }:108 let109 inherit (lib.attrsets) mapAttrs';110 inherit (lib.lists) elem;111 # Can also be built for darwin, through it is not usual to deploy nixos systems from macos machines.112 # I have no hardware for such testing, thus only adding machines I actually have and use.113 #114 # It is not possible to deploy any host from armv6/armv7 hardware, and I don't think it even makes sense.115 deployerSystems = [116 "aarch64-linux"117 "x86_64-linux"118 ];119 deployerSystem = elem system deployerSystems;120 lib = pkgs.lib;121 rust = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;122 craneLib = (inputs.crane.mkLib pkgs).overrideToolchain rust;123 treefmt = (inputs.treefmt-nix.lib.evalModule pkgs ./treefmt.nix).config.build;124 in125 {126 _module.args.pkgs = import inputs.nixpkgs {127 inherit system;128 overlays = [129 (inputs.rust-overlay.overlays.default)130 (final: prev: {131 boehmgc = prev.boehmgc.overrideAttrs (prevAttrs: {132 configureFlags = prevAttrs.configureFlags ++ [133 "--enable-gc-assertions"134 ];135 });136 })137 ];138 };139 # Reference fleet package should be built with nightly rust, specified in rust-toolchain.toml.140 packages = lib.mkIf deployerSystem (141 let142 packages = pkgs.callPackages ./pkgs {143 inherit craneLib inputs';144 };145 in146 packages // { default = packages.fleet; }147 );148 # fleet-install-secrets will not be built normally, because they are not ran directly by user most of the time.149 # checks there build packages for default nixpkgs rustPlatform packages.150 checks =151 let152 nixpkgsCraneLib = inputs.crane.mkLib pkgs;153 packages = pkgs.callPackages ./pkgs {154 craneLib = nixpkgsCraneLib;155 inherit inputs;156 };157 prefixAttrs =158 prefix: attrs:159 mapAttrs' (name: value: {160 name = "${prefix}${name}";161 value = value.overrideAttrs (prev: {162 pname = "${prefix}${prev.pname}";163 });164 }) attrs;165 in166 # fleet-install-secrets is installed to remote systems, thus needs to work167 # with rust in nixpkgs.168 (prefixAttrs "nixpkgs-" {169 inherit (packages) fleet-install-secrets;170 })171 // {172 formatting = treefmt.check self;173 };174 # TODO: It should be possible to move lib.mkIf to default attribute, instead of disabling the whole175 # devShells block, yet nix flake check fails here, due to no default shell found. It is nix or flake-parts bug?176 shelly.shells.default = lib.mkIf deployerSystem {177 factory = craneLib.devShell;178 packages = with pkgs; [179 rust180181 pkg-config182 openssl183 rustPlatform.bindgenHook184 inputs'.nix.packages.nix-expr-c185 inputs'.nix.packages.nix-flake-c186 inputs'.nix.packages.nix-fetchers-c187 inputs'.nix.packages.nix-store-c188 inputs'.nix.packages.nix189190 (rage.overrideAttrs { cargoFeatures = [ "plugin" ]; })191 ];192 environment.PROTOC = "${pkgs.protobuf}/bin/protoc";193 };194 formatter = treefmt.wrapper;195 };196 };197}