difftreelog
refactor rework fleet.nix secret bookkeeping
in: trunk
17 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -70,7 +70,7 @@
"aes",
"aes-gcm",
"age-core",
- "base64",
+ "base64 0.21.7",
"bcrypt-pbkdf",
"bech32",
"cbc",
@@ -102,7 +102,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5f11899bc2bbddd135edbc30c36b1924fa59d0746bb45beb5933fafe3fe509b"
dependencies = [
- "base64",
+ "base64 0.21.7",
"chacha20poly1305",
"cookie-factory",
"hkdf",
@@ -253,6 +253,12 @@
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
name = "base64ct"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -534,6 +540,7 @@
dependencies = [
"bitflags",
"crossterm_winapi",
+ "filedescriptor",
"libc",
"mio",
"parking_lot",
@@ -695,6 +702,17 @@
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
[[package]]
+name = "filedescriptor"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7199d965852c3bac31f779ef99cbb4537f80e952e2d6aa0ffeb30cce00f4f46e"
+dependencies = [
+ "libc",
+ "thiserror",
+ "winapi",
+]
+
+[[package]]
name = "find-crate"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -712,11 +730,12 @@
"age-core",
"anyhow",
"async-trait",
- "base64",
+ "base64 0.22.1",
"better-command",
"chrono",
"clap",
"crossterm",
+ "fleet-shared",
"futures",
"hostname",
"human-repr",
@@ -741,7 +760,6 @@
"tracing-indicatif",
"tracing-subscriber",
"unindent",
- "z85",
]
[[package]]
@@ -751,12 +769,22 @@
"age",
"anyhow",
"clap",
+ "fleet-shared",
"nix",
"serde",
"serde_json",
"tempfile",
"tracing",
"tracing-subscriber",
+]
+
+[[package]]
+name = "fleet-shared"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "serde",
+ "unicode_categories",
"z85",
]
@@ -1824,7 +1852,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
dependencies = [
- "base64",
+ "base64 0.21.7",
"bitflags",
"serde",
"serde_derive",
@@ -2013,9 +2041,9 @@
[[package]]
name = "serde"
-version = "1.0.201"
+version = "1.0.202"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c"
+checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395"
dependencies = [
"serde_derive",
]
@@ -2031,9 +2059,9 @@
[[package]]
name = "serde_derive"
-version = "1.0.201"
+version = "1.0.202"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865"
+checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838"
dependencies = [
"proc-macro2",
"quote",
@@ -2571,6 +2599,12 @@
checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
+[[package]]
name = "unindent"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,4 +9,4 @@
bifrostlink = "0.1.0"
uuid = { version = "1.7.0", features = ["v4"] }
tokio = { version = "1.36.0", features = ["fs", "rt", "macros", "sync", "time", "rt-multi-thread"] }
-
+fleet-shared = { path = "./crates/fleet-shared" }
cmds/fleet/Cargo.tomldiffbeforeafterboth--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -19,10 +19,15 @@
age-core = "0.10"
peg = "0.8"
age = { version = "0.10", features = ["ssh", "armor"] }
-base64 = "0.21"
+base64 = "0.22.1"
chrono = { version = "0.4", features = ["serde"] }
-z85 = "3.0"
-clap = { version = ">=4.4, <4.5", features = ["derive", "env", "wrap_help", "unicode"] }
+# Using fixed version for rust on stable nixos branches.
+clap = { version = ">=4.4, <4.5", features = [
+ "derive",
+ "env",
+ "wrap_help",
+ "unicode",
+] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
tokio-util = { version = "0.7", features = ["codec"] }
@@ -40,7 +45,8 @@
unindent = "0.2"
regex = "1.10"
openssh = "0.10"
-crossterm = "0.27"
+crossterm = { version = "0.27.0", features = ["use-dev-tty"] }
+fleet-shared.workspace = true
tracing-indicatif = { version = "0.3", optional = true }
human-repr = { version = "1.1", optional = true }
@@ -48,4 +54,9 @@
[features]
# Not quite stable
-indicatif = ["tracing-indicatif", "dep:indicatif", "human-repr", "better-command/indicatif"]
+indicatif = [
+ "tracing-indicatif",
+ "dep:indicatif",
+ "human-repr",
+ "better-command/indicatif",
+]
cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth1use crate::{1use std::{2 better_nix_eval::Field,2 collections::{BTreeMap, BTreeSet, HashSet},3 fleetdata::{FleetSecret, FleetSharedSecret, SecretData},3 ffi::OsString,4 io::{self, stdin, stdout, Read, Write},4 host::Config,5 path::PathBuf,5 nix_go, nix_go_json,6};6};77use anyhow::{anyhow, bail, ensure, Context, Result};8use anyhow::{anyhow, bail, ensure, Context, Result};8use chrono::{DateTime, Utc};9use chrono::{DateTime, Utc};9use clap::{error::ErrorKind, Parser};10use clap::Parser;10use crossterm::{terminal, tty::IsTty};11use crossterm::{terminal, tty::IsTty};12use fleet_shared::SecretData;11use itertools::Itertools;13use itertools::Itertools;12use owo_colors::OwoColorize;14use owo_colors::OwoColorize;13use serde::Deserialize;15use serde::Deserialize;14use std::{15 collections::{BTreeSet, HashSet},16 ffi::OsString,17 io::{self, stdin, Cursor, Read, Write},18 path::PathBuf,19};20use tabled::{Table, Tabled};16use tabled::{Table, Tabled};21use tempfile::NamedTempFile;17use tokio::{fs::read, process::Command};22use tokio::{fs::read_to_string, process::Command};23use tracing::{error, info, info_span, warn, Instrument};18use tracing::{error, info, info_span, warn, Instrument};241920use crate::{21 better_nix_eval::Field,22 fleetdata::{encrypt_secret_data, FleetSecret, FleetSecretPart, FleetSharedSecret},23 host::Config,24 nix_go, nix_go_json,25};2625#[derive(Parser)]27#[derive(Parser)]26pub enum Secret {28pub enum Secret {27 /// Force load host keys for all defined hosts29 /// Force load host keys for all defined hosts38 /// Secret public part40 /// Secret public part39 #[clap(long)]41 #[clap(long)]40 public: Option<String>,42 public: Option<String>,43 /// How to name public secret part44 #[clap(long, default_value = "public")]45 public_name: String,41 /// Load public part from specified file46 /// Load public part from specified file42 #[clap(long)]47 #[clap(long)]43 public_file: Option<PathBuf>,48 public_file: Option<PathBuf>,49 /// Secret with this name already exists, override its value while keeping the same owners.54 /// Secret with this name already exists, override its value while keeping the same owners.50 #[clap(long)]55 #[clap(long)]51 re_add: bool,56 re_add: bool,5758 #[clap(default_value = "secret")]59 part_name: String,52 },60 },53 /// Add secret, data should be provided in stdin61 /// Add secret, data should be provided in stdin54 Add {62 Add {59 /// Override secret if already present67 /// Override secret if already present60 #[clap(long)]68 #[clap(long)]61 force: bool,69 force: bool,70 /// Secret public part62 #[clap(long)]71 #[clap(long)]63 public: Option<String>,72 public: Option<String>,73 /// How to name public secret part74 #[clap(long, default_value = "public")]75 public_name: String,76 /// Load public part from specified file64 #[clap(long)]77 #[clap(long)]65 public_file: Option<PathBuf>,78 public_file: Option<PathBuf>,7980 #[clap(default_value = "secret")]81 part_name: String,66 },82 },67 /// Read secret from remote host, requires sudo on said host83 /// Read secret from remote host, requires sudo on said host68 Read {84 Read {69 name: String,85 name: String,70 machine: String,86 machine: String,8771 #[clap(long)]88 #[clap(default_value = "secret")]72 plaintext: bool,89 part_name: String,73 },90 },74 ReadPublic {75 name: String,76 machine: String,77 },78 UpdateShared {91 UpdateShared {79 name: String,92 name: String,809389 /// Which host should we use to decrypt102 /// Which host should we use to decrypt90 #[clap(long)]103 #[clap(long)]91 prefer_identities: Vec<String>,104 prefer_identities: Vec<String>,105106 #[clap(default_value = "secret")]107 part_name: String,92 },108 },93 Regenerate {109 Regenerate {94 /// Which host should we use to decrypt, in case if reencryption is required, without110 /// Which host should we use to decrypt, in case if reencryption is required, without97 prefer_identities: Vec<String>,113 prefer_identities: Vec<String>,98 },114 },99 List {},115 List {},116 Edit {117 name: String,118 machine: String,119120 #[clap(default_value = "secret")]121 part: String,122123 #[clap(long)]124 add: bool,125 },100}126}101127102#[tracing::instrument(skip(config, secret, field, prefer_identities))]128#[tracing::instrument(skip(config, secret, field, prefer_identities))]144 bail!("no available holder found");170 bail!("no available holder found");145 };171 };146172147 if let Some(data) = secret.secret.secret {173 for (part_name, part) in secret.secret.parts.iter_mut() {174 let _span = info_span!("part reencryption", part_name);175 if !part.raw.encrypted {176 continue;177 }148 let host = config.host(identity_holder).await?;178 let host = config.host(identity_holder).await?;149 let encrypted = host.reencrypt(data, updated_set.to_vec()).await?;179 let encrypted = host180 .reencrypt(part.raw.clone(), updated_set.to_vec())181 .await?;150 secret.secret.secret = Some(encrypted);182 part.raw = encrypted;151 }183 }152184153 secret.owners = updated_set.to_vec();185 secret.owners = updated_set.to_vec();232 ensure!(marker == "SUCCESS", "generation not succeeded");264 ensure!(marker == "SUCCESS", "generation not succeeded");233 }265 }234266235 let public = host.read_file_text(format!("{out}/public")).await.ok();267 let mut parts = BTreeMap::new();268 for part in host.read_dir(&out).await? {269 if part == "created_at" || part == "expired_at" || part == "marker" {270 continue;236 let secret = host.read_file_bin(format!("{out}/secret")).await.ok();271 }272 let contents: SecretData = host273 .read_file_text(format!("{out}/{part}"))274 .await?275 .parse()237 if let Some(secret) = &secret {276 .map_err(|e| anyhow!("failed to decode secret {out:?} part {part:?}: {e}"))?;238 ensure!(239 age::Decryptor::new(Cursor::new(&secret)).is_ok(),277 parts.insert(part.to_owned(), FleetSecretPart { raw: contents });240 "builder produced non-encrypted value as secret, this is highly insecure, and not allowed."241 );242 }278 }243279244 let created_at = host.read_file_value(format!("{out}/created_at")).await?;280 let created_at = host.read_file_value(format!("{out}/created_at")).await?;247 Ok(FleetSecret {283 Ok(FleetSecret {248 created_at,284 created_at,249 expires_at,285 expires_at,250 public,286 parts,251 secret: secret.map(SecretData),252 })287 })253}288}254async fn generate(289async fn generate(310async fn parse_public(345async fn parse_public(311 public: Option<String>,346 public: Option<String>,312 public_file: Option<PathBuf>,347 public_file: Option<PathBuf>,313) -> Result<Option<String>> {348) -> Result<Option<SecretData>> {314 Ok(match (public, public_file) {349 Ok(match (public, public_file) {315 (Some(v), None) => Some(v),350 (Some(v), None) => Some(SecretData {351 data: v.into(),316 (None, Some(v)) => Some(read_to_string(v).await?),352 encrypted: false,353 }),354 (None, Some(v)) => Some(SecretData {355 data: read(v).await?,356 encrypted: false,357 }),317 (Some(_), Some(_)) => {358 (Some(_), Some(_)) => {318 bail!("only public or public_file should be set")359 bail!("only public or public_file should be set")319 }360 }320 (None, None) => None,361 (None, None) => None,321 })362 })322}363}323364365async fn parse_secret() -> Result<Option<Vec<u8>>> {366 let mut input = vec![];367 io::stdin().read_to_end(&mut input)?;368 if input.is_empty() {369 Ok(None)370 } else {371 Ok(Some(input))372 }373}374324fn parse_machines(375fn parse_machines(325 initial: Vec<String>,376 initial: Vec<String>,326 machines: Option<Vec<String>>,377 machines: Option<Vec<String>>,391 name,442 name,392 force,443 force,393 public,444 public,445 public_name,394 public_file,446 public_file,395 expires_at,447 expires_at,396 re_add,448 re_add,449 part_name,397 } => {450 } => {398 // TODO: Forbid updating secrets with set expectedOwners (= not user-managed).451 // TODO: Forbid updating secrets with set expectedOwners (= not user-managed).399452415468416 let recipients = config.recipients(machines.clone()).await?;469 let recipients = config.recipients(machines.clone()).await?;417470418 let secret = {471 let mut parts = BTreeMap::new();419 let mut input = vec![];420 io::stdin().read_to_end(&mut input)?;421472422 if input.is_empty() {473 let mut input = vec![];423 None474 io::stdin().read_to_end(&mut input)?;475424 } else {476 if !input.is_empty() {425 Some(477 let encrypted = encrypt_secret_data(recipients, input)426 SecretData::encrypt(recipients, input)427 .ok_or_else(|| anyhow!("no recipients provided"))?,478 .ok_or_else(|| anyhow!("no recipients provided"))?;428 )479 parts.insert(part_name, FleetSecretPart { raw: encrypted });429 }480 }430 };481431 let public = parse_public(public, public_file).await?;482 if let Some(public) = parse_public(public, public_file).await? {483 parts.insert(public_name, FleetSecretPart { raw: public });484 }485432 config.replace_shared(486 config.replace_shared(433 name,487 name,436 secret: FleetSecret {490 secret: FleetSecret {437 created_at: Utc::now(),491 created_at: Utc::now(),438 expires_at,492 expires_at,439 secret,493 parts,440 public,441 },494 },442 },495 },443 );496 );447 name,500 name,448 force,501 force,449 public,502 public,503 public_name,450 public_file,504 public_file,505 part_name,451 } => {506 } => {452 let recipient = config.recipient(&machine).await?;507 if config.has_secret(&machine, &name) && !force {508 bail!("secret already defined");509 }453510454 let secret = {511 let mut parts = BTreeMap::new();455 let mut input = vec![];456 io::stdin().read_to_end(&mut input)?;457 if input.is_empty() {458 bail!("no data provided")459 }460512461 Some(SecretData::encrypt(vec![recipient], input).expect("recipient provided"))513 if let Some(secret) = parse_secret().await? {514 let recipient = config.recipient(&machine).await?;515 let encrypted =516 encrypt_secret_data(vec![recipient], secret).expect("recipient provided");517 parts.insert(part_name, FleetSecretPart { raw: encrypted });518 }519520 if let Some(public) = parse_public(public, public_file).await? {521 parts.insert(public_name, FleetSecretPart { raw: public });462 };522 };463523464 if config.has_secret(&machine, &name) && !force {465 bail!("secret already defined");466 }467 let public = parse_public(public, public_file).await?;468469 config.insert_secret(524 config.insert_secret(470 &machine,525 &machine,471 name,526 name,472 FleetSecret {527 FleetSecret {473 created_at: Utc::now(),528 created_at: Utc::now(),474 expires_at: None,529 expires_at: None,475 secret,530 parts,476 public,477 },531 },478 );532 );479 }533 }480 #[allow(clippy::await_holding_refcell_ref)]534 #[allow(clippy::await_holding_refcell_ref)]481 Secret::Read {535 Secret::Read {482 name,536 name,483 machine,537 machine,484 plaintext,538 part_name,485 } => {539 } => {486 let secret = config.host_secret(&machine, &name)?;540 let secret = config.host_secret(&machine, &name)?;487 let Some(secret) = secret.secret else {541 let Some(secret) = secret.parts.get(&part_name) else {488 bail!("no secret {name}");542 bail!("no part {part_name} in secret {name}");489 };543 };490 let host = config.host(&machine).await?;544 let data = if secret.raw.encrypted {491 let data = host.decrypt(secret).await?;545 let host = config.host(&machine).await?;492 if plaintext {546 host.decrypt(secret.raw.clone()).await?493 let s = String::from_utf8(data).context("output is not utf8")?;494 print!("{s}");495 } else {547 } else {496 println!("{}", z85::encode(&data));548 secret.raw.data.clone()497 }498 }499 Secret::ReadPublic { name, machine } => {500 let secret = config.host_secret(&machine, &name)?;501 let Some(public) = secret.public else {502 bail!("no secret {name}");503 };549 };550504 print!("{public}");551 stdout().write_all(&data)?;505 }552 }506 Secret::UpdateShared {553 Secret::UpdateShared {507 name,554 name,508 machines,555 machines,509 add_machines,556 add_machines,510 remove_machines,557 remove_machines,511 prefer_identities,558 prefer_identities,559 part_name,512 } => {560 } => {513 // TODO: Forbid updating secrets with set expectedOwners (= not user-managed).561 // TODO: Forbid updating secrets with set expectedOwners (= not user-managed).514562515 let secret = config.shared_secret(&name)?;563 let secret = config.shared_secret(&name)?;516 if secret.secret.secret.is_none() {564 if secret.secret.parts.get(&part_name).is_none() {517 bail!("no secret");565 bail!("no secret");518 }566 }519567572 }620 }573 }621 }574 for host in config.list_hosts().await? {622 for host in config.list_hosts().await? {623 if config.should_skip(&host.name) {624 continue;625 }626575 let _span = info_span!("host", host = host.name).entered();627 let _span = info_span!("host", host = host.name).entered();576 let expected_set = host628 let expected_set = host577 .list_configured_secrets()629 .list_configured_secrets()664 }716 }665 info!("loaded\n{}", Table::new(table).to_string())717 info!("loaded\n{}", Table::new(table).to_string())666 }718 }719 Secret::Edit {720 name,721 machine,722 part,723 add,724 } => {725 let secret = config.host_secret(&machine, &name)?;726 if let Some(data) = secret.parts.get(&part) {727 let host = config.host(&machine).await?;728 let secret = host.decrypt(data.raw.clone()).await?;729 String::from_utf8(secret).context("secret is not utf8")?730 } else if add {731 String::new()732 } else {733 bail!("part {part} not found in secret {name}. Did you mean to `--add` it?");734 };735 }667 }736 }668 Ok(())737 Ok(())669 }738 }739}740741async fn edit_temp_file(742 builder: tempfile::Builder<'_, '_>,743 r: Vec<u8>,744 header: &str,745 comment: &str,746) -> Result<(Vec<u8>, Option<String>), anyhow::Error> {747 if !stdin().is_tty() {748 // TODO: Also try to open /dev/tty directly?749 bail!("stdin is not tty, can't open editor");750 }751752 use std::fmt::Write;753 let mut file = builder.tempfile()?;754755 let mut full_header = String::new();756 let mut had = false;757 for line in header.trim_end().lines() {758 had = true;759 writeln!(&mut full_header, "{comment}{line}")?;760 }761 if had {762 writeln!(&mut full_header, "{}", comment.trim_end())?;763 }764 writeln!(765 &mut full_header,766 "{comment}Do not touch this header! It will be removed automatically"767 )?;768769 file.write_all(full_header.as_bytes())?;770 file.write_all(&r)?;771772 let abs_path = file.into_temp_path();773 let editor = std::env::var_os("VISUAL")774 .or_else(|| std::env::var_os("EDITOR"))775 .unwrap_or_else(|| "vi".into());776 let editor_args = shlex::bytes::split(editor.as_encoded_bytes())777 .ok_or_else(|| anyhow!("EDITOR env var has wrong syntax"))?;778 let editor_args = editor_args779 .into_iter()780 .map(|v| {781 // Only ASCII subsequences are replaced782 unsafe { OsString::from_encoded_bytes_unchecked(v) }783 })784 .collect_vec();785 let Some((editor, args)) = editor_args.split_first() else {786 bail!("EDITOR env var has no command");787 };788 let mut command = Command::new(editor);789 command.args(args);790791 let path_arg = abs_path.canonicalize()?;792793 // TODO: Save full state, using tcget/_getmode/_setmode794 let was_raw = terminal::is_raw_mode_enabled()?;795 terminal::enable_raw_mode()?;796797 let status = command.arg(path_arg).status().await;798799 if !was_raw {800 terminal::disable_raw_mode()?;801 }802803 let success = match status {804 Ok(s) => s.success(),805 Err(e) if e.kind() == io::ErrorKind::NotFound => {806 bail!("editor not found")807 }808 Err(e) => bail!("editor spawn error: {e}"),809 };810811 let mut file = std::fs::read(&abs_path).context("read editor output")?;812 let Some(v) = file.strip_prefix(full_header.as_bytes()) else {813 todo!();814 };815 todo!();816817 // Ok((success, abs_path))670}818}671819cmds/fleet/src/fleetdata.rsdiffbeforeafterboth--- a/cmds/fleet/src/fleetdata.rs
+++ b/cmds/fleet/src/fleetdata.rs
@@ -1,20 +1,14 @@
-use age::Recipient;
-use anyhow::Result;
-use chrono::{DateTime, Utc};
-use itertools::Itertools;
-use nixlike::format_nix;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::{
collections::BTreeMap,
io::{self, Cursor},
};
-use tempfile::TempDir;
-use tokio::{
- fs::{self, File},
- io::AsyncWriteExt,
- process::Command,
-};
+use age::Recipient;
+use chrono::{DateTime, Utc};
+use fleet_shared::SecretData;
+use itertools::Itertools;
+use serde::{de::Error, Deserialize, Serialize};
+
#[derive(Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct HostData {
@@ -23,9 +17,36 @@
pub encryption_key: String,
}
+const VERSION: &str = "0.1.0";
+pub struct FleetDataVersion;
+impl Serialize for FleetDataVersion {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ VERSION.serialize(serializer)
+ }
+}
+impl<'de> Deserialize<'de> for FleetDataVersion {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ let version = String::deserialize(deserializer)?;
+ if version != VERSION {
+ return Err(D::Error::custom(format!(
+ "fleet.nix data version mismatch, expected {VERSION}, got {version}.\nFollow the docs for migration instruction"
+ )));
+ }
+ Ok(Self)
+ }
+}
+
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct FleetData {
+ pub version: FleetDataVersion,
+
#[serde(default)]
pub hosts: BTreeMap<String, HostData>,
#[serde(default)]
@@ -45,41 +66,30 @@
pub secret: FleetSecret,
}
+/// Returns None if recipients.is_empty()
+pub fn encrypt_secret_data(
+ recipients: impl IntoIterator<Item = impl Recipient + Send + 'static>,
+ data: Vec<u8>,
+) -> Option<SecretData> {
+ let mut encrypted = vec![];
+ let recipients = recipients
+ .into_iter()
+ .map(|v| Box::new(v) as Box<dyn Recipient + Send>)
+ .collect_vec();
+ let mut encryptor = age::Encryptor::with_recipients(recipients)?
+ .wrap_output(&mut encrypted)
+ .expect("in memory write");
+ io::copy(&mut Cursor::new(data), &mut encryptor).expect("in memory copy");
+ encryptor.finish().expect("in memory flush");
+ Some(SecretData {
+ data: encrypted,
+ encrypted: true,
+ })
+}
+
#[derive(Serialize, Deserialize, Clone)]
-pub struct SecretData(
- #[serde(
- default,
- skip_serializing_if = "Vec::is_empty",
- serialize_with = "as_z85",
- deserialize_with = "from_z85"
- )]
- pub Vec<u8>,
-);
-impl SecretData {
- /// Returns None if recipients.is_empty()
- pub fn encrypt(
- recipients: impl IntoIterator<Item = impl Recipient + Send + 'static>,
- data: Vec<u8>,
- ) -> Option<Self> {
- let mut encrypted = vec![];
- let recipients = recipients
- .into_iter()
- .map(|v| Box::new(v) as Box<dyn Recipient + Send>)
- .collect_vec();
- let mut encryptor = age::Encryptor::with_recipients(recipients)?
- .wrap_output(&mut encrypted)
- .expect("in memory write");
- io::copy(&mut Cursor::new(data), &mut encryptor).expect("in memory copy");
- encryptor.finish().expect("in memory flush");
- Some(Self(encrypted))
- }
- pub fn encode_z85(&self) -> String {
- z85::encode(&self.0)
- }
- pub fn decode_z85(v: &str) -> Result<Self> {
- let v = z85::decode(v)?;
- Ok(Self(v))
- }
+pub struct FleetSecretPart {
+ pub raw: SecretData,
}
#[derive(Serialize, Deserialize, Clone)]
@@ -91,60 +101,7 @@
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none", alias = "expire_at")]
pub expires_at: Option<DateTime<Utc>>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub public: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub secret: Option<SecretData>,
-}
-
-fn as_z85<S>(key: &[u8], serializer: S) -> Result<S::Ok, S::Error>
-where
- S: Serializer,
-{
- serializer.serialize_str(&z85::encode(key))
-}
-fn from_z85<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
-where
- D: Deserializer<'de>,
-{
- use serde::de::Error;
- String::deserialize(deserializer)
- .and_then(|string| z85::decode(string).map_err(|err| Error::custom(err.to_string())))
-}
-
-/// Isn't used yet
-#[allow(dead_code)]
-pub async fn dummy_flake() -> Result<TempDir> {
- let data_str = fs::read_to_string("fleet.nix").await?;
-
- let mut cmd = Command::new("nix");
- cmd.arg("flake").arg("metadata").arg("--json");
-
- let flake_dir = tempfile::tempdir()?;
- let mut flake_nix = flake_dir.path().to_path_buf();
- flake_nix.push("flake.nix");
- // flake_dir
-
- File::create(&flake_nix)
- .await?
- .write_all(
- format_nix(&format!(
- "
- {{
- outputs = {{self, ...}}: {{
- data = {data_str};
- }};
- }}
- "
- ))
- .as_bytes(),
- )
- .await?;
-
- // std::thread::sleep(Duration::MAX);
- // flake_dir.close()
- // FIXME
- dbg!(&flake_nix);
- Ok(flake_dir)
+ #[serde(flatten)]
+ pub parts: BTreeMap<String, FleetSecretPart>,
}
cmds/fleet/src/host.rsdiffbeforeafterboth--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -9,8 +9,9 @@
sync::{Arc, Mutex, MutexGuard, OnceLock},
};
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
use clap::{ArgGroup, Parser};
+use fleet_shared::SecretData;
use openssh::SessionBuilder;
use serde::de::DeserializeOwned;
use tempfile::NamedTempFile;
@@ -18,7 +19,7 @@
use crate::{
better_nix_eval::{Field, NixSessionPool},
command::MyCommand,
- fleetdata::{FleetData, FleetSecret, FleetSharedSecret, SecretData},
+ fleetdata::{FleetData, FleetSecret, FleetSharedSecret},
nix_go, nix_go_json,
};
@@ -89,6 +90,16 @@
cmd.arg(path);
cmd.run_string().await
}
+ pub async fn read_dir(&self, path: impl AsRef<OsStr>) -> Result<Vec<String>> {
+ let mut cmd = self.cmd("ls").await?;
+ cmd.arg(path);
+ let out = cmd.run_string().await?;
+ let mut lines = out.split('\n');
+ if let Some(last) = lines.next_back() {
+ ensure!(last == "", "output of ls should end with newline");
+ }
+ Ok(lines.map(ToOwned::to_owned).collect())
+ }
#[allow(dead_code)]
pub async fn read_file_json<D: DeserializeOwned>(&self, path: impl AsRef<OsStr>) -> Result<D> {
let text = self.read_file_text(path).await?;
@@ -111,18 +122,22 @@
}
pub async fn decrypt(&self, data: SecretData) -> Result<Vec<u8>> {
+ ensure!(data.encrypted, "secret is not encrypted");
let mut cmd = self.cmd("fleet-install-secrets").await?;
- cmd.arg("decrypt").eqarg("--secret", data.encode_z85());
+ cmd.arg("decrypt").eqarg("--secret", data.to_string());
let encoded = cmd
.sudo()
.run_string()
.await
.context("failed to call remote host for decrypt")?;
- z85::decode(encoded.trim_end()).context("bad encoded data? outdated host?")
+ let data: SecretData = encoded.parse().map_err(|e| anyhow!("{e}"))?;
+ ensure!(!data.encrypted, "didn't decrypted secret");
+ Ok(data.data)
}
pub async fn reencrypt(&self, data: SecretData, targets: Vec<String>) -> Result<SecretData> {
+ ensure!(data.encrypted, "secret is not encrypted");
let mut cmd = self.cmd("fleet-install-secrets").await?;
- cmd.arg("reencrypt").eqarg("--secret", data.encode_z85());
+ cmd.arg("reencrypt").eqarg("--secret", data.to_string());
for target in targets {
let key = self.config.key(&target).await?;
cmd.eqarg("--targets", key);
@@ -132,7 +147,9 @@
.run_string()
.await
.context("failed to call remote host for decrypt")?;
- SecretData::decode_z85(encoded.trim_end()).context("bad encoded data? outdated host?")
+ let data: SecretData = encoded.parse().map_err(|e| anyhow!("{e}"))?;
+ ensure!(!data.encrypted, "didn't decrypted secret");
+ Ok(data)
}
/// Returns path for futureproofing, as path might change i.e on conversion to CA
pub async fn remote_derivation(&self, path: &PathBuf) -> Result<PathBuf> {
@@ -324,7 +341,7 @@
}
pub fn save(&self) -> Result<()> {
- let mut tempfile = NamedTempFile::new_in(self.directory.clone())?;
+ let mut tempfile = NamedTempFile::new_in(self.directory.clone()).context("failed to create updated version of fleet.nix in the same directory as original.\nDo you have write access to it? Access only to the fleet.nix won't be enough, the directory is used for atomic overwrite operation.\nIt is not recommended to use fleet by root anyway, move fleet project to your home directory.")?;
let data = nixlike::serialize(&self.data() as &FleetData)?;
tempfile.write_all(
format!(
cmds/install-secrets/Cargo.tomldiffbeforeafterboth--- a/cmds/install-secrets/Cargo.toml
+++ b/cmds/install-secrets/Cargo.toml
@@ -20,4 +20,4 @@
"unicode",
] }
tempfile = "3.10.0"
-z85 = "3.0.5"
+fleet-shared.workspace = true
cmds/install-secrets/src/main.rsdiffbeforeafterboth--- a/cmds/install-secrets/src/main.rs
+++ b/cmds/install-secrets/src/main.rs
@@ -1,64 +1,41 @@
-use age::{ssh::Identity as SshIdentity, ssh::Recipient as SshRecipient, Decryptor};
-use age::{Encryptor, Identity, Recipient};
-use anyhow::{anyhow, bail, Context, Result};
+use std::{
+ collections::{BTreeMap, HashMap},
+ fs::{self, File},
+ io::{self, Cursor, Read, Write},
+ iter,
+ os::unix::prelude::PermissionsExt,
+ path::{Path, PathBuf},
+ str::{from_utf8, FromStr},
+};
+
+use age::{
+ ssh::{Identity as SshIdentity, Recipient as SshRecipient},
+ Decryptor, Encryptor, Identity, Recipient,
+};
+use anyhow::{anyhow, bail, ensure, Context, Result};
use clap::Parser;
-use nix::sys::stat::Mode;
+use fleet_shared::SecretData;
use nix::unistd::{chown, Group, User};
-use serde::{Deserialize, Deserializer};
-use std::fmt::{self, Display};
-use std::fs::{self, File};
-use std::io::{self, Cursor, Read, Write};
-use std::iter;
-use std::os::unix::prelude::PermissionsExt;
-use std::path::Path;
-use std::str::{from_utf8, FromStr};
-use std::{collections::HashMap, path::PathBuf};
-use tracing::{error, info, info_span, warn};
-use tracing_subscriber::filter::LevelFilter;
-use tracing_subscriber::EnvFilter;
-
-#[derive(Clone, Debug)]
-struct SecretWrapper(Vec<u8>);
-impl Display for SecretWrapper {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let encoded = z85::encode(&self.0);
- write!(f, "{encoded}")
- }
-}
-impl FromStr for SecretWrapper {
- type Err = z85::DecodeError;
-
- fn from_str(s: &str) -> Result<Self, Self::Err> {
- z85::decode(s).map(Self)
- }
-}
-impl<'de> Deserialize<'de> for SecretWrapper {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: Deserializer<'de>,
- {
- let v = String::deserialize(deserializer)?;
- let de = z85::decode(v).map_err(|err| serde::de::Error::custom(err.to_string()))?;
- Ok(Self(de))
- }
-}
+use serde::Deserialize;
+use tracing::{error, info_span};
+use tracing_subscriber::{filter::LevelFilter, EnvFilter};
#[derive(Parser)]
#[clap(author)]
enum Opts {
/// Install secrets from json specification
Install { data: PathBuf },
- /// Reencrypt secret using host key, outputting in z85 encoded string
+ /// Reencrypt secret using host key, outputting in fleet encoded string
Reencrypt {
#[clap(long)]
- secret: SecretWrapper,
+ secret: SecretData,
#[clap(long)]
targets: Vec<String>,
},
- /// Decrypt secret using host key, outputting in z85 encoded string
+ /// Decrypt secret using host key, outputting in fleet encoded string
Decrypt {
#[clap(long)]
- secret: SecretWrapper,
+ secret: SecretData,
/// Shoult decoded output be printed as plaintext, instead of z85?
#[clap(long)]
plaintext: bool,
@@ -67,25 +44,29 @@
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
+struct Part {
+ raw: SecretData,
+ path: PathBuf,
+ stable_path: PathBuf,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
struct DataItem {
group: String,
mode: String,
owner: String,
-
- secret: Option<SecretWrapper>,
- public: Option<String>,
-
- public_path: PathBuf,
- stable_public_path: PathBuf,
+ root_path: Option<PathBuf>,
- secret_path: PathBuf,
- stable_secret_path: PathBuf,
+ #[serde(flatten)]
+ parts: BTreeMap<String, Part>,
}
type Data = HashMap<String, DataItem>;
-fn decrypt(input: &SecretWrapper, identity: &dyn Identity) -> Result<Vec<u8>> {
- let mut input = Cursor::new(&input.0);
+fn decrypt(input: &SecretData, identity: &dyn Identity) -> Result<Vec<u8>> {
+ ensure!(input.encrypted, "passed data is not encrypted!");
+ let mut input = Cursor::new(&input.data);
let decryptor = Decryptor::new(&mut input).context("failed to init decryptor")?;
let decryptor = match decryptor {
Decryptor::Recipients(r) => r,
@@ -101,7 +82,7 @@
.context("failed to decrypt")?;
Ok(decrypted)
}
-fn encrypt(input: &[u8], targets: Vec<String>) -> Result<SecretWrapper> {
+fn encrypt(input: &[u8], targets: Vec<String>) -> Result<SecretData> {
let recipients = targets
.into_iter()
.map(|t| {
@@ -119,70 +100,79 @@
.expect("constructor should not fail");
io::copy(&mut Cursor::new(input), &mut encryptor).expect("copy should not fail");
encryptor.finish().context("failed to finish encryption")?;
- Ok(SecretWrapper(encrypted))
+ Ok(SecretData {
+ data: encrypted,
+ encrypted: true,
+ })
}
-fn init_secret(identity: &age::ssh::Identity, value: DataItem) -> Result<()> {
- if let Some(public) = &value.public {
- let mut hashed = File::create(&value.public_path)?;
- let stable_dir = value.stable_public_path.parent().expect("not root");
- let mut stable_temp =
- tempfile::NamedTempFile::new_in(stable_dir).context("failed to create tempfile")?;
- hashed.write_all(public.as_bytes())?;
- stable_temp.write_all(public.as_bytes())?;
- stable_temp.flush()?;
- fs::set_permissions(stable_temp.path(), fs::Permissions::from_mode(0o444))
- .context("perm")?;
- fs::set_permissions(&value.public_path, fs::Permissions::from_mode(0o444))
- .context("perm")?;
+fn init_part(identity: &dyn Identity, item: &DataItem, value: &Part) -> Result<()> {
+ let stable_dir = value.stable_path.parent().expect("not root");
- stable_temp
- .persist(value.stable_public_path)
- .context("failed to persist")?;
- }
- if value.secret.is_none() {
- info!("no secret data found");
- return Ok(());
- }
- let secret = value.secret.as_ref().unwrap();
-
- let mode = Mode::from_bits(
- u32::from_str_radix(&value.mode, 8).context("failed to parse mode as octal")?,
- )
- .context("failed to parse mode")?;
- let user = User::from_name(&value.owner)
- .context("failed to get user")?
- .ok_or_else(|| anyhow!("user not found"))?;
- let group = Group::from_name(&value.group)
- .context("failed to get group")?
- .ok_or_else(|| anyhow!("group not found"))?;
+ // Right now stable & non-stable data are both located in this dir.
+ std::fs::create_dir_all(stable_dir)?;
- let stable_dir = value.stable_secret_path.parent().expect("not root");
let mut stable_temp =
tempfile::NamedTempFile::new_in(stable_dir).context("failed to create tempfile")?;
- let mut hashed = File::create(&value.secret_path)?;
+ let mut hashed = File::create(&value.path)?;
- // File is owned by root, and only root can modify it
- let decrypted = decrypt(secret, identity)?;
- if decrypted.is_empty() {
- warn!("secret is decoded as empty, something is broken?");
- }
+ let private = value.raw.encrypted;
+ let data = if private {
+ decrypt(&value.raw, identity)?
+ } else {
+ value.raw.data.to_owned()
+ };
- io::copy(&mut Cursor::new(&decrypted), &mut stable_temp)
- .context("failed to write decrypted file")?;
- io::copy(&mut Cursor::new(decrypted), &mut hashed).context("failed to write decrypted file")?;
+ hashed.write_all(&data)?;
+ hashed.flush()?;
+ stable_temp.write_all(&data)?;
+ stable_temp.flush()?;
- // Make file owned by specified user and group, then change mode
- chown(stable_temp.path(), Some(user.uid), Some(group.gid))
- .context("failed to apply user/group")?;
- chown(&value.secret_path, Some(user.uid), Some(group.gid))
- .context("failed to apply user/group")?;
- fs::set_permissions(stable_temp.path(), fs::Permissions::from_mode(mode.bits())).unwrap();
- fs::set_permissions(&value.secret_path, fs::Permissions::from_mode(mode.bits())).unwrap();
+ let mode = if private {
+ fs::Permissions::from_mode(
+ u32::from_str_radix(&item.mode, 8).context("failed to parse mode as octal")?,
+ )
+ } else {
+ fs::Permissions::from_mode(0o444)
+ };
+ fs::set_permissions(stable_temp.path(), mode.clone()).context("stable temp mode")?;
+ fs::set_permissions(&value.path, mode).context("hashed mode")?;
+
+ // Files are initially owned by root, thus making set mode first inaccessible to user, and then
+ // altering user/group.
+ if private {
+ let user = User::from_name(&item.owner)
+ .context("failed to get user")?
+ .ok_or_else(|| anyhow!("user not found"))?;
+ let group = Group::from_name(&item.group)
+ .context("failed to get group")?
+ .ok_or_else(|| anyhow!("group not found"))?;
+
+ chown(stable_temp.path(), Some(user.uid), Some(group.gid))
+ .context("failed to apply user/group")?;
+ chown(&value.path, Some(user.uid), Some(group.gid))
+ .context("failed to apply user/group")?;
+ }
+
stable_temp
- .persist(value.stable_secret_path)
- .context("failed to persist")?;
+ .persist(&value.stable_path)
+ .context("stable persist")?;
+ Ok(())
+}
+fn init_secret(identity: &age::ssh::Identity, value: &DataItem) -> Result<()> {
+ if let Some(root_path) = &value.root_path {
+ if !fs::metadata(root_path).map(|m| m.is_dir()).unwrap_or(false) {
+ fs::create_dir(root_path).context("failed to create secret directory")?;
+ }
+ }
+ for (part_id, part) in value.parts.iter() {
+ let _span = info_span!("part", part_id = part_id);
+ if let Err(e) = init_part(identity, value, part) {
+ error!("failed to init part {part_id}: {e}");
+ }
+ }
+
Ok(())
}
@@ -214,8 +204,8 @@
let mut failed = false;
for (name, value) in data {
let _span = info_span!("init", name = name);
- if let Err(e) = init_secret(&identity, value) {
- error!("{e}");
+ if let Err(e) = init_secret(&identity, &value) {
+ error!("secret failed to initialize: {e}");
failed = true;
}
}
@@ -257,7 +247,13 @@
let s = String::from_utf8(decrypted).context("output is not utf8")?;
print!("{s}");
} else {
- println!("{}", SecretWrapper(decrypted));
+ println!(
+ "{}",
+ SecretData {
+ data: decrypted,
+ encrypted: false
+ }
+ );
}
Ok(())
}
crates/better-command/src/handler.rsdiffbeforeafterboth--- a/crates/better-command/src/handler.rs
+++ b/crates/better-command/src/handler.rs
@@ -1,7 +1,9 @@
//! Collection of handlers, which transform program-specific stdout format to tracing
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex};
+use std::{
+ collections::HashMap,
+ sync::{Arc, Mutex},
+};
use once_cell::sync::Lazy;
use regex::Regex;
crates/fleet-shared/Cargo.tomldiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-shared/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+name = "fleet-shared"
+edition = "2021"
+version.workspace = true
+
+[dependencies]
+base64 = "0.22.1"
+serde = "1.0.202"
+unicode_categories = "0.1.1"
+z85 = "3.0.5"
crates/fleet-shared/src/lib.rsdiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-shared/src/lib.rs
@@ -0,0 +1,156 @@
+use std::{
+ fmt::{self, Display},
+ str::FromStr,
+};
+
+use base64::engine::{general_purpose::STANDARD_NO_PAD, Engine};
+use serde::{de::Error, Deserialize, Deserializer, Serialize};
+use unicode_categories::UnicodeCategories;
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct SecretData {
+ pub data: Vec<u8>,
+ pub encrypted: bool,
+}
+
+const BASE64_ENCODED_PREFIX: &str = "<BASE64-ENCODED>\n";
+const Z85_ENCODED_PREFIX: &str = "<Z85-ENCODED>\n";
+// Multiline text in Nix can only end with \n, which is not cool for actual single-line strings.
+const PLAINTEXT_NEWLINE_PREFIX: &str = "<PLAINTEXT-NL>\n";
+const PLAINTEXT_PREFIX: &str = "<PLAINTEXT>";
+
+const SECRET_PREFIX: &str = "<ENCRYPTED>";
+
+impl<'de> Deserialize<'de> for SecretData {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let string = String::deserialize(deserializer)?;
+ string.parse().map_err(D::Error::custom)
+ }
+}
+
+impl Serialize for SecretData {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ self.to_string().serialize(serializer)
+ }
+}
+
+impl FromStr for SecretData {
+ type Err = String;
+
+ fn from_str(string: &str) -> Result<Self, Self::Err> {
+ let (encrypted, string) = if let Some(unprefixed) = string.strip_prefix(SECRET_PREFIX) {
+ (true, unprefixed)
+ } else {
+ (false, string)
+ };
+ let data = if let Some(unprefixed) = string.strip_prefix(BASE64_ENCODED_PREFIX) {
+ STANDARD_NO_PAD
+ .decode(unprefixed.replace(|v| matches!(v, '\n' | '\t' | ' '), ""))
+ .map_err(|e| format!("base64-encoded failed: {e}"))?
+ } else if let Some(unprefixed) = string.strip_prefix(Z85_ENCODED_PREFIX) {
+ z85::decode(unprefixed.replace(|v| matches!(v, '\n' | '\t' | ' '), ""))
+ .map_err(|e| format!("z85-encoded failed: {e}"))?
+ } else if let Some(unprefixed) = string.strip_prefix(PLAINTEXT_NEWLINE_PREFIX) {
+ unprefixed.as_bytes().to_owned()
+ } else if let Some(unprefixed) = string.strip_prefix(PLAINTEXT_PREFIX) {
+ unprefixed.as_bytes().to_owned()
+ } else {
+ let secret_prefix = format!("{SECRET_PREFIX}{Z85_ENCODED_PREFIX}");
+ return Err(format!(
+ "unknown secret encoding. If you're migrating from old version of fleet, prefix public secret fields with {PLAINTEXT_PREFIX:?}, and encrypted data with {secret_prefix:?}: {string}"
+ ));
+ };
+ Ok(Self { data, encrypted })
+ }
+}
+
+impl Display for SecretData {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let mut readable = std::str::from_utf8(&self.data).ok();
+ if self.encrypted {
+ write!(f, "{SECRET_PREFIX}")?;
+ // Always base64-encode encrypted fields.
+ readable = None;
+ }
+ if Some(false) == readable.map(is_printable) {
+ readable = None
+ };
+ // TODO: Check if text is readable, and has no unprintable characters?..
+ if let Some(plaintext) = readable {
+ if plaintext.ends_with('\n') {
+ write!(f, "{PLAINTEXT_NEWLINE_PREFIX}")?;
+ } else {
+ write!(f, "{PLAINTEXT_PREFIX}")?;
+ }
+ write!(f, "{plaintext}")?;
+ } else {
+ write!(f, "{BASE64_ENCODED_PREFIX}")?;
+ let encoded = STANDARD_NO_PAD.encode(&self.data);
+ for ele in encoded.as_bytes().chunks(64) {
+ let chunk = std::str::from_utf8(ele).expect(
+ "any slice of base64-encoded text is utf-8 compatible, as it is ascii-based",
+ );
+ writeln!(f, "{chunk}")?;
+ }
+ };
+ Ok(())
+ }
+}
+
+fn is_printable(text: &str) -> bool {
+ text.chars().all(|c| {
+ c.is_letter()
+ || c.is_mark()
+ || c.is_number()
+ || c.is_punctuation()
+ || c.is_separator()
+ || c == '\n' || c == '\t'
+ // Complete base64 alphabet
+ || c == '/' || c == '+'
+ || c == '='
+ })
+}
+
+#[test]
+fn test() {
+ fn check_roundtrip(data: SecretData, expected: &str) {
+ let string = data.to_string();
+ assert_eq!(string, expected, "unexpected encoding");
+ let roundtrip: SecretData = string.parse().expect("roundtrip parse");
+ assert_eq!(data, roundtrip, "roundtrip didn't match");
+ }
+ check_roundtrip(
+ SecretData {
+ data: vec![1, 2, 3, 4, 5, 6],
+ encrypted: false,
+ },
+ "<BASE64-ENCODED>\nAQIDBAUG\n",
+ );
+ check_roundtrip(
+ SecretData {
+ data: vec![1, 2, 3, 4, 5, 6],
+ encrypted: true,
+ },
+ "<ENCRYPTED><BASE64-ENCODED>\nAQIDBAUG\n",
+ );
+ check_roundtrip(
+ SecretData {
+ data: "Привет, мир!\n".to_owned().into(),
+ encrypted: false,
+ },
+ "<PLAINTEXT-NL>\nПривет, мир!\n",
+ );
+ check_roundtrip(
+ SecretData {
+ data: "Привет, мир!".to_owned().into(),
+ encrypted: false,
+ },
+ "<PLAINTEXT>Привет, мир!",
+ );
+}
crates/nixlike/src/lib.rsdiffbeforeafterboth--- a/crates/nixlike/src/lib.rs
+++ b/crates/nixlike/src/lib.rs
@@ -38,6 +38,57 @@
Null,
}
+fn count_spaces(l: &str) -> usize {
+ l.chars().take_while(|&c| c == ' ').count()
+}
+fn is_significant(l: &str) -> bool {
+ count_spaces(l) != l.len()
+}
+
+fn dedent(l: &str, by: usize) -> &str {
+ assert!(
+ l[0..by.min(l.len())].chars().all(|c| c == ' '),
+ "dedent calculation is wrong"
+ );
+ &l[by.min(l.len())..]
+}
+
+fn process_multiline(lines: Vec<&str>) -> String {
+ // Even when parsing '''', there is single "line" between those '' delimiters.
+ // unwrap_or is for case where there is no significant lines
+ let dedent_by = lines
+ .iter()
+ .copied()
+ .filter(|c| is_significant(c))
+ .map(count_spaces)
+ .min()
+ .unwrap_or(0);
+
+ let mut out = String::new();
+
+ let mut had_first = false;
+ for (i, line) in lines.into_iter().enumerate() {
+ // Newline after '' is ignored, if there is no text.
+ if i == 0 && !is_significant(line) {
+ continue;
+ }
+ if had_first {
+ out.push('\n');
+ }
+ had_first = true;
+ // ''' is hard escape
+ for (i, part) in dedent(line, dedent_by).split("'''").enumerate() {
+ if i != 0 {
+ out.push_str(r#"""""#);
+ }
+ // This is the only replacements done by nixlike writer, no need to support more.
+ out.push_str(&part.replace("''${", "${").replace("''\\t", "\t"));
+ }
+ }
+
+ out
+}
+
peg::parser! {
pub grammar nixlike() for str {
rule number() -> i64
@@ -50,8 +101,17 @@
/ "\\r" { "\r" }
/ "\\$" { "$" }
/ c:$([_]) { c }
- rule string() -> String
+ rule string() -> String = singleline_string() / multiline_string();
+ rule singleline_string() -> String
= quiet! { "\"" v:(!"\"" c:string_char() {c})* "\"" { v.into_iter().collect() } } / expected!("<string>")
+ pub rule multiline_string() -> String
+ = "''"
+ // First line may also contain text, and whitespace for it is counted, but if it is empty - then it is'nt counted as full line...
+ // This logic is complicated, see `parse_multiline` test.
+ lines:$(("'''" / !"''" [_])*) "''"
+ {
+ process_multiline(lines.split('\n').collect())
+ }
rule boolean() -> bool
= quiet! { "true" {true}
/ "false" {false} } / expected!("<boolean>")
@@ -135,3 +195,12 @@
let (_, out) = alejandra::format::in_memory("".to_owned(), value.to_owned());
out
}
+
+#[test]
+fn parse_multiline() {
+ assert_eq!(nixlike::multiline_string("''\n''").expect("parse"), "");
+ assert_eq!(nixlike::multiline_string("''\n\n''").expect("parse"), "\n");
+ assert_eq!(nixlike::multiline_string("''t\n''").expect("parse"), "t\n");
+ assert_eq!(nixlike::multiline_string("''''").expect("parse"), "");
+ assert_eq!(nixlike::multiline_string("'' ''").expect("parse"), "");
+}
crates/nixlike/src/to_string.rsdiffbeforeafterboth--- a/crates/nixlike/src/to_string.rs
+++ b/crates/nixlike/src/to_string.rs
@@ -38,7 +38,28 @@
}
pub fn write_nix_str(str: &str, out: &mut String) {
- out.push_str(&escape_string(str))
+ if str.ends_with('\n') {
+ out.push_str("''");
+ for ele in str.split('\n') {
+ out.push('\n');
+ out.push_str(
+ &ele
+ // '' is escaped with '
+ .replace("''", "'''")
+ // ${ is escaped wth ''
+ .replace("${", "''${")
+ // \t is not counted as whitespace for dedent
+ // to avoid confusion, it is printed literally.
+ //
+ // ...Escaped \t literal should be prefixed with '' for... Idk, this logic is complicated.
+ .replace('\t', "''\\t"),
+ );
+ }
+ // Final newline is assumed due to str.ends_with condition
+ out.push_str("''");
+ } else {
+ out.push_str(&escape_string(str))
+ }
}
fn write_nix_buf(value: &Value, out: &mut String) {
crates/remowt-fs/src/lib.rsdiffbeforeafterboth--- a/crates/remowt-fs/src/lib.rs
+++ b/crates/remowt-fs/src/lib.rs
@@ -1,2 +1 @@
trait RemowtFS {}
-
modules/fleet/secrets.nixdiffbeforeafterboth--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -7,14 +7,8 @@
with lib;
with fleetLib; let
sharedSecret = with types; ({config, ...}: {
+ freeformType = types.lazyAttrsOf unspecified;
options = {
- managed = mkOption {
- type = bool;
- description = ''
- Is this secret managed by configuration (I.e will work with reencrypt/etc), or it is configured by user
- '';
- };
-
expectedOwners = mkOption {
type = nullOr (listOf str);
description = ''
@@ -71,23 +65,10 @@
'';
default = [];
};
- # TODO: Make secret generator generate arbitrary number of secret/public parts?
- # Make it generate a folder, where all files except suffixed by .enc are public, and the rest are secret?
- # How should modules refer to those files then?
- public = mkOption {
- type = nullOr str;
- description = "Secret public data. Imported from fleet.nix";
- default = null;
- };
- secret = mkOption {
- type = nullOr str;
- description = "Encrypted secret data. Imported from fleet.nix";
- default = null;
- internal = true;
- };
};
});
hostSecret = with types; {
+ freeformType = types.lazyAttrsOf unspecified;
options = {
createdAt = mkOption {
type = nullOr str;
@@ -97,21 +78,15 @@
type = nullOr str;
default = null;
};
- public = mkOption {
- type = nullOr str;
- description = "Secret public data. Imported from fleet.nix";
- default = null;
- };
- secret = mkOption {
- type = nullOr str;
- description = "Encrypted secret data. Imported from fleet.nix";
- default = null;
- internal = true;
- };
};
};
in {
options = with types; {
+ version = mkOption {
+ type = str;
+ default = "";
+ internal = true;
+ };
sharedSecrets = mkOption {
type = attrsOf (submodule sharedSecret);
default = {};
@@ -134,18 +109,20 @@
config.sharedSecrets;
hosts = hostsToAttrs (host: {
nixosModules = let
- cleanupSecret = secretName: v: {
- inherit (v) public secret;
- shared = true;
- };
+ # processPart
+ processSecret = v:
+ (removeAttrs v ["createdAt" "expiresAt" "expectedOwners" "owners" "regenerateOnOwnerAdded" "regenerateOnOwnerRemoved"])
+ // {
+ shared = true;
+ };
in [
{
secrets =
(
- mapAttrs cleanupSecret
+ mapAttrs (_: processSecret)
(filterAttrs (_: v: builtins.elem host v.owners) config.sharedSecrets)
)
- // (mapAttrs cleanupSecret (config.hostSecrets.${host} or {}));
+ // (mapAttrs (_: processSecret) (config.hostSecrets.${host} or {}));
}
];
});
nixos/secrets.nixdiffbeforeafterboth--- a/nixos/secrets.nix
+++ b/nixos/secrets.nix
@@ -1,39 +1,72 @@
-{ lib, config, pkgs, ... }:
-
-with lib;
+{
+ lib,
+ config,
+ pkgs,
+ ...
+}:
+with lib; let
+ inherit (lib.strings) hasPrefix stripPrefix;
+ plaintextPrefix = "<PLAINTEXT>";
+ plaintextNewlinePrefix = "<PLAINTEXT-NL>";
-let
sysConfig = config;
- secretType = types.submodule ({ config, ... }: {
- config = let secretName = config._module.args.name; in {
- stableSecretPath = mkOptionDefault "/run/secrets/secret-stable-${secretName}";
- secretPath = mkOptionDefault "/run/secrets/secret-${config.secretHash}-${secretName}";
- secretHash = mkOptionDefault (if config.secret != null then (builtins.hashString "sha1" config.secret) else throw "secret is not defined for secret ${secretName}");
-
- stablePublicPath = mkOptionDefault "/run/secrets/public-stable-${secretName}";
- publicPath = mkOptionDefault "/run/secrets/public-${config.publicHash}-${secretName}";
- publicHash = mkOptionDefault (if config.public != null then (builtins.hashString "sha1" config.public) else throw "public is not defined for secret ${secretName}");
- };
+ secretPartType = secretName:
+ types.submodule ({config, ...}: {
+ options = with types; {
+ raw = mkOption {
+ description = "Secret in fleet-specific undocumented format, do not use. Import from fleet.nix";
+ internal = true;
+ };
+ hash = mkOption {
+ type = str;
+ description = "Hash of secret in encoded format";
+ };
+ path = mkOption {
+ type = str;
+ description = "Path to secret part, incorporating data hash (thus it will be updated on secret change)";
+ };
+ stablePath = mkOption {
+ type = str;
+ description = "Path to secret part, incorporating data hash (thus it will be updated on secret change)";
+ };
+ data = mkOption {
+ type = str;
+ description = "Secret public data (only available for plaintext)";
+ };
+ };
+ config = let
+ partName = config._module.args.name;
+ in {
+ hash = mkOptionDefault (builtins.hashString "sha1" config.raw);
+ data = mkOptionDefault (
+ if hasPrefix plaintextPrefix config.raw
+ then stripPrefix plaintextPrefix config.raw
+ else if hasPrefix plaintextNewlinePrefix config.raw
+ then stripPrefix plaintextNewlinePrefix config.raw
+ else throw "secret.part.data attribute only works for public plaintext secret parts, got ${config.raw}"
+ );
+ path = mkOptionDefault "/run/secrets/${secretName}/${config.hash}-${partName}";
+ stablePath = mkOptionDefault "/run/secrets/${secretName}/${partName}";
+ };
+ });
+ secretType = types.submodule ({config, ...}: let
+ secretName = config._module.args.name;
+ in {
+ freeformType = types.lazyAttrsOf (secretPartType secretName);
options = with types; {
shared = mkOption {
description = "Is this secret owned by this machine, or propagated from shared secrets";
default = false;
};
-
- generator = mkOption {
+ expectedOwners = mkOption {
type = nullOr unspecified;
- description = "Derivation to evaluate for secret generation";
default = null;
+ internal = true;
};
- public = mkOption {
- type = nullOr str;
- description = "Secret public data";
- default = null;
- };
- secret = mkOption {
- type = nullOr str;
- description = "Encrypted secret data";
+ generator = mkOption {
+ type = nullOr unspecified;
+ description = "Derivation to evaluate for secret generation";
default = null;
};
mode = mkOption {
@@ -50,64 +83,43 @@
type = str;
description = "Group of the secret";
default = sysConfig.users.users.${config.owner}.group;
- };
-
- secretHash = mkOption {
- type = str;
- description = "Hash of .secret field";
- };
- publicHash = mkOption {
- type = str;
- description = "Hash of .public field";
- };
-
- stableSecretPath = mkOption {
- type = str;
- description = ''
- Use this, if target process supports re-reading of secret from disk,
- and doesn't needs to be restarted when secret is updated in file
- '';
- };
- secretPath = mkOption {
- type = str;
- description = "Path to decrypted secret, suffixed with contents hash";
- };
-
- stablePublicPath = mkOption {
- type = str;
- description = ''
- Use this, if target process supports re-reading of secret from disk,
- and doesn't needs to be restarted when secret is updated in file
- '';
- };
- publicPath = mkOption {
- type = str;
- description = "Path to the public part of secret";
};
};
});
+ processPart = part: {
+ inherit (part) raw path stablePath;
+ };
+ processSecret = secret:
+ {
+ inherit (secret) group mode owner;
+ }
+ // (mapAttrs (_: processPart) (removeAttrs secret [
+ "shared"
+ "generator"
+ "mode"
+ "group"
+ "owner"
+
+ # FIXME: Some of those removed attributes shouldn't be here, but there is some error in passing shared secrets from fleet to nixos.
+ "expectedOwners"
+ ]));
secretsFile = pkgs.writeTextFile {
name = "secrets.json";
- text = builtins.toJSON (mapAttrs (_: value: rec {
- inherit (value) group mode owner secret public;
- publicPath = if public != null then value.publicPath else "/missingno";
- stablePublicPath = if public != null then value.stablePublicPath else "/missingno";
- secretPath = if secret != null then value.secretPath else "/missingno";
- stableSecretPath = if secret != null then value.stableSecretPath else "/missingno";
- }) config.secrets);
+ text =
+ builtins.toJSON (mapAttrs (_: processSecret)
+ config.secrets);
};
-in
-{
+in {
options = {
secrets = mkOption {
type = types.attrsOf secretType;
- default = { };
+ default = {};
description = "Host-local secrets";
};
};
config = {
- environment.systemPackages = with pkgs; [pkgs.fleet-install-secrets];
- system.activationScripts.decryptSecrets = stringAfter [ "users" "groups" "specialfs" ] ''
+ environment.systemPackages = [pkgs.fleet-install-secrets];
+ system.activationScripts.decryptSecrets = stringAfter ["users" "groups" "specialfs"] ''
1>&2 echo "setting up secrets"
${pkgs.fleet-install-secrets}/bin/fleet-install-secrets install ${secretsFile}
'';
rustfmt.tomldiffbeforeafterboth--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1 +1,3 @@
hard_tabs = true
+imports_granularity = "Crate"
+group_imports = "StdExternalCrate"