difftreelog
refactor rework fleet.nix secret bookkeeping
in: trunk
17 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -70,7 +70,7 @@
"aes",
"aes-gcm",
"age-core",
- "base64",
+ "base64 0.21.7",
"bcrypt-pbkdf",
"bech32",
"cbc",
@@ -102,7 +102,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5f11899bc2bbddd135edbc30c36b1924fa59d0746bb45beb5933fafe3fe509b"
dependencies = [
- "base64",
+ "base64 0.21.7",
"chacha20poly1305",
"cookie-factory",
"hkdf",
@@ -253,6 +253,12 @@
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
name = "base64ct"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -534,6 +540,7 @@
dependencies = [
"bitflags",
"crossterm_winapi",
+ "filedescriptor",
"libc",
"mio",
"parking_lot",
@@ -695,6 +702,17 @@
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
[[package]]
+name = "filedescriptor"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7199d965852c3bac31f779ef99cbb4537f80e952e2d6aa0ffeb30cce00f4f46e"
+dependencies = [
+ "libc",
+ "thiserror",
+ "winapi",
+]
+
+[[package]]
name = "find-crate"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -712,11 +730,12 @@
"age-core",
"anyhow",
"async-trait",
- "base64",
+ "base64 0.22.1",
"better-command",
"chrono",
"clap",
"crossterm",
+ "fleet-shared",
"futures",
"hostname",
"human-repr",
@@ -741,7 +760,6 @@
"tracing-indicatif",
"tracing-subscriber",
"unindent",
- "z85",
]
[[package]]
@@ -751,12 +769,22 @@
"age",
"anyhow",
"clap",
+ "fleet-shared",
"nix",
"serde",
"serde_json",
"tempfile",
"tracing",
"tracing-subscriber",
+]
+
+[[package]]
+name = "fleet-shared"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "serde",
+ "unicode_categories",
"z85",
]
@@ -1824,7 +1852,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
dependencies = [
- "base64",
+ "base64 0.21.7",
"bitflags",
"serde",
"serde_derive",
@@ -2013,9 +2041,9 @@
[[package]]
name = "serde"
-version = "1.0.201"
+version = "1.0.202"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c"
+checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395"
dependencies = [
"serde_derive",
]
@@ -2031,9 +2059,9 @@
[[package]]
name = "serde_derive"
-version = "1.0.201"
+version = "1.0.202"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865"
+checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838"
dependencies = [
"proc-macro2",
"quote",
@@ -2571,6 +2599,12 @@
checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
+[[package]]
name = "unindent"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,4 +9,4 @@
bifrostlink = "0.1.0"
uuid = { version = "1.7.0", features = ["v4"] }
tokio = { version = "1.36.0", features = ["fs", "rt", "macros", "sync", "time", "rt-multi-thread"] }
-
+fleet-shared = { path = "./crates/fleet-shared" }
cmds/fleet/Cargo.tomldiffbeforeafterboth--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -19,10 +19,15 @@
age-core = "0.10"
peg = "0.8"
age = { version = "0.10", features = ["ssh", "armor"] }
-base64 = "0.21"
+base64 = "0.22.1"
chrono = { version = "0.4", features = ["serde"] }
-z85 = "3.0"
-clap = { version = ">=4.4, <4.5", features = ["derive", "env", "wrap_help", "unicode"] }
+# Using fixed version for rust on stable nixos branches.
+clap = { version = ">=4.4, <4.5", features = [
+ "derive",
+ "env",
+ "wrap_help",
+ "unicode",
+] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
tokio-util = { version = "0.7", features = ["codec"] }
@@ -40,7 +45,8 @@
unindent = "0.2"
regex = "1.10"
openssh = "0.10"
-crossterm = "0.27"
+crossterm = { version = "0.27.0", features = ["use-dev-tty"] }
+fleet-shared.workspace = true
tracing-indicatif = { version = "0.3", optional = true }
human-repr = { version = "1.1", optional = true }
@@ -48,4 +54,9 @@
[features]
# Not quite stable
-indicatif = ["tracing-indicatif", "dep:indicatif", "human-repr", "better-command/indicatif"]
+indicatif = [
+ "tracing-indicatif",
+ "dep:indicatif",
+ "human-repr",
+ "better-command/indicatif",
+]
cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -1,27 +1,29 @@
-use crate::{
- better_nix_eval::Field,
- fleetdata::{FleetSecret, FleetSharedSecret, SecretData},
- host::Config,
- nix_go, nix_go_json,
+use std::{
+ collections::{BTreeMap, BTreeSet, HashSet},
+ ffi::OsString,
+ io::{self, stdin, stdout, Read, Write},
+ path::PathBuf,
};
+
use anyhow::{anyhow, bail, ensure, Context, Result};
use chrono::{DateTime, Utc};
-use clap::{error::ErrorKind, Parser};
+use clap::Parser;
use crossterm::{terminal, tty::IsTty};
+use fleet_shared::SecretData;
use itertools::Itertools;
use owo_colors::OwoColorize;
use serde::Deserialize;
-use std::{
- collections::{BTreeSet, HashSet},
- ffi::OsString,
- io::{self, stdin, Cursor, Read, Write},
- path::PathBuf,
-};
use tabled::{Table, Tabled};
-use tempfile::NamedTempFile;
-use tokio::{fs::read_to_string, process::Command};
+use tokio::{fs::read, process::Command};
use tracing::{error, info, info_span, warn, Instrument};
+use crate::{
+ better_nix_eval::Field,
+ fleetdata::{encrypt_secret_data, FleetSecret, FleetSecretPart, FleetSharedSecret},
+ host::Config,
+ nix_go, nix_go_json,
+};
+
#[derive(Parser)]
pub enum Secret {
/// Force load host keys for all defined hosts
@@ -38,6 +40,9 @@
/// Secret public part
#[clap(long)]
public: Option<String>,
+ /// How to name public secret part
+ #[clap(long, default_value = "public")]
+ public_name: String,
/// Load public part from specified file
#[clap(long)]
public_file: Option<PathBuf>,
@@ -49,6 +54,9 @@
/// Secret with this name already exists, override its value while keeping the same owners.
#[clap(long)]
re_add: bool,
+
+ #[clap(default_value = "secret")]
+ part_name: String,
},
/// Add secret, data should be provided in stdin
Add {
@@ -59,22 +67,27 @@
/// Override secret if already present
#[clap(long)]
force: bool,
+ /// Secret public part
#[clap(long)]
public: Option<String>,
+ /// How to name public secret part
+ #[clap(long, default_value = "public")]
+ public_name: String,
+ /// Load public part from specified file
#[clap(long)]
public_file: Option<PathBuf>,
+
+ #[clap(default_value = "secret")]
+ part_name: String,
},
/// Read secret from remote host, requires sudo on said host
Read {
name: String,
machine: String,
- #[clap(long)]
- plaintext: bool,
+
+ #[clap(default_value = "secret")]
+ part_name: String,
},
- ReadPublic {
- name: String,
- machine: String,
- },
UpdateShared {
name: String,
@@ -89,6 +102,9 @@
/// Which host should we use to decrypt
#[clap(long)]
prefer_identities: Vec<String>,
+
+ #[clap(default_value = "secret")]
+ part_name: String,
},
Regenerate {
/// Which host should we use to decrypt, in case if reencryption is required, without
@@ -97,6 +113,16 @@
prefer_identities: Vec<String>,
},
List {},
+ Edit {
+ name: String,
+ machine: String,
+
+ #[clap(default_value = "secret")]
+ part: String,
+
+ #[clap(long)]
+ add: bool,
+ },
}
#[tracing::instrument(skip(config, secret, field, prefer_identities))]
@@ -144,10 +170,16 @@
bail!("no available holder found");
};
- if let Some(data) = secret.secret.secret {
+ for (part_name, part) in secret.secret.parts.iter_mut() {
+ let _span = info_span!("part reencryption", part_name);
+ if !part.raw.encrypted {
+ continue;
+ }
let host = config.host(identity_holder).await?;
- let encrypted = host.reencrypt(data, updated_set.to_vec()).await?;
- secret.secret.secret = Some(encrypted);
+ let encrypted = host
+ .reencrypt(part.raw.clone(), updated_set.to_vec())
+ .await?;
+ part.raw = encrypted;
}
secret.owners = updated_set.to_vec();
@@ -232,13 +264,17 @@
ensure!(marker == "SUCCESS", "generation not succeeded");
}
- let public = host.read_file_text(format!("{out}/public")).await.ok();
- let secret = host.read_file_bin(format!("{out}/secret")).await.ok();
- if let Some(secret) = &secret {
- ensure!(
- age::Decryptor::new(Cursor::new(&secret)).is_ok(),
- "builder produced non-encrypted value as secret, this is highly insecure, and not allowed."
- );
+ let mut parts = BTreeMap::new();
+ for part in host.read_dir(&out).await? {
+ if part == "created_at" || part == "expired_at" || part == "marker" {
+ continue;
+ }
+ let contents: SecretData = host
+ .read_file_text(format!("{out}/{part}"))
+ .await?
+ .parse()
+ .map_err(|e| anyhow!("failed to decode secret {out:?} part {part:?}: {e}"))?;
+ parts.insert(part.to_owned(), FleetSecretPart { raw: contents });
}
let created_at = host.read_file_value(format!("{out}/created_at")).await?;
@@ -247,8 +283,7 @@
Ok(FleetSecret {
created_at,
expires_at,
- public,
- secret: secret.map(SecretData),
+ parts,
})
}
async fn generate(
@@ -310,10 +345,16 @@
async fn parse_public(
public: Option<String>,
public_file: Option<PathBuf>,
-) -> Result<Option<String>> {
+) -> Result<Option<SecretData>> {
Ok(match (public, public_file) {
- (Some(v), None) => Some(v),
- (None, Some(v)) => Some(read_to_string(v).await?),
+ (Some(v), None) => Some(SecretData {
+ data: v.into(),
+ encrypted: false,
+ }),
+ (None, Some(v)) => Some(SecretData {
+ data: read(v).await?,
+ encrypted: false,
+ }),
(Some(_), Some(_)) => {
bail!("only public or public_file should be set")
}
@@ -321,6 +362,16 @@
})
}
+async fn parse_secret() -> Result<Option<Vec<u8>>> {
+ let mut input = vec![];
+ io::stdin().read_to_end(&mut input)?;
+ if input.is_empty() {
+ Ok(None)
+ } else {
+ Ok(Some(input))
+ }
+}
+
fn parse_machines(
initial: Vec<String>,
machines: Option<Vec<String>>,
@@ -391,9 +442,11 @@
name,
force,
public,
+ public_name,
public_file,
expires_at,
re_add,
+ part_name,
} => {
// TODO: Forbid updating secrets with set expectedOwners (= not user-managed).
@@ -415,20 +468,21 @@
let recipients = config.recipients(machines.clone()).await?;
- let secret = {
- let mut input = vec![];
- io::stdin().read_to_end(&mut input)?;
+ let mut parts = BTreeMap::new();
- if input.is_empty() {
- None
- } else {
- Some(
- SecretData::encrypt(recipients, input)
- .ok_or_else(|| anyhow!("no recipients provided"))?,
- )
- }
- };
- let public = parse_public(public, public_file).await?;
+ let mut input = vec![];
+ io::stdin().read_to_end(&mut input)?;
+
+ if !input.is_empty() {
+ let encrypted = encrypt_secret_data(recipients, input)
+ .ok_or_else(|| anyhow!("no recipients provided"))?;
+ parts.insert(part_name, FleetSecretPart { raw: encrypted });
+ }
+
+ if let Some(public) = parse_public(public, public_file).await? {
+ parts.insert(public_name, FleetSecretPart { raw: public });
+ }
+
config.replace_shared(
name,
FleetSharedSecret {
@@ -436,8 +490,7 @@
secret: FleetSecret {
created_at: Utc::now(),
expires_at,
- secret,
- public,
+ parts,
},
},
);
@@ -447,33 +500,34 @@
name,
force,
public,
+ public_name,
public_file,
+ part_name,
} => {
- let recipient = config.recipient(&machine).await?;
+ if config.has_secret(&machine, &name) && !force {
+ bail!("secret already defined");
+ }
- let secret = {
- let mut input = vec![];
- io::stdin().read_to_end(&mut input)?;
- if input.is_empty() {
- bail!("no data provided")
- }
+ let mut parts = BTreeMap::new();
- Some(SecretData::encrypt(vec![recipient], input).expect("recipient provided"))
- };
-
- if config.has_secret(&machine, &name) && !force {
- bail!("secret already defined");
+ if let Some(secret) = parse_secret().await? {
+ let recipient = config.recipient(&machine).await?;
+ let encrypted =
+ encrypt_secret_data(vec![recipient], secret).expect("recipient provided");
+ parts.insert(part_name, FleetSecretPart { raw: encrypted });
}
- let public = parse_public(public, public_file).await?;
+ if let Some(public) = parse_public(public, public_file).await? {
+ parts.insert(public_name, FleetSecretPart { raw: public });
+ };
+
config.insert_secret(
&machine,
name,
FleetSecret {
created_at: Utc::now(),
expires_at: None,
- secret,
- public,
+ parts,
},
);
}
@@ -481,27 +535,20 @@
Secret::Read {
name,
machine,
- plaintext,
+ part_name,
} => {
let secret = config.host_secret(&machine, &name)?;
- let Some(secret) = secret.secret else {
- bail!("no secret {name}");
+ let Some(secret) = secret.parts.get(&part_name) else {
+ bail!("no part {part_name} in secret {name}");
};
- let host = config.host(&machine).await?;
- let data = host.decrypt(secret).await?;
- if plaintext {
- let s = String::from_utf8(data).context("output is not utf8")?;
- print!("{s}");
+ let data = if secret.raw.encrypted {
+ let host = config.host(&machine).await?;
+ host.decrypt(secret.raw.clone()).await?
} else {
- println!("{}", z85::encode(&data));
- }
- }
- Secret::ReadPublic { name, machine } => {
- let secret = config.host_secret(&machine, &name)?;
- let Some(public) = secret.public else {
- bail!("no secret {name}");
+ secret.raw.data.clone()
};
- print!("{public}");
+
+ stdout().write_all(&data)?;
}
Secret::UpdateShared {
name,
@@ -509,11 +556,12 @@
add_machines,
remove_machines,
prefer_identities,
+ part_name,
} => {
// TODO: Forbid updating secrets with set expectedOwners (= not user-managed).
let secret = config.shared_secret(&name)?;
- if secret.secret.secret.is_none() {
+ if secret.secret.parts.get(&part_name).is_none() {
bail!("no secret");
}
@@ -572,6 +620,10 @@
}
}
for host in config.list_hosts().await? {
+ if config.should_skip(&host.name) {
+ continue;
+ }
+
let _span = info_span!("host", host = host.name).entered();
let expected_set = host
.list_configured_secrets()
@@ -664,7 +716,103 @@
}
info!("loaded\n{}", Table::new(table).to_string())
}
+ Secret::Edit {
+ name,
+ machine,
+ part,
+ add,
+ } => {
+ let secret = config.host_secret(&machine, &name)?;
+ if let Some(data) = secret.parts.get(&part) {
+ let host = config.host(&machine).await?;
+ let secret = host.decrypt(data.raw.clone()).await?;
+ String::from_utf8(secret).context("secret is not utf8")?
+ } else if add {
+ String::new()
+ } else {
+ bail!("part {part} not found in secret {name}. Did you mean to `--add` it?");
+ };
+ }
}
Ok(())
}
}
+
+async fn edit_temp_file(
+ builder: tempfile::Builder<'_, '_>,
+ r: Vec<u8>,
+ header: &str,
+ comment: &str,
+) -> Result<(Vec<u8>, Option<String>), anyhow::Error> {
+ if !stdin().is_tty() {
+ // TODO: Also try to open /dev/tty directly?
+ bail!("stdin is not tty, can't open editor");
+ }
+
+ use std::fmt::Write;
+ let mut file = builder.tempfile()?;
+
+ let mut full_header = String::new();
+ let mut had = false;
+ for line in header.trim_end().lines() {
+ had = true;
+ writeln!(&mut full_header, "{comment}{line}")?;
+ }
+ if had {
+ writeln!(&mut full_header, "{}", comment.trim_end())?;
+ }
+ writeln!(
+ &mut full_header,
+ "{comment}Do not touch this header! It will be removed automatically"
+ )?;
+
+ file.write_all(full_header.as_bytes())?;
+ file.write_all(&r)?;
+
+ let abs_path = file.into_temp_path();
+ let editor = std::env::var_os("VISUAL")
+ .or_else(|| std::env::var_os("EDITOR"))
+ .unwrap_or_else(|| "vi".into());
+ let editor_args = shlex::bytes::split(editor.as_encoded_bytes())
+ .ok_or_else(|| anyhow!("EDITOR env var has wrong syntax"))?;
+ let editor_args = editor_args
+ .into_iter()
+ .map(|v| {
+ // Only ASCII subsequences are replaced
+ unsafe { OsString::from_encoded_bytes_unchecked(v) }
+ })
+ .collect_vec();
+ let Some((editor, args)) = editor_args.split_first() else {
+ bail!("EDITOR env var has no command");
+ };
+ let mut command = Command::new(editor);
+ command.args(args);
+
+ let path_arg = abs_path.canonicalize()?;
+
+ // TODO: Save full state, using tcget/_getmode/_setmode
+ let was_raw = terminal::is_raw_mode_enabled()?;
+ terminal::enable_raw_mode()?;
+
+ let status = command.arg(path_arg).status().await;
+
+ if !was_raw {
+ terminal::disable_raw_mode()?;
+ }
+
+ let success = match status {
+ Ok(s) => s.success(),
+ Err(e) if e.kind() == io::ErrorKind::NotFound => {
+ bail!("editor not found")
+ }
+ Err(e) => bail!("editor spawn error: {e}"),
+ };
+
+ let mut file = std::fs::read(&abs_path).context("read editor output")?;
+ let Some(v) = file.strip_prefix(full_header.as_bytes()) else {
+ todo!();
+ };
+ todo!();
+
+ // Ok((success, abs_path))
+}
cmds/fleet/src/fleetdata.rsdiffbeforeafterboth--- a/cmds/fleet/src/fleetdata.rs
+++ b/cmds/fleet/src/fleetdata.rs
@@ -1,20 +1,14 @@
-use age::Recipient;
-use anyhow::Result;
-use chrono::{DateTime, Utc};
-use itertools::Itertools;
-use nixlike::format_nix;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::{
collections::BTreeMap,
io::{self, Cursor},
};
-use tempfile::TempDir;
-use tokio::{
- fs::{self, File},
- io::AsyncWriteExt,
- process::Command,
-};
+use age::Recipient;
+use chrono::{DateTime, Utc};
+use fleet_shared::SecretData;
+use itertools::Itertools;
+use serde::{de::Error, Deserialize, Serialize};
+
#[derive(Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct HostData {
@@ -23,9 +17,36 @@
pub encryption_key: String,
}
+const VERSION: &str = "0.1.0";
+pub struct FleetDataVersion;
+impl Serialize for FleetDataVersion {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ VERSION.serialize(serializer)
+ }
+}
+impl<'de> Deserialize<'de> for FleetDataVersion {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ let version = String::deserialize(deserializer)?;
+ if version != VERSION {
+ return Err(D::Error::custom(format!(
+ "fleet.nix data version mismatch, expected {VERSION}, got {version}.\nFollow the docs for migration instruction"
+ )));
+ }
+ Ok(Self)
+ }
+}
+
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct FleetData {
+ pub version: FleetDataVersion,
+
#[serde(default)]
pub hosts: BTreeMap<String, HostData>,
#[serde(default)]
@@ -45,41 +66,30 @@
pub secret: FleetSecret,
}
+/// Returns None if recipients.is_empty()
+pub fn encrypt_secret_data(
+ recipients: impl IntoIterator<Item = impl Recipient + Send + 'static>,
+ data: Vec<u8>,
+) -> Option<SecretData> {
+ let mut encrypted = vec![];
+ let recipients = recipients
+ .into_iter()
+ .map(|v| Box::new(v) as Box<dyn Recipient + Send>)
+ .collect_vec();
+ let mut encryptor = age::Encryptor::with_recipients(recipients)?
+ .wrap_output(&mut encrypted)
+ .expect("in memory write");
+ io::copy(&mut Cursor::new(data), &mut encryptor).expect("in memory copy");
+ encryptor.finish().expect("in memory flush");
+ Some(SecretData {
+ data: encrypted,
+ encrypted: true,
+ })
+}
+
#[derive(Serialize, Deserialize, Clone)]
-pub struct SecretData(
- #[serde(
- default,
- skip_serializing_if = "Vec::is_empty",
- serialize_with = "as_z85",
- deserialize_with = "from_z85"
- )]
- pub Vec<u8>,
-);
-impl SecretData {
- /// Returns None if recipients.is_empty()
- pub fn encrypt(
- recipients: impl IntoIterator<Item = impl Recipient + Send + 'static>,
- data: Vec<u8>,
- ) -> Option<Self> {
- let mut encrypted = vec![];
- let recipients = recipients
- .into_iter()
- .map(|v| Box::new(v) as Box<dyn Recipient + Send>)
- .collect_vec();
- let mut encryptor = age::Encryptor::with_recipients(recipients)?
- .wrap_output(&mut encrypted)
- .expect("in memory write");
- io::copy(&mut Cursor::new(data), &mut encryptor).expect("in memory copy");
- encryptor.finish().expect("in memory flush");
- Some(Self(encrypted))
- }
- pub fn encode_z85(&self) -> String {
- z85::encode(&self.0)
- }
- pub fn decode_z85(v: &str) -> Result<Self> {
- let v = z85::decode(v)?;
- Ok(Self(v))
- }
+pub struct FleetSecretPart {
+ pub raw: SecretData,
}
#[derive(Serialize, Deserialize, Clone)]
@@ -91,60 +101,7 @@
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none", alias = "expire_at")]
pub expires_at: Option<DateTime<Utc>>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub public: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- pub secret: Option<SecretData>,
-}
-
-fn as_z85<S>(key: &[u8], serializer: S) -> Result<S::Ok, S::Error>
-where
- S: Serializer,
-{
- serializer.serialize_str(&z85::encode(key))
-}
-fn from_z85<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
-where
- D: Deserializer<'de>,
-{
- use serde::de::Error;
- String::deserialize(deserializer)
- .and_then(|string| z85::decode(string).map_err(|err| Error::custom(err.to_string())))
-}
-
-/// Isn't used yet
-#[allow(dead_code)]
-pub async fn dummy_flake() -> Result<TempDir> {
- let data_str = fs::read_to_string("fleet.nix").await?;
-
- let mut cmd = Command::new("nix");
- cmd.arg("flake").arg("metadata").arg("--json");
-
- let flake_dir = tempfile::tempdir()?;
- let mut flake_nix = flake_dir.path().to_path_buf();
- flake_nix.push("flake.nix");
- // flake_dir
-
- File::create(&flake_nix)
- .await?
- .write_all(
- format_nix(&format!(
- "
- {{
- outputs = {{self, ...}}: {{
- data = {data_str};
- }};
- }}
- "
- ))
- .as_bytes(),
- )
- .await?;
-
- // std::thread::sleep(Duration::MAX);
- // flake_dir.close()
- // FIXME
- dbg!(&flake_nix);
- Ok(flake_dir)
+ #[serde(flatten)]
+ pub parts: BTreeMap<String, FleetSecretPart>,
}
cmds/fleet/src/host.rsdiffbeforeafterboth1use std::{2 env::current_dir,3 ffi::{OsStr, OsString},4 fmt::Display,5 io::Write,6 ops::Deref,7 path::PathBuf,8 str::FromStr,9 sync::{Arc, Mutex, MutexGuard, OnceLock},10};1112use anyhow::{anyhow, bail, Context, Result};13use clap::{ArgGroup, Parser};14use openssh::SessionBuilder;15use serde::de::DeserializeOwned;16use tempfile::NamedTempFile;1718use crate::{19 better_nix_eval::{Field, NixSessionPool},20 command::MyCommand,21 fleetdata::{FleetData, FleetSecret, FleetSharedSecret, SecretData},22 nix_go, nix_go_json,23};2425pub struct FleetConfigInternals {26 pub local_system: String,27 pub directory: PathBuf,28 pub opts: FleetOpts,29 pub data: Mutex<FleetData>,30 pub nix_args: Vec<OsString>,31 /// fleet_config.config32 pub config_field: Field,33 /// fleet_config.unchecked.config34 pub config_unchecked_field: Field,3536 /// import nixpkgs {system = local};37 pub default_pkgs: Field,38}3940#[derive(Clone)]41pub struct Config(Arc<FleetConfigInternals>);4243impl Deref for Config {44 type Target = FleetConfigInternals;4546 fn deref(&self) -> &Self::Target {47 &self.048 }49}5051pub struct ConfigHost {52 config: Config,53 pub name: String,54 pub local: bool,55 pub session: OnceLock<Arc<openssh::Session>>,5657 pub nixos_config: Option<Field>,58}59impl ConfigHost {60 async fn open_session(&self) -> Result<Arc<openssh::Session>> {61 assert!(!self.local, "do not open ssh connection to local session");62 // FIXME: TOCTOU63 if let Some(session) = &self.session.get() {64 return Ok((*session).clone());65 };66 let session = SessionBuilder::default();6768 let session = session69 .connect(&self.name)70 .await71 .map_err(|e| anyhow!("ssh error while connecting to {}: {e}", self.name))?;72 let session = Arc::new(session);73 self.session.set(session.clone()).expect("TOCTOU happened");74 Ok(session)75 }76 pub async fn mktemp_dir(&self) -> Result<String> {77 let mut cmd = self.cmd("mktemp").await?;78 cmd.arg("-d");79 let path = cmd.run_string().await?;80 Ok(path.trim_end().to_owned())81 }82 pub async fn read_file_bin(&self, path: impl AsRef<OsStr>) -> Result<Vec<u8>> {83 let mut cmd = self.cmd("cat").await?;84 cmd.arg(path);85 cmd.run_bytes().await86 }87 pub async fn read_file_text(&self, path: impl AsRef<OsStr>) -> Result<String> {88 let mut cmd = self.cmd("cat").await?;89 cmd.arg(path);90 cmd.run_string().await91 }92 #[allow(dead_code)]93 pub async fn read_file_json<D: DeserializeOwned>(&self, path: impl AsRef<OsStr>) -> Result<D> {94 let text = self.read_file_text(path).await?;95 Ok(serde_json::from_str(&text)?)96 }97 pub async fn read_file_value<D: FromStr>(&self, path: impl AsRef<OsStr>) -> Result<D>98 where99 <D as FromStr>::Err: Display,100 {101 let text = self.read_file_text(path).await?;102 D::from_str(&text).map_err(|e| anyhow!("failed to parse value: {e}"))103 }104 pub async fn cmd(&self, cmd: impl AsRef<OsStr>) -> Result<MyCommand> {105 if self.local {106 Ok(MyCommand::new(cmd))107 } else {108 let session = self.open_session().await?;109 Ok(MyCommand::new_on(cmd, session))110 }111 }112113 pub async fn decrypt(&self, data: SecretData) -> Result<Vec<u8>> {114 let mut cmd = self.cmd("fleet-install-secrets").await?;115 cmd.arg("decrypt").eqarg("--secret", data.encode_z85());116 let encoded = cmd117 .sudo()118 .run_string()119 .await120 .context("failed to call remote host for decrypt")?;121 z85::decode(encoded.trim_end()).context("bad encoded data? outdated host?")122 }123 pub async fn reencrypt(&self, data: SecretData, targets: Vec<String>) -> Result<SecretData> {124 let mut cmd = self.cmd("fleet-install-secrets").await?;125 cmd.arg("reencrypt").eqarg("--secret", data.encode_z85());126 for target in targets {127 let key = self.config.key(&target).await?;128 cmd.eqarg("--targets", key);129 }130 let encoded = cmd131 .sudo()132 .run_string()133 .await134 .context("failed to call remote host for decrypt")?;135 SecretData::decode_z85(encoded.trim_end()).context("bad encoded data? outdated host?")136 }137 /// Returns path for futureproofing, as path might change i.e on conversion to CA138 pub async fn remote_derivation(&self, path: &PathBuf) -> Result<PathBuf> {139 if self.local {140 // Path is located locally, thus already trusted.141 return Ok(path.to_owned());142 }143 let mut nix = MyCommand::new("nix");144 nix.arg("copy")145 .arg("--substitute-on-destination")146 .comparg("--to", format!("ssh-ng://{}", self.name))147 .arg(path);148 nix.run_nix().await.context("nix copy")?;149 Ok(path.to_owned())150 }151 pub async fn systemctl_stop(&self, name: &str) -> Result<()> {152 let mut cmd = self.cmd("systemctl").await?;153 cmd.arg("stop").arg(name);154 cmd.sudo().run().await155 }156 pub async fn systemctl_start(&self, name: &str) -> Result<()> {157 let mut cmd = self.cmd("systemctl").await?;158 cmd.arg("start").arg(name);159 cmd.sudo().run().await160 }161162 pub async fn rm_file(&self, path: impl AsRef<OsStr>, sudo: bool) -> Result<()> {163 let mut cmd = self.cmd("rm").await?;164 cmd.arg("-f").arg(path);165 if sudo {166 cmd = cmd.sudo()167 }168 cmd.run().await169 }170171 pub async fn list_configured_secrets(&self) -> Result<Vec<String>> {172 let Some(nixos) = &self.nixos_config else {173 return Ok(vec![]);174 };175 let secrets = nix_go!(nixos.secrets);176 let mut out = Vec::new();177 for name in secrets.list_fields().await? {178 let secret = nix_go!(secrets[{ name }]);179 let is_shared: bool = nix_go_json!(secret.shared);180 if is_shared {181 continue;182 }183 out.push(name);184 }185 Ok(out)186 }187 pub async fn secret_field(&self, name: &str) -> Result<Field> {188 let Some(nixos) = &self.nixos_config else {189 bail!("host is virtual and has no secrets");190 };191 Ok(nix_go!(nixos.secrets[{ name }]))192 }193194 /// Packages for this host, resolved with nixpkgs overlays195 pub async fn pkgs(&self) -> Result<Field> {196 let Some(nixos) = &self.nixos_config else {197 return Ok(self.config.default_pkgs.clone());198 };199 Ok(nix_go!(nixos.nixpkgs.resolvedPkgs))200 }201}202203impl Config {204 pub fn should_skip(&self, host: &str) -> bool {205 if !self.opts.skip.is_empty() {206 self.opts.skip.iter().any(|h| h as &str == host)207 } else if !self.opts.only.is_empty() {208 !self.opts.only.iter().any(|h| h as &str == host)209 } else {210 false211 }212 }213 pub fn is_local(&self, host: &str) -> bool {214 self.opts.localhost.as_ref().map(|s| s as &str) == Some(host)215 }216217 pub fn local_host(&self) -> ConfigHost {218 ConfigHost {219 config: self.clone(),220 name: "<virtual localhost>".to_owned(),221 local: true,222 session: OnceLock::new(),223 nixos_config: None,224 }225 }226227 pub async fn host(&self, name: &str) -> Result<ConfigHost> {228 let config = &self.config_unchecked_field;229 let nixos_config = nix_go!(config.hosts[{ name }].nixosSystem.config);230 Ok(ConfigHost {231 config: self.clone(),232 name: name.to_owned(),233 local: self.is_local(name),234 session: OnceLock::new(),235 nixos_config: Some(nixos_config),236 })237 }238 pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {239 let config = &self.config_unchecked_field;240 let names = nix_go!(config.hosts).list_fields().await?;241 let mut out = vec![];242 for name in names {243 out.push(self.host(&name).await?);244 }245 Ok(out)246 }247 pub async fn system_config(&self, host: &str) -> Result<Field> {248 let fleet_field = &self.config_unchecked_field;249 Ok(nix_go!(fleet_field.hosts[{ host }].nixosSystem.config))250 }251252 pub(super) fn data(&self) -> MutexGuard<FleetData> {253 self.data.lock().unwrap()254 }255 pub(super) fn data_mut(&self) -> MutexGuard<FleetData> {256 self.data.lock().unwrap()257 }258 /// Shared secrets configured in fleet.nix or in flake259 pub async fn list_configured_shared(&self) -> Result<Vec<String>> {260 let config_field = &self.config_unchecked_field;261 nix_go!(config_field.sharedSecrets).list_fields().await262 }263 /// Shared secrets configured in fleet.nix264 pub fn list_shared(&self) -> Vec<String> {265 let data = self.data();266 data.shared_secrets.keys().cloned().collect()267 }268 pub fn has_shared(&self, name: &str) -> bool {269 let data = self.data();270 data.shared_secrets.contains_key(name)271 }272 pub fn replace_shared(&self, name: String, shared: FleetSharedSecret) {273 let mut data = self.data_mut();274 data.shared_secrets.insert(name.to_owned(), shared);275 }276 pub fn remove_shared(&self, secret: &str) {277 let mut data = self.data_mut();278 data.shared_secrets.remove(secret);279 }280281 pub fn list_secrets(&self, host: &str) -> Vec<String> {282 let data = self.data();283 let Some(secrets) = data.host_secrets.get(host) else {284 return Vec::new();285 };286 secrets.keys().cloned().collect()287 }288289 pub fn has_secret(&self, host: &str, secret: &str) -> bool {290 let data = self.data();291 let Some(host_secrets) = data.host_secrets.get(host) else {292 return false;293 };294 host_secrets.contains_key(secret)295 }296 pub fn insert_secret(&self, host: &str, secret: String, value: FleetSecret) {297 let mut data = self.data_mut();298 let host_secrets = data.host_secrets.entry(host.to_owned()).or_default();299 host_secrets.insert(secret, value);300 }301302 pub fn host_secret(&self, host: &str, secret: &str) -> Result<FleetSecret> {303 let data = self.data();304 let Some(host_secrets) = data.host_secrets.get(host) else {305 bail!("no secrets for machine {host}");306 };307 let Some(secret) = host_secrets.get(secret) else {308 bail!("machine {host} has no secret {secret}");309 };310 Ok(secret.clone())311 }312 pub fn shared_secret(&self, secret: &str) -> Result<FleetSharedSecret> {313 let data = self.data();314 let Some(secret) = data.shared_secrets.get(secret) else {315 bail!("no shared secret {secret}");316 };317 Ok(secret.clone())318 }319 pub async fn shared_secret_expected_owners(&self, secret: &str) -> Result<Vec<String>> {320 let config_field = &self.config_unchecked_field;321 Ok(nix_go_json!(322 config_field.sharedSecrets[{ secret }].expectedOwners323 ))324 }325326 pub fn save(&self) -> Result<()> {327 let mut tempfile = NamedTempFile::new_in(self.directory.clone())?;328 let data = nixlike::serialize(&self.data() as &FleetData)?;329 tempfile.write_all(330 format!(331 "# This file contains fleet state and shouldn't be edited by hand\n\n{}\n\n# vim: ts=2 et nowrap\n",332 data333 )334 .as_bytes(),335 )?;336 let mut fleet_data_path = self.directory.clone();337 fleet_data_path.push("fleet.nix");338 tempfile.persist(fleet_data_path)?;339 Ok(())340 }341}342343#[derive(Parser, Clone)]344#[clap(group = ArgGroup::new("target_hosts"))]345pub struct FleetOpts {346 /// All hosts except those would be skipped347 #[clap(long, number_of_values = 1, group = "target_hosts")]348 only: Vec<String>,349350 /// Hosts to skip351 #[clap(long, number_of_values = 1, group = "target_hosts")]352 skip: Vec<String>,353354 /// Host, which should be threaten as current machine355 #[clap(long)]356 pub localhost: Option<String>,357358 /// Override detected system for host, to perform builds via359 /// binfmt-declared qemu instead of trying to crosscompile360 #[clap(long, default_value = "detect")]361 pub local_system: String,362}363364impl FleetOpts {365 pub async fn build(mut self, nix_args: Vec<OsString>) -> Result<Config> {366 if self.localhost.is_none() {367 self.localhost368 .replace(hostname::get().unwrap().to_str().unwrap().to_owned());369 }370 let directory = current_dir()?;371372 let pool = NixSessionPool::new(directory.as_os_str().to_owned(), nix_args.clone()).await?;373 let root_field = pool.get().await?;374375 let builtins_field = Field::field(root_field.clone(), "builtins").await?;376 if self.local_system == "detect" {377 self.local_system = nix_go_json!(builtins_field.currentSystem);378 }379 let local_system = self.local_system.clone();380381 let fleet_root = Field::field(root_field, "fleetConfigurations").await?;382 let fleet_field = nix_go!(fleet_root.default);383384 let config_field = nix_go!(fleet_field.config);385 let config_unchecked_field = nix_go!(fleet_field.unchecked.config);386387 let import = nix_go!(builtins_field.import);388 let overlays = nix_go!(config_unchecked_field.overlays);389 let nixpkgs = nix_go!(fleet_field.nixpkgs | import);390391 let default_pkgs = nix_go!(nixpkgs(Obj {392 overlays,393 system: { self.local_system.clone() },394 }));395396 let mut fleet_data_path = directory.clone();397 fleet_data_path.push("fleet.nix");398 let bytes = std::fs::read_to_string(fleet_data_path)?;399 let data = nixlike::parse_str(&bytes)?;400401 Ok(Config(Arc::new(FleetConfigInternals {402 opts: self,403 directory,404 data,405 local_system,406 nix_args,407 config_field,408 config_unchecked_field,409 default_pkgs,410 })))411 }412}cmds/install-secrets/Cargo.tomldiffbeforeafterboth--- a/cmds/install-secrets/Cargo.toml
+++ b/cmds/install-secrets/Cargo.toml
@@ -20,4 +20,4 @@
"unicode",
] }
tempfile = "3.10.0"
-z85 = "3.0.5"
+fleet-shared.workspace = true
cmds/install-secrets/src/main.rsdiffbeforeafterboth--- a/cmds/install-secrets/src/main.rs
+++ b/cmds/install-secrets/src/main.rs
@@ -1,64 +1,41 @@
-use age::{ssh::Identity as SshIdentity, ssh::Recipient as SshRecipient, Decryptor};
-use age::{Encryptor, Identity, Recipient};
-use anyhow::{anyhow, bail, Context, Result};
+use std::{
+ collections::{BTreeMap, HashMap},
+ fs::{self, File},
+ io::{self, Cursor, Read, Write},
+ iter,
+ os::unix::prelude::PermissionsExt,
+ path::{Path, PathBuf},
+ str::{from_utf8, FromStr},
+};
+
+use age::{
+ ssh::{Identity as SshIdentity, Recipient as SshRecipient},
+ Decryptor, Encryptor, Identity, Recipient,
+};
+use anyhow::{anyhow, bail, ensure, Context, Result};
use clap::Parser;
-use nix::sys::stat::Mode;
+use fleet_shared::SecretData;
use nix::unistd::{chown, Group, User};
-use serde::{Deserialize, Deserializer};
-use std::fmt::{self, Display};
-use std::fs::{self, File};
-use std::io::{self, Cursor, Read, Write};
-use std::iter;
-use std::os::unix::prelude::PermissionsExt;
-use std::path::Path;
-use std::str::{from_utf8, FromStr};
-use std::{collections::HashMap, path::PathBuf};
-use tracing::{error, info, info_span, warn};
-use tracing_subscriber::filter::LevelFilter;
-use tracing_subscriber::EnvFilter;
-
-#[derive(Clone, Debug)]
-struct SecretWrapper(Vec<u8>);
-impl Display for SecretWrapper {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let encoded = z85::encode(&self.0);
- write!(f, "{encoded}")
- }
-}
-impl FromStr for SecretWrapper {
- type Err = z85::DecodeError;
-
- fn from_str(s: &str) -> Result<Self, Self::Err> {
- z85::decode(s).map(Self)
- }
-}
-impl<'de> Deserialize<'de> for SecretWrapper {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: Deserializer<'de>,
- {
- let v = String::deserialize(deserializer)?;
- let de = z85::decode(v).map_err(|err| serde::de::Error::custom(err.to_string()))?;
- Ok(Self(de))
- }
-}
+use serde::Deserialize;
+use tracing::{error, info_span};
+use tracing_subscriber::{filter::LevelFilter, EnvFilter};
#[derive(Parser)]
#[clap(author)]
enum Opts {
/// Install secrets from json specification
Install { data: PathBuf },
- /// Reencrypt secret using host key, outputting in z85 encoded string
+ /// Reencrypt secret using host key, outputting in fleet encoded string
Reencrypt {
#[clap(long)]
- secret: SecretWrapper,
+ secret: SecretData,
#[clap(long)]
targets: Vec<String>,
},
- /// Decrypt secret using host key, outputting in z85 encoded string
+ /// Decrypt secret using host key, outputting in fleet encoded string
Decrypt {
#[clap(long)]
- secret: SecretWrapper,
+ secret: SecretData,
/// Shoult decoded output be printed as plaintext, instead of z85?
#[clap(long)]
plaintext: bool,
@@ -67,25 +44,29 @@
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
+struct Part {
+ raw: SecretData,
+ path: PathBuf,
+ stable_path: PathBuf,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
struct DataItem {
group: String,
mode: String,
owner: String,
-
- secret: Option<SecretWrapper>,
- public: Option<String>,
-
- public_path: PathBuf,
- stable_public_path: PathBuf,
+ root_path: Option<PathBuf>,
- secret_path: PathBuf,
- stable_secret_path: PathBuf,
+ #[serde(flatten)]
+ parts: BTreeMap<String, Part>,
}
type Data = HashMap<String, DataItem>;
-fn decrypt(input: &SecretWrapper, identity: &dyn Identity) -> Result<Vec<u8>> {
- let mut input = Cursor::new(&input.0);
+fn decrypt(input: &SecretData, identity: &dyn Identity) -> Result<Vec<u8>> {
+ ensure!(input.encrypted, "passed data is not encrypted!");
+ let mut input = Cursor::new(&input.data);
let decryptor = Decryptor::new(&mut input).context("failed to init decryptor")?;
let decryptor = match decryptor {
Decryptor::Recipients(r) => r,
@@ -101,7 +82,7 @@
.context("failed to decrypt")?;
Ok(decrypted)
}
-fn encrypt(input: &[u8], targets: Vec<String>) -> Result<SecretWrapper> {
+fn encrypt(input: &[u8], targets: Vec<String>) -> Result<SecretData> {
let recipients = targets
.into_iter()
.map(|t| {
@@ -119,70 +100,79 @@
.expect("constructor should not fail");
io::copy(&mut Cursor::new(input), &mut encryptor).expect("copy should not fail");
encryptor.finish().context("failed to finish encryption")?;
- Ok(SecretWrapper(encrypted))
+ Ok(SecretData {
+ data: encrypted,
+ encrypted: true,
+ })
}
-fn init_secret(identity: &age::ssh::Identity, value: DataItem) -> Result<()> {
- if let Some(public) = &value.public {
- let mut hashed = File::create(&value.public_path)?;
- let stable_dir = value.stable_public_path.parent().expect("not root");
- let mut stable_temp =
- tempfile::NamedTempFile::new_in(stable_dir).context("failed to create tempfile")?;
- hashed.write_all(public.as_bytes())?;
- stable_temp.write_all(public.as_bytes())?;
- stable_temp.flush()?;
- fs::set_permissions(stable_temp.path(), fs::Permissions::from_mode(0o444))
- .context("perm")?;
- fs::set_permissions(&value.public_path, fs::Permissions::from_mode(0o444))
- .context("perm")?;
+fn init_part(identity: &dyn Identity, item: &DataItem, value: &Part) -> Result<()> {
+ let stable_dir = value.stable_path.parent().expect("not root");
- stable_temp
- .persist(value.stable_public_path)
- .context("failed to persist")?;
- }
- if value.secret.is_none() {
- info!("no secret data found");
- return Ok(());
- }
- let secret = value.secret.as_ref().unwrap();
-
- let mode = Mode::from_bits(
- u32::from_str_radix(&value.mode, 8).context("failed to parse mode as octal")?,
- )
- .context("failed to parse mode")?;
- let user = User::from_name(&value.owner)
- .context("failed to get user")?
- .ok_or_else(|| anyhow!("user not found"))?;
- let group = Group::from_name(&value.group)
- .context("failed to get group")?
- .ok_or_else(|| anyhow!("group not found"))?;
+ // Right now stable & non-stable data are both located in this dir.
+ std::fs::create_dir_all(stable_dir)?;
- let stable_dir = value.stable_secret_path.parent().expect("not root");
let mut stable_temp =
tempfile::NamedTempFile::new_in(stable_dir).context("failed to create tempfile")?;
- let mut hashed = File::create(&value.secret_path)?;
+ let mut hashed = File::create(&value.path)?;
- // File is owned by root, and only root can modify it
- let decrypted = decrypt(secret, identity)?;
- if decrypted.is_empty() {
- warn!("secret is decoded as empty, something is broken?");
- }
+ let private = value.raw.encrypted;
+ let data = if private {
+ decrypt(&value.raw, identity)?
+ } else {
+ value.raw.data.to_owned()
+ };
- io::copy(&mut Cursor::new(&decrypted), &mut stable_temp)
- .context("failed to write decrypted file")?;
- io::copy(&mut Cursor::new(decrypted), &mut hashed).context("failed to write decrypted file")?;
+ hashed.write_all(&data)?;
+ hashed.flush()?;
+ stable_temp.write_all(&data)?;
+ stable_temp.flush()?;
- // Make file owned by specified user and group, then change mode
- chown(stable_temp.path(), Some(user.uid), Some(group.gid))
- .context("failed to apply user/group")?;
- chown(&value.secret_path, Some(user.uid), Some(group.gid))
- .context("failed to apply user/group")?;
- fs::set_permissions(stable_temp.path(), fs::Permissions::from_mode(mode.bits())).unwrap();
- fs::set_permissions(&value.secret_path, fs::Permissions::from_mode(mode.bits())).unwrap();
+ let mode = if private {
+ fs::Permissions::from_mode(
+ u32::from_str_radix(&item.mode, 8).context("failed to parse mode as octal")?,
+ )
+ } else {
+ fs::Permissions::from_mode(0o444)
+ };
+ fs::set_permissions(stable_temp.path(), mode.clone()).context("stable temp mode")?;
+ fs::set_permissions(&value.path, mode).context("hashed mode")?;
+
+ // Files are initially owned by root, thus making set mode first inaccessible to user, and then
+ // altering user/group.
+ if private {
+ let user = User::from_name(&item.owner)
+ .context("failed to get user")?
+ .ok_or_else(|| anyhow!("user not found"))?;
+ let group = Group::from_name(&item.group)
+ .context("failed to get group")?
+ .ok_or_else(|| anyhow!("group not found"))?;
+
+ chown(stable_temp.path(), Some(user.uid), Some(group.gid))
+ .context("failed to apply user/group")?;
+ chown(&value.path, Some(user.uid), Some(group.gid))
+ .context("failed to apply user/group")?;
+ }
+
stable_temp
- .persist(value.stable_secret_path)
- .context("failed to persist")?;
+ .persist(&value.stable_path)
+ .context("stable persist")?;
+ Ok(())
+}
+fn init_secret(identity: &age::ssh::Identity, value: &DataItem) -> Result<()> {
+ if let Some(root_path) = &value.root_path {
+ if !fs::metadata(root_path).map(|m| m.is_dir()).unwrap_or(false) {
+ fs::create_dir(root_path).context("failed to create secret directory")?;
+ }
+ }
+ for (part_id, part) in value.parts.iter() {
+ let _span = info_span!("part", part_id = part_id);
+ if let Err(e) = init_part(identity, value, part) {
+ error!("failed to init part {part_id}: {e}");
+ }
+ }
+
Ok(())
}
@@ -214,8 +204,8 @@
let mut failed = false;
for (name, value) in data {
let _span = info_span!("init", name = name);
- if let Err(e) = init_secret(&identity, value) {
- error!("{e}");
+ if let Err(e) = init_secret(&identity, &value) {
+ error!("secret failed to initialize: {e}");
failed = true;
}
}
@@ -257,7 +247,13 @@
let s = String::from_utf8(decrypted).context("output is not utf8")?;
print!("{s}");
} else {
- println!("{}", SecretWrapper(decrypted));
+ println!(
+ "{}",
+ SecretData {
+ data: decrypted,
+ encrypted: false
+ }
+ );
}
Ok(())
}
crates/better-command/src/handler.rsdiffbeforeafterboth--- a/crates/better-command/src/handler.rs
+++ b/crates/better-command/src/handler.rs
@@ -1,7 +1,9 @@
//! Collection of handlers, which transform program-specific stdout format to tracing
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex};
+use std::{
+ collections::HashMap,
+ sync::{Arc, Mutex},
+};
use once_cell::sync::Lazy;
use regex::Regex;
crates/fleet-shared/Cargo.tomldiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-shared/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+name = "fleet-shared"
+edition = "2021"
+version.workspace = true
+
+[dependencies]
+base64 = "0.22.1"
+serde = "1.0.202"
+unicode_categories = "0.1.1"
+z85 = "3.0.5"
crates/fleet-shared/src/lib.rsdiffbeforeafterboth--- /dev/null
+++ b/crates/fleet-shared/src/lib.rs
@@ -0,0 +1,156 @@
+use std::{
+ fmt::{self, Display},
+ str::FromStr,
+};
+
+use base64::engine::{general_purpose::STANDARD_NO_PAD, Engine};
+use serde::{de::Error, Deserialize, Deserializer, Serialize};
+use unicode_categories::UnicodeCategories;
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct SecretData {
+ pub data: Vec<u8>,
+ pub encrypted: bool,
+}
+
+const BASE64_ENCODED_PREFIX: &str = "<BASE64-ENCODED>\n";
+const Z85_ENCODED_PREFIX: &str = "<Z85-ENCODED>\n";
+// Multiline text in Nix can only end with \n, which is not cool for actual single-line strings.
+const PLAINTEXT_NEWLINE_PREFIX: &str = "<PLAINTEXT-NL>\n";
+const PLAINTEXT_PREFIX: &str = "<PLAINTEXT>";
+
+const SECRET_PREFIX: &str = "<ENCRYPTED>";
+
+impl<'de> Deserialize<'de> for SecretData {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let string = String::deserialize(deserializer)?;
+ string.parse().map_err(D::Error::custom)
+ }
+}
+
+impl Serialize for SecretData {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ self.to_string().serialize(serializer)
+ }
+}
+
+impl FromStr for SecretData {
+ type Err = String;
+
+ fn from_str(string: &str) -> Result<Self, Self::Err> {
+ let (encrypted, string) = if let Some(unprefixed) = string.strip_prefix(SECRET_PREFIX) {
+ (true, unprefixed)
+ } else {
+ (false, string)
+ };
+ let data = if let Some(unprefixed) = string.strip_prefix(BASE64_ENCODED_PREFIX) {
+ STANDARD_NO_PAD
+ .decode(unprefixed.replace(|v| matches!(v, '\n' | '\t' | ' '), ""))
+ .map_err(|e| format!("base64-encoded failed: {e}"))?
+ } else if let Some(unprefixed) = string.strip_prefix(Z85_ENCODED_PREFIX) {
+ z85::decode(unprefixed.replace(|v| matches!(v, '\n' | '\t' | ' '), ""))
+ .map_err(|e| format!("z85-encoded failed: {e}"))?
+ } else if let Some(unprefixed) = string.strip_prefix(PLAINTEXT_NEWLINE_PREFIX) {
+ unprefixed.as_bytes().to_owned()
+ } else if let Some(unprefixed) = string.strip_prefix(PLAINTEXT_PREFIX) {
+ unprefixed.as_bytes().to_owned()
+ } else {
+ let secret_prefix = format!("{SECRET_PREFIX}{Z85_ENCODED_PREFIX}");
+ return Err(format!(
+ "unknown secret encoding. If you're migrating from old version of fleet, prefix public secret fields with {PLAINTEXT_PREFIX:?}, and encrypted data with {secret_prefix:?}: {string}"
+ ));
+ };
+ Ok(Self { data, encrypted })
+ }
+}
+
+impl Display for SecretData {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let mut readable = std::str::from_utf8(&self.data).ok();
+ if self.encrypted {
+ write!(f, "{SECRET_PREFIX}")?;
+ // Always base64-encode encrypted fields.
+ readable = None;
+ }
+ if Some(false) == readable.map(is_printable) {
+ readable = None
+ };
+ // TODO: Check if text is readable, and has no unprintable characters?..
+ if let Some(plaintext) = readable {
+ if plaintext.ends_with('\n') {
+ write!(f, "{PLAINTEXT_NEWLINE_PREFIX}")?;
+ } else {
+ write!(f, "{PLAINTEXT_PREFIX}")?;
+ }
+ write!(f, "{plaintext}")?;
+ } else {
+ write!(f, "{BASE64_ENCODED_PREFIX}")?;
+ let encoded = STANDARD_NO_PAD.encode(&self.data);
+ for ele in encoded.as_bytes().chunks(64) {
+ let chunk = std::str::from_utf8(ele).expect(
+ "any slice of base64-encoded text is utf-8 compatible, as it is ascii-based",
+ );
+ writeln!(f, "{chunk}")?;
+ }
+ };
+ Ok(())
+ }
+}
+
+fn is_printable(text: &str) -> bool {
+ text.chars().all(|c| {
+ c.is_letter()
+ || c.is_mark()
+ || c.is_number()
+ || c.is_punctuation()
+ || c.is_separator()
+ || c == '\n' || c == '\t'
+ // Complete base64 alphabet
+ || c == '/' || c == '+'
+ || c == '='
+ })
+}
+
+#[test]
+fn test() {
+ fn check_roundtrip(data: SecretData, expected: &str) {
+ let string = data.to_string();
+ assert_eq!(string, expected, "unexpected encoding");
+ let roundtrip: SecretData = string.parse().expect("roundtrip parse");
+ assert_eq!(data, roundtrip, "roundtrip didn't match");
+ }
+ check_roundtrip(
+ SecretData {
+ data: vec![1, 2, 3, 4, 5, 6],
+ encrypted: false,
+ },
+ "<BASE64-ENCODED>\nAQIDBAUG\n",
+ );
+ check_roundtrip(
+ SecretData {
+ data: vec![1, 2, 3, 4, 5, 6],
+ encrypted: true,
+ },
+ "<ENCRYPTED><BASE64-ENCODED>\nAQIDBAUG\n",
+ );
+ check_roundtrip(
+ SecretData {
+ data: "Привет, мир!\n".to_owned().into(),
+ encrypted: false,
+ },
+ "<PLAINTEXT-NL>\nПривет, мир!\n",
+ );
+ check_roundtrip(
+ SecretData {
+ data: "Привет, мир!".to_owned().into(),
+ encrypted: false,
+ },
+ "<PLAINTEXT>Привет, мир!",
+ );
+}
crates/nixlike/src/lib.rsdiffbeforeafterboth--- a/crates/nixlike/src/lib.rs
+++ b/crates/nixlike/src/lib.rs
@@ -38,6 +38,57 @@
Null,
}
+fn count_spaces(l: &str) -> usize {
+ l.chars().take_while(|&c| c == ' ').count()
+}
+fn is_significant(l: &str) -> bool {
+ count_spaces(l) != l.len()
+}
+
+fn dedent(l: &str, by: usize) -> &str {
+ assert!(
+ l[0..by.min(l.len())].chars().all(|c| c == ' '),
+ "dedent calculation is wrong"
+ );
+ &l[by.min(l.len())..]
+}
+
+fn process_multiline(lines: Vec<&str>) -> String {
+ // Even when parsing '''', there is single "line" between those '' delimiters.
+ // unwrap_or is for case where there is no significant lines
+ let dedent_by = lines
+ .iter()
+ .copied()
+ .filter(|c| is_significant(c))
+ .map(count_spaces)
+ .min()
+ .unwrap_or(0);
+
+ let mut out = String::new();
+
+ let mut had_first = false;
+ for (i, line) in lines.into_iter().enumerate() {
+ // Newline after '' is ignored, if there is no text.
+ if i == 0 && !is_significant(line) {
+ continue;
+ }
+ if had_first {
+ out.push('\n');
+ }
+ had_first = true;
+ // ''' is hard escape
+ for (i, part) in dedent(line, dedent_by).split("'''").enumerate() {
+ if i != 0 {
+ out.push_str(r#"""""#);
+ }
+ // This is the only replacements done by nixlike writer, no need to support more.
+ out.push_str(&part.replace("''${", "${").replace("''\\t", "\t"));
+ }
+ }
+
+ out
+}
+
peg::parser! {
pub grammar nixlike() for str {
rule number() -> i64
@@ -50,8 +101,17 @@
/ "\\r" { "\r" }
/ "\\$" { "$" }
/ c:$([_]) { c }
- rule string() -> String
+ rule string() -> String = singleline_string() / multiline_string();
+ rule singleline_string() -> String
= quiet! { "\"" v:(!"\"" c:string_char() {c})* "\"" { v.into_iter().collect() } } / expected!("<string>")
+ pub rule multiline_string() -> String
+ = "''"
+ // First line may also contain text, and whitespace for it is counted, but if it is empty - then it is'nt counted as full line...
+ // This logic is complicated, see `parse_multiline` test.
+ lines:$(("'''" / !"''" [_])*) "''"
+ {
+ process_multiline(lines.split('\n').collect())
+ }
rule boolean() -> bool
= quiet! { "true" {true}
/ "false" {false} } / expected!("<boolean>")
@@ -135,3 +195,12 @@
let (_, out) = alejandra::format::in_memory("".to_owned(), value.to_owned());
out
}
+
+#[test]
+fn parse_multiline() {
+ assert_eq!(nixlike::multiline_string("''\n''").expect("parse"), "");
+ assert_eq!(nixlike::multiline_string("''\n\n''").expect("parse"), "\n");
+ assert_eq!(nixlike::multiline_string("''t\n''").expect("parse"), "t\n");
+ assert_eq!(nixlike::multiline_string("''''").expect("parse"), "");
+ assert_eq!(nixlike::multiline_string("'' ''").expect("parse"), "");
+}
crates/nixlike/src/to_string.rsdiffbeforeafterboth--- a/crates/nixlike/src/to_string.rs
+++ b/crates/nixlike/src/to_string.rs
@@ -38,7 +38,28 @@
}
pub fn write_nix_str(str: &str, out: &mut String) {
- out.push_str(&escape_string(str))
+ if str.ends_with('\n') {
+ out.push_str("''");
+ for ele in str.split('\n') {
+ out.push('\n');
+ out.push_str(
+ &ele
+ // '' is escaped with '
+ .replace("''", "'''")
+ // ${ is escaped wth ''
+ .replace("${", "''${")
+ // \t is not counted as whitespace for dedent
+ // to avoid confusion, it is printed literally.
+ //
+ // ...Escaped \t literal should be prefixed with '' for... Idk, this logic is complicated.
+ .replace('\t', "''\\t"),
+ );
+ }
+ // Final newline is assumed due to str.ends_with condition
+ out.push_str("''");
+ } else {
+ out.push_str(&escape_string(str))
+ }
}
fn write_nix_buf(value: &Value, out: &mut String) {
crates/remowt-fs/src/lib.rsdiffbeforeafterboth--- a/crates/remowt-fs/src/lib.rs
+++ b/crates/remowt-fs/src/lib.rs
@@ -1,2 +1 @@
trait RemowtFS {}
-
modules/fleet/secrets.nixdiffbeforeafterboth--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -7,14 +7,8 @@
with lib;
with fleetLib; let
sharedSecret = with types; ({config, ...}: {
+ freeformType = types.lazyAttrsOf unspecified;
options = {
- managed = mkOption {
- type = bool;
- description = ''
- Is this secret managed by configuration (I.e will work with reencrypt/etc), or it is configured by user
- '';
- };
-
expectedOwners = mkOption {
type = nullOr (listOf str);
description = ''
@@ -71,23 +65,10 @@
'';
default = [];
};
- # TODO: Make secret generator generate arbitrary number of secret/public parts?
- # Make it generate a folder, where all files except suffixed by .enc are public, and the rest are secret?
- # How should modules refer to those files then?
- public = mkOption {
- type = nullOr str;
- description = "Secret public data. Imported from fleet.nix";
- default = null;
- };
- secret = mkOption {
- type = nullOr str;
- description = "Encrypted secret data. Imported from fleet.nix";
- default = null;
- internal = true;
- };
};
});
hostSecret = with types; {
+ freeformType = types.lazyAttrsOf unspecified;
options = {
createdAt = mkOption {
type = nullOr str;
@@ -97,21 +78,15 @@
type = nullOr str;
default = null;
};
- public = mkOption {
- type = nullOr str;
- description = "Secret public data. Imported from fleet.nix";
- default = null;
- };
- secret = mkOption {
- type = nullOr str;
- description = "Encrypted secret data. Imported from fleet.nix";
- default = null;
- internal = true;
- };
};
};
in {
options = with types; {
+ version = mkOption {
+ type = str;
+ default = "";
+ internal = true;
+ };
sharedSecrets = mkOption {
type = attrsOf (submodule sharedSecret);
default = {};
@@ -134,18 +109,20 @@
config.sharedSecrets;
hosts = hostsToAttrs (host: {
nixosModules = let
- cleanupSecret = secretName: v: {
- inherit (v) public secret;
- shared = true;
- };
+ # processPart
+ processSecret = v:
+ (removeAttrs v ["createdAt" "expiresAt" "expectedOwners" "owners" "regenerateOnOwnerAdded" "regenerateOnOwnerRemoved"])
+ // {
+ shared = true;
+ };
in [
{
secrets =
(
- mapAttrs cleanupSecret
+ mapAttrs (_: processSecret)
(filterAttrs (_: v: builtins.elem host v.owners) config.sharedSecrets)
)
- // (mapAttrs cleanupSecret (config.hostSecrets.${host} or {}));
+ // (mapAttrs (_: processSecret) (config.hostSecrets.${host} or {}));
}
];
});
nixos/secrets.nixdiffbeforeafterboth--- a/nixos/secrets.nix
+++ b/nixos/secrets.nix
@@ -1,39 +1,72 @@
-{ lib, config, pkgs, ... }:
-
-with lib;
+{
+ lib,
+ config,
+ pkgs,
+ ...
+}:
+with lib; let
+ inherit (lib.strings) hasPrefix stripPrefix;
+ plaintextPrefix = "<PLAINTEXT>";
+ plaintextNewlinePrefix = "<PLAINTEXT-NL>";
-let
sysConfig = config;
- secretType = types.submodule ({ config, ... }: {
- config = let secretName = config._module.args.name; in {
- stableSecretPath = mkOptionDefault "/run/secrets/secret-stable-${secretName}";
- secretPath = mkOptionDefault "/run/secrets/secret-${config.secretHash}-${secretName}";
- secretHash = mkOptionDefault (if config.secret != null then (builtins.hashString "sha1" config.secret) else throw "secret is not defined for secret ${secretName}");
-
- stablePublicPath = mkOptionDefault "/run/secrets/public-stable-${secretName}";
- publicPath = mkOptionDefault "/run/secrets/public-${config.publicHash}-${secretName}";
- publicHash = mkOptionDefault (if config.public != null then (builtins.hashString "sha1" config.public) else throw "public is not defined for secret ${secretName}");
- };
+ secretPartType = secretName:
+ types.submodule ({config, ...}: {
+ options = with types; {
+ raw = mkOption {
+ description = "Secret in fleet-specific undocumented format, do not use. Import from fleet.nix";
+ internal = true;
+ };
+ hash = mkOption {
+ type = str;
+ description = "Hash of secret in encoded format";
+ };
+ path = mkOption {
+ type = str;
+ description = "Path to secret part, incorporating data hash (thus it will be updated on secret change)";
+ };
+ stablePath = mkOption {
+ type = str;
+ description = "Path to secret part, incorporating data hash (thus it will be updated on secret change)";
+ };
+ data = mkOption {
+ type = str;
+ description = "Secret public data (only available for plaintext)";
+ };
+ };
+ config = let
+ partName = config._module.args.name;
+ in {
+ hash = mkOptionDefault (builtins.hashString "sha1" config.raw);
+ data = mkOptionDefault (
+ if hasPrefix plaintextPrefix config.raw
+ then stripPrefix plaintextPrefix config.raw
+ else if hasPrefix plaintextNewlinePrefix config.raw
+ then stripPrefix plaintextNewlinePrefix config.raw
+ else throw "secret.part.data attribute only works for public plaintext secret parts, got ${config.raw}"
+ );
+ path = mkOptionDefault "/run/secrets/${secretName}/${config.hash}-${partName}";
+ stablePath = mkOptionDefault "/run/secrets/${secretName}/${partName}";
+ };
+ });
+ secretType = types.submodule ({config, ...}: let
+ secretName = config._module.args.name;
+ in {
+ freeformType = types.lazyAttrsOf (secretPartType secretName);
options = with types; {
shared = mkOption {
description = "Is this secret owned by this machine, or propagated from shared secrets";
default = false;
};
-
- generator = mkOption {
+ expectedOwners = mkOption {
type = nullOr unspecified;
- description = "Derivation to evaluate for secret generation";
default = null;
+ internal = true;
};
- public = mkOption {
- type = nullOr str;
- description = "Secret public data";
- default = null;
- };
- secret = mkOption {
- type = nullOr str;
- description = "Encrypted secret data";
+ generator = mkOption {
+ type = nullOr unspecified;
+ description = "Derivation to evaluate for secret generation";
default = null;
};
mode = mkOption {
@@ -50,64 +83,43 @@
type = str;
description = "Group of the secret";
default = sysConfig.users.users.${config.owner}.group;
- };
-
- secretHash = mkOption {
- type = str;
- description = "Hash of .secret field";
- };
- publicHash = mkOption {
- type = str;
- description = "Hash of .public field";
- };
-
- stableSecretPath = mkOption {
- type = str;
- description = ''
- Use this, if target process supports re-reading of secret from disk,
- and doesn't needs to be restarted when secret is updated in file
- '';
- };
- secretPath = mkOption {
- type = str;
- description = "Path to decrypted secret, suffixed with contents hash";
- };
-
- stablePublicPath = mkOption {
- type = str;
- description = ''
- Use this, if target process supports re-reading of secret from disk,
- and doesn't needs to be restarted when secret is updated in file
- '';
- };
- publicPath = mkOption {
- type = str;
- description = "Path to the public part of secret";
};
};
});
+ processPart = part: {
+ inherit (part) raw path stablePath;
+ };
+ processSecret = secret:
+ {
+ inherit (secret) group mode owner;
+ }
+ // (mapAttrs (_: processPart) (removeAttrs secret [
+ "shared"
+ "generator"
+ "mode"
+ "group"
+ "owner"
+
+ # FIXME: Some of those removed attributes shouldn't be here, but there is some error in passing shared secrets from fleet to nixos.
+ "expectedOwners"
+ ]));
secretsFile = pkgs.writeTextFile {
name = "secrets.json";
- text = builtins.toJSON (mapAttrs (_: value: rec {
- inherit (value) group mode owner secret public;
- publicPath = if public != null then value.publicPath else "/missingno";
- stablePublicPath = if public != null then value.stablePublicPath else "/missingno";
- secretPath = if secret != null then value.secretPath else "/missingno";
- stableSecretPath = if secret != null then value.stableSecretPath else "/missingno";
- }) config.secrets);
+ text =
+ builtins.toJSON (mapAttrs (_: processSecret)
+ config.secrets);
};
-in
-{
+in {
options = {
secrets = mkOption {
type = types.attrsOf secretType;
- default = { };
+ default = {};
description = "Host-local secrets";
};
};
config = {
- environment.systemPackages = with pkgs; [pkgs.fleet-install-secrets];
- system.activationScripts.decryptSecrets = stringAfter [ "users" "groups" "specialfs" ] ''
+ environment.systemPackages = [pkgs.fleet-install-secrets];
+ system.activationScripts.decryptSecrets = stringAfter ["users" "groups" "specialfs"] ''
1>&2 echo "setting up secrets"
${pkgs.fleet-install-secrets}/bin/fleet-install-secrets install ${secretsFile}
'';
rustfmt.tomldiffbeforeafterboth--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1 +1,3 @@
hard_tabs = true
+imports_granularity = "Crate"
+group_imports = "StdExternalCrate"