difftreelog
feat postgres secret generator
10 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1045,7 +1045,10 @@
"ed25519-dalek",
"fleet-shared",
"hex",
+ "hmac",
+ "pbkdf2",
"rand 0.10.1",
+ "sha2",
"x25519-dalek",
]
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,6 +31,7 @@
futures = "0.3.31"
futures-util = { version = "0.3.31", features = ["sink"] }
hex = "0.4.3"
+hmac = "0.12"
hostname = "0.4.1"
human-repr = "1.1"
hyper = "1.8.1"
@@ -40,11 +41,12 @@
linked-hash-map = "0.5.6"
nix = { version = "0.31.2", features = ["fs", "user"] }
nom = "8.0.0"
+openssh = "0.11.5"
opentelemetry = "0.31.0"
+opentelemetry-appender-tracing = "0.31.1"
opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "gzip-tonic", "http-json", "reqwest-rustls"] }
opentelemetry_sdk = "0.31.0"
-opentelemetry-appender-tracing = "0.31.1"
-openssh = "0.11.5"
+pbkdf2 = "0.12"
peg = "0.8.5"
pkg-config = "0.3.30"
rand = "0.10.0"
@@ -52,6 +54,7 @@
serde = { version = "1.0", features = ["derive"] }
serde-transcode = "1.1.1"
serde_json = "1.0"
+sha2 = "0.10"
shlex = "1.3"
tabled = "0.20.0"
tempfile = "3.20"
cmds/fleet/Cargo.tomldiffbeforeafterboth--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -44,12 +44,12 @@
indicatif = { workspace = true, optional = true }
nom.workspace = true
opentelemetry.workspace = true
+opentelemetry-appender-tracing.workspace = true
+opentelemetry-exporter-env.workspace = true
opentelemetry_sdk.workspace = true
thiserror.workspace = true
tracing-indicatif = { workspace = true, optional = true }
tracing-opentelemetry.workspace = true
-opentelemetry-exporter-env.workspace = true
-opentelemetry-appender-tracing.workspace = true
[features]
default = ["indicatif"]
cmds/generator-helper/Cargo.tomldiffbeforeafterboth--- a/cmds/generator-helper/Cargo.toml
+++ b/cmds/generator-helper/Cargo.toml
@@ -13,5 +13,8 @@
base64.workspace = true
ed25519-dalek.workspace = true
hex.workspace = true
+hmac.workspace = true
+pbkdf2.workspace = true
rand.workspace = true
+sha2.workspace = true
x25519-dalek.workspace = true
cmds/generator-helper/src/main.rsdiffbeforeafterboth1use std::{2 env,3 fs::{File, OpenOptions},4 io::{self, Read, Write, copy, stdin, stdout},5 str::FromStr,6};78use age::{9 Encryptor, Recipient,10 ssh::{ParseRecipientKeyError, Recipient as SshRecipient},11};12use anyhow::{Context, Result, anyhow, bail, ensure};13use base64::{Engine as _, engine::general_purpose::STANDARD, write::EncoderWriter};14use clap::{Parser, ValueEnum};15use ed25519_dalek::SecretKey;16use fleet_shared::SecretData;17use hmac::Mac as _;18use rand::{19 Rng as _,20 distr::{Alphanumeric, Distribution, SampleString, Uniform},21 rng,22};23use sha2::Digest as _;2425fn gen_password(rng: &mut impl rand::Rng, size: usize, no_symbols: bool) -> String {26 if no_symbols {27 Alphanumeric.sample_string(rng, size)28 } else {29 const GEN_ASCII_SYMBOLS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";30 let uniform = Uniform::new(0, GEN_ASCII_SYMBOLS.len()).expect("range is valid");31 (0..size)32 .map(|_| uniform.sample(rng))33 .map(|i| GEN_ASCII_SYMBOLS[i] as char)34 .collect::<String>()35 }36}3738fn write_output_file(out: &str) -> Result<File> {39 let file = OpenOptions::new()40 .create_new(true)41 .write(true)42 .open(out)43 .with_context(|| format!("failed to open output {out:?}"))?;44 Ok(file)45}46fn write_public(out: &str, mut input: impl Read, encoding: OutputEncoding) -> Result<()> {47 let mut output = write_output_file(out)?;4849 let mut data = Vec::new();50 copy(&mut input, &mut wrap_encoder(&mut data, encoding))?;5152 output.write_all(53 SecretData {54 data,55 encrypted: false,56 }57 .to_string()58 .as_bytes(),59 )?;60 Ok(())61}62fn write_private(63 identities: &Identities,64 out: &str,65 mut input: impl Read,66 encoding: OutputEncoding,67) -> Result<()> {68 let mut output = write_output_file(out)?;69 let encryptor = make_encryptor(identities)?;7071 let mut data = Vec::new();72 {73 let mut encrypted_writer = encryptor.wrap_output(&mut data)?;74 copy(75 &mut input,76 &mut wrap_encoder(&mut encrypted_writer, encoding),77 )?;78 encrypted_writer.finish()?;79 };8081 output.write_all(82 SecretData {83 data,84 encrypted: true,85 }86 .to_string()87 .as_bytes(),88 )?;89 Ok(())90}9192type Identities = Vec<SshRecipient>;93fn load_identities() -> Result<Identities> {94 let list = env::var("GENERATOR_HELPER_IDENTITIES");95 let list = match list {96 Ok(v) => v,97 Err(env::VarError::NotPresent) => {98 bail!(99 "gh is only intended to be used from secret generator scripts, but if you really want to use it somewhere else - set GENERATOR_HELPER_IDENTITIES to list of newline-delimited ssh identities"100 );101 }102 Err(e) => bail!("somehow, identities list is not utf-8: {e}"),103 };104 let list = list.trim();105 ensure!(!list.is_empty(), "no identities passed, can't encrypt data");106 list.lines()107 .map(age::ssh::Recipient::from_str)108 .collect::<Result<Identities, ParseRecipientKeyError>>()109 .map_err(|e| anyhow!("parse recipients: {e:?}"))110}111fn make_encryptor(r: &Identities) -> Result<Encryptor> {112 Ok(113 Encryptor::with_recipients(r.iter().map(|v| v as &dyn Recipient))114 .expect("list is not empty"),115 )116}117fn wrap_encoder<'t>(w: impl Write + 't, encoding: OutputEncoding) -> impl Write + 't {118 fn coerce<'t>(w: impl Write + 't) -> Box<dyn Write + 't> {119 Box::new(w)120 }121 match encoding {122 OutputEncoding::Raw => coerce(w),123 OutputEncoding::Base64 => {124 let writer = EncoderWriter::new(w, &STANDARD);125 coerce(writer)126 }127 OutputEncoding::Hex => {128 struct HexWriter<W>(W);129 impl<W> Write for HexWriter<W>130 where131 W: Write,132 {133 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {134 let encoded = hex::encode(buf);135 self.0.write_all(encoded.as_bytes())?;136 Ok(buf.len())137 }138139 fn flush(&mut self) -> io::Result<()> {140 self.0.flush()141 }142 }143 coerce(HexWriter(w))144 }145 }146}147148#[derive(Clone, Copy, ValueEnum, Default)]149enum OutputEncoding {150 /// Do not encode data, store as is.151 #[default]152 Raw,153 /// Encode as base64 (with padding).154 Base64,155 /// Encode as hex (without leading 0x)156 Hex,157}158159#[derive(Parser)]160enum Generate {161 /// Generate public, private keys without wrapping, in standard ed25519 schema162 /// (64 bytes private (due to merge with private), 32 bytes public)163 Ed25519 {164 #[arg(long, short = 'p')]165 public: String,166 #[arg(long, short = 's')]167 private: String,168 /// Private key should be just the private key (32 bytes), not standard private+public.169 #[arg(long)]170 no_embed_public: bool,171 #[arg(long, short = 'e', value_enum, default_value_t)]172 encoding: OutputEncoding,173 },174 /// Generate public, private keys without wrapping, in standard x25519 schema175 /// (32 bytes private, 32 bytes public)176 X25519 {177 #[arg(long, short = 'p')]178 public: String,179 #[arg(long, short = 's')]180 private: String,181 #[arg(long, short = 'e', value_enum, default_value_t)]182 encoding: OutputEncoding,183 },184 Password {185 #[arg(long, short = 'o')]186 output: String,187 #[arg(long)]188 size: usize,189 #[arg(long, short = 'n')]190 no_symbols: bool,191 #[arg(long, short = 'e', value_enum, default_value_t)]192 encoding: OutputEncoding,193 },194 PostgresPassword {195 #[arg(long, short = 's')]196 secret: String,197 #[arg(long, short = 'H')]198 hash: String,199 #[arg(long, default_value_t = 24)]200 size: usize,201 #[arg(long, default_value_t = 4096)]202 iterations: u32,203 #[arg(long, short = 'n')]204 no_symbols: bool,205 },206 Bytes {207 #[arg(long, short = 'o')]208 output: String,209 #[arg(long, short = 'c')]210 count: usize,211 /// Ensure there is no NULs in bytestring.212 #[arg(long)]213 no_nuls: bool,214 #[arg(long, short = 'e', value_enum, default_value_t)]215 encoding: OutputEncoding,216 },217}218219#[derive(Parser)]220enum Opts {221 /// Encode public part from stdin.222 Public {223 #[arg(long, short = 'o')]224 output: String,225 #[arg(long, short = 'e', value_enum, default_value_t)]226 encoding: OutputEncoding,227 },228 /// Encrypt private part from stdin.229 Private {230 #[arg(long, short = 'o')]231 output: String,232 #[arg(long, short = 'e', value_enum, default_value_t)]233 encoding: OutputEncoding,234 },235 /// Sometimes you also need to reencode secret, this command allows you to get raw data from236 /// secret encoded using `gh public`... I would like if I knew a better design for some sort of237 /// such thing. Ideally there should be no need to decode secrets back, but garage wants both238 /// raw pubkey and raw secret key, yet also requires node id which is hex-reencoded public key.239 Decode {240 #[arg(long, short = 'i')]241 input: String,242 },243 /// Generate keys in well-known schemas.244 ///245 /// Note that this command is only intended to be used in fleet secret generator,246 /// otherwise you should ensure noone is able to read generated files, they don't have any mode set by default.247 ///248 /// Fleet also doesn't zeroize memory/assumes good OsRng/makes other assumptions, which makes it only suitable to249 /// be used in nix sandbox.250 #[command(subcommand)]251 Generate(Generate),252}253254fn main() -> Result<()> {255 let opts = Opts::parse();256 // Assumed to be secure, seeded from secure OsRng+reseeded.257 let mut rng = rng();258259 match opts {260 Opts::Public { output, encoding } => {261 write_public(&output, stdin(), encoding)?;262 }263 Opts::Private { output, encoding } => {264 let recipients = load_identities()?;265 write_private(&recipients, &output, stdin(), encoding)?;266 }267 Opts::Generate(generate) => {268 match generate {269 Generate::Ed25519 {270 public,271 private,272 no_embed_public,273 encoding,274 } => {275 use ed25519_dalek::SigningKey;276277 let recipients = load_identities()?;278 let mut secret = SecretKey::default();279 rng.fill_bytes(&mut secret);280 // TODO: Use SigningKey::generate after https://github.com/dalek-cryptography/curve25519-dalek/pull/762281 let key = SigningKey::from_bytes(&secret).to_keypair_bytes();282 write_public(&public, &key[32..], encoding)?;283 write_private(284 &recipients,285 &private,286 &key[..{ if no_embed_public { 32 } else { 64 } }],287 encoding,288 )?;289 }290 Generate::X25519 {291 public,292 private,293 encoding,294 } => {295 use x25519_dalek::{PublicKey, StaticSecret};296297 let recipients = load_identities()?;298 // TODO: Use random_from_rng after https://github.com/dalek-cryptography/curve25519-dalek/pull/762299 let key = StaticSecret::random();300 let public_key: PublicKey = (&key).into();301 write_public(&public, public_key.as_bytes().as_slice(), encoding)?;302 write_private(&recipients, &private, key.as_bytes().as_slice(), encoding)?;303 }304 Generate::Password {305 size,306 no_symbols,307 output,308 encoding,309 } => {310 ensure!(311 size >= 6,312 "misconfiguration? password is shorter than 6 chars"313 );314 let recipients = load_identities()?;315 let out = gen_password(&mut rng, size, no_symbols);316 write_private(&recipients, &output, out.as_bytes(), encoding)?;317 }318 Generate::PostgresPassword {319 secret,320 hash,321 size,322 iterations,323 no_symbols,324 } => {325 ensure!(326 size >= 6,327 "misconfiguration? password is shorter than 6 chars"328 );329 let recipients = load_identities()?;330 let password = gen_password(&mut rng, size, no_symbols);331332 let mut salt = [0u8; 16];333 rng.fill_bytes(&mut salt);334 let salted = pbkdf2::pbkdf2_hmac_array::<sha2::Sha256, 32>(335 password.as_bytes(),336 &salt,337 iterations,338 );339340 type HmacSha256 = hmac::Hmac<sha2::Sha256>;341 let mut mac = <HmacSha256 as hmac::Mac>::new_from_slice(&salted)342 .expect("HMAC accepts any key length");343 mac.update(b"Client Key");344 let client_key = mac.finalize().into_bytes();345346 let mut hasher = sha2::Sha256::new();347 hasher.update(client_key);348 let stored_key = hasher.finalize();349350 let mut mac = <HmacSha256 as hmac::Mac>::new_from_slice(&salted)351 .expect("HMAC accepts any key length");352 mac.update(b"Server Key");353 let server_key = mac.finalize().into_bytes();354355 let hash_str = format!(356 "SCRAM-SHA-256${}:{}${}:{}",357 iterations,358 STANDARD.encode(salt),359 STANDARD.encode(stored_key),360 STANDARD.encode(server_key),361 );362363 write_private(364 &recipients,365 &secret,366 password.as_bytes(),367 OutputEncoding::Raw,368 )?;369 write_public(&hash, hash_str.as_bytes(), OutputEncoding::Raw)?;370 }371 Generate::Bytes {372 output,373 count,374 no_nuls,375 encoding,376 } => {377 ensure!(378 count >= 6,379 "misconfiguration? bytestring is shorter than 6 chars"380 );381 let recipients = load_identities()?;382 let mut bytes = vec![0u8; count];383 if no_nuls {384 let rand = Uniform::new_inclusive(0x1u8, 0xffu8)385 .expect("range is valid")386 .sample_iter(&mut rng);387 for (byte, rand) in bytes.iter_mut().zip(rand) {388 *byte = rand;389 }390 } else {391 rng.fill_bytes(&mut bytes);392 };393 write_private(&recipients, &output, bytes.as_slice(), encoding)?;394 }395 }396 }397 Opts::Decode { input } => {398 let mut data = Vec::new();399 File::open(input)?.read_to_end(&mut data)?;400 let data = String::from_utf8(data).context(401 "encoded data is always utf-8, you are trying to use decode the wrong way.",402 )?;403 let data =404 SecretData::from_str(&data).map_err(|e| anyhow!("failed to decode data: {e}"))?;405 ensure!(406 !data.encrypted,407 "you can not decrypt secret data, only decode public."408 );409 stdout().write_all(&data.data)?;410 }411 }412 Ok(())413}crates/nix-eval/src/lib.rsdiffbeforeafterboth--- a/crates/nix-eval/src/lib.rs
+++ b/crates/nix-eval/src/lib.rs
@@ -26,9 +26,8 @@
clear_err, copy_value, err_NIX_ERR_KEY, err_NIX_ERR_NIX_ERROR, err_NIX_ERR_OVERFLOW,
err_NIX_ERR_UNKNOWN, err_code, err_info_msg, err_msg, eval_state_build,
eval_state_builder_load, eval_state_builder_new, eval_state_builder_set_eval_setting,
- expr_eval_from_string, fetchers_settings,
- fetchers_settings_free, fetchers_settings_new, flake_lock, flake_lock_flags,
- flake_lock_flags_free, flake_lock_flags_new, flake_reference,
+ expr_eval_from_string, fetchers_settings, fetchers_settings_free, fetchers_settings_new,
+ flake_lock, flake_lock_flags, flake_lock_flags_free, flake_lock_flags_new, flake_reference,
flake_reference_and_fragment_from_string, flake_reference_parse_flags,
flake_reference_parse_flags_free, flake_reference_parse_flags_new,
flake_reference_parse_flags_set_base_directory, flake_settings, flake_settings_free,
@@ -323,7 +322,9 @@
thread_local! {
static THREAD_STATE: RefCell<ThreadState> = RefCell::new(ThreadState::new().expect("thread state init shouldn't fail"));
}
-pub(crate) fn with_default_context<T>(f: impl FnOnce(*mut c_context, *mut c_eval_state) -> T) -> Result<T> {
+pub(crate) fn with_default_context<T>(
+ f: impl FnOnce(*mut c_context, *mut c_eval_state) -> T,
+) -> Result<T> {
let global = &GLOBAL_STATE.state;
let (ctx, state) = THREAD_STATE.with_borrow_mut(|w| (w.ctx.0, global.0));
let mut ctx = NixContext(ctx);
@@ -439,7 +440,11 @@
}
}
-pub(crate) unsafe extern "C" fn copy_nix_str(start: *const c_char, n: c_uint, user_data: *mut c_void) {
+pub(crate) unsafe extern "C" fn copy_nix_str(
+ start: *const c_char,
+ n: c_uint,
+ user_data: *mut c_void,
+) {
let s = unsafe { slice::from_raw_parts(start.cast::<u8>(), n as usize) };
let s = std::str::from_utf8(s).expect("c string has invalid utf-8");
unsafe { *user_data.cast::<String>() = s.to_owned() };
@@ -1128,8 +1133,8 @@
let test_result: String = nix_go_json!(builtins.uppercaseSuffix2("test")("suffix"));
assert_eq!(test_result, "TESTsuffix");
- let drv_path = nix_go!(attrs.packages["x86_64-linux"]["fleet-install-secrets"].drvPath)
- .to_string()?;
+ let drv_path =
+ nix_go!(attrs.packages["x86_64-linux"]["fleet-install-secrets"].drvPath).to_string()?;
let graph = drv::DrvGraph::resolve(&drv_path)?;
eprintln!(
"fleet-install-secrets dependency graph: {} nodes",
crates/nix-eval/src/logging.rsdiffbeforeafterboth--- a/crates/nix-eval/src/logging.rs
+++ b/crates/nix-eval/src/logging.rs
@@ -351,12 +351,15 @@
if let Some(entry) = drv_graph.get_mut(dep_path) {
entry.refcount += 1;
} else {
- drv_graph.insert(dep_path.clone(), DrvGraphEntry {
- name: dep_node.name.clone(),
- parent: Some(path.clone()),
- span: None,
- refcount: 1,
- });
+ drv_graph.insert(
+ dep_path.clone(),
+ DrvGraphEntry {
+ name: dep_node.name.clone(),
+ parent: Some(path.clone()),
+ span: None,
+ refcount: 1,
+ },
+ );
}
paths.push(dep_path.clone());
queue.push_back(dep_path.clone());
crates/opentelemetry-exporter-env/src/lib.rsdiffbeforeafterboth--- a/crates/opentelemetry-exporter-env/src/lib.rs
+++ b/crates/opentelemetry-exporter-env/src/lib.rs
@@ -11,10 +11,7 @@
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("environment variable {env} contains invalid UTF-8: {value:?}")]
- InvalidUtf8 {
- env: &'static str,
- value: OsString,
- },
+ InvalidUtf8 { env: &'static str, value: OsString },
#[error("environment variable {env}={value:?}: {error}")]
EnvParse {
env: &'static str,
@@ -103,13 +100,30 @@
#[cfg_attr(feature = "clap", derive(clap::Parser))]
pub struct SignalExporterSettings {
/// Traces exporter to be used.
- #[cfg_attr(feature = "clap", arg(long = "otel-traces-exporter", env = "OTEL_TRACES_EXPORTER", value_enum))]
+ #[cfg_attr(
+ feature = "clap",
+ arg(
+ long = "otel-traces-exporter",
+ env = "OTEL_TRACES_EXPORTER",
+ value_enum
+ )
+ )]
pub traces: Option<ExporterKind>,
/// Metrics exporter to be used.
- #[cfg_attr(feature = "clap", arg(long = "otel-metrics-exporter", env = "OTEL_METRICS_EXPORTER", value_enum))]
+ #[cfg_attr(
+ feature = "clap",
+ arg(
+ long = "otel-metrics-exporter",
+ env = "OTEL_METRICS_EXPORTER",
+ value_enum
+ )
+ )]
pub metrics: Option<ExporterKind>,
/// Logs exporter to be used.
- #[cfg_attr(feature = "clap", arg(long = "otel-logs-exporter", env = "OTEL_LOGS_EXPORTER", value_enum))]
+ #[cfg_attr(
+ feature = "clap",
+ arg(long = "otel-logs-exporter", env = "OTEL_LOGS_EXPORTER", value_enum)
+ )]
pub logs: Option<ExporterKind>,
}
crates/opentelemetry-exporter-env/src/otlp.rsdiffbeforeafterboth--- a/crates/opentelemetry-exporter-env/src/otlp.rs
+++ b/crates/opentelemetry-exporter-env/src/otlp.rs
@@ -55,15 +55,13 @@
}
builder.build()
}
- OtlpProtocol::HttpProtobuf | OtlpProtocol::HttpJson => {
- <$exporter>::builder()
- .with_http()
- .with_endpoint(&s.endpoint)
- .with_headers(to_hashmap(s.headers.as_deref()))
- .with_protocol(s.protocol.into())
- .with_timeout(s.timeout)
- .build()
- }
+ OtlpProtocol::HttpProtobuf | OtlpProtocol::HttpJson => <$exporter>::builder()
+ .with_http()
+ .with_endpoint(&s.endpoint)
+ .with_headers(to_hashmap(s.headers.as_deref()))
+ .with_protocol(s.protocol.into())
+ .with_timeout(s.timeout)
+ .build(),
}
}};
}
lib/default.nixdiffbeforeafterboth--- a/lib/default.nix
+++ b/lib/default.nix
@@ -89,6 +89,44 @@
);
/**
+ Generate a random password suitable for PostgreSQL role authentication.
+
+ Options:
+ size: generated password length in ascii characters (bytes).
+ iterations: PBKDF2 iteration count (PG default: 4096).
+ noSymbols: by default, character set includes various special characters ($ , ! + * : ~), and might
+ not be accepted in some contexts, this option switches charset to just [A-Za-z0-9].
+
+ Output:
+ secret: encrypted password.
+ hash: SCRAM-SHA-256 hash of the password to be used by postgres.
+ */
+ # It is not possible to extract it into a scram-sha-256 generic generator because there is no estabilished format
+ # for it, and mongodb for example encodes it differently.
+ mkPostgresPassword =
+ {
+ size ? 32,
+ iterations ? 4096,
+ noSymbols ? false,
+ }:
+ (
+ { mkSecretGenerator }:
+ mkSecretGenerator {
+ script = ''
+ mkdir $out
+ gh generate postgres-password \
+ -s $out/secret \
+ -H $out/hash \
+ --size ${toString size} \
+ --iterations ${toString iterations} \
+ ${optionalString noSymbols "--no-symbols"}
+ '';
+ parts.secret.encrypted = true;
+ parts.hash.encrypted = false;
+ }
+ );
+
+ /**
Generate a random ed25519 keypair
Options:
@@ -205,8 +243,7 @@
mkAskFile {
inherit part;
header = builtins.concatStringsSep "\n" (
- (map (l: "# ${l}") (lib.splitString "\n" header))
- ++ (map (v: "${v}=") variables)
+ (map (l: "# ${l}") (lib.splitString "\n" header)) ++ (map (v: "${v}=") variables)
);
};
@@ -306,6 +343,7 @@
inherit (secrets)
mkPassword
+ mkPostgresPassword
mkEd25519
mkX25519
mkRsa