difftreelog
refactor nix secret module
in: trunk
17 files changed
Cargo.tomldiffbeforeafterboth--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,7 @@
[workspace]
members = ["crates/*", "cmds/*"]
resolver = "2"
+package.version = "0.1.0"
[workspace.dependencies]
nixlike = { path = "./crates/nixlike" }
cmds/fleet/src/cmds/secrets/mod.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/secrets/mod.rs
+++ b/cmds/fleet/src/cmds/secrets/mod.rs
@@ -1,6 +1,5 @@
use crate::{
better_nix_eval::Field,
- command::MyCommand,
fleetdata::{FleetSecret, FleetSharedSecret, SecretData},
host::Config,
nix_go, nix_go_json,
@@ -9,16 +8,14 @@
use chrono::{DateTime, Utc};
use clap::Parser;
use owo_colors::OwoColorize;
-use serde::{de::DeserializeOwned, Deserialize};
+use serde::Deserialize;
use std::{
collections::{BTreeSet, HashSet},
io::{self, Cursor, Read},
- path::{Path, PathBuf},
- str::FromStr,
+ path::PathBuf,
};
use tabled::{Table, Tabled};
-use tempfile::tempdir;
-use tokio::fs::{self, read_to_string};
+use tokio::fs::read_to_string;
use tracing::{error, info, info_span, warn, Instrument};
#[derive(Parser)]
@@ -162,84 +159,13 @@
}
async fn generate_pure(
- config: &Config,
+ _config: &Config,
_display_name: &str,
- secret: Field,
- default_generator: Field,
- owners: &[String],
+ _secret: Field,
+ _default_generator: Field,
+ _owners: &[String],
) -> Result<FleetSecret> {
- // TODO: pure secrets are supposed to be generated by nix daemon itself,
- // inside of a sandbox... But we aren't here yet.
- let config_field = &config.config_unchecked_field;
- let generator = nix_go!(secret.generator);
- let default_pkgs = &config.default_pkgs;
-
- let call_package = nix_go!(default_pkgs.callPackage);
-
- let generator = nix_go!(call_package(generator)(Obj {}));
- let generator = generator.build().await?;
- let generator = generator
- .get("out")
- .ok_or_else(|| anyhow!("missing generate out"))?;
-
- let mut recipients = String::new();
- for owner in owners {
- let key = config.key(owner).await?;
- recipients.push_str(&format!("-r \"{key}\" "));
- }
- recipients.push_str("-e");
-
- let out = tempdir()?;
-
- let mut gen = MyCommand::new(generator);
- gen.env("rageArgs", recipients);
- gen.env(
- "out",
- out.path().to_str().expect("sane tempdir should be utf-8"),
- );
- gen.run().await.context("impure generator")?;
-
- {
- let mut marker_path = out.path().to_owned();
- marker_path.push("marker");
- let marker = fs::read_to_string(&marker_path).await?;
- ensure!(marker == "SUCCESS", "generation not succeeded");
- }
-
- let mut public_path = out.path().to_owned();
- public_path.push("public");
- let mut secret_path = out.path().to_owned();
- secret_path.push("secret");
- let public = fs::read_to_string(&public_path).await.ok();
- let secret = fs::read(&secret_path).await.ok();
- if let Some(secret) = &secret {
- ensure!(
- age::Decryptor::new(Cursor::new(&secret)).is_ok(),
- "builder produced non-encrypted value as secret, this is highly insecure, and not allowed."
- );
- }
-
- let mut created_at_path = out.path().to_owned();
- created_at_path.push("created_at");
- let mut expires_at_path = out.path().to_owned();
- expires_at_path.push("expires_at");
-
- async fn read_value<T: FromStr>(path: &Path) -> Result<T> {
- dbg!(path);
- let raw = fs::read(path).await?;
- let raw = String::from_utf8(raw)?;
- raw.parse().map_err(|_| anyhow!("fromStr failed"))
- }
-
- let created_at = read_value(&created_at_path).await?;
- let expires_at = read_value(&expires_at_path).await.ok();
-
- Ok(FleetSecret {
- created_at,
- expires_at,
- public,
- secret: secret.map(SecretData),
- })
+ bail!("pure generators are broken for now")
}
async fn generate_impure(
config: &Config,
@@ -248,39 +174,53 @@
default_generator: Field,
owners: &[String],
) -> Result<FleetSecret> {
- let config_field = &config.config_unchecked_field;
let generator = nix_go!(secret.generator);
+ let on: Option<String> = nix_go_json!(default_generator.impureOn);
- let on: String = nix_go_json!(default_generator.impureOn);
- let call_package = nix_go!(
- config_field.hosts[{ on }]
- .nixosSystem
- .config
- .nixpkgs
- .resolvedPkgs
- .callPackage
- );
+ let host = if let Some(on) = &on {
+ config.host(on).await?
+ } else {
+ config.local_host()
+ };
+ let on_pkgs = host.pkgs().await?;
+ let call_package = nix_go!(on_pkgs.callPackage);
+ let mk_encrypt_secret = nix_go!(on_pkgs.mkEncryptSecret);
- let host = config.host(&on).await?;
+ let mut recipients = Vec::new();
+ for owner in owners {
+ let key = config.key(owner).await?;
+ recipients.push(key);
+ }
+ let encrypt = nix_go!(mk_encrypt_secret(Obj {
+ recipients: { recipients },
+ }));
+
+ let generator = nix_go!(call_package(generator)(Obj {
+ encrypt,
+ rustfmt_please_newline: { true },
+ }));
- let generator = nix_go!(call_package(generator)(Obj {}));
let generator = generator.build().await?;
let generator = generator
.get("out")
.ok_or_else(|| anyhow!("missing generateImpure out"))?;
let generator = host.remote_derivation(generator).await?;
- let mut recipients = String::new();
- for owner in owners {
- let key = config.key(owner).await?;
- recipients.push_str(&format!("-r \"{key}\" "));
- }
- recipients.push_str("-e");
-
- let out = host.mktemp_dir().await?;
+ let out_parent = host.mktemp_dir().await?;
+ let out = format!("{out_parent}/out");
let mut gen = host.cmd(generator).await?;
- gen.env("rageArgs", recipients).env("out", &out);
+ gen.env("out", &out);
+ if on.is_none() {
+ // This path is local, thus we can feed `OsString` directly to env var... But I don't think that's necessary to handle.
+ let project_path: String = config
+ .directory
+ .clone()
+ .into_os_string()
+ .into_string()
+ .map_err(|s| anyhow!("fleet project path is not utf-8: {s:?}"))?;
+ gen.env("FLEET_PROJECT", project_path);
+ }
gen.run().await.context("impure generator")?;
{
@@ -549,10 +489,7 @@
println!("{}", z85::encode(&data));
}
}
- Secret::ReadPublic {
- name,
- machine,
- } => {
+ Secret::ReadPublic { name, machine } => {
let secret = config.host_secret(&machine, &name)?;
let Some(public) = secret.public else {
bail!("no secret {name}");
cmds/fleet/src/host.rsdiffbeforeafterboth--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -54,7 +54,7 @@
pub local: bool,
pub session: OnceLock<Arc<openssh::Session>>,
- pub nixos_config: Field,
+ pub nixos_config: Option<Field>,
}
impl ConfigHost {
async fn open_session(&self) -> Result<Arc<openssh::Session>> {
@@ -169,7 +169,9 @@
}
pub async fn list_configured_secrets(&self) -> Result<Vec<String>> {
- let nixos = &self.nixos_config;
+ let Some(nixos) = &self.nixos_config else {
+ return Ok(vec![]);
+ };
let secrets = nix_go!(nixos.secrets);
let mut out = Vec::new();
for name in secrets.list_fields().await? {
@@ -183,9 +185,19 @@
Ok(out)
}
pub async fn secret_field(&self, name: &str) -> Result<Field> {
- let nixos = &self.nixos_config;
+ let Some(nixos) = &self.nixos_config else {
+ bail!("host is virtual and has no secrets");
+ };
Ok(nix_go!(nixos.secrets[{ name }]))
}
+
+ /// Packages for this host, resolved with nixpkgs overlays
+ pub async fn pkgs(&self) -> Result<Field> {
+ let Some(nixos) = &self.nixos_config else {
+ return Ok(self.config.default_pkgs.clone());
+ };
+ Ok(nix_go!(nixos.nixpkgs.resolvedPkgs))
+ }
}
impl Config {
@@ -202,6 +214,16 @@
self.opts.localhost.as_ref().map(|s| s as &str) == Some(host)
}
+ pub fn local_host(&self) -> ConfigHost {
+ ConfigHost {
+ config: self.clone(),
+ name: "<virtual localhost>".to_owned(),
+ local: true,
+ session: OnceLock::new(),
+ nixos_config: None,
+ }
+ }
+
pub async fn host(&self, name: &str) -> Result<ConfigHost> {
let config = &self.config_unchecked_field;
let nixos_config = nix_go!(config.hosts[{ name }].nixosSystem.config);
@@ -210,7 +232,7 @@
name: name.to_owned(),
local: self.is_local(name),
session: OnceLock::new(),
- nixos_config,
+ nixos_config: Some(nixos_config),
})
}
pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
cmds/fleet/src/main.rsdiffbeforeafterboth--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -11,7 +11,6 @@
mod fleetdata;
-use std::time::Duration;
use std::{ffi::OsString, process::ExitCode};
use anyhow::{bail, Result};
@@ -158,7 +157,7 @@
let reg = tracing_subscriber::registry().with({
let sub = tracing_subscriber::fmt::layer()
.without_time()
- .with_target(true);
+ .with_target(false);
#[cfg(feature = "indicatif")]
let sub = sub.with_writer(indicatif_layer.get_stdout_writer());
sub.with_filter(filter) // .withou,
cmds/install-secrets/src/main.rsdiffbeforeafterboth--- a/cmds/install-secrets/src/main.rs
+++ b/cmds/install-secrets/src/main.rs
@@ -13,7 +13,7 @@
use std::path::Path;
use std::str::{from_utf8, FromStr};
use std::{collections::HashMap, path::PathBuf};
-use tracing::{error, info, warn};
+use tracing::{error, info, info_span, warn};
use tracing_subscriber::filter::LevelFilter;
use tracing_subscriber::EnvFilter;
@@ -213,12 +213,9 @@
let mut failed = false;
for (name, value) in data {
- info!("initializing secret {name}");
+ let _span = info_span!("init", name = name);
if let Err(e) = init_secret(&identity, value) {
- error!(
- "{:?}",
- e.context(format!("failed to initialize secret {}", name))
- );
+ error!("{e}");
failed = true;
}
}
@@ -237,6 +234,7 @@
.from_env_lossy(),
)
.without_time()
+ .with_target(false)
.init();
let opts = Opts::parse();
crates/better-command/src/handler.rsdiffbeforeafterboth1//! Collection of handlers, which transform program-specific stdout format to tracing23use std::collections::HashMap;4use std::sync::{Arc, Mutex};56use once_cell::sync::Lazy;7use regex::Regex;8use serde::Deserialize;9use tracing::{info, info_span, warn, Span};10#[cfg(feature = "indicatif")]11use tracing_indicatif::span_ext::IndicatifSpanExt as _;1213pub trait Handler: Send {14 fn handle_line(&mut self, e: &str);15}1617/// Handler wrapper, which can be cloned.18pub struct ClonableHandler<H>(Arc<Mutex<H>>);19impl<H> Clone for ClonableHandler<H> {20 fn clone(&self) -> Self {21 Self(self.0.clone())22 }23}24impl<H> ClonableHandler<H> {25 pub fn new(inner: H) -> Self {26 Self(Arc::new(Mutex::new(inner)))27 }28}29impl<H: Handler> Handler for ClonableHandler<H> {30 fn handle_line(&mut self, e: &str) {31 self.0.lock().unwrap().handle_line(e)32 }33}3435/// Converts command output to tracing lines36pub struct PlainHandler;37impl Handler for PlainHandler {38 fn handle_line(&mut self, e: &str) {39 info!(target: "log", "{e}");40 }41}4243/// Ignores output44pub struct NoopHandler;45impl Handler for NoopHandler {46 fn handle_line(&mut self, _e: &str) {}47}4849/// Transform nix internal-json logs to tracing spans.50#[derive(Default)]51pub struct NixHandler {52 spans: HashMap<u64, Span>,53}54#[derive(Deserialize, Debug)]55#[serde(untagged)]56enum LogField {57 String(String),58 Num(u64),59}6061/// Nix internal-json log line type62#[derive(Deserialize, Debug)]63#[serde(rename_all = "camelCase", tag = "action")]64#[allow(dead_code)]65enum NixLog {66 Msg {67 level: u32,68 msg: String,69 raw_msg: Option<String>,70 },71 Start {72 id: u64,73 level: u32,74 #[serde(default)]75 fields: Vec<LogField>,76 text: String,77 #[serde(rename = "type")]78 typ: u32,79 },80 Stop {81 id: u64,82 },83 Result {84 id: u64,85 #[serde(rename = "type")]86 typ: u32,87 #[serde(default)]88 fields: Vec<LogField>,89 },90}91fn process_message(m: &str) -> String {92 // Supposed to remove formatting characters except colors, as some programs try to reset cursor position etc.93 static OSC_CLEANER: Lazy<Regex> =94 Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());95 static DETABBER: Lazy<Regex> = Lazy::new(|| Regex::new(r"\t").unwrap());96 let m = OSC_CLEANER.replace_all(m, "");97 // Indicatif can't format tabs. This is not the correct tab formatting, as correct one should be aligned,98 // and not just be replaced with the constant number of spaces, but it's ok for now, as statuses are single-line.99 DETABBER.replace_all(m.as_ref(), " ").to_string()100}101impl Handler for NixHandler {102 fn handle_line(&mut self, e: &str) {103 if let Some(e) = e.strip_prefix("@nix ") {104 let log: NixLog = match serde_json::from_str(e) {105 Ok(l) => l,106 Err(err) => {107 warn!("failed to parse nix log line {:?}: {}", e, err);108 return;109 }110 };111 match log {112 NixLog::Msg { msg, raw_msg, .. } => {113 #[allow(clippy::nonminimal_bool)]114 if !(msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m Git tree '") && msg.ends_with("' is dirty"))115 && !msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m not writing modified lock file of flake")116 && msg != "\u{1b}[35;1mwarning:\u{1b}[0m \u{1b}[31;1merror:\u{1b}[0m SQLite database '\u{1b}[35;1m/nix/var/nix/db/db.sqlite\u{1b}[0m' is busy" {117 if let Some(raw_msg) = raw_msg {118 if !msg.is_empty() {119 info!(target: "nix", "{}\n{}", raw_msg.trim_end(), msg.trim_end())120 } else {121 info!(target: "nix", "{}", raw_msg.trim_end())122 }123 } else {124 info!(target: "nix", "{}", msg.trim_end())125 }126 }127 }128 NixLog::Start {129 ref fields,130 typ,131 id,132 ..133 } if typ == 105 && !fields.is_empty() => {134 if let [LogField::String(drv), ..] = &fields[..] {135 let mut drv = drv.as_str();136 if let Some(pkg) = drv.strip_prefix("/nix/store/") {137 let mut it = pkg.splitn(2, '-');138 it.next();139 if let Some(pkg) = it.next() {140 drv = pkg;141 }142 }143 info!(target: "nix","building {}", drv);144 let span = info_span!("build", drv);145 #[cfg(feature = "indicatif")]146 span.pb_start();147 self.spans.insert(id, span);148 } else {149 warn!("bad build log: {:?}", log)150 }151 }152 NixLog::Start {153 ref fields,154 typ,155 id,156 ..157 } if typ == 100 && fields.len() >= 3 => {158 if let [LogField::String(drv), LogField::String(from), LogField::String(to), ..] =159 &fields[..]160 {161 let mut drv = drv.as_str();162163 if let Some(pkg) = drv.strip_prefix("/nix/store/") {164 let mut it = pkg.splitn(2, '-');165 it.next();166 if let Some(pkg) = it.next() {167 drv = pkg;168 }169 }170 info!(target: "nix","copying {} {} -> {}", drv, from, to);171 let span = info_span!("copy", from, to, drv);172 #[cfg(feature = "indicatif")]173 span.pb_start();174 self.spans.insert(id, span);175 } else {176 warn!("bad copy log: {:?}", log)177 }178 }179 NixLog::Start { text, typ, id, .. }180 if typ == 0 || typ == 102 || typ == 103 || typ == 104 =>181 {182 if !text.is_empty()183 && text != "querying info about missing paths"184 && text != "copying 0 paths"185 // Too much spam on lazy-trees branch186 && !(text.starts_with("copying '") && text.ends_with("' to the store"))187 {188 let span = info_span!("job");189 #[cfg(feature = "indicatif")]190 {191 span.pb_start();192 span.pb_set_message(&process_message(text.trim()));193 }194 self.spans.insert(id, span);195 info!(target: "nix", "{}", text);196 }197 }198 NixLog::Start {199 text,200 level: 0,201 typ: 108,202 ..203 } if text.is_empty() => {204 // Cache lookup? Coupled with copy log205 }206 NixLog::Start {207 text,208 level: 4,209 typ: 109,210 ..211 } if text.starts_with("querying info about ") => {212 // Cache lookup213 }214 NixLog::Start {215 text,216 level: 4,217 typ: 101,218 ..219 } if text.starts_with("downloading ") => {220 // NAR downloading, coupled with copy log221 }222 NixLog::Start {223 text,224 level: 1,225 typ: 111,226 ..227 } if text.starts_with("waiting for a machine to build ") => {228 // Useless repeating notification about build229 }230 NixLog::Start {231 text,232 level: 3,233 typ: 111,234 ..235 } if text.starts_with("resolved derivation: ") => {236 // CA resolved237 }238 NixLog::Start {239 text,240 level: 1,241 typ: 111,242 id,243 ..244 } if text.starts_with("waiting for lock on ") => {245 let mut drv = text.strip_prefix("waiting for lock on ").unwrap();246 if let Some(txt) = drv.strip_prefix("\u{1b}[35;1m'") {247 drv = txt;248 }249 if let Some(txt) = drv.strip_suffix("'\u{1b}[0m") {250 drv = txt;251 }252 if let Some(txt) = drv.split("', '").next() {253 drv = txt;254 }255 if let Some(pkg) = drv.strip_prefix("/nix/store/") {256 let mut it = pkg.splitn(2, '-');257 it.next();258 if let Some(pkg) = it.next() {259 drv = pkg;260 }261 }262 let span = info_span!("waiting on drv", drv);263 #[cfg(feature = "indicatif")]264 span.pb_start();265 self.spans.insert(id, span);266 // Concurrent build of the same message267 }268 NixLog::Stop { id, .. } => {269 self.spans.remove(&id);270 }271 NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {272 if let Some(span) = self.spans.get(&id) {273 if let LogField::String(s) = &fields[0] {274 #[cfg(feature = "indicatif")]275 span.pb_set_message(&process_message(s.trim()));276 #[cfg(not(feature = "indicatif"))]277 info!("{}", process_message(s));278 } else {279 warn!("bad fields: {fields:?}");280 }281 } else {282 warn!("unknown result id: {id} {typ} {fields:?}");283 }284 // dbg!(fields, id, typ);285 }286 NixLog::Result { fields, id, typ } if typ == 105 && fields.len() >= 4 => {287 if let Some(span) = self.spans.get(&id) {288 if let [LogField::Num(done), LogField::Num(expected), LogField::Num(_running), LogField::Num(_failed)] =289 &fields[..4]290 {291 #[cfg(feature = "indicatif")]292 {293 span.pb_set_length(*expected);294 span.pb_set_position(*done);295 }296 let _ = (span, done, expected);297 } else {298 warn!("bad fields: {fields:?}");299 }300 } else {301 // warn!("unknown result id: {id} {typ} {fields:?}");302 // Unaccounted progress.303 }304 // dbg!(fields, id, typ);305 }306 NixLog::Result { typ, .. } if typ == 104 || typ == 106 => {307 // Set phase, expected308 }309 _ => warn!("unknown log: {:?}", log),310 };311 } else {312 let e = e.trim();313 if e.starts_with("Failed tcsetattr(TCSADRAIN): ") {314 return;315 }316 info!("{e}")317 }318 }319}1//! Collection of handlers, which transform program-specific stdout format to tracing23use std::collections::HashMap;4use std::sync::{Arc, Mutex};56use once_cell::sync::Lazy;7use regex::Regex;8use serde::Deserialize;9use tracing::{info, info_span, warn, Span};10#[cfg(feature = "indicatif")]11use tracing_indicatif::span_ext::IndicatifSpanExt as _;1213pub trait Handler: Send {14 fn handle_line(&mut self, e: &str);15}1617/// Handler wrapper, which can be cloned.18pub struct ClonableHandler<H>(Arc<Mutex<H>>);19impl<H> Clone for ClonableHandler<H> {20 fn clone(&self) -> Self {21 Self(self.0.clone())22 }23}24impl<H> ClonableHandler<H> {25 pub fn new(inner: H) -> Self {26 Self(Arc::new(Mutex::new(inner)))27 }28}29impl<H: Handler> Handler for ClonableHandler<H> {30 fn handle_line(&mut self, e: &str) {31 self.0.lock().unwrap().handle_line(e)32 }33}3435/// Converts command output to tracing lines36pub struct PlainHandler;37impl Handler for PlainHandler {38 fn handle_line(&mut self, e: &str) {39 info!(target: "log", "{e}");40 }41}4243/// Ignores output44pub struct NoopHandler;45impl Handler for NoopHandler {46 fn handle_line(&mut self, _e: &str) {}47}4849/// Transform nix internal-json logs to tracing spans.50#[derive(Default)]51pub struct NixHandler {52 spans: HashMap<u64, Span>,53}54#[derive(Deserialize, Debug)]55#[serde(untagged)]56enum LogField {57 String(String),58 Num(u64),59}6061/// Nix internal-json log line type62#[derive(Deserialize, Debug)]63#[serde(rename_all = "camelCase", tag = "action")]64#[allow(dead_code)]65enum NixLog {66 Msg {67 level: u32,68 msg: String,69 raw_msg: Option<String>,70 },71 Start {72 id: u64,73 level: u32,74 #[serde(default)]75 fields: Vec<LogField>,76 text: String,77 #[serde(rename = "type")]78 typ: u32,79 },80 Stop {81 id: u64,82 },83 Result {84 id: u64,85 #[serde(rename = "type")]86 typ: u32,87 #[serde(default)]88 fields: Vec<LogField>,89 },90}91fn process_message(m: &str) -> String {92 // Supposed to remove formatting characters except colors, as some programs try to reset cursor position etc.93 static OSC_CLEANER: Lazy<Regex> =94 Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());95 static DETABBER: Lazy<Regex> = Lazy::new(|| Regex::new(r"\t").unwrap());96 let m = OSC_CLEANER.replace_all(m, "");97 // Indicatif can't format tabs. This is not the correct tab formatting, as correct one should be aligned,98 // and not just be replaced with the constant number of spaces, but it's ok for now, as statuses are single-line.99 DETABBER.replace_all(m.as_ref(), " ").to_string()100}101impl Handler for NixHandler {102 fn handle_line(&mut self, e: &str) {103 if let Some(e) = e.strip_prefix("@nix ") {104 let log: NixLog = match serde_json::from_str(e) {105 Ok(l) => l,106 Err(err) => {107 warn!("failed to parse nix log line {:?}: {}", e, err);108 return;109 }110 };111 match log {112 NixLog::Msg { msg, raw_msg, .. } => {113 #[allow(clippy::nonminimal_bool)]114 if !(msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m Git tree '") && msg.ends_with("' is dirty"))115 && !msg.starts_with("\u{1b}[35;1mwarning:\u{1b}[0m not writing modified lock file of flake")116 && msg != "\u{1b}[35;1mwarning:\u{1b}[0m \u{1b}[31;1merror:\u{1b}[0m SQLite database '\u{1b}[35;1m/nix/var/nix/db/db.sqlite\u{1b}[0m' is busy" {117 if let Some(raw_msg) = raw_msg {118 if !msg.is_empty() {119 info!(target: "nix", "{}\n{}", raw_msg.trim_end(), msg.trim_end())120 } else {121 info!(target: "nix", "{}", raw_msg.trim_end())122 }123 } else {124 info!(target: "nix", "{}", msg.trim_end())125 }126 }127 }128 NixLog::Start {129 ref fields,130 typ,131 id,132 ..133 } if typ == 105 && !fields.is_empty() => {134 if let [LogField::String(drv), ..] = &fields[..] {135 let mut drv = drv.as_str();136 if let Some(pkg) = drv.strip_prefix("/nix/store/") {137 let mut it = pkg.splitn(2, '-');138 it.next();139 if let Some(pkg) = it.next() {140 drv = pkg;141 }142 }143 info!(target: "nix","building {}", drv);144 let span = info_span!("build", drv);145 #[cfg(feature = "indicatif")]146 span.pb_start();147 self.spans.insert(id, span);148 } else {149 warn!("bad build log: {:?}", log)150 }151 }152 NixLog::Start {153 ref fields,154 typ,155 id,156 ..157 } if typ == 100 && fields.len() >= 3 => {158 if let [LogField::String(drv), LogField::String(from), LogField::String(to), ..] =159 &fields[..]160 {161 let mut drv = drv.as_str();162163 if let Some(pkg) = drv.strip_prefix("/nix/store/") {164 let mut it = pkg.splitn(2, '-');165 it.next();166 if let Some(pkg) = it.next() {167 drv = pkg;168 }169 }170 info!(target: "nix","copying {} {} -> {}", drv, from, to);171 let span = info_span!("copy", from, to, drv);172 #[cfg(feature = "indicatif")]173 span.pb_start();174 self.spans.insert(id, span);175 } else {176 warn!("bad copy log: {:?}", log)177 }178 }179 NixLog::Start { text, typ, id, .. }180 if typ == 0 || typ == 102 || typ == 103 || typ == 104 =>181 {182 if !text.is_empty()183 && text != "querying info about missing paths"184 && text != "copying 0 paths"185 // Too much spam on lazy-trees branch186 && !(text.starts_with("copying '") && text.ends_with("' to the store"))187 {188 let span = info_span!("job");189 #[cfg(feature = "indicatif")]190 {191 span.pb_start();192 span.pb_set_message(&process_message(text.trim()));193 }194 self.spans.insert(id, span);195 info!(target: "nix", "{}", text);196 }197 }198 NixLog::Start {199 text,200 level: 0,201 typ: 108,202 ..203 } if text.is_empty() => {204 // Cache lookup? Coupled with copy log205 }206 NixLog::Start {207 text,208 level: 4,209 typ: 109,210 ..211 } if text.starts_with("querying info about ") => {212 // Cache lookup213 }214 NixLog::Start {215 text,216 level: 4,217 typ: 101,218 ..219 } if text.starts_with("downloading ") => {220 // NAR downloading, coupled with copy log221 }222 NixLog::Start {223 text,224 level: 1,225 typ: 111,226 ..227 } if text.starts_with("waiting for a machine to build ") => {228 // Useless repeating notification about build229 }230 NixLog::Start {231 text,232 level: 3,233 typ: 111,234 ..235 } if text.starts_with("resolved derivation: ") => {236 // CA resolved237 }238 NixLog::Start {239 text,240 level: 1,241 typ: 111,242 id,243 ..244 } if text.starts_with("waiting for lock on ") => {245 let mut drv = text.strip_prefix("waiting for lock on ").unwrap();246 if let Some(txt) = drv.strip_prefix("\u{1b}[35;1m'") {247 drv = txt;248 }249 if let Some(txt) = drv.strip_suffix("'\u{1b}[0m") {250 drv = txt;251 }252 if let Some(txt) = drv.split("', '").next() {253 drv = txt;254 }255 if let Some(pkg) = drv.strip_prefix("/nix/store/") {256 let mut it = pkg.splitn(2, '-');257 it.next();258 if let Some(pkg) = it.next() {259 drv = pkg;260 }261 }262 let span = info_span!("waiting on drv", drv);263 #[cfg(feature = "indicatif")]264 span.pb_start();265 self.spans.insert(id, span);266 // Concurrent build of the same message267 }268 NixLog::Stop { id, .. } => {269 self.spans.remove(&id);270 }271 NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {272 if let Some(span) = self.spans.get(&id) {273 if let LogField::String(s) = &fields[0] {274 #[cfg(feature = "indicatif")]275 span.pb_set_message(&process_message(s.trim()));276 #[cfg(not(feature = "indicatif"))]277 {278 let _span = span.enter();279 info!("{}", process_message(s));280 }281 } else {282 warn!("bad fields: {fields:?}");283 }284 } else {285 warn!("unknown result id: {id} {typ} {fields:?}");286 }287 // dbg!(fields, id, typ);288 }289 NixLog::Result { fields, id, typ } if typ == 105 && fields.len() >= 4 => {290 if let Some(span) = self.spans.get(&id) {291 if let [LogField::Num(done), LogField::Num(expected), LogField::Num(_running), LogField::Num(_failed)] =292 &fields[..4]293 {294 #[cfg(feature = "indicatif")]295 {296 span.pb_set_length(*expected);297 span.pb_set_position(*done);298 }299 let _ = (span, done, expected);300 } else {301 warn!("bad fields: {fields:?}");302 }303 } else {304 // warn!("unknown result id: {id} {typ} {fields:?}");305 // Unaccounted progress.306 }307 // dbg!(fields, id, typ);308 }309 NixLog::Result { typ, .. } if typ == 104 || typ == 106 => {310 // Set phase, expected311 }312 _ => warn!("unknown log: {:?}", log),313 };314 } else {315 let e = e.trim();316 if e.starts_with("Failed tcsetattr(TCSADRAIN): ") {317 return;318 }319 info!("{e}")320 }321 }322}crates/better-command/src/lib.rsdiffbeforeafterboth--- a/crates/better-command/src/lib.rs
+++ b/crates/better-command/src/lib.rs
@@ -1,5 +1,5 @@
mod handler;
-pub use handler::{Handler, PlainHandler, NoopHandler, NixHandler, ClonableHandler};
+pub use handler::{ClonableHandler, Handler, NixHandler, NoopHandler, PlainHandler};
pub fn add(left: usize, right: usize) -> usize {
left + right
flake.lockdiffbeforeafterboth--- a/flake.lock
+++ b/flake.lock
@@ -1,26 +1,28 @@
{
"nodes": {
- "flake-utils": {
+ "crane": {
"inputs": {
- "systems": "systems"
+ "nixpkgs": [
+ "nixpkgs"
+ ]
},
"locked": {
- "lastModified": 1705309234,
- "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
- "owner": "numtide",
- "repo": "flake-utils",
- "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+ "lastModified": 1712681629,
+ "narHash": "sha256-bMDXn4AkTXLCpoZbII6pDGoSeSe9gI87jxPsHRXgu/E=",
+ "owner": "ipetkov",
+ "repo": "crane",
+ "rev": "220387ac8e99cbee0ca4c95b621c4bc782b6a235",
"type": "github"
},
"original": {
- "owner": "numtide",
- "repo": "flake-utils",
+ "owner": "ipetkov",
+ "repo": "crane",
"type": "github"
}
},
- "flake-utils_2": {
+ "flake-utils": {
"inputs": {
- "systems": "systems_2"
+ "systems": "systems"
},
"locked": {
"lastModified": 1705309234,
@@ -54,6 +56,7 @@
},
"root": {
"inputs": {
+ "crane": "crane",
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"rust-overlay": "rust-overlay"
@@ -61,7 +64,9 @@
},
"rust-overlay": {
"inputs": {
- "flake-utils": "flake-utils_2",
+ "flake-utils": [
+ "flake-utils"
+ ],
"nixpkgs": [
"nixpkgs"
]
@@ -81,21 +86,6 @@
}
},
"systems": {
- "locked": {
- "lastModified": 1681028828,
- "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
- "owner": "nix-systems",
- "repo": "default",
- "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
- "type": "github"
- },
- "original": {
- "owner": "nix-systems",
- "repo": "default",
- "type": "github"
- }
- },
- "systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
flake.nixdiffbeforeafterboth--- a/flake.nix
+++ b/flake.nix
@@ -5,15 +5,23 @@
nixpkgs.url = "github:nixos/nixpkgs/master";
rust-overlay = {
url = "github:oxalica/rust-overlay";
+ inputs = {
+ nixpkgs.follows = "nixpkgs";
+ flake-utils.follows = "flake-utils";
+ };
+ };
+ flake-utils.url = "github:numtide/flake-utils";
+ crane = {
+ url = "github:ipetkov/crane";
inputs.nixpkgs.follows = "nixpkgs";
};
- flake-utils = {url = "github:numtide/flake-utils";};
};
outputs = {
self,
rust-overlay,
flake-utils,
nixpkgs,
+ crane,
}:
with nixpkgs.lib;
{
@@ -26,20 +34,16 @@
inherit system;
overlays = [(import rust-overlay)];
};
- llvmPkgs = pkgs.buildPackages.llvmPackages_11;
- rust =
- (pkgs.rustChannelOf {
- date = "2024-02-10";
- channel = "nightly";
- })
- .default
- .override {extensions = ["rust-src" "rust-analyzer"];};
+ rust = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
+ craneLib = (crane.mkLib pkgs).overrideToolchain rust;
in {
- packages = (import ./pkgs) pkgs pkgs;
- devShell = (pkgs.mkShell.override {stdenv = llvmPkgs.stdenv;}) {
+ packages = import ./pkgs {
+ inherit (pkgs) callPackage;
+ inherit craneLib;
+ };
+ devShell = craneLib.devShell {
nativeBuildInputs = with pkgs; [
alejandra
- rust
lld
cargo-edit
cargo-udeps
lib/default.nixdiffbeforeafterboth--- a/lib/default.nix
+++ b/lib/default.nix
@@ -10,11 +10,14 @@
hosts,
modules,
globalModules ? [],
+ extraFleetLib ? {},
}: let
hostNames = nixpkgs.lib.attrNames hosts;
- fleetLib = import ./fleetLib.nix {
- inherit nixpkgs hostNames;
- };
+ fleetLib =
+ (import ./fleetLib.nix {
+ inherit nixpkgs hostNames;
+ })
+ // extraFleetLib;
in let
root = nixpkgs.lib.evalModules {
modules =
lib/fleetLib.nixdiffbeforeafterboth--- a/lib/fleetLib.nix
+++ b/lib/fleetLib.nix
@@ -39,4 +39,32 @@
mkFleetDefault = mkOverride 999;
# Some generators use mkDefault, but optionDefault is set by nixpkgs.
mkFleetGeneratorDefault = mkOverride 1001;
+
+ mkPassword = {size ? 32}: {
+ coreutils,
+ encrypt,
+ mkSecretGenerator,
+ }:
+ mkSecretGenerator {
+ script = ''
+ ${coreutils}/bin/tr -dc 'A-Za-z0-9!?%=' < /dev/random \
+ | ${coreutils}/bin/head -c ${toString size} \
+ | ${encrypt} > $out/secret
+ '';
+ };
+
+ mkRsa = {size ? 4096}: {
+ openssl,
+ encrypt,
+ mkSecretGenerator,
+ }:
+ mkSecretGenerator {
+ script = ''
+ ${openssl}/bin/openssl genrsa -out rsa_private.key ${toString size}
+ ${openssl}/bin/openssl rsa -in rsa_private.key -pubout -out rsa_public.key
+
+ sudo cat rsa_private.key | ${encrypt} > $out/secret
+ sudo cat rsa_public.key > $out/public
+ '';
+ };
}
modules/fleet/secrets.nixdiffbeforeafterboth--- a/modules/fleet/secrets.nix
+++ b/modules/fleet/secrets.nix
@@ -8,6 +8,13 @@
with fleetLib; let
sharedSecret = with types; ({config, ...}: {
options = {
+ managed = mkOption {
+ type = bool;
+ description = ''
+ Is this secret managed by configuration (I.e will work with reencrypt/etc), or it is configured by user
+ '';
+ };
+
expectedOwners = mkOption {
type = nullOr (listOf str);
description = ''
@@ -146,77 +153,81 @@
overlays = [
(final: prev: let
lib = final.lib;
+ inherit (lib) strings;
+ inherit (strings) escapeShellArgs;
in {
- mkPassword = {size ? 32}:
- final.mkSecretGenerator ''
- ${final.coreutils}/bin/tr -dc 'A-Za-z0-9!?%=' < /dev/random \
- | ${final.coreutils}/bin/head -c ${toString size} \
- | encrypt > $out/secret
- '';
- mkRsa = {size ? 4096}:
- final.mkSecretGenerator ''
- ${final.openssl}/bin/openssl genrsa -out rsa_private.key ${toString size}
- ${final.openssl}/bin/openssl rsa -in rsa_private.key -pubout -out rsa_public.key
-
- sudo cat rsa_private.key | encrypt > $out/secret
- sudo cat rsa_public.key > $out/public
+ mkEncryptSecret = {
+ rage ? prev.rage,
+ recipients,
+ }:
+ prev.writeShellScript "encryptor" ''
+ #!/bin/sh
+ exec ${rage}/bin/rage ${escapeShellArgs recipients} -e "$@"
'';
# TODO: Move to fleet
# TODO: Merge both generators to one with consistent options syntax?
# Impure generator is built on local machine, then built closure is copied to remote machine,
# and then it is ran in inpure context, so that this generator may access HSMs and other things.
- mkImpureSecretGenerator = generatorText: machine:
+ mkImpureSecretGenerator = {
+ script,
+ # If set - script will be run on remote machine, otherwise it will be run with fleet project in CWD
+ # (Some secrets-encryption-in-git/managed PKI solution is expected)
+ impureOn ? null,
+ }:
(prev.writeShellScript "impureGenerator.sh" ''
#!/bin/sh
set -eu
-
- # TODO: Provide encryption function as script passed to `callPackage generator {encrypt = ...;}`
- function encrypt() {
- eval ${final.rage}/bin/rage $rageArgs
- }
+ cd /var/empty
created_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ")
- echo -n $created_at > $out/created_at
- ${generatorText}
+ ${script}
+ if ! test -d $out; then
+ echo "impure generator script did not produce expected \$out output"
+ exit 1
+ fi
+
+ echo -n $created_at > $out/created_at
echo -n SUCCESS > $out/marker
'')
.overrideAttrs (old: {
passthru = {
+ inherit impureOn;
generatorKind = "impure";
- impureOn = machine;
};
});
+ # Pure generators are disabled for now
+ mkSecretGenerator = {script}: final.mkImpureSecretGenerator {inherit script;};
+
# TODO: Implement consistent naming
# Pure secret generator is supposed to be run entirely by nix, using `__impure` derivation type...
# But for now, it is ran the same way as `impureSecretGenerator`, but on the local machine.
- mkSecretGenerator = generatorText:
- (prev.writeShellScript "generator.sh" ''
- #!/bin/sh
- set -eu
- # TODO: User should create output directory by themselves.
- cd $out
-
- # TODO: Provide encryption function as script passed to `callPackage generator {encrypt = ...;}`
- function encrypt() {
- eval ${final.rage}/bin/rage $rageArgs
- }
-
- created_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ")
- echo -n $created_at > $out/created_at
-
- ${generatorText}
-
- echo -n SUCCESS > $out/marker
- '')
- .overrideAttrs (old: {
- passthru = {
- generatorKind = "pure";
- };
- # TODO: make nix daemon build secret, not just the script.
- # __impure = true;
- });
+ # mkSecretGenerator = {script}:
+ # (prev.writeShellScript "generator.sh" ''
+ # #!/bin/sh
+ # set -eu
+ # # TODO: make nix daemon build secret, not just the script.
+ # cd /var/empty
+ #
+ # created_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ")
+ #
+ # ${script}
+ # if ! test -d $out; then
+ # echo "impure generator script did not produce expected \$out output"
+ # exit 1
+ # fi
+ #
+ # echo -n $created_at > $out/created_at
+ # echo -n SUCCESS > $out/marker
+ # '')
+ # .overrideAttrs (old: {
+ # passthru = {
+ # generatorKind = "pure";
+ # };
+ # # TODO: make nix daemon build secret, not just the script.
+ # # __impure = true;
+ # });
})
];
};
nixos/fleetPkgs.nixdiffbeforeafterboth--- a/nixos/fleetPkgs.nix
+++ b/nixos/fleetPkgs.nix
@@ -1,3 +1,24 @@
-{ ... }: {
- nixpkgs.overlays = [ (import ../pkgs) ];
+{...}: {
+ nixpkgs.overlays = [
+ # Not using craneLib here, because we don't want to have two different rust versions for some platforms.
+ (final: prev: {
+ fleet-install-secrets = prev.callPackage ({rustPlatform}:
+ rustPlatform.buildRustPackage rec {
+ pname = "fleet-install-secrets";
+ name = "${pname}";
+
+ src = ../.;
+ strictDeps = true;
+
+ buildAndTestSubdir = "cmds/install-secrets";
+
+ cargoLock = {
+ lockFile = ../Cargo.lock;
+ outputHashes = {
+ "alejandra-3.0.0" = "sha256-lStDIPizbJipd1JpNKX1olBKzyIosyC2U/mVFwJPcZE=";
+ };
+ };
+ }) {};
+ })
+ ];
}
pkgs/default.nixdiffbeforeafterboth--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -1,6 +1,9 @@
-pkgs: super:
-with pkgs;
{
- fleet-install-secrets = callPackage ./fleet-install-secrets.nix { };
- fleet = callPackage ./fleet.nix { };
+ callPackage,
+ craneLib,
+}: rec {
+ default = fleet;
+
+ fleet-install-secrets = callPackage ./fleet-install-secrets.nix {inherit craneLib;};
+ fleet = callPackage ./fleet.nix {inherit craneLib;};
}
pkgs/fleet-install-secrets.nixdiffbeforeafterboth--- a/pkgs/fleet-install-secrets.nix
+++ b/pkgs/fleet-install-secrets.nix
@@ -1,16 +1,9 @@
-{ rustPlatform, lib }:
-
-rustPlatform.buildRustPackage rec {
+{craneLib}:
+craneLib.buildPackage rec {
pname = "fleet-install-secrets";
- version = "0.0.1";
- name = "${pname}-${version}";
- src = ../.;
- buildAndTestSubdir = "cmds/install-secrets";
- cargoLock = {
- lockFile = ../Cargo.lock;
- outputHashes = {
- "alejandra-3.0.0" = "sha256-lStDIPizbJipd1JpNKX1olBKzyIosyC2U/mVFwJPcZE=";
- };
- };
+ src = craneLib.cleanCargoSource (craneLib.path ../.);
+ strictDeps = true;
+
+ cargoExtraArgs = "--locked -p ${pname}";
}
pkgs/fleet.nixdiffbeforeafterboth--- a/pkgs/fleet.nix
+++ b/pkgs/fleet.nix
@@ -1,16 +1,9 @@
-{ rustPlatform }:
-
-rustPlatform.buildRustPackage rec {
+{craneLib}:
+craneLib.buildPackage rec {
pname = "fleet";
- version = "0.0.1";
- name = "${pname}-${version}";
- src = ../.;
- cargoBuildFlags = "-p ${pname}";
- cargoLock = {
- lockFile = ../Cargo.lock;
- outputHashes = {
- "alejandra-3.0.0" = "sha256-YSdHsJ73G7TEFzbmpZ2peuMefIa9/vNB2g+xdiyma3U=";
- };
- };
+ src = craneLib.cleanCargoSource (craneLib.path ../.);
+ strictDeps = true;
+
+ cargoExtraArgs = "--locked -p ${pname}";
}
rust-toolchain.tomldiffbeforeafterboth--- /dev/null
+++ b/rust-toolchain.toml
@@ -0,0 +1,3 @@
+[toolchain]
+channel = "nightly-2024-02-10"
+components = ["rustfmt", "clippy", "rust-analyzer", "rust-src"]