git.delta.rocks / jrsonnet / refs/commits / fb6d3038c4a7

difftreelog

refactor drop old db

Yaroslav Bolyukin2021-09-18parent: #4ad5065.patch.diff
in: trunk

8 files changed

deletedsrc/cmds/generate_secrets.rsdiffbeforeafterboth
--- a/src/cmds/generate_secrets.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-use std::collections::HashSet;
-
-use anyhow::Result;
-use clap::Clap;
-use log::info;
-
-use crate::{
-	db::{
-		secret::{list_secrets, SecretDb},
-		Db, DbData,
-	},
-	host::FleetOpts,
-};
-
-#[derive(Clap)]
-pub struct GenerateSecrets {
-	#[clap(flatten)]
-	fleet_opts: FleetOpts,
-
-	/// If set - remove orphaned secrets
-	#[clap(long)]
-	cleanup: bool,
-}
-
-impl GenerateSecrets {
-	pub fn run(self) -> Result<()> {
-		let db = Db::new(".fleet")?;
-		let mut secrets = SecretDb::open(&db)?;
-
-		let defined_secrets = list_secrets()?;
-		for (secret, data) in defined_secrets.iter() {
-			//let keys = KeyDb::open(&db)?;
-			secrets.ensure_generated(&self.fleet_opts, secret, data)?;
-		}
-		let key_names = defined_secrets
-			.keys()
-			.filter(|s| !secrets.has_secret(s))
-			.cloned()
-			.collect::<HashSet<_>>();
-		if !key_names.is_empty() {
-			if self.cleanup {
-				info!("Removed orphan secrets:");
-			} else {
-				info!("Orphan secrets found, run with --cleanup to remove them from db:");
-			}
-			for key in key_names {
-				info!("- {}", key);
-				if self.cleanup {
-					secrets.remove_secret(&key)
-				}
-			}
-		}
-
-		Ok(())
-	}
-}
modifiedsrc/cmds/mod.rsdiffbeforeafterboth
--- a/src/cmds/mod.rs
+++ b/src/cmds/mod.rs
@@ -1,4 +1,2 @@
 pub mod build_systems;
-// pub mod fetch_keys;
-pub mod generate_secrets;
 pub mod secrets;
deletedsrc/db/dbr.rsdiffbeforeafterboth
--- a/src/db/dbr.rs
+++ /dev/null
@@ -1,117 +0,0 @@
-//! Small .toml based readable data store
-
-use anyhow::{Context, Result};
-use serde::{de::DeserializeOwned, Serialize};
-use std::{
-	cell::Cell,
-	collections::HashSet,
-	io::Write,
-	ops::{Deref, DerefMut},
-	path::Path,
-	path::PathBuf,
-	sync::{Arc, Mutex},
-};
-
-struct DbInternal {
-	root: PathBuf,
-	locked_paths: HashSet<PathBuf>,
-}
-
-pub trait DbData: DeserializeOwned + Serialize + Default {
-	const DB_NAME: &'static str;
-
-	fn open(db: &Db) -> Result<DbFile<Self>> {
-		db.db::<Self>()
-	}
-}
-
-#[derive(Clone)]
-pub struct Db(Arc<Mutex<DbInternal>>);
-impl Db {
-	pub fn new(root: impl AsRef<Path>) -> Result<Self> {
-		let root: &Path = root.as_ref();
-		std::fs::create_dir_all(&root).context("db root")?;
-		Ok(Db(Arc::new(Mutex::new(DbInternal {
-			root: root.to_owned(),
-			locked_paths: HashSet::new(),
-		}))))
-	}
-
-	pub fn db<T: DbData>(&self) -> Result<DbFile<T>> {
-		let name = T::DB_NAME;
-		assert!(!name.contains('/') && !name.contains('\\'));
-		let mut db = self.0.lock().unwrap();
-		let mut data_path = db.root.clone();
-		data_path.push(format!("{}.toml", name));
-
-		if !db.locked_paths.insert(data_path.clone()) {
-			anyhow::bail!("file is already open");
-		}
-
-		let data = if data_path.exists() {
-			let raw_data = std::fs::read(&data_path).context("reading file")?;
-			toml::from_slice(&raw_data).context("parsing file")?
-		} else {
-			T::default()
-		};
-
-		Ok(DbFile {
-			db: self.clone(),
-			root: db.root.clone(),
-			path: data_path,
-			data,
-			dirty: Cell::new(false),
-		})
-	}
-}
-
-pub struct DbFile<T: DbData> {
-	db: Db,
-	root: PathBuf,
-	path: PathBuf,
-	data: T,
-	dirty: Cell<bool>,
-}
-
-impl<T: DbData> Deref for DbFile<T> {
-	type Target = T;
-
-	fn deref(&self) -> &Self::Target {
-		&self.data
-	}
-}
-
-impl<T: DbData> DerefMut for DbFile<T> {
-	fn deref_mut(&mut self) -> &mut Self::Target {
-		self.dirty.set(true);
-		&mut self.data
-	}
-}
-
-impl<T: DbData> DbFile<T> {
-	pub fn write(&self) -> Result<()> {
-		if !self.dirty.get() {
-			return Ok(());
-		}
-		let mut temp = tempfile::Builder::new()
-			.prefix("~")
-			.suffix(".toml")
-			.tempfile_in(&self.root)?;
-		let mut out = String::new();
-		let mut serializer = toml::Serializer::new(&mut out);
-		serializer.pretty_array(true).pretty_string(true);
-		self.data.serialize(&mut serializer)?;
-		temp.write_all(out.as_bytes())?;
-		temp.persist(&self.path)?;
-		self.dirty.set(false);
-		Ok(())
-	}
-}
-
-impl<T: DbData> Drop for DbFile<T> {
-	fn drop(&mut self) {
-		let mut db = self.db.0.lock().unwrap();
-		self.write().unwrap();
-		db.locked_paths.remove(&self.path);
-	}
-}
deletedsrc/db/mod.rsdiffbeforeafterboth
--- a/src/db/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-mod dbr;
-pub mod secret;
-
-pub use dbr::*;
deletedsrc/db/secret.rsdiffbeforeafterboth
before · src/db/secret.rs
1use crate::{command::CommandExt, host::FleetOpts, nix::SECRETS_ATTRIBUTE};2use anyhow::{bail, Context, Result};3use log::info;4use serde::{Deserialize, Deserializer, Serialize, Serializer};5use std::{6	collections::{BTreeMap, BTreeSet, HashMap},7	process::Command,8	time::Instant,9	time::SystemTime,10};11use time::{Duration, PrimitiveDateTime};1213use super::DbData;1415#[derive(Serialize, Deserialize, Debug)]16pub struct SecretListData {17	pub owners: BTreeSet<String>,18	#[serde(rename = "expireIn")]19	renew_in: Option<u64>,20}21pub fn list_secrets() -> Result<HashMap<String, SecretListData>> {22	Command::new("nix")23		.inherit_stdio()24		.arg("eval")25		.arg(SECRETS_ATTRIBUTE)26		.arg("--apply")27		.arg(28			r#"29				s: (builtins.mapAttrs (n: {owners, expireIn, ...}: {30					inherit owners expireIn;31				}) s)32			"#,33		)34		.arg("--json")35		.run_json()36		.context("while getting secret list")37}3839struct ReadableDate(PrimitiveDateTime);40impl Serialize for ReadableDate {41	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>42	where43		S: Serializer,44	{45		serializer.serialize_str(&self.0.to_string())46	}47}48impl<'de> Deserialize<'de> for ReadableDate {49	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>50	where51		D: Deserializer<'de>,52	{53		Ok(Self(54			PrimitiveDateTime::parse(String::deserialize(deserializer)?, "%F %T").unwrap(),55		))56	}57}58impl From<PrimitiveDateTime> for ReadableDate {59	fn from(d: PrimitiveDateTime) -> Self {60		Self(d)61	}62}63impl From<ReadableDate> for PrimitiveDateTime {64	fn from(d: ReadableDate) -> Self {65		d.066	}67}6869#[derive(serde::Serialize, serde::Deserialize)]70struct SecretData {71	created_at: ReadableDate,72	renew_at: Option<ReadableDate>,73	owners: BTreeSet<String>,7475	public_data: BTreeMap<String, String>,76	private_files: BTreeMap<String, String>,77}78impl SecretData {79	fn should_renew(&self) -> bool {80		if let Some(renew_at) = &self.renew_at {81			let now: PrimitiveDateTime = SystemTime::now().into();82			renew_at.0 <= now83		} else {84			false85		}86	}87	fn is_valid(&self, data: &SecretListData) -> bool {88		self.owners == data.owners89	}90}9192#[derive(serde::Serialize, serde::Deserialize)]93struct NixDataValue {94	data: BTreeMap<String, String>,95}9697#[derive(serde::Serialize, serde::Deserialize)]98struct NixData {99	secrets: BTreeMap<String, NixDataValue>,100}101102#[derive(serde::Serialize, serde::Deserialize, Default)]103pub struct SecretDb {104	secrets: BTreeMap<String, SecretData>,105}106impl DbData for SecretDb {107	const DB_NAME: &'static str = "secrets";108}109110impl SecretDb {111	// Secrets are generated on machine running fleet command112	pub fn generate_secret(113		&mut self,114		_fleet_config: &FleetOpts,115		secret: &str,116		data: &SecretListData,117	) -> Result<()> {118		let mut rage_keys = String::new();119		for (i, _owner) in data.owners.iter().enumerate() {120			if i != 0 {121				rage_keys.push(' ');122			}123			rage_keys.push_str("--recipient \"");124			// rage_keys.push_str(&fleet_config.clone().build()?.host(owner)?.key()?);125			//rage_keys.push_str(&keys.get_host_key(&owner)?);126			rage_keys.push('"')127		}128		let created_at: PrimitiveDateTime = SystemTime::now().into();129		let renew_at = data130			.renew_in131			.map(|hours| created_at + Duration::hours(hours as i64));132		let built = tempfile::tempdir()?;133		Command::new("nix")134			.inherit_stdio()135			.arg("build")136			.arg(format!("{}.{}.generator", SECRETS_ATTRIBUTE, secret))137			.arg("--no-link")138			.arg("--out-link")139			.arg(built.path())140			.arg("--impure")141			.env("RAGE_KEYS", rage_keys)142			.env("IMPURITY_SOURCE", format!("{:?}", Instant::now()))143			.run()?;144		let path = built.path().to_owned();145		let mut secret_data = SecretData {146			created_at: created_at.into(),147			renew_at: renew_at.map(|v| v.into()),148			owners: data.owners.clone(),149			public_data: BTreeMap::new(),150			private_files: BTreeMap::new(),151		};152		for file in std::fs::read_dir(path)? {153			let entry = file?;154			if !entry.file_type()?.is_file() {155				bail!("Secret generator should produce files, not directories");156			}157			let name = entry.file_name();158			let name = name159				.to_str()160				.ok_or_else(|| anyhow::anyhow!("file name should be utf-8"))?;161			let value = String::from_utf8(std::fs::read(entry.path())?)?;162			if let Some(name) = name.strip_prefix("pub_") {163				secret_data.public_data.insert(name.into(), value);164			} else {165				secret_data.private_files.insert(name.into(), value);166			}167		}168		self.secrets.insert(secret.into(), secret_data);169		Ok(())170	}171	pub fn need_to_generate(&self, secret: &str, data: &SecretListData) -> Result<bool> {172		let secret = self.secrets.get(secret);173		if secret.is_none() {174			return Ok(true);175		}176		let secret = secret.unwrap();177178		if secret.should_renew() {179			return Ok(true);180		}181182		if !secret.is_valid(data) {183			return Ok(true);184		}185186		Ok(false)187	}188	pub fn ensure_generated(189		&mut self,190		// keys: &KeyDb,191		fleet_config: &FleetOpts,192		secret: &str,193		data: &SecretListData,194	) -> Result<()> {195		if self.need_to_generate(secret, data)? {196			info!("Generating secret {}", secret);197			self.generate_secret(fleet_config, secret, data)?;198		}199200		Ok(())201	}202	pub fn generate_nix_data(&self) -> Result<String> {203		let mut out = BTreeMap::new();204		for (host, secrets) in &self.secrets {205			out.insert(206				host.to_owned(),207				NixDataValue {208					data: secrets209						.public_data210						.clone()211						.iter()212						.map(|(k, v)| (k.to_owned(), v.trim().to_owned()))213						.collect(),214				},215			);216		}217		Ok(serde_json::to_string(&out)?)218	}219220	pub fn has_secret(&self, secret: &str) -> bool {221		self.secrets.contains_key(secret)222	}223224	pub fn remove_secret(&mut self, secret: &str) {225		self.secrets.remove(secret);226	}227}
modifiedsrc/fleetdata.rsdiffbeforeafterboth
--- a/src/fleetdata.rs
+++ b/src/fleetdata.rs
@@ -1,16 +1,30 @@
+use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 
 #[derive(Serialize, Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
 pub struct HostData {
 	#[serde(default)]
+	#[serde(skip_serializing_if = "String::is_empty")]
 	pub encryption_key: String,
-	#[serde(default)]
-	pub encrypted_secrets: BTreeMap<String, String>,
 }
 
 #[derive(Serialize, Deserialize)]
 pub struct FleetData {
 	#[serde(default)]
 	pub hosts: BTreeMap<String, HostData>,
+	#[serde(default)]
+	#[serde(skip_serializing_if = "BTreeMap::is_empty")]
+	pub secrets: BTreeMap<String, FleetSecret>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FleetSecret {
+	pub owners: Vec<String>,
+	#[serde(default)]
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub expire_at: Option<DateTime<Utc>>,
+	pub data: BTreeMap<String, String>,
 }
modifiedsrc/keys.rsdiffbeforeafterboth
--- a/src/keys.rs
+++ b/src/keys.rs
@@ -20,14 +20,6 @@
 		let host = data.hosts.entry(host.to_string()).or_default();
 		host.encryption_key = key.trim().to_string();
 	}
-	pub fn update_secret(&self, host: &str, name: &str, value: &[u8]) {
-		let mut data = self.data_mut();
-		let host = data.hosts.entry(host.to_string()).or_default();
-		host.encrypted_secrets.insert(
-			name.to_string(),
-			format!("[ENCRYPTED:{}]", base64::encode(value)),
-		);
-	}
 
 	pub fn key(&self, host: &str) -> anyhow::Result<String> {
 		if let Some(key) = self.cached_key(host) {
@@ -35,7 +27,7 @@
 		} else {
 			warn!("Loading key for {}", host);
 			let key = self
-				.command_on("host", "cat", false)
+				.command_on(&host, "cat", false)
 				.arg("/etc/ssh/ssh_host_ed25519_key.pub")
 				.run_string()?;
 			self.update_key(host, key.clone());
modifiedsrc/main.rsdiffbeforeafterboth
--- a/src/main.rs
+++ b/src/main.rs
@@ -5,7 +5,6 @@
 pub mod keys;
 
 pub mod cmds;
-pub mod db;
 pub mod nix;
 
 mod fleetdata;
@@ -13,14 +12,12 @@
 use anyhow::Result;
 use clap::Clap;
 
-use cmds::{build_systems::BuildSystems, generate_secrets::GenerateSecrets, secrets::Secrets};
+use cmds::{build_systems::BuildSystems, secrets::Secrets};
 use host::{Config, FleetOpts};
 
 #[derive(Clap)]
 #[clap(version = "1.0", author = "CertainLach <iam@lach.pw>")]
 enum Opts {
-	/// Force generation of missing secrets
-	GenerateSecrets(GenerateSecrets),
 	/// Prepare systems for deployments
 	BuildSystems(BuildSystems),
 	/// Secret management
@@ -38,7 +35,6 @@
 fn run_command(config: &Config, command: Opts) -> Result<()> {
 	match command {
 		Opts::BuildSystems(c) => c.run(config)?,
-		Opts::GenerateSecrets(c) => c.run()?,
 		Opts::Secrets(s) => s.run(config)?,
 	};
 	Ok(())