git.delta.rocks / jrsonnet / refs/commits / fb6d3038c4a7

difftreelog

refactor drop old db

Yaroslav Bolyukin2021-09-18parent: #4ad5065.patch.diff
in: trunk

8 files changed

deletedsrc/cmds/generate_secrets.rsdiffbeforeafterboth
--- a/src/cmds/generate_secrets.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-use std::collections::HashSet;
-
-use anyhow::Result;
-use clap::Clap;
-use log::info;
-
-use crate::{
-	db::{
-		secret::{list_secrets, SecretDb},
-		Db, DbData,
-	},
-	host::FleetOpts,
-};
-
-#[derive(Clap)]
-pub struct GenerateSecrets {
-	#[clap(flatten)]
-	fleet_opts: FleetOpts,
-
-	/// If set - remove orphaned secrets
-	#[clap(long)]
-	cleanup: bool,
-}
-
-impl GenerateSecrets {
-	pub fn run(self) -> Result<()> {
-		let db = Db::new(".fleet")?;
-		let mut secrets = SecretDb::open(&db)?;
-
-		let defined_secrets = list_secrets()?;
-		for (secret, data) in defined_secrets.iter() {
-			//let keys = KeyDb::open(&db)?;
-			secrets.ensure_generated(&self.fleet_opts, secret, data)?;
-		}
-		let key_names = defined_secrets
-			.keys()
-			.filter(|s| !secrets.has_secret(s))
-			.cloned()
-			.collect::<HashSet<_>>();
-		if !key_names.is_empty() {
-			if self.cleanup {
-				info!("Removed orphan secrets:");
-			} else {
-				info!("Orphan secrets found, run with --cleanup to remove them from db:");
-			}
-			for key in key_names {
-				info!("- {}", key);
-				if self.cleanup {
-					secrets.remove_secret(&key)
-				}
-			}
-		}
-
-		Ok(())
-	}
-}
modifiedsrc/cmds/mod.rsdiffbeforeafterboth
--- a/src/cmds/mod.rs
+++ b/src/cmds/mod.rs
@@ -1,4 +1,2 @@
 pub mod build_systems;
-// pub mod fetch_keys;
-pub mod generate_secrets;
 pub mod secrets;
deletedsrc/db/dbr.rsdiffbeforeafterboth
--- a/src/db/dbr.rs
+++ /dev/null
@@ -1,117 +0,0 @@
-//! Small .toml based readable data store
-
-use anyhow::{Context, Result};
-use serde::{de::DeserializeOwned, Serialize};
-use std::{
-	cell::Cell,
-	collections::HashSet,
-	io::Write,
-	ops::{Deref, DerefMut},
-	path::Path,
-	path::PathBuf,
-	sync::{Arc, Mutex},
-};
-
-struct DbInternal {
-	root: PathBuf,
-	locked_paths: HashSet<PathBuf>,
-}
-
-pub trait DbData: DeserializeOwned + Serialize + Default {
-	const DB_NAME: &'static str;
-
-	fn open(db: &Db) -> Result<DbFile<Self>> {
-		db.db::<Self>()
-	}
-}
-
-#[derive(Clone)]
-pub struct Db(Arc<Mutex<DbInternal>>);
-impl Db {
-	pub fn new(root: impl AsRef<Path>) -> Result<Self> {
-		let root: &Path = root.as_ref();
-		std::fs::create_dir_all(&root).context("db root")?;
-		Ok(Db(Arc::new(Mutex::new(DbInternal {
-			root: root.to_owned(),
-			locked_paths: HashSet::new(),
-		}))))
-	}
-
-	pub fn db<T: DbData>(&self) -> Result<DbFile<T>> {
-		let name = T::DB_NAME;
-		assert!(!name.contains('/') && !name.contains('\\'));
-		let mut db = self.0.lock().unwrap();
-		let mut data_path = db.root.clone();
-		data_path.push(format!("{}.toml", name));
-
-		if !db.locked_paths.insert(data_path.clone()) {
-			anyhow::bail!("file is already open");
-		}
-
-		let data = if data_path.exists() {
-			let raw_data = std::fs::read(&data_path).context("reading file")?;
-			toml::from_slice(&raw_data).context("parsing file")?
-		} else {
-			T::default()
-		};
-
-		Ok(DbFile {
-			db: self.clone(),
-			root: db.root.clone(),
-			path: data_path,
-			data,
-			dirty: Cell::new(false),
-		})
-	}
-}
-
-pub struct DbFile<T: DbData> {
-	db: Db,
-	root: PathBuf,
-	path: PathBuf,
-	data: T,
-	dirty: Cell<bool>,
-}
-
-impl<T: DbData> Deref for DbFile<T> {
-	type Target = T;
-
-	fn deref(&self) -> &Self::Target {
-		&self.data
-	}
-}
-
-impl<T: DbData> DerefMut for DbFile<T> {
-	fn deref_mut(&mut self) -> &mut Self::Target {
-		self.dirty.set(true);
-		&mut self.data
-	}
-}
-
-impl<T: DbData> DbFile<T> {
-	pub fn write(&self) -> Result<()> {
-		if !self.dirty.get() {
-			return Ok(());
-		}
-		let mut temp = tempfile::Builder::new()
-			.prefix("~")
-			.suffix(".toml")
-			.tempfile_in(&self.root)?;
-		let mut out = String::new();
-		let mut serializer = toml::Serializer::new(&mut out);
-		serializer.pretty_array(true).pretty_string(true);
-		self.data.serialize(&mut serializer)?;
-		temp.write_all(out.as_bytes())?;
-		temp.persist(&self.path)?;
-		self.dirty.set(false);
-		Ok(())
-	}
-}
-
-impl<T: DbData> Drop for DbFile<T> {
-	fn drop(&mut self) {
-		let mut db = self.db.0.lock().unwrap();
-		self.write().unwrap();
-		db.locked_paths.remove(&self.path);
-	}
-}
deletedsrc/db/mod.rsdiffbeforeafterboth
--- a/src/db/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-mod dbr;
-pub mod secret;
-
-pub use dbr::*;
deletedsrc/db/secret.rsdiffbeforeafterboth
--- a/src/db/secret.rs
+++ /dev/null
@@ -1,227 +0,0 @@
-use crate::{command::CommandExt, host::FleetOpts, nix::SECRETS_ATTRIBUTE};
-use anyhow::{bail, Context, Result};
-use log::info;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
-use std::{
-	collections::{BTreeMap, BTreeSet, HashMap},
-	process::Command,
-	time::Instant,
-	time::SystemTime,
-};
-use time::{Duration, PrimitiveDateTime};
-
-use super::DbData;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct SecretListData {
-	pub owners: BTreeSet<String>,
-	#[serde(rename = "expireIn")]
-	renew_in: Option<u64>,
-}
-pub fn list_secrets() -> Result<HashMap<String, SecretListData>> {
-	Command::new("nix")
-		.inherit_stdio()
-		.arg("eval")
-		.arg(SECRETS_ATTRIBUTE)
-		.arg("--apply")
-		.arg(
-			r#"
-				s: (builtins.mapAttrs (n: {owners, expireIn, ...}: {
-					inherit owners expireIn;
-				}) s)
-			"#,
-		)
-		.arg("--json")
-		.run_json()
-		.context("while getting secret list")
-}
-
-struct ReadableDate(PrimitiveDateTime);
-impl Serialize for ReadableDate {
-	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-	where
-		S: Serializer,
-	{
-		serializer.serialize_str(&self.0.to_string())
-	}
-}
-impl<'de> Deserialize<'de> for ReadableDate {
-	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-	where
-		D: Deserializer<'de>,
-	{
-		Ok(Self(
-			PrimitiveDateTime::parse(String::deserialize(deserializer)?, "%F %T").unwrap(),
-		))
-	}
-}
-impl From<PrimitiveDateTime> for ReadableDate {
-	fn from(d: PrimitiveDateTime) -> Self {
-		Self(d)
-	}
-}
-impl From<ReadableDate> for PrimitiveDateTime {
-	fn from(d: ReadableDate) -> Self {
-		d.0
-	}
-}
-
-#[derive(serde::Serialize, serde::Deserialize)]
-struct SecretData {
-	created_at: ReadableDate,
-	renew_at: Option<ReadableDate>,
-	owners: BTreeSet<String>,
-
-	public_data: BTreeMap<String, String>,
-	private_files: BTreeMap<String, String>,
-}
-impl SecretData {
-	fn should_renew(&self) -> bool {
-		if let Some(renew_at) = &self.renew_at {
-			let now: PrimitiveDateTime = SystemTime::now().into();
-			renew_at.0 <= now
-		} else {
-			false
-		}
-	}
-	fn is_valid(&self, data: &SecretListData) -> bool {
-		self.owners == data.owners
-	}
-}
-
-#[derive(serde::Serialize, serde::Deserialize)]
-struct NixDataValue {
-	data: BTreeMap<String, String>,
-}
-
-#[derive(serde::Serialize, serde::Deserialize)]
-struct NixData {
-	secrets: BTreeMap<String, NixDataValue>,
-}
-
-#[derive(serde::Serialize, serde::Deserialize, Default)]
-pub struct SecretDb {
-	secrets: BTreeMap<String, SecretData>,
-}
-impl DbData for SecretDb {
-	const DB_NAME: &'static str = "secrets";
-}
-
-impl SecretDb {
-	// Secrets are generated on machine running fleet command
-	pub fn generate_secret(
-		&mut self,
-		_fleet_config: &FleetOpts,
-		secret: &str,
-		data: &SecretListData,
-	) -> Result<()> {
-		let mut rage_keys = String::new();
-		for (i, _owner) in data.owners.iter().enumerate() {
-			if i != 0 {
-				rage_keys.push(' ');
-			}
-			rage_keys.push_str("--recipient \"");
-			// rage_keys.push_str(&fleet_config.clone().build()?.host(owner)?.key()?);
-			//rage_keys.push_str(&keys.get_host_key(&owner)?);
-			rage_keys.push('"')
-		}
-		let created_at: PrimitiveDateTime = SystemTime::now().into();
-		let renew_at = data
-			.renew_in
-			.map(|hours| created_at + Duration::hours(hours as i64));
-		let built = tempfile::tempdir()?;
-		Command::new("nix")
-			.inherit_stdio()
-			.arg("build")
-			.arg(format!("{}.{}.generator", SECRETS_ATTRIBUTE, secret))
-			.arg("--no-link")
-			.arg("--out-link")
-			.arg(built.path())
-			.arg("--impure")
-			.env("RAGE_KEYS", rage_keys)
-			.env("IMPURITY_SOURCE", format!("{:?}", Instant::now()))
-			.run()?;
-		let path = built.path().to_owned();
-		let mut secret_data = SecretData {
-			created_at: created_at.into(),
-			renew_at: renew_at.map(|v| v.into()),
-			owners: data.owners.clone(),
-			public_data: BTreeMap::new(),
-			private_files: BTreeMap::new(),
-		};
-		for file in std::fs::read_dir(path)? {
-			let entry = file?;
-			if !entry.file_type()?.is_file() {
-				bail!("Secret generator should produce files, not directories");
-			}
-			let name = entry.file_name();
-			let name = name
-				.to_str()
-				.ok_or_else(|| anyhow::anyhow!("file name should be utf-8"))?;
-			let value = String::from_utf8(std::fs::read(entry.path())?)?;
-			if let Some(name) = name.strip_prefix("pub_") {
-				secret_data.public_data.insert(name.into(), value);
-			} else {
-				secret_data.private_files.insert(name.into(), value);
-			}
-		}
-		self.secrets.insert(secret.into(), secret_data);
-		Ok(())
-	}
-	pub fn need_to_generate(&self, secret: &str, data: &SecretListData) -> Result<bool> {
-		let secret = self.secrets.get(secret);
-		if secret.is_none() {
-			return Ok(true);
-		}
-		let secret = secret.unwrap();
-
-		if secret.should_renew() {
-			return Ok(true);
-		}
-
-		if !secret.is_valid(data) {
-			return Ok(true);
-		}
-
-		Ok(false)
-	}
-	pub fn ensure_generated(
-		&mut self,
-		// keys: &KeyDb,
-		fleet_config: &FleetOpts,
-		secret: &str,
-		data: &SecretListData,
-	) -> Result<()> {
-		if self.need_to_generate(secret, data)? {
-			info!("Generating secret {}", secret);
-			self.generate_secret(fleet_config, secret, data)?;
-		}
-
-		Ok(())
-	}
-	pub fn generate_nix_data(&self) -> Result<String> {
-		let mut out = BTreeMap::new();
-		for (host, secrets) in &self.secrets {
-			out.insert(
-				host.to_owned(),
-				NixDataValue {
-					data: secrets
-						.public_data
-						.clone()
-						.iter()
-						.map(|(k, v)| (k.to_owned(), v.trim().to_owned()))
-						.collect(),
-				},
-			);
-		}
-		Ok(serde_json::to_string(&out)?)
-	}
-
-	pub fn has_secret(&self, secret: &str) -> bool {
-		self.secrets.contains_key(secret)
-	}
-
-	pub fn remove_secret(&mut self, secret: &str) {
-		self.secrets.remove(secret);
-	}
-}
modifiedsrc/fleetdata.rsdiffbeforeafterboth
--- a/src/fleetdata.rs
+++ b/src/fleetdata.rs
@@ -1,16 +1,30 @@
+use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 
 #[derive(Serialize, Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
 pub struct HostData {
 	#[serde(default)]
+	#[serde(skip_serializing_if = "String::is_empty")]
 	pub encryption_key: String,
-	#[serde(default)]
-	pub encrypted_secrets: BTreeMap<String, String>,
 }
 
 #[derive(Serialize, Deserialize)]
 pub struct FleetData {
 	#[serde(default)]
 	pub hosts: BTreeMap<String, HostData>,
+	#[serde(default)]
+	#[serde(skip_serializing_if = "BTreeMap::is_empty")]
+	pub secrets: BTreeMap<String, FleetSecret>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FleetSecret {
+	pub owners: Vec<String>,
+	#[serde(default)]
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub expire_at: Option<DateTime<Utc>>,
+	pub data: BTreeMap<String, String>,
 }
modifiedsrc/keys.rsdiffbeforeafterboth
--- a/src/keys.rs
+++ b/src/keys.rs
@@ -20,14 +20,6 @@
 		let host = data.hosts.entry(host.to_string()).or_default();
 		host.encryption_key = key.trim().to_string();
 	}
-	pub fn update_secret(&self, host: &str, name: &str, value: &[u8]) {
-		let mut data = self.data_mut();
-		let host = data.hosts.entry(host.to_string()).or_default();
-		host.encrypted_secrets.insert(
-			name.to_string(),
-			format!("[ENCRYPTED:{}]", base64::encode(value)),
-		);
-	}
 
 	pub fn key(&self, host: &str) -> anyhow::Result<String> {
 		if let Some(key) = self.cached_key(host) {
@@ -35,7 +27,7 @@
 		} else {
 			warn!("Loading key for {}", host);
 			let key = self
-				.command_on("host", "cat", false)
+				.command_on(&host, "cat", false)
 				.arg("/etc/ssh/ssh_host_ed25519_key.pub")
 				.run_string()?;
 			self.update_key(host, key.clone());
modifiedsrc/main.rsdiffbeforeafterboth
5pub mod keys;5pub mod keys;
66
7pub mod cmds;7pub mod cmds;
8pub mod db;
9pub mod nix;8pub mod nix;
109
11mod fleetdata;10mod fleetdata;
1211
13use anyhow::Result;12use anyhow::Result;
14use clap::Clap;13use clap::Clap;
1514
16use cmds::{build_systems::BuildSystems, generate_secrets::GenerateSecrets, secrets::Secrets};15use cmds::{build_systems::BuildSystems, secrets::Secrets};
17use host::{Config, FleetOpts};16use host::{Config, FleetOpts};
1817
19#[derive(Clap)]18#[derive(Clap)]
20#[clap(version = "1.0", author = "CertainLach <iam@lach.pw>")]19#[clap(version = "1.0", author = "CertainLach <iam@lach.pw>")]
21enum Opts {20enum Opts {
22 /// Force generation of missing secrets
23 GenerateSecrets(GenerateSecrets),
24 /// Prepare systems for deployments21 /// Prepare systems for deployments
25 BuildSystems(BuildSystems),22 BuildSystems(BuildSystems),
26 /// Secret management23 /// Secret management
38fn run_command(config: &Config, command: Opts) -> Result<()> {35fn run_command(config: &Config, command: Opts) -> Result<()> {
39 match command {36 match command {
40 Opts::BuildSystems(c) => c.run(config)?,37 Opts::BuildSystems(c) => c.run(config)?,
41 Opts::GenerateSecrets(c) => c.run()?,
42 Opts::Secrets(s) => s.run(config)?,38 Opts::Secrets(s) => s.run(config)?,
43 };39 };
44 Ok(())40 Ok(())