From fb6d3038c4a7f6a08410f1f4f5466d326dfb4e79 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sat, 18 Sep 2021 18:11:49 +0000 Subject: [PATCH] refactor: drop old db --- --- a/src/cmds/generate_secrets.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::collections::HashSet; - -use anyhow::Result; -use clap::Clap; -use log::info; - -use crate::{ - db::{ - secret::{list_secrets, SecretDb}, - Db, DbData, - }, - host::FleetOpts, -}; - -#[derive(Clap)] -pub struct GenerateSecrets { - #[clap(flatten)] - fleet_opts: FleetOpts, - - /// If set - remove orphaned secrets - #[clap(long)] - cleanup: bool, -} - -impl GenerateSecrets { - pub fn run(self) -> Result<()> { - let db = Db::new(".fleet")?; - let mut secrets = SecretDb::open(&db)?; - - let defined_secrets = list_secrets()?; - for (secret, data) in defined_secrets.iter() { - //let keys = KeyDb::open(&db)?; - secrets.ensure_generated(&self.fleet_opts, secret, data)?; - } - let key_names = defined_secrets - .keys() - .filter(|s| !secrets.has_secret(s)) - .cloned() - .collect::>(); - if !key_names.is_empty() { - if self.cleanup { - info!("Removed orphan secrets:"); - } else { - info!("Orphan secrets found, run with --cleanup to remove them from db:"); - } - for key in key_names { - info!("- {}", key); - if self.cleanup { - secrets.remove_secret(&key) - } - } - } - - Ok(()) - } -} --- a/src/cmds/mod.rs +++ b/src/cmds/mod.rs @@ -1,4 +1,2 @@ pub mod build_systems; -// pub mod fetch_keys; -pub mod generate_secrets; pub mod secrets; --- a/src/db/dbr.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Small .toml based readable data store - -use anyhow::{Context, Result}; -use serde::{de::DeserializeOwned, Serialize}; -use std::{ - cell::Cell, - collections::HashSet, - io::Write, - ops::{Deref, DerefMut}, - path::Path, - path::PathBuf, - sync::{Arc, Mutex}, -}; - -struct DbInternal { - root: PathBuf, - locked_paths: HashSet, -} - -pub trait DbData: DeserializeOwned + Serialize + Default { - const DB_NAME: &'static str; - - fn open(db: &Db) -> Result> { - db.db::() - } -} - -#[derive(Clone)] -pub struct Db(Arc>); -impl Db { - pub fn new(root: impl AsRef) -> Result { - let root: &Path = root.as_ref(); - std::fs::create_dir_all(&root).context("db root")?; - Ok(Db(Arc::new(Mutex::new(DbInternal { - root: root.to_owned(), - locked_paths: HashSet::new(), - })))) - } - - pub fn db(&self) -> Result> { - let name = T::DB_NAME; - assert!(!name.contains('/') && !name.contains('\\')); - let mut db = self.0.lock().unwrap(); - let mut data_path = db.root.clone(); - data_path.push(format!("{}.toml", name)); - - if !db.locked_paths.insert(data_path.clone()) { - anyhow::bail!("file is already open"); - } - - let data = if data_path.exists() { - let raw_data = std::fs::read(&data_path).context("reading file")?; - toml::from_slice(&raw_data).context("parsing file")? - } else { - T::default() - }; - - Ok(DbFile { - db: self.clone(), - root: db.root.clone(), - path: data_path, - data, - dirty: Cell::new(false), - }) - } -} - -pub struct DbFile { - db: Db, - root: PathBuf, - path: PathBuf, - data: T, - dirty: Cell, -} - -impl Deref for DbFile { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.data - } -} - -impl DerefMut for DbFile { - fn deref_mut(&mut self) -> &mut Self::Target { - self.dirty.set(true); - &mut self.data - } -} - -impl DbFile { - pub fn write(&self) -> Result<()> { - if !self.dirty.get() { - return Ok(()); - } - let mut temp = tempfile::Builder::new() - .prefix("~") - .suffix(".toml") - .tempfile_in(&self.root)?; - let mut out = String::new(); - let mut serializer = toml::Serializer::new(&mut out); - serializer.pretty_array(true).pretty_string(true); - self.data.serialize(&mut serializer)?; - temp.write_all(out.as_bytes())?; - temp.persist(&self.path)?; - self.dirty.set(false); - Ok(()) - } -} - -impl Drop for DbFile { - fn drop(&mut self) { - let mut db = self.db.0.lock().unwrap(); - self.write().unwrap(); - db.locked_paths.remove(&self.path); - } -} --- a/src/db/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod dbr; -pub mod secret; - -pub use dbr::*; --- a/src/db/secret.rs +++ /dev/null @@ -1,227 +0,0 @@ -use crate::{command::CommandExt, host::FleetOpts, nix::SECRETS_ATTRIBUTE}; -use anyhow::{bail, Context, Result}; -use log::info; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - process::Command, - time::Instant, - time::SystemTime, -}; -use time::{Duration, PrimitiveDateTime}; - -use super::DbData; - -#[derive(Serialize, Deserialize, Debug)] -pub struct SecretListData { - pub owners: BTreeSet, - #[serde(rename = "expireIn")] - renew_in: Option, -} -pub fn list_secrets() -> Result> { - Command::new("nix") - .inherit_stdio() - .arg("eval") - .arg(SECRETS_ATTRIBUTE) - .arg("--apply") - .arg( - r#" - s: (builtins.mapAttrs (n: {owners, expireIn, ...}: { - inherit owners expireIn; - }) s) - "#, - ) - .arg("--json") - .run_json() - .context("while getting secret list") -} - -struct ReadableDate(PrimitiveDateTime); -impl Serialize for ReadableDate { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.0.to_string()) - } -} -impl<'de> Deserialize<'de> for ReadableDate { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(Self( - PrimitiveDateTime::parse(String::deserialize(deserializer)?, "%F %T").unwrap(), - )) - } -} -impl From for ReadableDate { - fn from(d: PrimitiveDateTime) -> Self { - Self(d) - } -} -impl From for PrimitiveDateTime { - fn from(d: ReadableDate) -> Self { - d.0 - } -} - -#[derive(serde::Serialize, serde::Deserialize)] -struct SecretData { - created_at: ReadableDate, - renew_at: Option, - owners: BTreeSet, - - public_data: BTreeMap, - private_files: BTreeMap, -} -impl SecretData { - fn should_renew(&self) -> bool { - if let Some(renew_at) = &self.renew_at { - let now: PrimitiveDateTime = SystemTime::now().into(); - renew_at.0 <= now - } else { - false - } - } - fn is_valid(&self, data: &SecretListData) -> bool { - self.owners == data.owners - } -} - -#[derive(serde::Serialize, serde::Deserialize)] -struct NixDataValue { - data: BTreeMap, -} - -#[derive(serde::Serialize, serde::Deserialize)] -struct NixData { - secrets: BTreeMap, -} - -#[derive(serde::Serialize, serde::Deserialize, Default)] -pub struct SecretDb { - secrets: BTreeMap, -} -impl DbData for SecretDb { - const DB_NAME: &'static str = "secrets"; -} - -impl SecretDb { - // Secrets are generated on machine running fleet command - pub fn generate_secret( - &mut self, - _fleet_config: &FleetOpts, - secret: &str, - data: &SecretListData, - ) -> Result<()> { - let mut rage_keys = String::new(); - for (i, _owner) in data.owners.iter().enumerate() { - if i != 0 { - rage_keys.push(' '); - } - rage_keys.push_str("--recipient \""); - // rage_keys.push_str(&fleet_config.clone().build()?.host(owner)?.key()?); - //rage_keys.push_str(&keys.get_host_key(&owner)?); - rage_keys.push('"') - } - let created_at: PrimitiveDateTime = SystemTime::now().into(); - let renew_at = data - .renew_in - .map(|hours| created_at + Duration::hours(hours as i64)); - let built = tempfile::tempdir()?; - Command::new("nix") - .inherit_stdio() - .arg("build") - .arg(format!("{}.{}.generator", SECRETS_ATTRIBUTE, secret)) - .arg("--no-link") - .arg("--out-link") - .arg(built.path()) - .arg("--impure") - .env("RAGE_KEYS", rage_keys) - .env("IMPURITY_SOURCE", format!("{:?}", Instant::now())) - .run()?; - let path = built.path().to_owned(); - let mut secret_data = SecretData { - created_at: created_at.into(), - renew_at: renew_at.map(|v| v.into()), - owners: data.owners.clone(), - public_data: BTreeMap::new(), - private_files: BTreeMap::new(), - }; - for file in std::fs::read_dir(path)? { - let entry = file?; - if !entry.file_type()?.is_file() { - bail!("Secret generator should produce files, not directories"); - } - let name = entry.file_name(); - let name = name - .to_str() - .ok_or_else(|| anyhow::anyhow!("file name should be utf-8"))?; - let value = String::from_utf8(std::fs::read(entry.path())?)?; - if let Some(name) = name.strip_prefix("pub_") { - secret_data.public_data.insert(name.into(), value); - } else { - secret_data.private_files.insert(name.into(), value); - } - } - self.secrets.insert(secret.into(), secret_data); - Ok(()) - } - pub fn need_to_generate(&self, secret: &str, data: &SecretListData) -> Result { - let secret = self.secrets.get(secret); - if secret.is_none() { - return Ok(true); - } - let secret = secret.unwrap(); - - if secret.should_renew() { - return Ok(true); - } - - if !secret.is_valid(data) { - return Ok(true); - } - - Ok(false) - } - pub fn ensure_generated( - &mut self, - // keys: &KeyDb, - fleet_config: &FleetOpts, - secret: &str, - data: &SecretListData, - ) -> Result<()> { - if self.need_to_generate(secret, data)? { - info!("Generating secret {}", secret); - self.generate_secret(fleet_config, secret, data)?; - } - - Ok(()) - } - pub fn generate_nix_data(&self) -> Result { - let mut out = BTreeMap::new(); - for (host, secrets) in &self.secrets { - out.insert( - host.to_owned(), - NixDataValue { - data: secrets - .public_data - .clone() - .iter() - .map(|(k, v)| (k.to_owned(), v.trim().to_owned())) - .collect(), - }, - ); - } - Ok(serde_json::to_string(&out)?) - } - - pub fn has_secret(&self, secret: &str) -> bool { - self.secrets.contains_key(secret) - } - - pub fn remove_secret(&mut self, secret: &str) { - self.secrets.remove(secret); - } -} --- a/src/fleetdata.rs +++ b/src/fleetdata.rs @@ -1,16 +1,30 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; #[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] pub struct HostData { #[serde(default)] + #[serde(skip_serializing_if = "String::is_empty")] pub encryption_key: String, - #[serde(default)] - pub encrypted_secrets: BTreeMap, } #[derive(Serialize, Deserialize)] pub struct FleetData { #[serde(default)] pub hosts: BTreeMap, + #[serde(default)] + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub secrets: BTreeMap, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FleetSecret { + pub owners: Vec, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub expire_at: Option>, + pub data: BTreeMap, } --- a/src/keys.rs +++ b/src/keys.rs @@ -20,14 +20,6 @@ let host = data.hosts.entry(host.to_string()).or_default(); host.encryption_key = key.trim().to_string(); } - pub fn update_secret(&self, host: &str, name: &str, value: &[u8]) { - let mut data = self.data_mut(); - let host = data.hosts.entry(host.to_string()).or_default(); - host.encrypted_secrets.insert( - name.to_string(), - format!("[ENCRYPTED:{}]", base64::encode(value)), - ); - } pub fn key(&self, host: &str) -> anyhow::Result { if let Some(key) = self.cached_key(host) { @@ -35,7 +27,7 @@ } else { warn!("Loading key for {}", host); let key = self - .command_on("host", "cat", false) + .command_on(&host, "cat", false) .arg("/etc/ssh/ssh_host_ed25519_key.pub") .run_string()?; self.update_key(host, key.clone()); --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,6 @@ pub mod keys; pub mod cmds; -pub mod db; pub mod nix; mod fleetdata; @@ -13,14 +12,12 @@ use anyhow::Result; use clap::Clap; -use cmds::{build_systems::BuildSystems, generate_secrets::GenerateSecrets, secrets::Secrets}; +use cmds::{build_systems::BuildSystems, secrets::Secrets}; use host::{Config, FleetOpts}; #[derive(Clap)] #[clap(version = "1.0", author = "CertainLach ")] enum Opts { - /// Force generation of missing secrets - GenerateSecrets(GenerateSecrets), /// Prepare systems for deployments BuildSystems(BuildSystems), /// Secret management @@ -38,7 +35,6 @@ fn run_command(config: &Config, command: Opts) -> Result<()> { match command { Opts::BuildSystems(c) => c.run(config)?, - Opts::GenerateSecrets(c) => c.run()?, Opts::Secrets(s) => s.run(config)?, }; Ok(()) -- gitstuff