git.delta.rocks / jrsonnet / refs/commits / f59b0688ddd0

difftreelog

feat fleet tf now executes terraform by itself

Yaroslav Bolyukin2024-09-01parent: #87c4900.patch.diff
in: trunk

3 files changed

modifiedcmds/fleet/src/cmds/tf.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/tf.rs
+++ b/cmds/fleet/src/cmds/tf.rs
@@ -1,17 +1,23 @@
 use std::{
 	collections::{BTreeMap, HashMap},
+	ffi::OsString,
 	path::PathBuf,
 };
 
-use anyhow::{bail, Context, Result};
+use anyhow::{Context, Result};
 use clap::Parser;
 use fleet_base::host::Config;
 use nix_eval::nix_go;
 use serde::Deserialize;
 use serde_json::Value;
-use tokio::{fs::copy, process::Command};
+use tempfile::NamedTempFile;
+use tokio::{
+	fs::{self, create_dir_all},
+	process::Command,
+};
+use tracing::debug;
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Debug)]
 pub struct TfData {
 	// Dummy
 	#[allow(dead_code)]
@@ -23,44 +29,56 @@
 }
 
 #[derive(Parser)]
-pub enum Tf {
-	/// Generate fleet.tf.json file for running terraform.
-	Generate,
-	/// Fetch data from terraform to fleet.
-	Refresh,
+pub struct Tf {
+	args: Vec<OsString>,
 }
 impl Tf {
 	pub async fn run(&self, config: &Config) -> Result<()> {
-		match self {
-			Tf::Generate => {
-				let system = &config.local_system;
-				let config = &config.config_field;
-				let data: HashMap<String, PathBuf> = nix_go!(config.tf({ system })).build().await?;
-				let data = &data["out"];
+		let dir = config.directory.join(".fleet/tf/default");
+		// TODO: consider postponing fleet init until this step, as it might be
+		// highly preferred to extract terraform configuration using multithreaded nix or
+		// lazy-trees nix. lazy-trees nix is very fast and perfect for this task.
+		{
+			debug!("generating terraform configs");
+			let system = &config.local_system;
+			let config = &config.config_field;
+			let data: HashMap<String, PathBuf> = nix_go!(config.tf({ system })).build().await?;
+			let data = &data["out"];
+			let data = fs::read(&data).await?;
+
+			create_dir_all(&dir).await?;
 
-				copy(data, "fleet.tf.json").await?;
-			}
-			Tf::Refresh => {
-				let cmd = Command::new("terraform").arg("refresh").status().await?;
-				if !cmd.success() {
-					bail!("terraform refresh failed")
-				}
+			let tmp = NamedTempFile::new_in(&dir)?;
+			fs::write(tmp.path(), data).await?;
+			tmp.persist(dir.join("fleet.tf.json"))?;
+		}
 
-				let data = Command::new("terraform")
-					.arg("output")
-					.arg("-json")
-					.arg("fleet")
-					.output()
-					.await?;
-				let tf_data: TfData = serde_json::from_slice(&data.stdout)
-					.context("failed to parse terraform fleet output")?;
+		{
+			debug!("running terraform command");
+			Command::new("terraform")
+				.current_dir(&dir)
+				.args(&self.args)
+				.status()
+				.await?;
+		}
+		{
+			debug!("syncing terraform data");
+			let data = Command::new("terraform")
+				.current_dir(dir)
+				.arg("output")
+				.arg("-json")
+				.arg("fleet")
+				.output()
+				.await?;
+			let tf_data: TfData = serde_json::from_slice(&data.stdout)
+				.context("failed to parse terraform fleet output")?;
 
-				let mut data = config.data();
-				data.extra.insert(
-					"terraformHosts".to_owned(),
-					serde_json::to_value(tf_data.hosts).expect("should be valid extra"),
-				);
-			}
+			let mut data = config.data();
+			debug!("synchronized done = {tf_data:?}");
+			data.extra.insert(
+				"terraformHosts".to_owned(),
+				serde_json::to_value(tf_data.hosts).expect("should be valid extra"),
+			);
 		}
 
 		Ok(())
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
before · cmds/fleet/src/main.rs
1#![recursion_limit = "512"]2#![feature(try_blocks)]34pub(crate) mod cmds;5// pub(crate) mod command;6pub(crate) mod extra_args;78use std::{ffi::OsString, process::ExitCode};910use anyhow::{bail, Result};11use clap::{CommandFactory, Parser};12use cmds::{13	build_systems::{BuildSystems, Deploy},14	complete::Complete,15	info::Info,16	secrets::Secret,17	tf::Tf,18};19use fleet_base::{host::Config, opts::FleetOpts};20use futures::{future::LocalBoxFuture, stream::FuturesUnordered, TryStreamExt};21// use host::Config;22#[cfg(feature = "indicatif")]23use human_repr::HumanCount;24#[cfg(feature = "indicatif")]25use indicatif::{ProgressState, ProgressStyle};26use tracing::{error, info, info_span, Instrument};27#[cfg(feature = "indicatif")]28use tracing_indicatif::IndicatifLayer;29use tracing_subscriber::{prelude::*, EnvFilter};3031#[derive(Parser)]32struct Prefetch {}33impl Prefetch {34	async fn run(&self, config: &Config) -> Result<()> {35		let mut prefetch_dir = config.directory.to_path_buf();36		prefetch_dir.push("prefetch");37		if !prefetch_dir.is_dir() {38			info!("nothing to prefetch: no prefetch directory");39			return Ok(());40		}41		let tasks = <FuturesUnordered<LocalBoxFuture<Result<()>>>>::new();42		for entry in std::fs::read_dir(&prefetch_dir)? {43			tasks.push(Box::pin(async {44				let entry = entry?;45				if !entry.metadata()?.is_file() {46					bail!("only files should exist in prefetch directory");47				}48				let span = info_span!(49					"prefetching",50					name = entry.file_name().to_string_lossy().as_ref()51				);52				let mut path = OsString::new();53				path.push("file://");54				path.push(entry.path());5556				let mut status = config.local_host().cmd("nix").await?;57				status.args(&config.nix_args);58				status.arg("store").arg("prefetch-file").arg(path);59				status.run_nix_string().instrument(span).await?;60				Ok(())61			}));62		}63		tasks.try_collect::<Vec<()>>().await?;64		Ok(())65	}66}6768#[derive(Parser)]69enum Opts {70	/// Prepare systems for deployments71	BuildSystems(BuildSystems),7273	Deploy(Deploy),74	/// Secret management75	#[clap(subcommand)]76	Secret(Secret),77	/// Upload prefetch directory to the nix store78	Prefetch(Prefetch),79	/// Config parsing80	Info(Info),81	/// Command completions82	#[clap(hide(true))]83	Complete(Complete),84	/// Compile and evaluate terranix configuration85	#[clap(subcommand)]86	Tf(Tf),87}8889#[derive(Parser)]90#[clap(version, author)]91struct RootOpts {92	#[clap(flatten)]93	fleet_opts: FleetOpts,94	#[clap(subcommand)]95	command: Opts,96}9798async fn run_command(config: &Config, opts: FleetOpts, command: Opts) -> Result<()> {99	match command {100		Opts::BuildSystems(c) => c.run(config, &opts).await?,101		Opts::Deploy(d) => d.run(config, &opts).await?,102		Opts::Secret(s) => s.run(config, &opts).await?,103		Opts::Info(i) => i.run(config).await?,104		Opts::Prefetch(p) => p.run(config).await?,105		Opts::Tf(t) => t.run(config).await?,106		// TODO: actually parse commands before starting the async runtime107		Opts::Complete(c) => {108			tokio::task::spawn_blocking(move || c.run(RootOpts::command())).await?109		}110	};111	Ok(())112}113114fn setup_logging() {115	#[cfg(feature = "indicatif")]116	let indicatif_layer = {117		use std::time::Duration;118119		IndicatifLayer::new().with_progress_style(120			ProgressStyle::with_template(121				"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",122			)123				.unwrap()124				.with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {125					let Some(len) = state.len() else {126						return;127					};128					let pos = state.pos();129					if pos > len {130						let _ = write!(writer, "{}", pos.human_count_bare());131					} else {132						let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());133					}134				})135				.with_key(136					"color_start",137					|state: &ProgressState, writer: &mut dyn std::fmt::Write| {138						let elapsed = state.elapsed();139140						if elapsed > Duration::from_secs(60) {141							// Red142							let _ = write!(writer, "\x1b[{}m", 1 + 30);143						} else if elapsed > Duration::from_secs(30) {144							// Yellow145							let _ = write!(writer, "\x1b[{}m", 3 + 30);146						}147					},148				)149				.with_key(150					"color_end",151					|state: &ProgressState, writer: &mut dyn std::fmt::Write| {152						if state.elapsed() > Duration::from_secs(30) {153							let _ = write!(writer, "\x1b[0m");154						}155					},156				),157		)158	};159160	let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));161162	let reg = tracing_subscriber::registry().with({163		let sub = tracing_subscriber::fmt::layer()164			.without_time()165			.with_target(false);166		#[cfg(feature = "indicatif")]167		let sub = sub.with_writer(indicatif_layer.get_stdout_writer());168		sub.with_filter(filter) // .without,169	});170	// #[cfg(feature = "indicatif")]171	#[cfg(feature = "indicatif")]172	let reg = reg.with(indicatif_layer);173	reg.init();174}175176fn main() -> ExitCode {177	let opts = RootOpts::parse();178	if let Opts::Complete(c) = &opts.command {179		c.run(RootOpts::command());180		return ExitCode::SUCCESS;181	}182183	setup_logging();184	async_main(opts)185}186187#[tokio::main]188async fn async_main(opts: RootOpts) -> ExitCode {189	if let Err(e) = main_real(opts).await {190		// If I remove this line, the next error!() line gets eaten.191		// This is a bug in indicatif, it needs to be fixed192		#[cfg(feature = "indicatif")]193		info!("fixme: this line gets eaten by tracing-indicatif on levels info+");194		error!("{e:#}");195		return ExitCode::FAILURE;196	}197	ExitCode::SUCCESS198}199200async fn main_real(opts: RootOpts) -> Result<()> {201	nix_eval::init_tokio();202203	let nix_args = std::env::var_os("NIX_ARGS")204		.map(|a| extra_args::parse_os(&a))205		.transpose()?206		.unwrap_or_default();207	let config = opts.fleet_opts.build(nix_args).await?;208209	match run_command(&config, opts.fleet_opts, opts.command).await {210		Ok(()) => {211			config.save()?;212			Ok(())213		}214		Err(e) => {215			let _ = config.save();216			Err(e)217		}218	}219}220221#[cfg(test)]222mod tests {223	use super::*;224225	#[test]226	fn verify_command() {227		use clap::CommandFactory;228		RootOpts::command().debug_assert();229	}230}
addedcmds/terraform-provider-fleet/Cargo.tomldiffbeforeafterboth
--- /dev/null
+++ b/cmds/terraform-provider-fleet/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "terraform-provider-fleet"
+edition = "2021"
+version.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+async-trait = "0.1.81"
+serde = { workspace = true, features = ["derive"] }
+tf-provider = "0.2.2"
+tokio.workspace = true