git.delta.rocks / jrsonnet / refs/commits / 213ad7de4b85

difftreelog

feat experimental opentofu integration

Yaroslav Bolyukin2024-08-18parent: #505f82e.patch.diff
in: trunk

9 files changed

modifiedCargo.lockdiffbeforeafterboth
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1438,6 +1438,7 @@
  "itertools",
  "nixlike",
  "r2d2",
+ "regex",
  "serde",
  "serde_json",
  "thiserror",
@@ -1866,9 +1867,9 @@
 
 [[package]]
 name = "regex"
-version = "1.10.4"
+version = "1.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
 dependencies = [
  "aho-corasick",
  "memchr",
modifiedcmds/fleet/src/cmds/mod.rsdiffbeforeafterboth
--- a/cmds/fleet/src/cmds/mod.rs
+++ b/cmds/fleet/src/cmds/mod.rs
@@ -2,3 +2,4 @@
 pub mod complete;
 pub mod info;
 pub mod secrets;
+pub mod tf;
addedcmds/fleet/src/cmds/tf.rsdiffbeforeafterboth
--- /dev/null
+++ b/cmds/fleet/src/cmds/tf.rs
@@ -0,0 +1,23 @@
+use anyhow::Result;
+use clap::Parser;
+use nix_eval::nix_go_json;
+use serde_json::Value;
+use tokio::fs::write;
+use tracing::info;
+
+use crate::host::Config;
+
+#[derive(Parser)]
+pub struct Tf;
+impl Tf {
+	pub async fn run(&self, config: &Config) -> Result<()> {
+		let system = &config.local_system;
+		let config = &config.config_field;
+		let data: Value = nix_go_json!(config.tf({ system }).config);
+		let str = serde_json::to_string_pretty(&data)?;
+
+		write("fleet.tf.json", str.as_bytes()).await?;
+
+		Ok(())
+	}
+}
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
before · cmds/fleet/src/main.rs
1#![recursion_limit = "512"]2#![feature(try_blocks)]34pub(crate) mod cmds;5pub(crate) mod command;6pub(crate) mod host;7pub(crate) mod keys;89pub(crate) mod extra_args;1011mod fleetdata;1213use std::{ffi::OsString, process::ExitCode};1415use anyhow::{bail, Result};16use clap::{CommandFactory, Parser};17use cmds::{18	build_systems::{BuildSystems, Deploy},19	complete::Complete,20	info::Info,21	secrets::Secret,22};23use futures::{future::LocalBoxFuture, stream::FuturesUnordered, TryStreamExt};24use host::{Config, FleetOpts};25#[cfg(feature = "indicatif")]26use human_repr::HumanCount;27#[cfg(feature = "indicatif")]28use indicatif::{ProgressState, ProgressStyle};29use tracing::{error, info, info_span, Instrument};30#[cfg(feature = "indicatif")]31use tracing_indicatif::IndicatifLayer;32use tracing_subscriber::{prelude::*, EnvFilter};3334use crate::command::MyCommand;3536#[derive(Parser)]37struct Prefetch {}38impl Prefetch {39	async fn run(&self, config: &Config) -> Result<()> {40		let mut prefetch_dir = config.directory.to_path_buf();41		prefetch_dir.push("prefetch");42		if !prefetch_dir.is_dir() {43			info!("nothing to prefetch: no prefetch directory");44			return Ok(());45		}46		let tasks = <FuturesUnordered<LocalBoxFuture<Result<()>>>>::new();47		for entry in std::fs::read_dir(&prefetch_dir)? {48			tasks.push(Box::pin(async {49				let entry = entry?;50				if !entry.metadata()?.is_file() {51					bail!("only files should exist in prefetch directory");52				}53				let span = info_span!(54					"prefetching",55					name = entry.file_name().to_string_lossy().as_ref()56				);57				let mut path = OsString::new();58				path.push("file://");59				path.push(entry.path());6061				let mut status = config.local_host().cmd("nix").await?;62				status.args(&config.nix_args);63				status.arg("store").arg("prefetch-file").arg(path);64				status.run_nix_string().instrument(span).await?;65				Ok(())66			}));67		}68		tasks.try_collect::<Vec<()>>().await?;69		Ok(())70	}71}7273#[derive(Parser)]74enum Opts {75	/// Prepare systems for deployments76	BuildSystems(BuildSystems),7778	Deploy(Deploy),79	/// Secret management80	#[clap(subcommand)]81	Secret(Secret),82	/// Upload prefetch directory to the nix store83	Prefetch(Prefetch),84	/// Config parsing85	Info(Info),86	/// Command completions87	#[clap(hide(true))]88	Complete(Complete),89}9091#[derive(Parser)]92#[clap(version, author)]93struct RootOpts {94	#[clap(flatten)]95	fleet_opts: FleetOpts,96	#[clap(subcommand)]97	command: Opts,98}99100async fn run_command(config: &Config, command: Opts) -> Result<()> {101	match command {102		Opts::BuildSystems(c) => c.run(config).await?,103		Opts::Deploy(d) => d.run(config).await?,104		Opts::Secret(s) => s.run(config).await?,105		Opts::Info(i) => i.run(config).await?,106		Opts::Prefetch(p) => p.run(config).await?,107		// TODO: actually parse commands before starting the async runtime108		Opts::Complete(c) => {109			tokio::task::spawn_blocking(move || c.run(RootOpts::command())).await?110		}111	};112	Ok(())113}114115fn setup_logging() {116	#[cfg(feature = "indicatif")]117	let indicatif_layer = {118		use std::time::Duration;119120		IndicatifLayer::new().with_progress_style(121			ProgressStyle::with_template(122				"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",123			)124				.unwrap()125				.with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {126					let Some(len) = state.len() else {127						return;128					};129					let pos = state.pos();130					if pos > len {131						let _ = write!(writer, "{}", pos.human_count_bare());132					} else {133						let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());134					}135				})136				.with_key(137					"color_start",138					|state: &ProgressState, writer: &mut dyn std::fmt::Write| {139						let elapsed = state.elapsed();140141						if elapsed > Duration::from_secs(60) {142							// Red143							let _ = write!(writer, "\x1b[{}m", 1 + 30);144						} else if elapsed > Duration::from_secs(30) {145							// Yellow146							let _ = write!(writer, "\x1b[{}m", 3 + 30);147						}148					},149				)150				.with_key(151					"color_end",152					|state: &ProgressState, writer: &mut dyn std::fmt::Write| {153						if state.elapsed() > Duration::from_secs(30) {154							let _ = write!(writer, "\x1b[0m");155						}156					},157				),158		)159	};160161	let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));162163	let reg = tracing_subscriber::registry().with({164		let sub = tracing_subscriber::fmt::layer()165			.without_time()166			.with_target(false);167		#[cfg(feature = "indicatif")]168		let sub = sub.with_writer(indicatif_layer.get_stdout_writer());169		sub.with_filter(filter) // .without,170	});171	// #[cfg(feature = "indicatif")]172	#[cfg(feature = "indicatif")]173	let reg = reg.with(indicatif_layer);174	reg.init();175}176177fn main() -> ExitCode {178	let opts = RootOpts::parse();179	if let Opts::Complete(c) = &opts.command {180		c.run(RootOpts::command());181		return ExitCode::SUCCESS;182	}183184	setup_logging();185	async_main(opts)186}187188#[tokio::main]189async fn async_main(opts: RootOpts) -> ExitCode {190	if let Err(e) = main_real(opts).await {191		// If I remove this line, the next error!() line gets eaten.192		// This is a bug in indicatif, it needs to be fixed193		#[cfg(feature = "indicatif")]194		info!("fixme: this line gets eaten by tracing-indicatif on levels info+");195		error!("{e:#}");196		return ExitCode::FAILURE;197	}198	ExitCode::SUCCESS199}200201async fn main_real(opts: RootOpts) -> Result<()> {202	nix_eval::init_tokio();203204	let nix_args = std::env::var_os("NIX_ARGS")205		.map(|a| extra_args::parse_os(&a))206		.transpose()?207		.unwrap_or_default();208	let config = opts.fleet_opts.build(nix_args).await?;209210	match run_command(&config, opts.command).await {211		Ok(()) => {212			config.save()?;213			Ok(())214		}215		Err(e) => {216			let _ = config.save();217			Err(e)218		}219	}220}221222#[cfg(test)]223mod tests {224	use super::*;225226	#[test]227	fn verify_command() {228		use clap::CommandFactory;229		RootOpts::command().debug_assert();230	}231}
after · cmds/fleet/src/main.rs
1#![recursion_limit = "512"]2#![feature(try_blocks)]34pub(crate) mod cmds;5pub(crate) mod command;6pub(crate) mod host;7pub(crate) mod keys;89pub(crate) mod extra_args;1011mod fleetdata;1213use std::{ffi::OsString, process::ExitCode};1415use anyhow::{bail, Result};16use clap::{CommandFactory, Parser};17use cmds::{18	build_systems::{BuildSystems, Deploy},19	complete::Complete,20	info::Info,21	secrets::Secret,22	tf::Tf,23};24use futures::{future::LocalBoxFuture, stream::FuturesUnordered, TryStreamExt};25use host::{Config, FleetOpts};26#[cfg(feature = "indicatif")]27use human_repr::HumanCount;28#[cfg(feature = "indicatif")]29use indicatif::{ProgressState, ProgressStyle};30use tracing::{error, info, info_span, Instrument};31#[cfg(feature = "indicatif")]32use tracing_indicatif::IndicatifLayer;33use tracing_subscriber::{prelude::*, EnvFilter};3435use crate::command::MyCommand;3637#[derive(Parser)]38struct Prefetch {}39impl Prefetch {40	async fn run(&self, config: &Config) -> Result<()> {41		let mut prefetch_dir = config.directory.to_path_buf();42		prefetch_dir.push("prefetch");43		if !prefetch_dir.is_dir() {44			info!("nothing to prefetch: no prefetch directory");45			return Ok(());46		}47		let tasks = <FuturesUnordered<LocalBoxFuture<Result<()>>>>::new();48		for entry in std::fs::read_dir(&prefetch_dir)? {49			tasks.push(Box::pin(async {50				let entry = entry?;51				if !entry.metadata()?.is_file() {52					bail!("only files should exist in prefetch directory");53				}54				let span = info_span!(55					"prefetching",56					name = entry.file_name().to_string_lossy().as_ref()57				);58				let mut path = OsString::new();59				path.push("file://");60				path.push(entry.path());6162				let mut status = config.local_host().cmd("nix").await?;63				status.args(&config.nix_args);64				status.arg("store").arg("prefetch-file").arg(path);65				status.run_nix_string().instrument(span).await?;66				Ok(())67			}));68		}69		tasks.try_collect::<Vec<()>>().await?;70		Ok(())71	}72}7374#[derive(Parser)]75enum Opts {76	/// Prepare systems for deployments77	BuildSystems(BuildSystems),7879	Deploy(Deploy),80	/// Secret management81	#[clap(subcommand)]82	Secret(Secret),83	/// Upload prefetch directory to the nix store84	Prefetch(Prefetch),85	/// Config parsing86	Info(Info),87	/// Command completions88	#[clap(hide(true))]89	Complete(Complete),90	/// Compile and evaluate terranix configuration91	Tf(Tf),92}9394#[derive(Parser)]95#[clap(version, author)]96struct RootOpts {97	#[clap(flatten)]98	fleet_opts: FleetOpts,99	#[clap(subcommand)]100	command: Opts,101}102103async fn run_command(config: &Config, command: Opts) -> Result<()> {104	match command {105		Opts::BuildSystems(c) => c.run(config).await?,106		Opts::Deploy(d) => d.run(config).await?,107		Opts::Secret(s) => s.run(config).await?,108		Opts::Info(i) => i.run(config).await?,109		Opts::Prefetch(p) => p.run(config).await?,110		Opts::Tf(t) => t.run(config).await?,111		// TODO: actually parse commands before starting the async runtime112		Opts::Complete(c) => {113			tokio::task::spawn_blocking(move || c.run(RootOpts::command())).await?114		}115	};116	Ok(())117}118119fn setup_logging() {120	#[cfg(feature = "indicatif")]121	let indicatif_layer = {122		use std::time::Duration;123124		IndicatifLayer::new().with_progress_style(125			ProgressStyle::with_template(126				"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",127			)128				.unwrap()129				.with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {130					let Some(len) = state.len() else {131						return;132					};133					let pos = state.pos();134					if pos > len {135						let _ = write!(writer, "{}", pos.human_count_bare());136					} else {137						let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());138					}139				})140				.with_key(141					"color_start",142					|state: &ProgressState, writer: &mut dyn std::fmt::Write| {143						let elapsed = state.elapsed();144145						if elapsed > Duration::from_secs(60) {146							// Red147							let _ = write!(writer, "\x1b[{}m", 1 + 30);148						} else if elapsed > Duration::from_secs(30) {149							// Yellow150							let _ = write!(writer, "\x1b[{}m", 3 + 30);151						}152					},153				)154				.with_key(155					"color_end",156					|state: &ProgressState, writer: &mut dyn std::fmt::Write| {157						if state.elapsed() > Duration::from_secs(30) {158							let _ = write!(writer, "\x1b[0m");159						}160					},161				),162		)163	};164165	let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));166167	let reg = tracing_subscriber::registry().with({168		let sub = tracing_subscriber::fmt::layer()169			.without_time()170			.with_target(false);171		#[cfg(feature = "indicatif")]172		let sub = sub.with_writer(indicatif_layer.get_stdout_writer());173		sub.with_filter(filter) // .without,174	});175	// #[cfg(feature = "indicatif")]176	#[cfg(feature = "indicatif")]177	let reg = reg.with(indicatif_layer);178	reg.init();179}180181fn main() -> ExitCode {182	let opts = RootOpts::parse();183	if let Opts::Complete(c) = &opts.command {184		c.run(RootOpts::command());185		return ExitCode::SUCCESS;186	}187188	setup_logging();189	async_main(opts)190}191192#[tokio::main]193async fn async_main(opts: RootOpts) -> ExitCode {194	if let Err(e) = main_real(opts).await {195		// If I remove this line, the next error!() line gets eaten.196		// This is a bug in indicatif, it needs to be fixed197		#[cfg(feature = "indicatif")]198		info!("fixme: this line gets eaten by tracing-indicatif on levels info+");199		error!("{e:#}");200		return ExitCode::FAILURE;201	}202	ExitCode::SUCCESS203}204205async fn main_real(opts: RootOpts) -> Result<()> {206	nix_eval::init_tokio();207208	let nix_args = std::env::var_os("NIX_ARGS")209		.map(|a| extra_args::parse_os(&a))210		.transpose()?211		.unwrap_or_default();212	let config = opts.fleet_opts.build(nix_args).await?;213214	match run_command(&config, opts.command).await {215		Ok(()) => {216			config.save()?;217			Ok(())218		}219		Err(e) => {220			let _ = config.save();221			Err(e)222		}223	}224}225226#[cfg(test)]227mod tests {228	use super::*;229230	#[test]231	fn verify_command() {232		use clap::CommandFactory;233		RootOpts::command().debug_assert();234	}235}
modifiedcrates/nix-eval/Cargo.tomldiffbeforeafterboth
--- a/crates/nix-eval/Cargo.toml
+++ b/crates/nix-eval/Cargo.toml
@@ -11,6 +11,7 @@
 itertools = "0.13.0"
 nixlike.workspace = true
 r2d2 = "0.8.10"
+regex = "1.10.6"
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 thiserror = "1.0.61"
modifiedcrates/nix-eval/src/session.rsdiffbeforeafterboth
--- a/crates/nix-eval/src/session.rs
+++ b/crates/nix-eval/src/session.rs
@@ -12,7 +12,7 @@
 	sync::{mpsc, oneshot, Mutex},
 };
 use tokio_util::codec::{FramedRead, LinesCodec};
-use tracing::{debug, error, warn, Level};
+use tracing::{debug, error, info, warn, Level};
 
 #[derive(Error, Debug)]
 pub enum Error {
@@ -327,8 +327,16 @@
 		n.parse::<u64>().map_err(Error::Int)
 	}
 	async fn execute_expression_string(&mut self, expr: impl AsRef<[u8]>) -> Result<String> {
+		// builtins.toJSON escapes some thing in incorrect way, e.g escaped "$" in "\${" is being outputed as "\$",
+		// while this escape should be removed as it is intended for nix itself, not for json output.
+		//
+		// This regex only allows \$ in the beginning of the string, it is easier to implement correctly.
+		// TODO: Add peg parser for nix-produced JSON?..
+		let regex = regex::Regex::new(r#"(?<prefix>[: {,\[]\\")\\\$"#).expect("fixup json");
+
 		let num = self.string_wrapping.clone();
 		let n = self.execute_expression_wrapping(expr, &num).await?;
+		let n = regex.replace_all(&n, "$prefix$$");
 		let str: String = serde_json::from_str(&n)?;
 		Ok(str)
 	}
@@ -339,6 +347,7 @@
 		let mut fexpr = b"builtins.toJSON (".to_vec();
 		fexpr.extend_from_slice(expr.as_ref());
 		fexpr.push(b')');
+		let s = String::from_utf8_lossy(expr.as_ref());
 		let v = self.execute_expression_string(fexpr).await?;
 		Ok(serde_json::from_str(&v)?)
 	}
modifiedflake.nixdiffbeforeafterboth
--- a/flake.nix
+++ b/flake.nix
@@ -37,6 +37,8 @@
         };
         flakeModule = flakeModules.default;
 
+        fleetModules.tf = ./modules/extras/tf.nix;
+
         # To be used with https://github.com/NixOS/nix/pull/8892
         schemas = let
           inherit (inputs.nixpkgs.lib) mapAttrs;
modifiedlib/flakePart.nixdiffbeforeafterboth
--- a/lib/flakePart.nix
+++ b/lib/flakePart.nix
@@ -2,6 +2,7 @@
   fleetLib,
   lib,
   config,
+  inputs ? {},
   ...
 }: let
   inherit (lib.options) mkOption;
@@ -58,8 +59,11 @@
                   };
                 }
               ];
-            specialArgs.fleetLib = import ../lib {
-              inherit (bootstrapNixpkgs) lib;
+            specialArgs = {
+              fleetLib = import ../lib {
+                inherit (bootstrapNixpkgs) lib;
+              };
+              inputs = inputs;
             };
           };
         in
addedmodules/extras/tf.nixdiffbeforeafterboth
--- /dev/null
+++ b/modules/extras/tf.nix
@@ -0,0 +1,26 @@
+{
+  config,
+  lib,
+  inputs,
+  ...
+}: let
+  inherit (lib) mkOption;
+  inherit (lib.types) deferredModule;
+in {
+  options.tf = mkOption {
+    type = deferredModule;
+    apply = module: system:
+      inputs.terranix.lib.terranixConfigurationAst {
+        inherit system;
+        pkgs = config.nixpkgs.buildUsing.legacyPackages.${system};
+        modules = [module];
+      };
+  };
+  config.tf.output.fleet = {
+    value = {
+      managed = true;
+    };
+    # Just to avoid printing this attribute on every apply.
+    sensitive = true;
+  };
+}