difftreelog
feat experimental opentofu integration
in: trunk
9 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1438,6 +1438,7 @@
"itertools",
"nixlike",
"r2d2",
+ "regex",
"serde",
"serde_json",
"thiserror",
@@ -1866,9 +1867,9 @@
[[package]]
name = "regex"
-version = "1.10.4"
+version = "1.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
dependencies = [
"aho-corasick",
"memchr",
cmds/fleet/src/cmds/mod.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/mod.rs
+++ b/cmds/fleet/src/cmds/mod.rs
@@ -2,3 +2,4 @@
pub mod complete;
pub mod info;
pub mod secrets;
+pub mod tf;
cmds/fleet/src/cmds/tf.rsdiffbeforeafterboth--- /dev/null
+++ b/cmds/fleet/src/cmds/tf.rs
@@ -0,0 +1,23 @@
+use anyhow::Result;
+use clap::Parser;
+use nix_eval::nix_go_json;
+use serde_json::Value;
+use tokio::fs::write;
+use tracing::info;
+
+use crate::host::Config;
+
+#[derive(Parser)]
+pub struct Tf;
+impl Tf {
+ pub async fn run(&self, config: &Config) -> Result<()> {
+ let system = &config.local_system;
+ let config = &config.config_field;
+ let data: Value = nix_go_json!(config.tf({ system }).config);
+ let str = serde_json::to_string_pretty(&data)?;
+
+ write("fleet.tf.json", str.as_bytes()).await?;
+
+ Ok(())
+ }
+}
cmds/fleet/src/main.rsdiffbeforeafterboth1#![recursion_limit = "512"]2#![feature(try_blocks)]34pub(crate) mod cmds;5pub(crate) mod command;6pub(crate) mod host;7pub(crate) mod keys;89pub(crate) mod extra_args;1011mod fleetdata;1213use std::{ffi::OsString, process::ExitCode};1415use anyhow::{bail, Result};16use clap::{CommandFactory, Parser};17use cmds::{18 build_systems::{BuildSystems, Deploy},19 complete::Complete,20 info::Info,21 secrets::Secret,22};23use futures::{future::LocalBoxFuture, stream::FuturesUnordered, TryStreamExt};24use host::{Config, FleetOpts};25#[cfg(feature = "indicatif")]26use human_repr::HumanCount;27#[cfg(feature = "indicatif")]28use indicatif::{ProgressState, ProgressStyle};29use tracing::{error, info, info_span, Instrument};30#[cfg(feature = "indicatif")]31use tracing_indicatif::IndicatifLayer;32use tracing_subscriber::{prelude::*, EnvFilter};3334use crate::command::MyCommand;3536#[derive(Parser)]37struct Prefetch {}38impl Prefetch {39 async fn run(&self, config: &Config) -> Result<()> {40 let mut prefetch_dir = config.directory.to_path_buf();41 prefetch_dir.push("prefetch");42 if !prefetch_dir.is_dir() {43 info!("nothing to prefetch: no prefetch directory");44 return Ok(());45 }46 let tasks = <FuturesUnordered<LocalBoxFuture<Result<()>>>>::new();47 for entry in std::fs::read_dir(&prefetch_dir)? {48 tasks.push(Box::pin(async {49 let entry = entry?;50 if !entry.metadata()?.is_file() {51 bail!("only files should exist in prefetch directory");52 }53 let span = info_span!(54 "prefetching",55 name = entry.file_name().to_string_lossy().as_ref()56 );57 let mut path = OsString::new();58 path.push("file://");59 path.push(entry.path());6061 let mut status = config.local_host().cmd("nix").await?;62 status.args(&config.nix_args);63 status.arg("store").arg("prefetch-file").arg(path);64 status.run_nix_string().instrument(span).await?;65 Ok(())66 }));67 }68 tasks.try_collect::<Vec<()>>().await?;69 Ok(())70 }71}7273#[derive(Parser)]74enum Opts {75 /// Prepare systems for deployments76 BuildSystems(BuildSystems),7778 Deploy(Deploy),79 /// Secret management80 #[clap(subcommand)]81 Secret(Secret),82 /// Upload prefetch directory to the nix store83 Prefetch(Prefetch),84 /// Config parsing85 Info(Info),86 /// Command completions87 #[clap(hide(true))]88 Complete(Complete),89}9091#[derive(Parser)]92#[clap(version, author)]93struct RootOpts {94 #[clap(flatten)]95 fleet_opts: FleetOpts,96 #[clap(subcommand)]97 command: Opts,98}99100async fn run_command(config: &Config, command: Opts) -> Result<()> {101 match command {102 Opts::BuildSystems(c) => c.run(config).await?,103 Opts::Deploy(d) => d.run(config).await?,104 Opts::Secret(s) => s.run(config).await?,105 Opts::Info(i) => i.run(config).await?,106 Opts::Prefetch(p) => p.run(config).await?,107 // TODO: actually parse commands before starting the async runtime108 Opts::Complete(c) => {109 tokio::task::spawn_blocking(move || c.run(RootOpts::command())).await?110 }111 };112 Ok(())113}114115fn setup_logging() {116 #[cfg(feature = "indicatif")]117 let indicatif_layer = {118 use std::time::Duration;119120 IndicatifLayer::new().with_progress_style(121 ProgressStyle::with_template(122 "{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",123 )124 .unwrap()125 .with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {126 let Some(len) = state.len() else {127 return;128 };129 let pos = state.pos();130 if pos > len {131 let _ = write!(writer, "{}", pos.human_count_bare());132 } else {133 let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());134 }135 })136 .with_key(137 "color_start",138 |state: &ProgressState, writer: &mut dyn std::fmt::Write| {139 let elapsed = state.elapsed();140141 if elapsed > Duration::from_secs(60) {142 // Red143 let _ = write!(writer, "\x1b[{}m", 1 + 30);144 } else if elapsed > Duration::from_secs(30) {145 // Yellow146 let _ = write!(writer, "\x1b[{}m", 3 + 30);147 }148 },149 )150 .with_key(151 "color_end",152 |state: &ProgressState, writer: &mut dyn std::fmt::Write| {153 if state.elapsed() > Duration::from_secs(30) {154 let _ = write!(writer, "\x1b[0m");155 }156 },157 ),158 )159 };160161 let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));162163 let reg = tracing_subscriber::registry().with({164 let sub = tracing_subscriber::fmt::layer()165 .without_time()166 .with_target(false);167 #[cfg(feature = "indicatif")]168 let sub = sub.with_writer(indicatif_layer.get_stdout_writer());169 sub.with_filter(filter) // .without,170 });171 // #[cfg(feature = "indicatif")]172 #[cfg(feature = "indicatif")]173 let reg = reg.with(indicatif_layer);174 reg.init();175}176177fn main() -> ExitCode {178 let opts = RootOpts::parse();179 if let Opts::Complete(c) = &opts.command {180 c.run(RootOpts::command());181 return ExitCode::SUCCESS;182 }183184 setup_logging();185 async_main(opts)186}187188#[tokio::main]189async fn async_main(opts: RootOpts) -> ExitCode {190 if let Err(e) = main_real(opts).await {191 // If I remove this line, the next error!() line gets eaten.192 // This is a bug in indicatif, it needs to be fixed193 #[cfg(feature = "indicatif")]194 info!("fixme: this line gets eaten by tracing-indicatif on levels info+");195 error!("{e:#}");196 return ExitCode::FAILURE;197 }198 ExitCode::SUCCESS199}200201async fn main_real(opts: RootOpts) -> Result<()> {202 nix_eval::init_tokio();203204 let nix_args = std::env::var_os("NIX_ARGS")205 .map(|a| extra_args::parse_os(&a))206 .transpose()?207 .unwrap_or_default();208 let config = opts.fleet_opts.build(nix_args).await?;209210 match run_command(&config, opts.command).await {211 Ok(()) => {212 config.save()?;213 Ok(())214 }215 Err(e) => {216 let _ = config.save();217 Err(e)218 }219 }220}221222#[cfg(test)]223mod tests {224 use super::*;225226 #[test]227 fn verify_command() {228 use clap::CommandFactory;229 RootOpts::command().debug_assert();230 }231}1#![recursion_limit = "512"]2#![feature(try_blocks)]34pub(crate) mod cmds;5pub(crate) mod command;6pub(crate) mod host;7pub(crate) mod keys;89pub(crate) mod extra_args;1011mod fleetdata;1213use std::{ffi::OsString, process::ExitCode};1415use anyhow::{bail, Result};16use clap::{CommandFactory, Parser};17use cmds::{18 build_systems::{BuildSystems, Deploy},19 complete::Complete,20 info::Info,21 secrets::Secret,22 tf::Tf,23};24use futures::{future::LocalBoxFuture, stream::FuturesUnordered, TryStreamExt};25use host::{Config, FleetOpts};26#[cfg(feature = "indicatif")]27use human_repr::HumanCount;28#[cfg(feature = "indicatif")]29use indicatif::{ProgressState, ProgressStyle};30use tracing::{error, info, info_span, Instrument};31#[cfg(feature = "indicatif")]32use tracing_indicatif::IndicatifLayer;33use tracing_subscriber::{prelude::*, EnvFilter};3435use crate::command::MyCommand;3637#[derive(Parser)]38struct Prefetch {}39impl Prefetch {40 async fn run(&self, config: &Config) -> Result<()> {41 let mut prefetch_dir = config.directory.to_path_buf();42 prefetch_dir.push("prefetch");43 if !prefetch_dir.is_dir() {44 info!("nothing to prefetch: no prefetch directory");45 return Ok(());46 }47 let tasks = <FuturesUnordered<LocalBoxFuture<Result<()>>>>::new();48 for entry in std::fs::read_dir(&prefetch_dir)? {49 tasks.push(Box::pin(async {50 let entry = entry?;51 if !entry.metadata()?.is_file() {52 bail!("only files should exist in prefetch directory");53 }54 let span = info_span!(55 "prefetching",56 name = entry.file_name().to_string_lossy().as_ref()57 );58 let mut path = OsString::new();59 path.push("file://");60 path.push(entry.path());6162 let mut status = config.local_host().cmd("nix").await?;63 status.args(&config.nix_args);64 status.arg("store").arg("prefetch-file").arg(path);65 status.run_nix_string().instrument(span).await?;66 Ok(())67 }));68 }69 tasks.try_collect::<Vec<()>>().await?;70 Ok(())71 }72}7374#[derive(Parser)]75enum Opts {76 /// Prepare systems for deployments77 BuildSystems(BuildSystems),7879 Deploy(Deploy),80 /// Secret management81 #[clap(subcommand)]82 Secret(Secret),83 /// Upload prefetch directory to the nix store84 Prefetch(Prefetch),85 /// Config parsing86 Info(Info),87 /// Command completions88 #[clap(hide(true))]89 Complete(Complete),90 /// Compile and evaluate terranix configuration91 Tf(Tf),92}9394#[derive(Parser)]95#[clap(version, author)]96struct RootOpts {97 #[clap(flatten)]98 fleet_opts: FleetOpts,99 #[clap(subcommand)]100 command: Opts,101}102103async fn run_command(config: &Config, command: Opts) -> Result<()> {104 match command {105 Opts::BuildSystems(c) => c.run(config).await?,106 Opts::Deploy(d) => d.run(config).await?,107 Opts::Secret(s) => s.run(config).await?,108 Opts::Info(i) => i.run(config).await?,109 Opts::Prefetch(p) => p.run(config).await?,110 Opts::Tf(t) => t.run(config).await?,111 // TODO: actually parse commands before starting the async runtime112 Opts::Complete(c) => {113 tokio::task::spawn_blocking(move || c.run(RootOpts::command())).await?114 }115 };116 Ok(())117}118119fn setup_logging() {120 #[cfg(feature = "indicatif")]121 let indicatif_layer = {122 use std::time::Duration;123124 IndicatifLayer::new().with_progress_style(125 ProgressStyle::with_template(126 "{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",127 )128 .unwrap()129 .with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {130 let Some(len) = state.len() else {131 return;132 };133 let pos = state.pos();134 if pos > len {135 let _ = write!(writer, "{}", pos.human_count_bare());136 } else {137 let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());138 }139 })140 .with_key(141 "color_start",142 |state: &ProgressState, writer: &mut dyn std::fmt::Write| {143 let elapsed = state.elapsed();144145 if elapsed > Duration::from_secs(60) {146 // Red147 let _ = write!(writer, "\x1b[{}m", 1 + 30);148 } else if elapsed > Duration::from_secs(30) {149 // Yellow150 let _ = write!(writer, "\x1b[{}m", 3 + 30);151 }152 },153 )154 .with_key(155 "color_end",156 |state: &ProgressState, writer: &mut dyn std::fmt::Write| {157 if state.elapsed() > Duration::from_secs(30) {158 let _ = write!(writer, "\x1b[0m");159 }160 },161 ),162 )163 };164165 let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));166167 let reg = tracing_subscriber::registry().with({168 let sub = tracing_subscriber::fmt::layer()169 .without_time()170 .with_target(false);171 #[cfg(feature = "indicatif")]172 let sub = sub.with_writer(indicatif_layer.get_stdout_writer());173 sub.with_filter(filter) // .without,174 });175 // #[cfg(feature = "indicatif")]176 #[cfg(feature = "indicatif")]177 let reg = reg.with(indicatif_layer);178 reg.init();179}180181fn main() -> ExitCode {182 let opts = RootOpts::parse();183 if let Opts::Complete(c) = &opts.command {184 c.run(RootOpts::command());185 return ExitCode::SUCCESS;186 }187188 setup_logging();189 async_main(opts)190}191192#[tokio::main]193async fn async_main(opts: RootOpts) -> ExitCode {194 if let Err(e) = main_real(opts).await {195 // If I remove this line, the next error!() line gets eaten.196 // This is a bug in indicatif, it needs to be fixed197 #[cfg(feature = "indicatif")]198 info!("fixme: this line gets eaten by tracing-indicatif on levels info+");199 error!("{e:#}");200 return ExitCode::FAILURE;201 }202 ExitCode::SUCCESS203}204205async fn main_real(opts: RootOpts) -> Result<()> {206 nix_eval::init_tokio();207208 let nix_args = std::env::var_os("NIX_ARGS")209 .map(|a| extra_args::parse_os(&a))210 .transpose()?211 .unwrap_or_default();212 let config = opts.fleet_opts.build(nix_args).await?;213214 match run_command(&config, opts.command).await {215 Ok(()) => {216 config.save()?;217 Ok(())218 }219 Err(e) => {220 let _ = config.save();221 Err(e)222 }223 }224}225226#[cfg(test)]227mod tests {228 use super::*;229230 #[test]231 fn verify_command() {232 use clap::CommandFactory;233 RootOpts::command().debug_assert();234 }235}crates/nix-eval/Cargo.tomldiffbeforeafterboth--- a/crates/nix-eval/Cargo.toml
+++ b/crates/nix-eval/Cargo.toml
@@ -11,6 +11,7 @@
itertools = "0.13.0"
nixlike.workspace = true
r2d2 = "0.8.10"
+regex = "1.10.6"
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
thiserror = "1.0.61"
crates/nix-eval/src/session.rsdiffbeforeafterboth--- a/crates/nix-eval/src/session.rs
+++ b/crates/nix-eval/src/session.rs
@@ -12,7 +12,7 @@
sync::{mpsc, oneshot, Mutex},
};
use tokio_util::codec::{FramedRead, LinesCodec};
-use tracing::{debug, error, warn, Level};
+use tracing::{debug, error, info, warn, Level};
#[derive(Error, Debug)]
pub enum Error {
@@ -327,8 +327,16 @@
n.parse::<u64>().map_err(Error::Int)
}
async fn execute_expression_string(&mut self, expr: impl AsRef<[u8]>) -> Result<String> {
+ // builtins.toJSON escapes some thing in incorrect way, e.g escaped "$" in "\${" is being outputed as "\$",
+ // while this escape should be removed as it is intended for nix itself, not for json output.
+ //
+ // This regex only allows \$ in the beginning of the string, it is easier to implement correctly.
+ // TODO: Add peg parser for nix-produced JSON?..
+ let regex = regex::Regex::new(r#"(?<prefix>[: {,\[]\\")\\\$"#).expect("fixup json");
+
let num = self.string_wrapping.clone();
let n = self.execute_expression_wrapping(expr, &num).await?;
+ let n = regex.replace_all(&n, "$prefix$$");
let str: String = serde_json::from_str(&n)?;
Ok(str)
}
@@ -339,6 +347,7 @@
let mut fexpr = b"builtins.toJSON (".to_vec();
fexpr.extend_from_slice(expr.as_ref());
fexpr.push(b')');
+ let s = String::from_utf8_lossy(expr.as_ref());
let v = self.execute_expression_string(fexpr).await?;
Ok(serde_json::from_str(&v)?)
}
flake.nixdiffbeforeafterboth--- a/flake.nix
+++ b/flake.nix
@@ -37,6 +37,8 @@
};
flakeModule = flakeModules.default;
+ fleetModules.tf = ./modules/extras/tf.nix;
+
# To be used with https://github.com/NixOS/nix/pull/8892
schemas = let
inherit (inputs.nixpkgs.lib) mapAttrs;
lib/flakePart.nixdiffbeforeafterboth--- a/lib/flakePart.nix
+++ b/lib/flakePart.nix
@@ -2,6 +2,7 @@
fleetLib,
lib,
config,
+ inputs ? {},
...
}: let
inherit (lib.options) mkOption;
@@ -58,8 +59,11 @@
};
}
];
- specialArgs.fleetLib = import ../lib {
- inherit (bootstrapNixpkgs) lib;
+ specialArgs = {
+ fleetLib = import ../lib {
+ inherit (bootstrapNixpkgs) lib;
+ };
+ inputs = inputs;
};
};
in
modules/extras/tf.nixdiffbeforeafterboth--- /dev/null
+++ b/modules/extras/tf.nix
@@ -0,0 +1,26 @@
+{
+ config,
+ lib,
+ inputs,
+ ...
+}: let
+ inherit (lib) mkOption;
+ inherit (lib.types) deferredModule;
+in {
+ options.tf = mkOption {
+ type = deferredModule;
+ apply = module: system:
+ inputs.terranix.lib.terranixConfigurationAst {
+ inherit system;
+ pkgs = config.nixpkgs.buildUsing.legacyPackages.${system};
+ modules = [module];
+ };
+ };
+ config.tf.output.fleet = {
+ value = {
+ managed = true;
+ };
+ # Just to avoid printing this attribute on every apply.
+ sensitive = true;
+ };
+}