git.delta.rocks / jrsonnet / refs/commits / 97d9be65842c

--- a/Cargo.lock
+++ b/Cargo.lock
@@ -651,6 +651,27 @@
 ]
 
 [[package]]
+name = "dirs"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
+dependencies = [
+ "libc",
+ "option-ext",
+ "redox_users",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
 name = "displaydoc"
 version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -731,13 +752,16 @@
  "clap",
  "futures",
  "hostname",
+ "human-repr",
  "indicatif",
  "itertools",
  "nixlike",
  "once_cell",
+ "openssh",
  "owo-colors",
  "peg",
  "r2d2",
+ "regex",
  "serde",
  "serde_json",
  "shlex",
@@ -1019,6 +1043,12 @@
 ]
 
 [[package]]
+name = "human-repr"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1"
+
+[[package]]
 name = "humantime"
 version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1257,6 +1287,17 @@
 ]
 
 [[package]]
+name = "libredox"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.1",
+ "libc",
+ "redox_syscall 0.4.1",
+]
+
+[[package]]
 name = "linked-hash-map"
 version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1480,6 +1521,28 @@
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
+name = "openssh"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfe68c42d6ee6bd9de175b7a5d9bb86aa99d4e2fa7cf2f2a44e97f60b6d2759"
+dependencies = [
+ "dirs",
+ "libc",
+ "once_cell",
+ "shell-escape",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tokio-pipe",
+]
+
+[[package]]
+name = "option-ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+
+[[package]]
 name = "overload"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1804,15 +1867,26 @@
 ]
 
 [[package]]
+name = "redox_users"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
+dependencies = [
+ "getrandom 0.2.10",
+ "libredox",
+ "thiserror",
+]
+
+[[package]]
 name = "regex"
-version = "1.9.5"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.3.8",
- "regex-syntax 0.7.5",
+ "regex-automata 0.4.3",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -1826,13 +1900,13 @@
 
 [[package]]
 name = "regex-automata"
-version = "0.3.8"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.5",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -1843,9 +1917,9 @@
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.5"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
 [[package]]
 name = "rnix"
@@ -2099,6 +2173,12 @@
 ]
 
 [[package]]
+name = "shell-escape"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
+
+[[package]]
 name = "shlex"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2379,6 +2459,16 @@
 ]
 
 [[package]]
+name = "tokio-pipe"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784"
+dependencies = [
+ "libc",
+ "tokio",
+]
+
+[[package]]
 name = "tokio-util"
 version = "0.7.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"

--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -11,7 +11,7 @@
 serde_json = "1.0"
 time = { version = "0.3.30", features = ["serde"] }
 tempfile = "3.8"
-once_cell = "1.18"
+once_cell = "1.18.0"
 hostname = "0.3.1"
 age-core = "0.9.0"
 peg = "0.8.2"
@@ -41,3 +41,6 @@
 r2d2 = "0.8.10"
 abort-on-drop = "0.2.2"
 unindent = "0.2.3"
+regex = "1.10.2"
+openssh = "0.10.1"
+human-repr = "1.1.0"

--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -1,4 +1,5 @@
 use std::{
+	borrow::Cow,
 	collections::HashMap,
 	ffi::OsStr,
 	process::Stdio,
@@ -6,8 +7,12 @@
 	task::Poll,
 };
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use futures::StreamExt;
+use itertools::Either;
+use once_cell::sync::Lazy;
+use openssh::{OverSsh, Session};
+use regex::Regex;
 use serde::{de::Visitor, Deserialize};
 use tokio::{io::AsyncRead, process::Command, select};
 use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};
@@ -37,6 +42,7 @@
 	command: String,
 	args: Vec<String>,
 	env: Vec<(String, String)>,
+	ssh_session: Option<Arc<Session>>,
 }
 impl MyCommand {
 	pub fn new(cmd: impl AsRef<OsStr>) -> Self {
@@ -45,6 +51,7 @@
 			command: ostoutf8(cmd),
 			args: vec![],
 			env: vec![],
+			ssh_session: None,
 		}
 	}
 	fn into_args(self) -> Vec<String> {
@@ -90,6 +97,18 @@
 		}
 		out
 	}
+	fn into_command_new(self) -> Result<Either<Command, openssh::OwningCommand<Arc<Session>>>> {
+		Ok(if let Some(session) = self.ssh_session.clone() {
+			let cmd = self.into_command();
+			Either::Right(
+				cmd.over_ssh(session)
+					.map_err(|e| anyhow!("ssh error: {e}"))?,
+			)
+		} else {
+			let cmd = self.into_command();
+			Either::Left(cmd)
+		})
+	}
 	pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
 		let arg = arg.as_ref();
 		self.args.push(ostoutf8(arg));
@@ -116,9 +135,15 @@
 		self
 	}
 	pub fn sudo(self) -> Self {
-		let mut out = Self::new("sudo");
-		out.args(self.into_args());
-		out
+		if std::env::var_os("NO_SUDO").is_some() {
+			let mut out = Self::new("su");
+			out.arg("-c").arg(self.into_string());
+			out
+		} else {
+			let mut out = Self::new("sudo");
+			out.args(self.into_args());
+			out
+		}
 	}
 	pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {
 		let mut out = Self::new("ssh");
@@ -126,6 +151,10 @@
 		out.arg(self.into_string());
 		out
 	}
+	pub fn over_ssh(mut self, session: Arc<Session>) -> Self {
+		self.ssh_session = Some(session);
+		self
+	}
 
 	pub async fn run(self) -> Result<()> {
 		let str = self.clone().into_string();
@@ -218,6 +247,11 @@
 pub struct NixHandler {
 	spans: HashMap<u64, Span>,
 }
+fn process_message(m: &str) -> Cow<'_, str> {
+	static OSC_CLEANER: Lazy<Regex> =
+		Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());
+	OSC_CLEANER.replace_all(m, "")
+}
 impl Handler for NixHandler {
 	fn handle_line(&mut self, e: &str) {
 		if let Some(e) = e.strip_prefix("@nix ") {
@@ -303,7 +337,7 @@
 					{
 						let span = info_span!("job");
 						span.pb_start();
-						span.pb_set_message(text.trim());
+						span.pb_set_message(&process_message(text.trim()));
 						self.spans.insert(id, span);
 						info!(target: "nix", "{}", text);
 					}
@@ -383,7 +417,7 @@
 				NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {
 					if let Some(span) = self.spans.get(&id) {
 						if let LogField::String(s) = &fields[0] {
-							span.pb_set_message(s.trim());
+							span.pb_set_message(&process_message(s.trim()));
 						} else {
 							warn!("bad fields: {fields:?}");
 						}

--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -7,8 +7,9 @@
 	sync::{Arc, Mutex, MutexGuard},
 };
 
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use clap::{ArgGroup, Parser};
+use openssh::SessionBuilder;
 use tempfile::NamedTempFile;
 
 use crate::{
@@ -43,6 +44,16 @@
 pub struct ConfigHost {
 	pub name: String,
 }
+impl ConfigHost {
+	async fn open_session(&self) -> Result<openssh::Session> {
+		let mut session = SessionBuilder::default();
+
+		session
+			.connect(&self.name)
+			.await
+			.map_err(|e| anyhow!("ssh error: {e}"))
+	}
+}
 
 impl Config {
 	pub fn should_skip(&self, host: &str) -> bool {
@@ -93,21 +104,22 @@
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
-		let names = self.fleet_field
+		let names = self
+			.fleet_field
 			.get_field_deep(["configuredHosts"])
 			.await?
 			.list_fields()
 			.await?;
-		 let mut out = vec![];
-		 for name in names {
-			out.push(ConfigHost {
-				name,
-			})
-		 }
-		 Ok(out)
+		let mut out = vec![];
+		for name in names {
+			out.push(ConfigHost { name })
+		}
+		Ok(out)
 	}
 	pub async fn system_config(&self, host: &str) -> Result<Field> {
-		self.fleet_field.get_field_deep(["configuredSystems", host, "config"]).await
+		self.fleet_field
+			.get_field_deep(["configuredSystems", host, "config"])
+			.await
 	}
 
 	pub(super) fn data(&self) -> MutexGuard<FleetData> {

--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -5,8 +5,8 @@
 pub(crate) mod host;
 pub(crate) mod keys;
 
+pub(crate) mod better_nix_eval;
 pub(crate) mod extra_args;
-pub(crate) mod better_nix_eval;
 
 mod fleetdata;
 
@@ -21,6 +21,7 @@
 use futures::stream::FuturesUnordered;
 use futures::TryStreamExt;
 use host::{Config, FleetOpts};
+use human_repr::HumanCount;
 use indicatif::{ProgressState, ProgressStyle};
 use tracing::{info, metadata::LevelFilter};
 use tracing::{info_span, Instrument};
@@ -121,9 +122,16 @@
 fn setup_logging() {
 	let indicatif_layer = IndicatifLayer::new().with_progress_style(
 		ProgressStyle::with_template(
-			"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{pos:>7}/{len:7}{elapsed}{color_end}",
+			"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",
 		)
 		.unwrap()
+		.with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {
+			let Some(len) = state.len() else {
+				return;
+			};
+			let pos = state.pos();
+			let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());
+		})
 		.with_key(
 			"color_start",
 			|state: &ProgressState, writer: &mut dyn std::fmt::Write| {

--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,44 +1,56 @@
-{ flake-utils }: {
-  fleetConfiguration = { data, nixpkgs, hosts, ... }@allConfig:
-    let
-      hostNames = nixpkgs.lib.attrNames hosts;
-      config = builtins.removeAttrs allConfig [ "nixpkgs" "data" ];
-      fleetLib = import ./fleetLib.nix {
-        inherit nixpkgs hostNames;
-      };
-    in
-    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system:
-      let
+{flake-utils}: {
+  fleetConfiguration = {
+    data,
+    nixpkgs,
+    hosts,
+    ...
+  } @ allConfig: let
+    hostNames = nixpkgs.lib.attrNames hosts;
+    config = builtins.removeAttrs allConfig ["nixpkgs" "data"];
+    fleetLib = import ./fleetLib.nix {
+      inherit nixpkgs hostNames;
+    };
+  in
+    # Top-level arg is the builder system (not the target system!)
+    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: let
+      withData = data: rec {
         root = nixpkgs.lib.evalModules {
-          modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];
+          modules = (import ../modules/fleet/_modules.nix) ++ [config data];
           specialArgs = {
             inherit nixpkgs fleetLib;
           };
         };
         failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
         rootAssertWarn =
-          if failedAssertions != [ ]
+          if failedAssertions != []
           then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
           else nixpkgs.lib.showWarnings root.config.warnings root;
         configuredHosts = rootAssertWarn.config.hosts;
         configuredSecrets = rootAssertWarn.config.secrets;
-        configuredSystems = configuredSystemsWithExtraModules [ ];
-        configuredSystemsWithExtraModules = extraModules: nixpkgs.lib.listToAttrs (
-          map
+        configuredSystems = configuredSystemsWithExtraModules [];
+        configuredSystemsWithExtraModules = extraModules:
+          nixpkgs.lib.listToAttrs (
+            map
             (
               name: {
                 inherit name;
                 value = nixpkgs.lib.nixosSystem {
                   system = configuredHosts.${name}.system;
-                  modules = configuredHosts.${name}.modules ++ extraModules ++ [
-                    ({ ... }: {
-                      nixpkgs.system = system;
-                      nixpkgs.localSystem.system = system;
-                      nixpkgs.crossSystem = if system == configuredHosts.${name}.system then null else {
-                        system = configuredHosts.${name}.system;
-                      };
-                    })
-                  ];
+                  modules =
+                    configuredHosts.${name}.modules
+                    ++ extraModules
+                    ++ [
+                      ({...}: {
+                        nixpkgs.system = system;
+                        nixpkgs.localSystem.system = system;
+                        nixpkgs.crossSystem =
+                          if system == configuredHosts.${name}.system
+                          then null
+                          else {
+                            system = configuredHosts.${name}.system;
+                          };
+                      })
+                    ];
                   specialArgs = {
                     inherit fleetLib;
                     fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
@@ -47,11 +59,7 @@
               }
             )
             (builtins.attrNames rootAssertWarn.config.hosts)
-        );
-      in
-      rec {
-        inherit configuredHosts configuredSecrets configuredSystems;
-        configUnchecked = root.config;
+          );
         buildSystems = {
           toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
             ({...}: {
@@ -66,12 +74,22 @@
           ]);
           installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
             (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
-            ({ lib, ... }: {
+            ({lib, ...}: {
               buildTarget = "installation-cd";
               # Needed for https://github.com/NixOS/nixpkgs/issues/58959
-              boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ];
+              boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
             })
           ]);
         };
-      });
+        configUnchecked = root.config;
+      };
+      defaultData = withData data;
+    in rec {
+      inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+      injectData = data: let
+        injectedData = withData data;
+      in {
+        inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+      };
+    });
 }