git.delta.rocks / jrsonnet / refs/commits / 97d9be65842c

650 "subtle",650 "subtle",
651]651]
652
653[[package]]
654name = "dirs"
655version = "5.0.1"
656source = "registry+https://github.com/rust-lang/crates.io-index"
657checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
658dependencies = [
659 "dirs-sys",
660]
661
662[[package]]
663name = "dirs-sys"
664version = "0.4.1"
665source = "registry+https://github.com/rust-lang/crates.io-index"
666checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
667dependencies = [
668 "libc",
669 "option-ext",
670 "redox_users",
671 "windows-sys 0.48.0",
672]
652673
653[[package]]674[[package]]
654name = "displaydoc"675name = "displaydoc"
731 "clap",752 "clap",
732 "futures",753 "futures",
733 "hostname",754 "hostname",
755 "human-repr",
734 "indicatif",756 "indicatif",
735 "itertools",757 "itertools",
736 "nixlike",758 "nixlike",
737 "once_cell",759 "once_cell",
760 "openssh",
738 "owo-colors",761 "owo-colors",
739 "peg",762 "peg",
740 "r2d2",763 "r2d2",
764 "regex",
741 "serde",765 "serde",
742 "serde_json",766 "serde_json",
743 "shlex",767 "shlex",
1018 "winapi",1042 "winapi",
1019]1043]
1044
1045[[package]]
1046name = "human-repr"
1047version = "1.1.0"
1048source = "registry+https://github.com/rust-lang/crates.io-index"
1049checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1"
10201050
1021[[package]]1051[[package]]
1022name = "humantime"1052name = "humantime"
1256 "libc",1286 "libc",
1257]1287]
1288
1289[[package]]
1290name = "libredox"
1291version = "0.0.1"
1292source = "registry+https://github.com/rust-lang/crates.io-index"
1293checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
1294dependencies = [
1295 "bitflags 2.4.1",
1296 "libc",
1297 "redox_syscall 0.4.1",
1298]
12581299
1259[[package]]1300[[package]]
1260name = "linked-hash-map"1301name = "linked-hash-map"
1479source = "registry+https://github.com/rust-lang/crates.io-index"1520source = "registry+https://github.com/rust-lang/crates.io-index"
1480checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"1521checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
1522
1523[[package]]
1524name = "openssh"
1525version = "0.10.1"
1526source = "registry+https://github.com/rust-lang/crates.io-index"
1527checksum = "3dfe68c42d6ee6bd9de175b7a5d9bb86aa99d4e2fa7cf2f2a44e97f60b6d2759"
1528dependencies = [
1529 "dirs",
1530 "libc",
1531 "once_cell",
1532 "shell-escape",
1533 "tempfile",
1534 "thiserror",
1535 "tokio",
1536 "tokio-pipe",
1537]
1538
1539[[package]]
1540name = "option-ext"
1541version = "0.2.0"
1542source = "registry+https://github.com/rust-lang/crates.io-index"
1543checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
14811544
1482[[package]]1545[[package]]
1483name = "overload"1546name = "overload"
1803 "bitflags 1.3.2",1866 "bitflags 1.3.2",
1804]1867]
1868
1869[[package]]
1870name = "redox_users"
1871version = "0.4.4"
1872source = "registry+https://github.com/rust-lang/crates.io-index"
1873checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
1874dependencies = [
1875 "getrandom 0.2.10",
1876 "libredox",
1877 "thiserror",
1878]
18051879
1806[[package]]1880[[package]]
1807name = "regex"1881name = "regex"
1808version = "1.9.5"1882version = "1.10.2"
1809source = "registry+https://github.com/rust-lang/crates.io-index"1883source = "registry+https://github.com/rust-lang/crates.io-index"
1810checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"1884checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
1811dependencies = [1885dependencies = [
1812 "aho-corasick",1886 "aho-corasick",
1813 "memchr",1887 "memchr",
1814 "regex-automata 0.3.8",1888 "regex-automata 0.4.3",
1815 "regex-syntax 0.7.5",1889 "regex-syntax 0.8.2",
1816]1890]
18171891
1818[[package]]1892[[package]]
18261900
1827[[package]]1901[[package]]
1828name = "regex-automata"1902name = "regex-automata"
1829version = "0.3.8"1903version = "0.4.3"
1830source = "registry+https://github.com/rust-lang/crates.io-index"1904source = "registry+https://github.com/rust-lang/crates.io-index"
1831checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"1905checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
1832dependencies = [1906dependencies = [
1833 "aho-corasick",1907 "aho-corasick",
1834 "memchr",1908 "memchr",
1835 "regex-syntax 0.7.5",1909 "regex-syntax 0.8.2",
1836]1910]
18371911
1838[[package]]1912[[package]]
18431917
1844[[package]]1918[[package]]
1845name = "regex-syntax"1919name = "regex-syntax"
1846version = "0.7.5"1920version = "0.8.2"
1847source = "registry+https://github.com/rust-lang/crates.io-index"1921source = "registry+https://github.com/rust-lang/crates.io-index"
1848checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"1922checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
18491923
1850[[package]]1924[[package]]
1851name = "rnix"1925name = "rnix"
2098 "lazy_static",2172 "lazy_static",
2099]2173]
2174
2175[[package]]
2176name = "shell-escape"
2177version = "0.1.5"
2178source = "registry+https://github.com/rust-lang/crates.io-index"
2179checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
21002180
2101[[package]]2181[[package]]
2102name = "shlex"2182name = "shlex"
2378 "syn 2.0.38",2458 "syn 2.0.38",
2379]2459]
2460
2461[[package]]
2462name = "tokio-pipe"
2463version = "0.2.12"
2464source = "registry+https://github.com/rust-lang/crates.io-index"
2465checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784"
2466dependencies = [
2467 "libc",
2468 "tokio",
2469]
23802470
2381[[package]]2471[[package]]
2382name = "tokio-util"2472name = "tokio-util"

--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -11,7 +11,7 @@
 serde_json = "1.0"
 time = { version = "0.3.30", features = ["serde"] }
 tempfile = "3.8"
-once_cell = "1.18"
+once_cell = "1.18.0"
 hostname = "0.3.1"
 age-core = "0.9.0"
 peg = "0.8.2"
@@ -41,3 +41,6 @@
 r2d2 = "0.8.10"
 abort-on-drop = "0.2.2"
 unindent = "0.2.3"
+regex = "1.10.2"
+openssh = "0.10.1"
+human-repr = "1.1.0"

--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -336,15 +336,14 @@
 				if !config.is_local(&host) {
 					info!("uploading system closure");
 					{
-						let mut sign = MyCommand::new("sudo");
+						let mut sign = MyCommand::new("nix");
 						// Private key for host machine is registered in nix-sign.nix
-						sign.arg("nix")
-							.arg("store")
+						sign.arg("store")
 							.arg("sign")
 							.comparg("--key-file", "/etc/nix/private-key")
 							.arg("-r")
 							.arg(&built);
-						if let Err(e) = sign.run_nix().await {
+						if let Err(e) = sign.sudo().run_nix().await {
 							warn!("Failed to sign store paths: {e}");
 						};
 					}

--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -1,4 +1,5 @@
 use std::{
+	borrow::Cow,
 	collections::HashMap,
 	ffi::OsStr,
 	process::Stdio,
@@ -6,8 +7,12 @@
 	task::Poll,
 };
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use futures::StreamExt;
+use itertools::Either;
+use once_cell::sync::Lazy;
+use openssh::{OverSsh, Session};
+use regex::Regex;
 use serde::{de::Visitor, Deserialize};
 use tokio::{io::AsyncRead, process::Command, select};
 use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};
@@ -37,6 +42,7 @@
 	command: String,
 	args: Vec<String>,
 	env: Vec<(String, String)>,
+	ssh_session: Option<Arc<Session>>,
 }
 impl MyCommand {
 	pub fn new(cmd: impl AsRef<OsStr>) -> Self {
@@ -45,6 +51,7 @@
 			command: ostoutf8(cmd),
 			args: vec![],
 			env: vec![],
+			ssh_session: None,
 		}
 	}
 	fn into_args(self) -> Vec<String> {
@@ -90,6 +97,18 @@
 		}
 		out
 	}
+	fn into_command_new(self) -> Result<Either<Command, openssh::OwningCommand<Arc<Session>>>> {
+		Ok(if let Some(session) = self.ssh_session.clone() {
+			let cmd = self.into_command();
+			Either::Right(
+				cmd.over_ssh(session)
+					.map_err(|e| anyhow!("ssh error: {e}"))?,
+			)
+		} else {
+			let cmd = self.into_command();
+			Either::Left(cmd)
+		})
+	}
 	pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
 		let arg = arg.as_ref();
 		self.args.push(ostoutf8(arg));
@@ -116,9 +135,15 @@
 		self
 	}
 	pub fn sudo(self) -> Self {
-		let mut out = Self::new("sudo");
-		out.args(self.into_args());
-		out
+		if std::env::var_os("NO_SUDO").is_some() {
+			let mut out = Self::new("su");
+			out.arg("-c").arg(self.into_string());
+			out
+		} else {
+			let mut out = Self::new("sudo");
+			out.args(self.into_args());
+			out
+		}
 	}
 	pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {
 		let mut out = Self::new("ssh");
@@ -126,6 +151,10 @@
 		out.arg(self.into_string());
 		out
 	}
+	pub fn over_ssh(mut self, session: Arc<Session>) -> Self {
+		self.ssh_session = Some(session);
+		self
+	}
 
 	pub async fn run(self) -> Result<()> {
 		let str = self.clone().into_string();
@@ -218,6 +247,11 @@
 pub struct NixHandler {
 	spans: HashMap<u64, Span>,
 }
+fn process_message(m: &str) -> Cow<'_, str> {
+	static OSC_CLEANER: Lazy<Regex> =
+		Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());
+	OSC_CLEANER.replace_all(m, "")
+}
 impl Handler for NixHandler {
 	fn handle_line(&mut self, e: &str) {
 		if let Some(e) = e.strip_prefix("@nix ") {
@@ -303,7 +337,7 @@
 					{
 						let span = info_span!("job");
 						span.pb_start();
-						span.pb_set_message(text.trim());
+						span.pb_set_message(&process_message(text.trim()));
 						self.spans.insert(id, span);
 						info!(target: "nix", "{}", text);
 					}
@@ -383,7 +417,7 @@
 				NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {
 					if let Some(span) = self.spans.get(&id) {
 						if let LogField::String(s) = &fields[0] {
-							span.pb_set_message(s.trim());
+							span.pb_set_message(&process_message(s.trim()));
 						} else {
 							warn!("bad fields: {fields:?}");
 						}

--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -7,8 +7,9 @@
 	sync::{Arc, Mutex, MutexGuard},
 };
 
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use clap::{ArgGroup, Parser};
+use openssh::SessionBuilder;
 use tempfile::NamedTempFile;
 
 use crate::{
@@ -43,6 +44,16 @@
 pub struct ConfigHost {
 	pub name: String,
 }
+impl ConfigHost {
+	async fn open_session(&self) -> Result<openssh::Session> {
+		let mut session = SessionBuilder::default();
+
+		session
+			.connect(&self.name)
+			.await
+			.map_err(|e| anyhow!("ssh error: {e}"))
+	}
+}
 
 impl Config {
 	pub fn should_skip(&self, host: &str) -> bool {
@@ -93,21 +104,22 @@
 	}
 
 	pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
-		let names = self.fleet_field
+		let names = self
+			.fleet_field
 			.get_field_deep(["configuredHosts"])
 			.await?
 			.list_fields()
 			.await?;
-		 let mut out = vec![];
-		 for name in names {
-			out.push(ConfigHost {
-				name,
-			})
-		 }
-		 Ok(out)
+		let mut out = vec![];
+		for name in names {
+			out.push(ConfigHost { name })
+		}
+		Ok(out)
 	}
 	pub async fn system_config(&self, host: &str) -> Result<Field> {
-		self.fleet_field.get_field_deep(["configuredSystems", host, "config"]).await
+		self.fleet_field
+			.get_field_deep(["configuredSystems", host, "config"])
+			.await
 	}
 
 	pub(super) fn data(&self) -> MutexGuard<FleetData> {

--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -5,8 +5,8 @@
 pub(crate) mod host;
 pub(crate) mod keys;
 
+pub(crate) mod better_nix_eval;
 pub(crate) mod extra_args;
-pub(crate) mod better_nix_eval;
 
 mod fleetdata;
 
@@ -21,6 +21,7 @@
 use futures::stream::FuturesUnordered;
 use futures::TryStreamExt;
 use host::{Config, FleetOpts};
+use human_repr::HumanCount;
 use indicatif::{ProgressState, ProgressStyle};
 use tracing::{info, metadata::LevelFilter};
 use tracing::{info_span, Instrument};
@@ -121,9 +122,16 @@
 fn setup_logging() {
 	let indicatif_layer = IndicatifLayer::new().with_progress_style(
 		ProgressStyle::with_template(
-			"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{pos:>7}/{len:7}{elapsed}{color_end}",
+			"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",
 		)
 		.unwrap()
+		.with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {
+			let Some(len) = state.len() else {
+				return;
+			};
+			let pos = state.pos();
+			let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());
+		})
 		.with_key(
 			"color_start",
 			|state: &ProgressState, writer: &mut dyn std::fmt::Write| {

--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,44 +1,56 @@
-{ flake-utils }: {
-  fleetConfiguration = { data, nixpkgs, hosts, ... }@allConfig:
-    let
-      hostNames = nixpkgs.lib.attrNames hosts;
-      config = builtins.removeAttrs allConfig [ "nixpkgs" "data" ];
-      fleetLib = import ./fleetLib.nix {
-        inherit nixpkgs hostNames;
-      };
-    in
-    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system:
-      let
+{flake-utils}: {
+  fleetConfiguration = {
+    data,
+    nixpkgs,
+    hosts,
+    ...
+  } @ allConfig: let
+    hostNames = nixpkgs.lib.attrNames hosts;
+    config = builtins.removeAttrs allConfig ["nixpkgs" "data"];
+    fleetLib = import ./fleetLib.nix {
+      inherit nixpkgs hostNames;
+    };
+  in
+    # Top-level arg is the builder system (not the target system!)
+    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: let
+      withData = data: rec {
         root = nixpkgs.lib.evalModules {
-          modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];
+          modules = (import ../modules/fleet/_modules.nix) ++ [config data];
           specialArgs = {
             inherit nixpkgs fleetLib;
           };
         };
         failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
         rootAssertWarn =
-          if failedAssertions != [ ]
+          if failedAssertions != []
           then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
           else nixpkgs.lib.showWarnings root.config.warnings root;
         configuredHosts = rootAssertWarn.config.hosts;
         configuredSecrets = rootAssertWarn.config.secrets;
-        configuredSystems = configuredSystemsWithExtraModules [ ];
-        configuredSystemsWithExtraModules = extraModules: nixpkgs.lib.listToAttrs (
-          map
+        configuredSystems = configuredSystemsWithExtraModules [];
+        configuredSystemsWithExtraModules = extraModules:
+          nixpkgs.lib.listToAttrs (
+            map
             (
               name: {
                 inherit name;
                 value = nixpkgs.lib.nixosSystem {
                   system = configuredHosts.${name}.system;
-                  modules = configuredHosts.${name}.modules ++ extraModules ++ [
-                    ({ ... }: {
-                      nixpkgs.system = system;
-                      nixpkgs.localSystem.system = system;
-                      nixpkgs.crossSystem = if system == configuredHosts.${name}.system then null else {
-                        system = configuredHosts.${name}.system;
-                      };
-                    })
-                  ];
+                  modules =
+                    configuredHosts.${name}.modules
+                    ++ extraModules
+                    ++ [
+                      ({...}: {
+                        nixpkgs.system = system;
+                        nixpkgs.localSystem.system = system;
+                        nixpkgs.crossSystem =
+                          if system == configuredHosts.${name}.system
+                          then null
+                          else {
+                            system = configuredHosts.${name}.system;
+                          };
+                      })
+                    ];
                   specialArgs = {
                     inherit fleetLib;
                     fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
@@ -47,11 +59,7 @@
               }
             )
             (builtins.attrNames rootAssertWarn.config.hosts)
-        );
-      in
-      rec {
-        inherit configuredHosts configuredSecrets configuredSystems;
-        configUnchecked = root.config;
+          );
         buildSystems = {
           toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
             ({...}: {
@@ -66,12 +74,22 @@
           ]);
           installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
             (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
-            ({ lib, ... }: {
+            ({lib, ...}: {
               buildTarget = "installation-cd";
               # Needed for https://github.com/NixOS/nixpkgs/issues/58959
-              boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ];
+              boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
             })
           ]);
         };
-      });
+        configUnchecked = root.config;
+      };
+      defaultData = withData data;
+    in rec {
+      inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+      injectData = data: let
+        injectedData = withData data;
+      in {
+        inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+      };
+    });
 }