git.delta.rocks / jrsonnet / refs/commits / 97d9be65842c

difftreelog

feat libssh preparation

Yaroslav Bolyukin2023-11-17parent: #3f73827.patch.diff
in: trunk

7 files changed

modifiedCargo.lockdiffbeforeafterboth
650 "subtle",650 "subtle",
651]651]
652
653[[package]]
654name = "dirs"
655version = "5.0.1"
656source = "registry+https://github.com/rust-lang/crates.io-index"
657checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
658dependencies = [
659 "dirs-sys",
660]
661
662[[package]]
663name = "dirs-sys"
664version = "0.4.1"
665source = "registry+https://github.com/rust-lang/crates.io-index"
666checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
667dependencies = [
668 "libc",
669 "option-ext",
670 "redox_users",
671 "windows-sys 0.48.0",
672]
652673
653[[package]]674[[package]]
654name = "displaydoc"675name = "displaydoc"
731 "clap",752 "clap",
732 "futures",753 "futures",
733 "hostname",754 "hostname",
755 "human-repr",
734 "indicatif",756 "indicatif",
735 "itertools",757 "itertools",
736 "nixlike",758 "nixlike",
737 "once_cell",759 "once_cell",
760 "openssh",
738 "owo-colors",761 "owo-colors",
739 "peg",762 "peg",
740 "r2d2",763 "r2d2",
764 "regex",
741 "serde",765 "serde",
742 "serde_json",766 "serde_json",
743 "shlex",767 "shlex",
1018 "winapi",1042 "winapi",
1019]1043]
1044
1045[[package]]
1046name = "human-repr"
1047version = "1.1.0"
1048source = "registry+https://github.com/rust-lang/crates.io-index"
1049checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1"
10201050
1021[[package]]1051[[package]]
1022name = "humantime"1052name = "humantime"
1256 "libc",1286 "libc",
1257]1287]
1288
1289[[package]]
1290name = "libredox"
1291version = "0.0.1"
1292source = "registry+https://github.com/rust-lang/crates.io-index"
1293checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
1294dependencies = [
1295 "bitflags 2.4.1",
1296 "libc",
1297 "redox_syscall 0.4.1",
1298]
12581299
1259[[package]]1300[[package]]
1260name = "linked-hash-map"1301name = "linked-hash-map"
1479source = "registry+https://github.com/rust-lang/crates.io-index"1520source = "registry+https://github.com/rust-lang/crates.io-index"
1480checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"1521checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
1522
1523[[package]]
1524name = "openssh"
1525version = "0.10.1"
1526source = "registry+https://github.com/rust-lang/crates.io-index"
1527checksum = "3dfe68c42d6ee6bd9de175b7a5d9bb86aa99d4e2fa7cf2f2a44e97f60b6d2759"
1528dependencies = [
1529 "dirs",
1530 "libc",
1531 "once_cell",
1532 "shell-escape",
1533 "tempfile",
1534 "thiserror",
1535 "tokio",
1536 "tokio-pipe",
1537]
1538
1539[[package]]
1540name = "option-ext"
1541version = "0.2.0"
1542source = "registry+https://github.com/rust-lang/crates.io-index"
1543checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
14811544
1482[[package]]1545[[package]]
1483name = "overload"1546name = "overload"
1803 "bitflags 1.3.2",1866 "bitflags 1.3.2",
1804]1867]
1868
1869[[package]]
1870name = "redox_users"
1871version = "0.4.4"
1872source = "registry+https://github.com/rust-lang/crates.io-index"
1873checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
1874dependencies = [
1875 "getrandom 0.2.10",
1876 "libredox",
1877 "thiserror",
1878]
18051879
1806[[package]]1880[[package]]
1807name = "regex"1881name = "regex"
1808version = "1.9.5"1882version = "1.10.2"
1809source = "registry+https://github.com/rust-lang/crates.io-index"1883source = "registry+https://github.com/rust-lang/crates.io-index"
1810checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"1884checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
1811dependencies = [1885dependencies = [
1812 "aho-corasick",1886 "aho-corasick",
1813 "memchr",1887 "memchr",
1814 "regex-automata 0.3.8",1888 "regex-automata 0.4.3",
1815 "regex-syntax 0.7.5",1889 "regex-syntax 0.8.2",
1816]1890]
18171891
1818[[package]]1892[[package]]
18261900
1827[[package]]1901[[package]]
1828name = "regex-automata"1902name = "regex-automata"
1829version = "0.3.8"1903version = "0.4.3"
1830source = "registry+https://github.com/rust-lang/crates.io-index"1904source = "registry+https://github.com/rust-lang/crates.io-index"
1831checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"1905checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
1832dependencies = [1906dependencies = [
1833 "aho-corasick",1907 "aho-corasick",
1834 "memchr",1908 "memchr",
1835 "regex-syntax 0.7.5",1909 "regex-syntax 0.8.2",
1836]1910]
18371911
1838[[package]]1912[[package]]
18431917
1844[[package]]1918[[package]]
1845name = "regex-syntax"1919name = "regex-syntax"
1846version = "0.7.5"1920version = "0.8.2"
1847source = "registry+https://github.com/rust-lang/crates.io-index"1921source = "registry+https://github.com/rust-lang/crates.io-index"
1848checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"1922checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
18491923
1850[[package]]1924[[package]]
1851name = "rnix"1925name = "rnix"
2098 "lazy_static",2172 "lazy_static",
2099]2173]
2174
2175[[package]]
2176name = "shell-escape"
2177version = "0.1.5"
2178source = "registry+https://github.com/rust-lang/crates.io-index"
2179checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
21002180
2101[[package]]2181[[package]]
2102name = "shlex"2182name = "shlex"
2378 "syn 2.0.38",2458 "syn 2.0.38",
2379]2459]
2460
2461[[package]]
2462name = "tokio-pipe"
2463version = "0.2.12"
2464source = "registry+https://github.com/rust-lang/crates.io-index"
2465checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784"
2466dependencies = [
2467 "libc",
2468 "tokio",
2469]
23802470
2381[[package]]2471[[package]]
2382name = "tokio-util"2472name = "tokio-util"
modifiedcmds/fleet/Cargo.tomldiffbeforeafterboth
11serde_json = "1.0"11serde_json = "1.0"
12time = { version = "0.3.30", features = ["serde"] }12time = { version = "0.3.30", features = ["serde"] }
13tempfile = "3.8"13tempfile = "3.8"
14once_cell = "1.18"14once_cell = "1.18.0"
15hostname = "0.3.1"15hostname = "0.3.1"
16age-core = "0.9.0"16age-core = "0.9.0"
17peg = "0.8.2"17peg = "0.8.2"
41r2d2 = "0.8.10"41r2d2 = "0.8.10"
42abort-on-drop = "0.2.2"42abort-on-drop = "0.2.2"
43unindent = "0.2.3"43unindent = "0.2.3"
44regex = "1.10.2"
45openssh = "0.10.1"
46human-repr = "1.1.0"
4447
modifiedcmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth
336				if !config.is_local(&host) {336				if !config.is_local(&host) {
337					info!("uploading system closure");337					info!("uploading system closure");
338					{338					{
339						let mut sign = MyCommand::new("sudo");339						let mut sign = MyCommand::new("nix");
340						// Private key for host machine is registered in nix-sign.nix340						// Private key for host machine is registered in nix-sign.nix
341						sign.arg("nix")341						sign.arg("store")
342							.arg("store")
343							.arg("sign")342							.arg("sign")
344							.comparg("--key-file", "/etc/nix/private-key")343							.comparg("--key-file", "/etc/nix/private-key")
345							.arg("-r")344							.arg("-r")
346							.arg(&built);345							.arg(&built);
347						if let Err(e) = sign.run_nix().await {346						if let Err(e) = sign.sudo().run_nix().await {
348							warn!("Failed to sign store paths: {e}");347							warn!("Failed to sign store paths: {e}");
349						};348						};
350					}349					}
modifiedcmds/fleet/src/command.rsdiffbeforeafterboth
1use std::{1use std::{
2	borrow::Cow,
2	collections::HashMap,3	collections::HashMap,
3	ffi::OsStr,4	ffi::OsStr,
4	process::Stdio,5	process::Stdio,
5	sync::{Arc, Mutex},6	sync::{Arc, Mutex},
6	task::Poll,7	task::Poll,
7};8};
89
9use anyhow::Result;10use anyhow::{anyhow, Result};
10use futures::StreamExt;11use futures::StreamExt;
12use itertools::Either;
13use once_cell::sync::Lazy;
14use openssh::{OverSsh, Session};
15use regex::Regex;
11use serde::{de::Visitor, Deserialize};16use serde::{de::Visitor, Deserialize};
12use tokio::{io::AsyncRead, process::Command, select};17use tokio::{io::AsyncRead, process::Command, select};
13use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};18use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};
37	command: String,42	command: String,
38	args: Vec<String>,43	args: Vec<String>,
39	env: Vec<(String, String)>,44	env: Vec<(String, String)>,
45	ssh_session: Option<Arc<Session>>,
40}46}
41impl MyCommand {47impl MyCommand {
42	pub fn new(cmd: impl AsRef<OsStr>) -> Self {48	pub fn new(cmd: impl AsRef<OsStr>) -> Self {
45			command: ostoutf8(cmd),51			command: ostoutf8(cmd),
46			args: vec![],52			args: vec![],
47			env: vec![],53			env: vec![],
54			ssh_session: None,
48		}55		}
49	}56	}
50	fn into_args(self) -> Vec<String> {57	fn into_args(self) -> Vec<String> {
90		}97		}
91		out98		out
92	}99	}
100	fn into_command_new(self) -> Result<Either<Command, openssh::OwningCommand<Arc<Session>>>> {
101		Ok(if let Some(session) = self.ssh_session.clone() {
102			let cmd = self.into_command();
103			Either::Right(
104				cmd.over_ssh(session)
105					.map_err(|e| anyhow!("ssh error: {e}"))?,
106			)
107		} else {
108			let cmd = self.into_command();
109			Either::Left(cmd)
110		})
111	}
93	pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {112	pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
94		let arg = arg.as_ref();113		let arg = arg.as_ref();
95		self.args.push(ostoutf8(arg));114		self.args.push(ostoutf8(arg));
116		self135		self
117	}136	}
118	pub fn sudo(self) -> Self {137	pub fn sudo(self) -> Self {
138		if std::env::var_os("NO_SUDO").is_some() {
139			let mut out = Self::new("su");
140			out.arg("-c").arg(self.into_string());
141			out
142		} else {
119		let mut out = Self::new("sudo");143			let mut out = Self::new("sudo");
120		out.args(self.into_args());144			out.args(self.into_args());
121		out145			out
122	}146		}
147	}
123	pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {148	pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {
124		let mut out = Self::new("ssh");149		let mut out = Self::new("ssh");
125		out.arg(on).arg("--");150		out.arg(on).arg("--");
126		out.arg(self.into_string());151		out.arg(self.into_string());
127		out152		out
128	}153	}
154	pub fn over_ssh(mut self, session: Arc<Session>) -> Self {
155		self.ssh_session = Some(session);
156		self
157	}
129158
130	pub async fn run(self) -> Result<()> {159	pub async fn run(self) -> Result<()> {
131		let str = self.clone().into_string();160		let str = self.clone().into_string();
218pub struct NixHandler {247pub struct NixHandler {
219	spans: HashMap<u64, Span>,248	spans: HashMap<u64, Span>,
220}249}
250fn process_message(m: &str) -> Cow<'_, str> {
251	static OSC_CLEANER: Lazy<Regex> =
252		Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());
253	OSC_CLEANER.replace_all(m, "")
254}
221impl Handler for NixHandler {255impl Handler for NixHandler {
222	fn handle_line(&mut self, e: &str) {256	fn handle_line(&mut self, e: &str) {
223		if let Some(e) = e.strip_prefix("@nix ") {257		if let Some(e) = e.strip_prefix("@nix ") {
303					{337					{
304						let span = info_span!("job");338						let span = info_span!("job");
305						span.pb_start();339						span.pb_start();
306						span.pb_set_message(text.trim());340						span.pb_set_message(&process_message(text.trim()));
307						self.spans.insert(id, span);341						self.spans.insert(id, span);
308						info!(target: "nix", "{}", text);342						info!(target: "nix", "{}", text);
309					}343					}
383				NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {417				NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {
384					if let Some(span) = self.spans.get(&id) {418					if let Some(span) = self.spans.get(&id) {
385						if let LogField::String(s) = &fields[0] {419						if let LogField::String(s) = &fields[0] {
386							span.pb_set_message(s.trim());420							span.pb_set_message(&process_message(s.trim()));
387						} else {421						} else {
388							warn!("bad fields: {fields:?}");422							warn!("bad fields: {fields:?}");
389						}423						}
modifiedcmds/fleet/src/host.rsdiffbeforeafterboth
7	sync::{Arc, Mutex, MutexGuard},7	sync::{Arc, Mutex, MutexGuard},
8};8};
99
10use anyhow::{bail, Context, Result};10use anyhow::{anyhow, bail, Context, Result};
11use clap::{ArgGroup, Parser};11use clap::{ArgGroup, Parser};
12use openssh::SessionBuilder;
12use tempfile::NamedTempFile;13use tempfile::NamedTempFile;
1314
14use crate::{15use crate::{
43pub struct ConfigHost {44pub struct ConfigHost {
44	pub name: String,45	pub name: String,
45}46}
47impl ConfigHost {
48	async fn open_session(&self) -> Result<openssh::Session> {
49		let mut session = SessionBuilder::default();
50
51		session
52			.connect(&self.name)
53			.await
54			.map_err(|e| anyhow!("ssh error: {e}"))
55	}
56}
4657
47impl Config {58impl Config {
48	pub fn should_skip(&self, host: &str) -> bool {59	pub fn should_skip(&self, host: &str) -> bool {
101		 let mut out = vec![];113		let mut out = vec![];
102		 for name in names {114		for name in names {
103			out.push(ConfigHost {115			out.push(ConfigHost { name })
104				name,
105			})
106		 }116		}
107		 Ok(out)117		Ok(out)
modifiedcmds/fleet/src/main.rsdiffbeforeafterboth
5pub(crate) mod host;5pub(crate) mod host;
6pub(crate) mod keys;6pub(crate) mod keys;
77
8pub(crate) mod extra_args;8pub(crate) mod better_nix_eval;
9pub(crate) mod better_nix_eval;9pub(crate) mod extra_args;
1010
11mod fleetdata;11mod fleetdata;
1212
21use futures::stream::FuturesUnordered;21use futures::stream::FuturesUnordered;
22use futures::TryStreamExt;22use futures::TryStreamExt;
23use host::{Config, FleetOpts};23use host::{Config, FleetOpts};
24use human_repr::HumanCount;
24use indicatif::{ProgressState, ProgressStyle};25use indicatif::{ProgressState, ProgressStyle};
25use tracing::{info, metadata::LevelFilter};26use tracing::{info, metadata::LevelFilter};
26use tracing::{info_span, Instrument};27use tracing::{info_span, Instrument};
121fn setup_logging() {122fn setup_logging() {
122	let indicatif_layer = IndicatifLayer::new().with_progress_style(123	let indicatif_layer = IndicatifLayer::new().with_progress_style(
123		ProgressStyle::with_template(124		ProgressStyle::with_template(
124			"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{pos:>7}/{len:7}{elapsed}{color_end}",125			"{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",
125		)126		)
127		.unwrap()
126		.unwrap()128		.with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {
129			let Some(len) = state.len() else {
130				return;
131			};
132			let pos = state.pos();
133			let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());
134		})
127		.with_key(135		.with_key(
128			"color_start",136			"color_start",
129			|state: &ProgressState, writer: &mut dyn std::fmt::Write| {137			|state: &ProgressState, writer: &mut dyn std::fmt::Write| {
modifiedlib/default.nixdiffbeforeafterboth
7        inherit nixpkgs hostNames;11      inherit nixpkgs hostNames;
8      };12    };
9    in13  in
14    # Top-level arg is the builder system (not the target system!)
10    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system:15    nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: let
11      let16      withData = data: rec {
12        root = nixpkgs.lib.evalModules {17        root = nixpkgs.lib.evalModules {
13          modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];18          modules = (import ../modules/fleet/_modules.nix) ++ [config data];
14          specialArgs = {19          specialArgs = {
48            )60            )
49            (builtins.attrNames rootAssertWarn.config.hosts)61            (builtins.attrNames rootAssertWarn.config.hosts)
50        );62          );
63        buildSystems = {
64          toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
65            ({...}: {
66              buildTarget = "toplevel";
67            })
68          ]);
69          sdImage = builtins.mapAttrs (_name: value: value.config.system.build.sdImage) (configuredSystemsWithExtraModules [
70            #(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64-installer.nix")
71            ({...}: {
72              buildTarget = "sd-image";
73            })
74          ]);
75          installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
76            (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
77            ({lib, ...}: {
78              buildTarget = "installation-cd";
79              # Needed for https://github.com/NixOS/nixpkgs/issues/58959
80              boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
81            })
82          ]);
83        };
84        configUnchecked = root.config;
85      };
86      defaultData = withData data;
51      in87    in rec {
52      rec {
53        inherit configuredHosts configuredSecrets configuredSystems;88      inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
54        configUnchecked = root.config;
55        buildSystems = {89      injectData = data: let
56          toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
57            ({...}: {
58              buildTarget = "toplevel";
59            })
60          ]);
61          sdImage = builtins.mapAttrs (_name: value: value.config.system.build.sdImage) (configuredSystemsWithExtraModules [90        injectedData = withData data;
62            #(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64-installer.nix")91      in {
63            ({...}: {
64              buildTarget = "sd-image";
65            })
66          ]);
67          installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [92        inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
68            (nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
69            ({ lib, ... }: {
70              buildTarget = "installation-cd";
71              # Needed for https://github.com/NixOS/nixpkgs/issues/58959
72              boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ];
73            })
74          ]);
75        };93      };
76      });94    });
77}95}
7896