difftreelog
feat libssh preparation
in: trunk
7 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -651,6 +651,27 @@
]
[[package]]
+name = "dirs"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
+dependencies = [
+ "libc",
+ "option-ext",
+ "redox_users",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
name = "displaydoc"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -731,13 +752,16 @@
"clap",
"futures",
"hostname",
+ "human-repr",
"indicatif",
"itertools",
"nixlike",
"once_cell",
+ "openssh",
"owo-colors",
"peg",
"r2d2",
+ "regex",
"serde",
"serde_json",
"shlex",
@@ -1019,6 +1043,12 @@
]
[[package]]
+name = "human-repr"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1"
+
+[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1257,6 +1287,17 @@
]
[[package]]
+name = "libredox"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.1",
+ "libc",
+ "redox_syscall 0.4.1",
+]
+
+[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1480,6 +1521,28 @@
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
+name = "openssh"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfe68c42d6ee6bd9de175b7a5d9bb86aa99d4e2fa7cf2f2a44e97f60b6d2759"
+dependencies = [
+ "dirs",
+ "libc",
+ "once_cell",
+ "shell-escape",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tokio-pipe",
+]
+
+[[package]]
+name = "option-ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+
+[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1804,15 +1867,26 @@
]
[[package]]
+name = "redox_users"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
+dependencies = [
+ "getrandom 0.2.10",
+ "libredox",
+ "thiserror",
+]
+
+[[package]]
name = "regex"
-version = "1.9.5"
+version = "1.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
dependencies = [
"aho-corasick",
"memchr",
- "regex-automata 0.3.8",
- "regex-syntax 0.7.5",
+ "regex-automata 0.4.3",
+ "regex-syntax 0.8.2",
]
[[package]]
@@ -1826,13 +1900,13 @@
[[package]]
name = "regex-automata"
-version = "0.3.8"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
dependencies = [
"aho-corasick",
"memchr",
- "regex-syntax 0.7.5",
+ "regex-syntax 0.8.2",
]
[[package]]
@@ -1843,9 +1917,9 @@
[[package]]
name = "regex-syntax"
-version = "0.7.5"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
[[package]]
name = "rnix"
@@ -2099,6 +2173,12 @@
]
[[package]]
+name = "shell-escape"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
+
+[[package]]
name = "shlex"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2379,6 +2459,16 @@
]
[[package]]
+name = "tokio-pipe"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784"
+dependencies = [
+ "libc",
+ "tokio",
+]
+
+[[package]]
name = "tokio-util"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
cmds/fleet/Cargo.tomldiffbeforeafterboth--- a/cmds/fleet/Cargo.toml
+++ b/cmds/fleet/Cargo.toml
@@ -11,7 +11,7 @@
serde_json = "1.0"
time = { version = "0.3.30", features = ["serde"] }
tempfile = "3.8"
-once_cell = "1.18"
+once_cell = "1.18.0"
hostname = "0.3.1"
age-core = "0.9.0"
peg = "0.8.2"
@@ -41,3 +41,6 @@
r2d2 = "0.8.10"
abort-on-drop = "0.2.2"
unindent = "0.2.3"
+regex = "1.10.2"
+openssh = "0.10.1"
+human-repr = "1.1.0"
cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -336,15 +336,14 @@
if !config.is_local(&host) {
info!("uploading system closure");
{
- let mut sign = MyCommand::new("sudo");
+ let mut sign = MyCommand::new("nix");
// Private key for host machine is registered in nix-sign.nix
- sign.arg("nix")
- .arg("store")
+ sign.arg("store")
.arg("sign")
.comparg("--key-file", "/etc/nix/private-key")
.arg("-r")
.arg(&built);
- if let Err(e) = sign.run_nix().await {
+ if let Err(e) = sign.sudo().run_nix().await {
warn!("Failed to sign store paths: {e}");
};
}
cmds/fleet/src/command.rsdiffbeforeafterboth1use std::{1use std::{2 borrow::Cow,2 collections::HashMap,3 collections::HashMap,3 ffi::OsStr,4 ffi::OsStr,4 process::Stdio,5 process::Stdio,5 sync::{Arc, Mutex},6 sync::{Arc, Mutex},6 task::Poll,7 task::Poll,7};8};899use anyhow::Result;10use anyhow::{anyhow, Result};10use futures::StreamExt;11use futures::StreamExt;12use itertools::Either;13use once_cell::sync::Lazy;14use openssh::{OverSsh, Session};15use regex::Regex;11use serde::{de::Visitor, Deserialize};16use serde::{de::Visitor, Deserialize};12use tokio::{io::AsyncRead, process::Command, select};17use tokio::{io::AsyncRead, process::Command, select};13use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};18use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};37 command: String,42 command: String,38 args: Vec<String>,43 args: Vec<String>,39 env: Vec<(String, String)>,44 env: Vec<(String, String)>,45 ssh_session: Option<Arc<Session>>,40}46}41impl MyCommand {47impl MyCommand {42 pub fn new(cmd: impl AsRef<OsStr>) -> Self {48 pub fn new(cmd: impl AsRef<OsStr>) -> Self {45 command: ostoutf8(cmd),51 command: ostoutf8(cmd),46 args: vec![],52 args: vec![],47 env: vec![],53 env: vec![],54 ssh_session: None,48 }55 }49 }56 }50 fn into_args(self) -> Vec<String> {57 fn into_args(self) -> Vec<String> {90 }97 }91 out98 out92 }99 }100 fn into_command_new(self) -> Result<Either<Command, openssh::OwningCommand<Arc<Session>>>> {101 Ok(if let Some(session) = self.ssh_session.clone() {102 let cmd = self.into_command();103 Either::Right(104 cmd.over_ssh(session)105 .map_err(|e| anyhow!("ssh error: {e}"))?,106 )107 } else {108 let cmd = self.into_command();109 Either::Left(cmd)110 })111 }93 pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {112 pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {94 let arg = arg.as_ref();113 let arg = arg.as_ref();95 self.args.push(ostoutf8(arg));114 self.args.push(ostoutf8(arg));116 self135 self117 }136 }118 pub fn sudo(self) -> Self {137 pub fn sudo(self) -> Self {138 if std::env::var_os("NO_SUDO").is_some() {139 let mut out = Self::new("su");140 out.arg("-c").arg(self.into_string());141 out142 } else {119 let mut out = Self::new("sudo");143 let mut out = Self::new("sudo");120 out.args(self.into_args());144 out.args(self.into_args());121 out145 out122 }146 }147 }123 pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {148 pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {124 let mut out = Self::new("ssh");149 let mut out = Self::new("ssh");125 out.arg(on).arg("--");150 out.arg(on).arg("--");126 out.arg(self.into_string());151 out.arg(self.into_string());127 out152 out128 }153 }154 pub fn over_ssh(mut self, session: Arc<Session>) -> Self {155 self.ssh_session = Some(session);156 self157 }129158130 pub async fn run(self) -> Result<()> {159 pub async fn run(self) -> Result<()> {131 let str = self.clone().into_string();160 let str = self.clone().into_string();218pub struct NixHandler {247pub struct NixHandler {219 spans: HashMap<u64, Span>,248 spans: HashMap<u64, Span>,220}249}250fn process_message(m: &str) -> Cow<'_, str> {251 static OSC_CLEANER: Lazy<Regex> =252 Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());253 OSC_CLEANER.replace_all(m, "")254}221impl Handler for NixHandler {255impl Handler for NixHandler {222 fn handle_line(&mut self, e: &str) {256 fn handle_line(&mut self, e: &str) {223 if let Some(e) = e.strip_prefix("@nix ") {257 if let Some(e) = e.strip_prefix("@nix ") {303 {337 {304 let span = info_span!("job");338 let span = info_span!("job");305 span.pb_start();339 span.pb_start();306 span.pb_set_message(text.trim());340 span.pb_set_message(&process_message(text.trim()));307 self.spans.insert(id, span);341 self.spans.insert(id, span);308 info!(target: "nix", "{}", text);342 info!(target: "nix", "{}", text);309 }343 }383 NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {417 NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {384 if let Some(span) = self.spans.get(&id) {418 if let Some(span) = self.spans.get(&id) {385 if let LogField::String(s) = &fields[0] {419 if let LogField::String(s) = &fields[0] {386 span.pb_set_message(s.trim());420 span.pb_set_message(&process_message(s.trim()));387 } else {421 } else {388 warn!("bad fields: {fields:?}");422 warn!("bad fields: {fields:?}");389 }423 }cmds/fleet/src/host.rsdiffbeforeafterboth--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -7,8 +7,9 @@
sync::{Arc, Mutex, MutexGuard},
};
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
use clap::{ArgGroup, Parser};
+use openssh::SessionBuilder;
use tempfile::NamedTempFile;
use crate::{
@@ -43,6 +44,16 @@
pub struct ConfigHost {
pub name: String,
}
+impl ConfigHost {
+ async fn open_session(&self) -> Result<openssh::Session> {
+ let mut session = SessionBuilder::default();
+
+ session
+ .connect(&self.name)
+ .await
+ .map_err(|e| anyhow!("ssh error: {e}"))
+ }
+}
impl Config {
pub fn should_skip(&self, host: &str) -> bool {
@@ -93,21 +104,22 @@
}
pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
- let names = self.fleet_field
+ let names = self
+ .fleet_field
.get_field_deep(["configuredHosts"])
.await?
.list_fields()
.await?;
- let mut out = vec![];
- for name in names {
- out.push(ConfigHost {
- name,
- })
- }
- Ok(out)
+ let mut out = vec![];
+ for name in names {
+ out.push(ConfigHost { name })
+ }
+ Ok(out)
}
pub async fn system_config(&self, host: &str) -> Result<Field> {
- self.fleet_field.get_field_deep(["configuredSystems", host, "config"]).await
+ self.fleet_field
+ .get_field_deep(["configuredSystems", host, "config"])
+ .await
}
pub(super) fn data(&self) -> MutexGuard<FleetData> {
cmds/fleet/src/main.rsdiffbeforeafterboth--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -5,8 +5,8 @@
pub(crate) mod host;
pub(crate) mod keys;
+pub(crate) mod better_nix_eval;
pub(crate) mod extra_args;
-pub(crate) mod better_nix_eval;
mod fleetdata;
@@ -21,6 +21,7 @@
use futures::stream::FuturesUnordered;
use futures::TryStreamExt;
use host::{Config, FleetOpts};
+use human_repr::HumanCount;
use indicatif::{ProgressState, ProgressStyle};
use tracing::{info, metadata::LevelFilter};
use tracing::{info_span, Instrument};
@@ -121,9 +122,16 @@
fn setup_logging() {
let indicatif_layer = IndicatifLayer::new().with_progress_style(
ProgressStyle::with_template(
- "{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{pos:>7}/{len:7}{elapsed}{color_end}",
+ "{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",
)
.unwrap()
+ .with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {
+ let Some(len) = state.len() else {
+ return;
+ };
+ let pos = state.pos();
+ let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());
+ })
.with_key(
"color_start",
|state: &ProgressState, writer: &mut dyn std::fmt::Write| {
lib/default.nixdiffbeforeafterboth--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,44 +1,56 @@
-{ flake-utils }: {
- fleetConfiguration = { data, nixpkgs, hosts, ... }@allConfig:
- let
- hostNames = nixpkgs.lib.attrNames hosts;
- config = builtins.removeAttrs allConfig [ "nixpkgs" "data" ];
- fleetLib = import ./fleetLib.nix {
- inherit nixpkgs hostNames;
- };
- in
- nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system:
- let
+{flake-utils}: {
+ fleetConfiguration = {
+ data,
+ nixpkgs,
+ hosts,
+ ...
+ } @ allConfig: let
+ hostNames = nixpkgs.lib.attrNames hosts;
+ config = builtins.removeAttrs allConfig ["nixpkgs" "data"];
+ fleetLib = import ./fleetLib.nix {
+ inherit nixpkgs hostNames;
+ };
+ in
+ # Top-level arg is the builder system (not the target system!)
+ nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: let
+ withData = data: rec {
root = nixpkgs.lib.evalModules {
- modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];
+ modules = (import ../modules/fleet/_modules.nix) ++ [config data];
specialArgs = {
inherit nixpkgs fleetLib;
};
};
failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
rootAssertWarn =
- if failedAssertions != [ ]
+ if failedAssertions != []
then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
else nixpkgs.lib.showWarnings root.config.warnings root;
configuredHosts = rootAssertWarn.config.hosts;
configuredSecrets = rootAssertWarn.config.secrets;
- configuredSystems = configuredSystemsWithExtraModules [ ];
- configuredSystemsWithExtraModules = extraModules: nixpkgs.lib.listToAttrs (
- map
+ configuredSystems = configuredSystemsWithExtraModules [];
+ configuredSystemsWithExtraModules = extraModules:
+ nixpkgs.lib.listToAttrs (
+ map
(
name: {
inherit name;
value = nixpkgs.lib.nixosSystem {
system = configuredHosts.${name}.system;
- modules = configuredHosts.${name}.modules ++ extraModules ++ [
- ({ ... }: {
- nixpkgs.system = system;
- nixpkgs.localSystem.system = system;
- nixpkgs.crossSystem = if system == configuredHosts.${name}.system then null else {
- system = configuredHosts.${name}.system;
- };
- })
- ];
+ modules =
+ configuredHosts.${name}.modules
+ ++ extraModules
+ ++ [
+ ({...}: {
+ nixpkgs.system = system;
+ nixpkgs.localSystem.system = system;
+ nixpkgs.crossSystem =
+ if system == configuredHosts.${name}.system
+ then null
+ else {
+ system = configuredHosts.${name}.system;
+ };
+ })
+ ];
specialArgs = {
inherit fleetLib;
fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
@@ -47,11 +59,7 @@
}
)
(builtins.attrNames rootAssertWarn.config.hosts)
- );
- in
- rec {
- inherit configuredHosts configuredSecrets configuredSystems;
- configUnchecked = root.config;
+ );
buildSystems = {
toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
({...}: {
@@ -66,12 +74,22 @@
]);
installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
(nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
- ({ lib, ... }: {
+ ({lib, ...}: {
buildTarget = "installation-cd";
# Needed for https://github.com/NixOS/nixpkgs/issues/58959
- boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ];
+ boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
})
]);
};
- });
+ configUnchecked = root.config;
+ };
+ defaultData = withData data;
+ in rec {
+ inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+ injectData = data: let
+ injectedData = withData data;
+ in {
+ inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+ };
+ });
}