difftreelog
feat libssh preparation
in: trunk
7 files changed
Cargo.lockdiffbeforeafterboth--- a/Cargo.lock
+++ b/Cargo.lock
@@ -651,6 +651,27 @@
]
[[package]]
+name = "dirs"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
+dependencies = [
+ "libc",
+ "option-ext",
+ "redox_users",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
name = "displaydoc"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -731,13 +752,16 @@
"clap",
"futures",
"hostname",
+ "human-repr",
"indicatif",
"itertools",
"nixlike",
"once_cell",
+ "openssh",
"owo-colors",
"peg",
"r2d2",
+ "regex",
"serde",
"serde_json",
"shlex",
@@ -1019,6 +1043,12 @@
]
[[package]]
+name = "human-repr"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1"
+
+[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1257,6 +1287,17 @@
]
[[package]]
+name = "libredox"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.1",
+ "libc",
+ "redox_syscall 0.4.1",
+]
+
+[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1480,6 +1521,28 @@
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
+name = "openssh"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfe68c42d6ee6bd9de175b7a5d9bb86aa99d4e2fa7cf2f2a44e97f60b6d2759"
+dependencies = [
+ "dirs",
+ "libc",
+ "once_cell",
+ "shell-escape",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "tokio-pipe",
+]
+
+[[package]]
+name = "option-ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+
+[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1804,15 +1867,26 @@
]
[[package]]
+name = "redox_users"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
+dependencies = [
+ "getrandom 0.2.10",
+ "libredox",
+ "thiserror",
+]
+
+[[package]]
name = "regex"
-version = "1.9.5"
+version = "1.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
dependencies = [
"aho-corasick",
"memchr",
- "regex-automata 0.3.8",
- "regex-syntax 0.7.5",
+ "regex-automata 0.4.3",
+ "regex-syntax 0.8.2",
]
[[package]]
@@ -1826,13 +1900,13 @@
[[package]]
name = "regex-automata"
-version = "0.3.8"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
dependencies = [
"aho-corasick",
"memchr",
- "regex-syntax 0.7.5",
+ "regex-syntax 0.8.2",
]
[[package]]
@@ -1843,9 +1917,9 @@
[[package]]
name = "regex-syntax"
-version = "0.7.5"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
[[package]]
name = "rnix"
@@ -2099,6 +2173,12 @@
]
[[package]]
+name = "shell-escape"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
+
+[[package]]
name = "shlex"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2379,6 +2459,16 @@
]
[[package]]
+name = "tokio-pipe"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784"
+dependencies = [
+ "libc",
+ "tokio",
+]
+
+[[package]]
name = "tokio-util"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
cmds/fleet/Cargo.tomldiffbeforeafterboth11serde_json = "1.0"11serde_json = "1.0"12time = { version = "0.3.30", features = ["serde"] }12time = { version = "0.3.30", features = ["serde"] }13tempfile = "3.8"13tempfile = "3.8"14once_cell = "1.18"14once_cell = "1.18.0"15hostname = "0.3.1"15hostname = "0.3.1"16age-core = "0.9.0"16age-core = "0.9.0"17peg = "0.8.2"17peg = "0.8.2"41r2d2 = "0.8.10"41r2d2 = "0.8.10"42abort-on-drop = "0.2.2"42abort-on-drop = "0.2.2"43unindent = "0.2.3"43unindent = "0.2.3"44regex = "1.10.2"45openssh = "0.10.1"46human-repr = "1.1.0"4447cmds/fleet/src/cmds/build_systems.rsdiffbeforeafterboth--- a/cmds/fleet/src/cmds/build_systems.rs
+++ b/cmds/fleet/src/cmds/build_systems.rs
@@ -336,15 +336,14 @@
if !config.is_local(&host) {
info!("uploading system closure");
{
- let mut sign = MyCommand::new("sudo");
+ let mut sign = MyCommand::new("nix");
// Private key for host machine is registered in nix-sign.nix
- sign.arg("nix")
- .arg("store")
+ sign.arg("store")
.arg("sign")
.comparg("--key-file", "/etc/nix/private-key")
.arg("-r")
.arg(&built);
- if let Err(e) = sign.run_nix().await {
+ if let Err(e) = sign.sudo().run_nix().await {
warn!("Failed to sign store paths: {e}");
};
}
cmds/fleet/src/command.rsdiffbeforeafterboth--- a/cmds/fleet/src/command.rs
+++ b/cmds/fleet/src/command.rs
@@ -1,4 +1,5 @@
use std::{
+ borrow::Cow,
collections::HashMap,
ffi::OsStr,
process::Stdio,
@@ -6,8 +7,12 @@
task::Poll,
};
-use anyhow::Result;
+use anyhow::{anyhow, Result};
use futures::StreamExt;
+use itertools::Either;
+use once_cell::sync::Lazy;
+use openssh::{OverSsh, Session};
+use regex::Regex;
use serde::{de::Visitor, Deserialize};
use tokio::{io::AsyncRead, process::Command, select};
use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec};
@@ -37,6 +42,7 @@
command: String,
args: Vec<String>,
env: Vec<(String, String)>,
+ ssh_session: Option<Arc<Session>>,
}
impl MyCommand {
pub fn new(cmd: impl AsRef<OsStr>) -> Self {
@@ -45,6 +51,7 @@
command: ostoutf8(cmd),
args: vec![],
env: vec![],
+ ssh_session: None,
}
}
fn into_args(self) -> Vec<String> {
@@ -90,6 +97,18 @@
}
out
}
+ fn into_command_new(self) -> Result<Either<Command, openssh::OwningCommand<Arc<Session>>>> {
+ Ok(if let Some(session) = self.ssh_session.clone() {
+ let cmd = self.into_command();
+ Either::Right(
+ cmd.over_ssh(session)
+ .map_err(|e| anyhow!("ssh error: {e}"))?,
+ )
+ } else {
+ let cmd = self.into_command();
+ Either::Left(cmd)
+ })
+ }
pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
let arg = arg.as_ref();
self.args.push(ostoutf8(arg));
@@ -116,9 +135,15 @@
self
}
pub fn sudo(self) -> Self {
- let mut out = Self::new("sudo");
- out.args(self.into_args());
- out
+ if std::env::var_os("NO_SUDO").is_some() {
+ let mut out = Self::new("su");
+ out.arg("-c").arg(self.into_string());
+ out
+ } else {
+ let mut out = Self::new("sudo");
+ out.args(self.into_args());
+ out
+ }
}
pub fn ssh(self, on: impl AsRef<OsStr>) -> Self {
let mut out = Self::new("ssh");
@@ -126,6 +151,10 @@
out.arg(self.into_string());
out
}
+ pub fn over_ssh(mut self, session: Arc<Session>) -> Self {
+ self.ssh_session = Some(session);
+ self
+ }
pub async fn run(self) -> Result<()> {
let str = self.clone().into_string();
@@ -218,6 +247,11 @@
pub struct NixHandler {
spans: HashMap<u64, Span>,
}
+fn process_message(m: &str) -> Cow<'_, str> {
+ static OSC_CLEANER: Lazy<Regex> =
+ Lazy::new(|| Regex::new(r"\x1B\]([^\x07\x1C]*[\x07\x1C])?|\r").unwrap());
+ OSC_CLEANER.replace_all(m, "")
+}
impl Handler for NixHandler {
fn handle_line(&mut self, e: &str) {
if let Some(e) = e.strip_prefix("@nix ") {
@@ -303,7 +337,7 @@
{
let span = info_span!("job");
span.pb_start();
- span.pb_set_message(text.trim());
+ span.pb_set_message(&process_message(text.trim()));
self.spans.insert(id, span);
info!(target: "nix", "{}", text);
}
@@ -383,7 +417,7 @@
NixLog::Result { fields, id, typ } if typ == 101 && !fields.is_empty() => {
if let Some(span) = self.spans.get(&id) {
if let LogField::String(s) = &fields[0] {
- span.pb_set_message(s.trim());
+ span.pb_set_message(&process_message(s.trim()));
} else {
warn!("bad fields: {fields:?}");
}
cmds/fleet/src/host.rsdiffbeforeafterboth--- a/cmds/fleet/src/host.rs
+++ b/cmds/fleet/src/host.rs
@@ -7,8 +7,9 @@
sync::{Arc, Mutex, MutexGuard},
};
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
use clap::{ArgGroup, Parser};
+use openssh::SessionBuilder;
use tempfile::NamedTempFile;
use crate::{
@@ -43,6 +44,16 @@
pub struct ConfigHost {
pub name: String,
}
+impl ConfigHost {
+ async fn open_session(&self) -> Result<openssh::Session> {
+ let mut session = SessionBuilder::default();
+
+ session
+ .connect(&self.name)
+ .await
+ .map_err(|e| anyhow!("ssh error: {e}"))
+ }
+}
impl Config {
pub fn should_skip(&self, host: &str) -> bool {
@@ -93,21 +104,22 @@
}
pub async fn list_hosts(&self) -> Result<Vec<ConfigHost>> {
- let names = self.fleet_field
+ let names = self
+ .fleet_field
.get_field_deep(["configuredHosts"])
.await?
.list_fields()
.await?;
- let mut out = vec![];
- for name in names {
- out.push(ConfigHost {
- name,
- })
- }
- Ok(out)
+ let mut out = vec![];
+ for name in names {
+ out.push(ConfigHost { name })
+ }
+ Ok(out)
}
pub async fn system_config(&self, host: &str) -> Result<Field> {
- self.fleet_field.get_field_deep(["configuredSystems", host, "config"]).await
+ self.fleet_field
+ .get_field_deep(["configuredSystems", host, "config"])
+ .await
}
pub(super) fn data(&self) -> MutexGuard<FleetData> {
cmds/fleet/src/main.rsdiffbeforeafterboth--- a/cmds/fleet/src/main.rs
+++ b/cmds/fleet/src/main.rs
@@ -5,8 +5,8 @@
pub(crate) mod host;
pub(crate) mod keys;
+pub(crate) mod better_nix_eval;
pub(crate) mod extra_args;
-pub(crate) mod better_nix_eval;
mod fleetdata;
@@ -21,6 +21,7 @@
use futures::stream::FuturesUnordered;
use futures::TryStreamExt;
use host::{Config, FleetOpts};
+use human_repr::HumanCount;
use indicatif::{ProgressState, ProgressStyle};
use tracing::{info, metadata::LevelFilter};
use tracing::{info_span, Instrument};
@@ -121,9 +122,16 @@
fn setup_logging() {
let indicatif_layer = IndicatifLayer::new().with_progress_style(
ProgressStyle::with_template(
- "{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{pos:>7}/{len:7}{elapsed}{color_end}",
+ "{color_start}{span_child_prefix} {span_name}{{{span_fields}}}{color_end} {wide_msg} {color_start}{download_progress} {elapsed}{color_end}",
)
.unwrap()
+ .with_key("download_progress", |state: &ProgressState, writer: &mut dyn std::fmt::Write| {
+ let Some(len) = state.len() else {
+ return;
+ };
+ let pos = state.pos();
+ let _ = write!(writer, "{} / {}", pos.human_count_bare(), len.human_count_bare());
+ })
.with_key(
"color_start",
|state: &ProgressState, writer: &mut dyn std::fmt::Write| {
lib/default.nixdiffbeforeafterboth--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,44 +1,56 @@
-{ flake-utils }: {
- fleetConfiguration = { data, nixpkgs, hosts, ... }@allConfig:
- let
- hostNames = nixpkgs.lib.attrNames hosts;
- config = builtins.removeAttrs allConfig [ "nixpkgs" "data" ];
- fleetLib = import ./fleetLib.nix {
- inherit nixpkgs hostNames;
- };
- in
- nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system:
- let
+{flake-utils}: {
+ fleetConfiguration = {
+ data,
+ nixpkgs,
+ hosts,
+ ...
+ } @ allConfig: let
+ hostNames = nixpkgs.lib.attrNames hosts;
+ config = builtins.removeAttrs allConfig ["nixpkgs" "data"];
+ fleetLib = import ./fleetLib.nix {
+ inherit nixpkgs hostNames;
+ };
+ in
+ # Top-level arg is the builder system (not the target system!)
+ nixpkgs.lib.genAttrs flake-utils.lib.defaultSystems (system: let
+ withData = data: rec {
root = nixpkgs.lib.evalModules {
- modules = (import ../modules/fleet/_modules.nix) ++ [ config data ];
+ modules = (import ../modules/fleet/_modules.nix) ++ [config data];
specialArgs = {
inherit nixpkgs fleetLib;
};
};
failedAssertions = map (x: x.message) (nixpkgs.lib.filter (x: !x.assertion) root.config.assertions);
rootAssertWarn =
- if failedAssertions != [ ]
+ if failedAssertions != []
then throw "Failed assertions:\n${nixpkgs.lib.concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
else nixpkgs.lib.showWarnings root.config.warnings root;
configuredHosts = rootAssertWarn.config.hosts;
configuredSecrets = rootAssertWarn.config.secrets;
- configuredSystems = configuredSystemsWithExtraModules [ ];
- configuredSystemsWithExtraModules = extraModules: nixpkgs.lib.listToAttrs (
- map
+ configuredSystems = configuredSystemsWithExtraModules [];
+ configuredSystemsWithExtraModules = extraModules:
+ nixpkgs.lib.listToAttrs (
+ map
(
name: {
inherit name;
value = nixpkgs.lib.nixosSystem {
system = configuredHosts.${name}.system;
- modules = configuredHosts.${name}.modules ++ extraModules ++ [
- ({ ... }: {
- nixpkgs.system = system;
- nixpkgs.localSystem.system = system;
- nixpkgs.crossSystem = if system == configuredHosts.${name}.system then null else {
- system = configuredHosts.${name}.system;
- };
- })
- ];
+ modules =
+ configuredHosts.${name}.modules
+ ++ extraModules
+ ++ [
+ ({...}: {
+ nixpkgs.system = system;
+ nixpkgs.localSystem.system = system;
+ nixpkgs.crossSystem =
+ if system == configuredHosts.${name}.system
+ then null
+ else {
+ system = configuredHosts.${name}.system;
+ };
+ })
+ ];
specialArgs = {
inherit fleetLib;
fleet = fleetLib.hostsToAttrs (host: configuredSystems.${host}.config);
@@ -47,11 +59,7 @@
}
)
(builtins.attrNames rootAssertWarn.config.hosts)
- );
- in
- rec {
- inherit configuredHosts configuredSecrets configuredSystems;
- configUnchecked = root.config;
+ );
buildSystems = {
toplevel = builtins.mapAttrs (_name: value: value.config.system.build.toplevel) (configuredSystemsWithExtraModules [
({...}: {
@@ -66,12 +74,22 @@
]);
installationCd = builtins.mapAttrs (_name: value: value.config.system.build.isoImage) (configuredSystemsWithExtraModules [
(nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
- ({ lib, ... }: {
+ ({lib, ...}: {
buildTarget = "installation-cd";
# Needed for https://github.com/NixOS/nixpkgs/issues/58959
- boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ];
+ boot.supportedFilesystems = lib.mkForce ["btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs"];
})
]);
};
- });
+ configUnchecked = root.config;
+ };
+ defaultData = withData data;
+ in rec {
+ inherit (defaultData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+ injectData = data: let
+ injectedData = withData data;
+ in {
+ inherit (injectedData) configuredHosts configuredSecrets configuredSystems buildSystems configUnchecked;
+ };
+ });
}