From 41ecf404fcd9e128642132f6899abeed05fddbab Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Mon, 20 Dec 2021 22:21:58 +0000 Subject: [PATCH] feat: better logs --- --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,15 @@ ] [[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array", +] + +[[package]] name = "aes" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -29,6 +38,19 @@ ] [[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if", + "cipher 0.3.0", + "cpufeatures 0.2.1", + "ctr 0.8.0", + "opaque-debug", +] + +[[package]] name = "aes-ctr" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -37,7 +59,7 @@ "aes-soft", "aesni", "cipher 0.2.5", - "ctr", + "ctr 0.6.0", ] [[package]] @@ -66,37 +88,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16d68559c3ef40bc0fd7c3d2b156743e9387d477a68733b61dff0f6a5004ad58" dependencies = [ - "aes", + "aes 0.6.0", "aes-ctr", - "age-core", + "age-core 0.6.0", "base64", - "bcrypt-pbkdf", + "bcrypt-pbkdf 0.6.1", "bech32", - "block-modes", + "block-modes 0.7.0", "c2-chacha", - "chacha20poly1305", + "chacha20poly1305 0.7.1", "cookie-factory", "curve25519-dalek", "hkdf", - "hmac", - "i18n-embed", - "i18n-embed-fl", + "hmac 0.11.0", + "i18n-embed 0.12.1", + "i18n-embed-fl 0.5.0", "lazy_static", - "nom", + "nom 6.2.1", "num-traits", "pin-project", "rand 0.7.3", - "rsa", - "rust-embed", - "scrypt", - "secrecy", - "sha2", + "rsa 0.3.0", + "rust-embed 5.9.0", + "scrypt 0.7.0", + "secrecy 0.7.0", + "sha2 0.9.5", "subtle", "x25519-dalek", "zeroize", ] [[package]] +name = "age" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf320f937ccd0eb7f63450be0f071586cd918cd86785303ec1d052a3e243b550" +dependencies = [ + "aes 0.7.5", + "age-core 0.7.0", + "base64", + "bcrypt-pbkdf 0.7.2", + "bech32", + "block-modes 0.8.1", + "chacha20poly1305 0.9.0", + "cookie-factory", + "curve25519-dalek", + "hkdf", + "hmac 0.11.0", + "i18n-embed 0.13.1", + "i18n-embed-fl 0.6.1", + "lazy_static", + "nom 7.1.0", + "num-traits", + "pin-project", + "rand 0.7.3", + "rand 0.8.4", + "rsa 0.5.0", + "rust-embed 6.3.0", + "scrypt 0.8.1", + "sha2 0.9.5", + "subtle", + "x25519-dalek", + "zeroize", +] + +[[package]] name = "age-core" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -104,16 +160,32 @@ dependencies = [ "base64", "c2-chacha", - "chacha20poly1305", + "chacha20poly1305 0.7.1", "cookie-factory", "hkdf", - "nom", + "nom 6.2.1", "rand 0.7.3", - "secrecy", - "sha2", + "secrecy 0.7.0", + "sha2 0.9.5", ] [[package]] +name = "age-core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a485102f6c7a23e0666b169ba77c9ff6c6d249c05395c379be3cbab48a948e84" +dependencies = [ + "base64", + "chacha20poly1305 0.9.0", + "cookie-factory", + "hkdf", + "nom 7.1.0", + "rand 0.8.4", + "secrecy 0.8.0", + "sha2 0.9.5", +] + +[[package]] name = "aho-corasick" version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -132,12 +204,32 @@ ] [[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] name = "anyhow" version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" [[package]] +name = "async-trait" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "atty" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -161,25 +253,49 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] +name = "base58" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6107fe1be6682a68940da878d9e9f5e90ca5745b3dec9fd1bb393c8777d4f581" + +[[package]] name = "base64" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] +name = "base64ct" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c" + +[[package]] name = "bcrypt-pbkdf" -version = "0.6.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c38c03b9506bd92bf1ef50665a81eda156f615438f7654bffba58907e6149d7" +checksum = "12621b8e87feb183a6e5dbb315e49026b2229c4398797ee0ae2d1bc00aef41b9" dependencies = [ "blowfish", "crypto-mac", - "pbkdf2", - "sha2", + "pbkdf2 0.8.0", + "sha2 0.9.5", "zeroize", ] [[package]] +name = "bcrypt-pbkdf" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bde65b3c84000288c0abe8aa601a4b7c40b0dbbb7d144dd6c712ed9796e1fd5" +dependencies = [ + "blowfish", + "hex-literal", + "pbkdf2 0.10.0", + "sha2 0.10.0", +] + +[[package]] name = "bech32" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -213,6 +329,15 @@ ] [[package]] +name = "block-buffer" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +dependencies = [ + "generic-array", +] + +[[package]] name = "block-modes" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -223,6 +348,16 @@ ] [[package]] +name = "block-modes" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" +dependencies = [ + "block-padding", + "cipher 0.3.0", +] + +[[package]] name = "block-padding" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -252,6 +387,12 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] name = "c2-chacha" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -274,14 +415,39 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] +name = "chacha20" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b72a433d0cf2aef113ba70f62634c56fddb0f244e6377185c56a7cadbd8f91" +dependencies = [ + "cfg-if", + "cipher 0.3.0", + "cpufeatures 0.2.1", + "zeroize", +] + +[[package]] name = "chacha20poly1305" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1fc18e6d90c40164bf6c317476f2a98f04661e310e79830366b7e914c58a8e" dependencies = [ - "aead", + "aead 0.3.2", "cipher 0.2.5", - "poly1305", + "poly1305 0.6.2", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b84ed6d1d5f7aa9bdde921a5090e0ca4d934d250ea3b402a5fab3a994e28a2a" +dependencies = [ + "aead 0.4.3", + "chacha20", + "cipher 0.3.0", + "poly1305 0.7.2", "zeroize", ] @@ -323,7 +489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ - "ansi_term", + "ansi_term 0.11.0", "atty", "bitflags", "strsim 0.8.0", @@ -333,6 +499,12 @@ ] [[package]] +name = "const-oid" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b" + +[[package]] name = "cookie-factory" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -348,12 +520,41 @@ ] [[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] name = "cpuid-bool" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" [[package]] +name = "crypto-bigint" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83bd3bb4314701c568e340cd8cf78c975aa0ca79e03d3f6d1677d5b0c9c0c03" +dependencies = [ + "generic-array", + "rand_core 0.6.2", + "subtle", +] + +[[package]] +name = "crypto-common" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" +dependencies = [ + "generic-array", +] + +[[package]] name = "crypto-mac" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -373,13 +574,22 @@ ] [[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher 0.3.0", +] + +[[package]] name = "curve25519-dalek" version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ "byteorder", - "digest", + "digest 0.9.0", "rand_core 0.5.1", "subtle", "zeroize", @@ -396,6 +606,16 @@ ] [[package]] +name = "der" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4" +dependencies = [ + "const-oid", + "crypto-bigint", +] + +[[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -405,6 +625,18 @@ ] [[package]] +name = "digest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +dependencies = [ + "block-buffer 0.10.0", + "crypto-common", + "generic-array", + "subtle", +] + +[[package]] name = "dprint-core" version = "0.46.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -441,14 +673,15 @@ name = "fleet" version = "0.1.0" dependencies = [ - "age", - "age-core", + "age 0.7.0", + "age-core 0.7.0", "anyhow", + "async-trait", + "base58", "base64", "chrono", - "env_logger", + "futures", "hostname", - "log", "nixlike", "once_cell", "peg", @@ -457,6 +690,10 @@ "structopt", "tempfile", "time 0.3.3", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", "z85", ] @@ -464,7 +701,7 @@ name = "fleet-install-secrets" version = "0.1.0" dependencies = [ - "age", + "age 0.6.0", "anyhow", "env_logger", "log", @@ -487,6 +724,16 @@ ] [[package]] +name = "fluent" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61f69378194459db76abd2ce3952b790db103ceb003008d3d50d97c41ff847a7" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] name = "fluent-bundle" version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -533,6 +780,100 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] +name = "futures" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445" + +[[package]] +name = "futures-executor" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11" + +[[package]] +name = "futures-macro" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +dependencies = [ + "autocfg 1.0.1", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af" + +[[package]] +name = "futures-task" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12" + +[[package]] +name = "futures-util" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +dependencies = [ + "autocfg 1.0.1", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] name = "generic-array" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -583,13 +924,19 @@ ] [[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] name = "hkdf" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01706d578d5c281058480e673ae4086a9f4710d8df1ad80a5b03e39ece5f886b" dependencies = [ - "digest", - "hmac", + "digest 0.9.0", + "hmac 0.11.0", ] [[package]] @@ -599,10 +946,19 @@ checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ "crypto-mac", - "digest", + "digest 0.9.0", ] [[package]] +name = "hmac" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddca131f3e7f2ce2df364b57949a9d47915cfbd35e46cfee355ccebbf794d6a2" +dependencies = [ + "digest 0.10.1", +] + +[[package]] name = "hostname" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -639,15 +995,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3794c3d7fea43e076281c9213cfaaa7a53c3f18b1613f12514b9f575a2908457" dependencies = [ - "fluent", + "fluent 0.15.0", + "fluent-langneg", + "fluent-syntax", + "i18n-embed-impl 0.7.0", + "intl-memoizer", + "lazy_static", + "log", + "parking_lot", + "rust-embed 5.9.0", + "thiserror", + "unic-langid", + "walkdir", +] + +[[package]] +name = "i18n-embed" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a0b4598fcd199eb5da38f70ece82903b178ad638839661c00612719bcfc0ad" +dependencies = [ + "fluent 0.16.0", "fluent-langneg", "fluent-syntax", - "i18n-embed-impl", + "i18n-embed-impl 0.8.0", "intl-memoizer", "lazy_static", "log", "parking_lot", - "rust-embed", + "rust-embed 6.3.0", "thiserror", "unic-langid", "walkdir", @@ -661,10 +1037,10 @@ dependencies = [ "dashmap", "find-crate", - "fluent", + "fluent 0.15.0", "fluent-syntax", "i18n-config", - "i18n-embed", + "i18n-embed 0.12.1", "lazy_static", "proc-macro-error", "proc-macro2", @@ -675,6 +1051,27 @@ ] [[package]] +name = "i18n-embed-fl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1d347d2f7f9f2f977385b43b204d59ebeb1b2f93ce73cd23622df2d2da1033" +dependencies = [ + "dashmap", + "find-crate", + "fluent 0.16.0", + "fluent-syntax", + "i18n-config", + "i18n-embed 0.13.1", + "lazy_static", + "proc-macro-error", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", + "unic-langid", +] + +[[package]] name = "i18n-embed-impl" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -688,6 +1085,19 @@ ] [[package]] +name = "i18n-embed-impl" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0db2330e035808eb064afb67e6743ddce353763af3e0f2bdfc2476e00ce76136" +dependencies = [ + "find-crate", + "i18n-config", + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "instant" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -774,6 +1184,15 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] name = "memchr" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -789,6 +1208,34 @@ ] [[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] name = "nix" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -825,6 +1272,26 @@ ] [[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] name = "num-bigint" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -855,6 +1322,24 @@ ] [[package]] +name = "num-bigint-dig" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" +dependencies = [ + "autocfg 0.1.7", + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.4", + "smallvec", + "zeroize", +] + +[[package]] name = "num-integer" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -882,6 +1367,7 @@ checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg 1.0.1", + "libm", ] [[package]] @@ -964,6 +1450,15 @@ ] [[package]] +name = "pbkdf2" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4628cc3cf953b82edcd3c1388c5715401420ce5524fedbab426bd5aba017434" +dependencies = [ + "digest 0.10.1", +] + +[[package]] name = "peg" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1002,6 +1497,15 @@ ] [[package]] +name = "pem-rfc7468" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84e93a3b1cc0510b03020f33f21e62acdde3dcaef432edc95bea377fbd4c2cd4" +dependencies = [ + "base64ct", +] + +[[package]] name = "pin-project" version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1022,6 +1526,42 @@ ] [[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "116bee8279d783c0cf370efa1a94632f2108e5ef0bb32df31f051647810a4e2c" +dependencies = [ + "der", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "pkcs8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447" +dependencies = [ + "der", + "pem-rfc7468", + "pkcs1", + "spki", + "zeroize", +] + +[[package]] name = "poly1305" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1032,6 +1572,17 @@ ] [[package]] +name = "poly1305" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede" +dependencies = [ + "cpufeatures 0.2.1", + "opaque-debug", + "universal-hash", +] + +[[package]] name = "ppv-lite86" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1062,6 +1613,18 @@ ] [[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1187,6 +1750,15 @@ ] [[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] name = "regex-syntax" version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1208,15 +1780,15 @@ checksum = "3648b669b10afeab18972c105e284a7b953a669b0be3514c27f9b17acab2f9cd" dependencies = [ "byteorder", - "digest", + "digest 0.9.0", "lazy_static", - "num-bigint-dig", + "num-bigint-dig 0.6.1", "num-integer", "num-iter", "num-traits", "pem", "rand 0.7.3", - "sha2", + "sha2 0.9.5", "simple_asn1", "subtle", "thiserror", @@ -1224,17 +1796,48 @@ ] [[package]] +name = "rsa" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c2603e2823634ab331437001b411b9ed11660fbc4066f3908c84a9439260d" +dependencies = [ + "byteorder", + "digest 0.9.0", + "lazy_static", + "num-bigint-dig 0.7.0", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8", + "rand 0.8.4", + "subtle", + "zeroize", +] + +[[package]] name = "rust-embed" version = "5.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fe1fe6aac5d6bb9e1ffd81002340363272a7648234ec7bdfac5ee202cb65523" dependencies = [ - "rust-embed-impl", - "rust-embed-utils", + "rust-embed-impl 5.9.0", + "rust-embed-utils 5.1.0", "walkdir", ] [[package]] +name = "rust-embed" +version = "6.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40377bff8cceee81e28ddb73ac97f5c2856ce5522f0b260b763f434cdfae602" +dependencies = [ + "rust-embed-impl 6.2.0", + "rust-embed-utils 7.1.0", + "walkdir", +] + +[[package]] name = "rust-embed-impl" version = "5.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1242,7 +1845,20 @@ dependencies = [ "proc-macro2", "quote", - "rust-embed-utils", + "rust-embed-utils 5.1.0", + "syn", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e763e24ba2bf0c72bc6be883f967f794a019fafd1b86ba1daff9c91a7edd30" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils 7.1.0", "syn", "walkdir", ] @@ -1257,6 +1873,16 @@ ] [[package]] +name = "rust-embed-utils" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad22c7226e4829104deab21df575e995bfbc4adfad13a595e387477f238c1aec" +dependencies = [ + "sha2 0.9.5", + "walkdir", +] + +[[package]] name = "rustc-hash" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1278,6 +1904,15 @@ ] [[package]] +name = "salsa20" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0fbb5f676da676c260ba276a8f43a8dc67cf02d1438423aeb1c677a7212686" +dependencies = [ + "cipher 0.3.0", +] + +[[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1298,13 +1933,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879588d8f90906e73302547e20fffefdd240eb3e0e744e142321f5d49dea0518" dependencies = [ - "hmac", - "pbkdf2", - "salsa20", - "sha2", + "hmac 0.11.0", + "pbkdf2 0.8.0", + "salsa20 0.8.1", + "sha2 0.9.5", ] [[package]] +name = "scrypt" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e73d6d7c6311ebdbd9184ad6c4447b2f36337e327bda107d3ba9e3c374f9d325" +dependencies = [ + "hmac 0.12.0", + "pbkdf2 0.10.0", + "salsa20 0.9.0", + "sha2 0.10.0", +] + +[[package]] name = "secrecy" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1314,6 +1961,15 @@ ] [[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "zeroize", +] + +[[package]] name = "serde" version = "1.0.130" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1350,14 +2006,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.1.5", + "digest 0.9.0", "opaque-debug", ] [[package]] +name = "sha2" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d964dd36bb15bcf2f2b35694c072feab74969a54f2bbeec7a2d725d2bdcb6" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.1", + "digest 0.10.1", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] name = "simple_asn1" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1369,6 +2054,12 @@ ] [[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] name = "smallvec" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1381,6 +2072,15 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] +name = "spki" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32" +dependencies = [ + "der", +] + +[[package]] name = "stable_deref_trait" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1510,6 +2210,15 @@ ] [[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] name = "time" version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1536,6 +2245,51 @@ checksum = "29738eedb4388d9ea620eeab9384884fc3f06f586a2eddb56bedc5885126c7c1" [[package]] +name = "tokio" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +dependencies = [ + "autocfg 1.0.1", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] name = "toml" version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1545,6 +2299,67 @@ ] [[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245da694cc7fc4729f3f418b304cb57789f1bed2a78c575407ab8a23f53cb4d3" +dependencies = [ + "ansi_term 0.12.1", + "lazy_static", + "matchers", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] name = "type-map" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1680,9 +2495,9 @@ [[package]] name = "x25519-dalek" -version = "1.2.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2392b6b94a576b4e2bf3c5b2757d63f10ada8020a2e4d08ac849ebcf6ea8e077" +checksum = "5a0c105152107e3b96f6a00a65e86ce82d9b125230e1c4302940eca58ff71f4f" dependencies = [ "curve25519-dalek", "rand_core 0.5.1", @@ -1697,9 +2512,9 @@ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" dependencies = [ "zeroize_derive", ] --- a/cmds/fleet/Cargo.toml +++ b/cmds/fleet/Cargo.toml @@ -7,19 +7,24 @@ [dependencies] anyhow = "1.0" -log = "0.4.14" -env_logger = "0.9.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" time = { version = "0.3.2", features = ["serde"] } tempfile = "3.2" once_cell = "1.5" hostname = "0.3.1" -age-core = "0.6.0" +age-core = "0.7.0" peg = "0.7.0" nixlike = {path = "../../crates/nixlike"} -age = { version = "0.6.0", features = ["ssh", "armor"] } +age = { version = "0.7.0", features = ["ssh", "armor"] } base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } z85 = "3.0.3" +base58 = "*" structopt = "0.3.23" +tokio = { version = "1.14.0", features = ["full"] } +tracing = "0.1.29" +tracing-subscriber = { version = "0.3.3", features = ["fmt", "env-filter"] } +tokio-util = { version = "0.6.9", features = ["codec"] } +async-trait = "0.1.52" +futures = "0.3.17" --- a/cmds/fleet/src/cmds/build_systems.rs +++ b/cmds/fleet/src/cmds/build_systems.rs @@ -1,11 +1,12 @@ -use std::{env::current_dir, process::Command}; +use std::{env::current_dir, time::Duration}; use crate::{command::CommandExt, host::Config, nix::SYSTEMS_ATTRIBUTE}; use anyhow::Result; -use log::info; use structopt::StructOpt; +use tokio::{process::Command, task::LocalSet, time::sleep}; +use tracing::{error, field, info, info_span, warn, Instrument}; -#[derive(StructOpt)] +#[derive(StructOpt, Clone)] pub struct BuildSystems { /// --builders arg for nix #[structopt(long)] @@ -78,117 +79,158 @@ } impl BuildSystems { - pub fn run(self, config: &Config) -> Result<()> { - let hosts = config.list_hosts()?; - + pub async fn run(self, config: &Config) -> Result<()> { + let hosts = config.list_hosts().await?; + let set = LocalSet::new(); + let this = &self; for host in hosts.iter() { if config.should_skip(host) { continue; } - info!("Building host {}", host); - let built = { - let dir = tempfile::tempdir()?; - dir.path().to_owned() - }; + let config = config.clone(); + let host = host.clone(); + let this = this.clone(); + let span = info_span!("deployment", host = field::display(&host)); + set.spawn_local( + (async move { + let res: Result<()> = try { + info!("building"); + let built = { + let dir = tempfile::tempdir()?; + dir.path().to_owned() + }; - let mut nix_build = if self.privileged_build { - let mut out = Command::new("sudo"); - out.arg("nix"); - out - } else { - Command::new("nix") - }; - nix_build - .args(&["build", "--impure", "--no-link", "--out-link"]) - .arg(&built) - .arg(format!( - "{}.{}.config.system.build.toplevel", - SYSTEMS_ATTRIBUTE, host, - )); + let mut nix_build = if this.privileged_build { + let mut out = Command::new("sudo"); + out.arg("nix"); + out + } else { + Command::new("nix") + }; + nix_build + .args(&[ + "build", + "--impure", + "--json", + // "--show-trace", + "--no-link", + "--out-link", + ]) + .arg(&built) + .arg(format!( + "{}.{}.config.system.build.toplevel", + SYSTEMS_ATTRIBUTE, host, + )); + + if let Some(builders) = &this.builders { + nix_build.arg("--builders").arg(builders); + } + if let Some(jobs) = &this.jobs { + nix_build.arg("--max-jobs"); + nix_build.arg(format!("{}", jobs)); + } + if !this.fail_fast { + nix_build.arg("--keep-going"); + } - if let Some(builders) = &self.builders { - nix_build.arg("--builders").arg(builders); - } - if let Some(jobs) = &self.jobs { - nix_build.arg("--max-jobs"); - nix_build.arg(format!("{}", jobs)); - } - if !self.fail_fast { - nix_build.arg("--keep-going"); - } + nix_build.run_nix().await?; + let built = std::fs::canonicalize(built)?; - nix_build.inherit_stdio().run()?; - let built = std::fs::canonicalize(built)?; - info!("Built closure: {:?}", built); + let action = Action::from(this.subcommand.clone()); - let action = Action::from(self.subcommand.clone()); + match action { + Action::Upload(action) => { + if !config.is_local(&host) { + info!("uploading system closure"); + let mut tries = 0; + loop { + match Command::new("nix") + .args(&["copy", "--to"]) + .arg(format!("ssh://root@{}", host)) + .arg(&built) + .inherit_stdio() + .run_nix() + .await + { + Ok(()) => break, + Err(e) if tries < 3 => { + tries += 1; + warn!("Copy failure ({}/3): {}", tries, e); + sleep(Duration::from_millis(5000)).await; + } + Err(e) => return Err(e), + } + } + } + if let Some(action) = action { + if action.should_switch_profile() { + info!("switching generation"); + config + .command_on(&host, "nix-env", true) + .args(&["-p", "/nix/var/nix/profiles/system", "--set"]) + .arg(&built) + .inherit_stdio() + .run() + .await?; + } + info!("executing activation script"); + let mut switch_script = built.clone(); + switch_script.push("bin"); + switch_script.push("switch-to-configuration"); + config + .command_on(&host, switch_script, true) + .arg(action.name()) + .inherit_stdio() + .run() + .await?; + } + } + Action::Package(PackageAction::SdImage) => { + let mut out = current_dir()?; + out.push(format!("sd-image-{}", host)); - match action { - Action::Upload(action) => { - if !config.is_local(host) { - info!("Uploading system closure"); - Command::new("nix") - .args(&["copy", "--to"]) - .arg(format!("ssh://root@{}", host)) - .arg(&built) - .inherit_stdio() - .run()?; - } - if let Some(action) = action { - if action.should_switch_profile() { - info!("Switching generation"); - config - .command_on(host, "nix-env", true) - .args(&["-p", "/nix/var/nix/profiles/system", "--set"]) - .arg(&built) - .inherit_stdio() - .run()?; - } - info!("Executing activation script"); - let mut switch_script = built.clone(); - switch_script.push("bin"); - switch_script.push("switch-to-configuration"); - config - .command_on(host, switch_script, true) - .arg(action.name()) - .inherit_stdio() - .run()?; - } - } - Action::Package(PackageAction::SdImage) => { - let mut out = current_dir()?; - out.push(format!("sd-image-{}", host)); + info!("building sd image to {:?}", out); + let mut nix_build = if this.privileged_build { + let mut out = Command::new("sudo"); + out.arg("nix"); + out + } else { + Command::new("nix") + }; + nix_build + .args(&["build", "--impure", "--no-link", "--out-link"]) + .arg(&out) + .arg(format!( + "{}.{}.config.system.build.sdImage", + SYSTEMS_ATTRIBUTE, host, + )); + if let Some(builders) = &this.builders { + nix_build.arg("--builders").arg(builders); + } + if let Some(jobs) = &this.jobs { + nix_build.arg("--max-jobs"); + nix_build.arg(format!("{}", jobs)); + } + if !this.fail_fast { + nix_build.arg("--keep-going"); + } - info!("Building sd image to {:?}", out); - let mut nix_build = if self.privileged_build { - let mut out = Command::new("sudo"); - out.arg("nix"); - out - } else { - Command::new("nix") + nix_build.inherit_stdio().run_nix().await?; + } + }; }; - nix_build - .args(&["build", "--impure", "--no-link", "--out-link"]) - .arg(&out) - .arg(format!( - "{}.{}.config.system.build.sdImage", - SYSTEMS_ATTRIBUTE, host, - )); - if let Some(builders) = &self.builders { - nix_build.arg("--builders").arg(builders); + match res { + Ok(_) => {} + Err(e) => { + error!("failed to deploy host: {}", e) + } } - if let Some(jobs) = &self.jobs { - nix_build.arg("--max-jobs"); - nix_build.arg(format!("{}", jobs)); - } - if !self.fail_fast { - nix_build.arg("--keep-going"); - } - - nix_build.inherit_stdio().run()?; - } - }; + Ok(()) + }) + .instrument(span), + ); } + set.await; Ok(()) } } --- a/cmds/fleet/src/cmds/info.rs +++ b/cmds/fleet/src/cmds/info.rs @@ -30,13 +30,13 @@ } impl Info { - pub fn run(self, config: &Config) -> Result<()> { + pub async fn run(self, config: &Config) -> Result<()> { let mut data = Vec::new(); match self.cmd { InfoCmd::ListHosts { ref tagged } => { - 'host: for host in config.list_hosts()? { + 'host: for host in config.list_hosts().await? { if !tagged.is_empty() { - let tags: Vec = config.config_attr(&host, "tags")?; + let tags: Vec = config.config_attr(&host, "tags").await?; for tag in tagged { if !tags.contains(&tag) { continue 'host; @@ -57,10 +57,18 @@ ); let mut out = >::new(); if external { - out.extend(config.config_attr::>(&host, "network.externalIps")?); + out.extend( + config + .config_attr::>(&host, "network.externalIps") + .await?, + ); } if internal { - out.extend(config.config_attr::>(&host, "network.internalIps")?); + out.extend( + config + .config_attr::>(&host, "network.internalIps") + .await?, + ); } for ip in out { data.push(ip); --- a/cmds/fleet/src/cmds/secrets/mod.rs +++ b/cmds/fleet/src/cmds/secrets/mod.rs @@ -3,6 +3,7 @@ host::Config, }; use anyhow::{bail, Result}; +use futures::{StreamExt, TryStreamExt}; use std::{ io::{self, Cursor, Read}, path::PathBuf, @@ -44,14 +45,14 @@ } impl Secrets { - pub fn run(self, config: &Config) -> Result<()> { + pub async fn run(self, config: &Config) -> Result<()> { match self { Secrets::ForceKeys => { - for host in config.list_hosts()? { + for host in config.list_hosts().await? { if config.should_skip(&host) { continue; } - config.key(&host)?; + config.key(&host).await?; } } Secrets::AddShared { @@ -61,10 +62,10 @@ public, public_file, } => { - let recipients = machines - .iter() - .map(|m| config.recipient(m)) - .collect::>>()?; + let recipients = futures::stream::iter(machines.iter()) + .then(|m| config.recipient(m)) + .try_collect::>() + .await?; let secret = { let mut input = vec![]; @@ -117,7 +118,7 @@ public, public_file, } => { - let recipient = config.recipient(&machine)?; + let recipient = config.recipient(&machine).await?; let secret = { let mut input = vec![]; --- a/cmds/fleet/src/command.rs +++ b/cmds/fleet/src/command.rs @@ -1,42 +1,204 @@ -use std::{ - ffi::OsStr, - process::{Command, Stdio}, -}; +use std::{ffi::OsStr, process::Stdio}; use anyhow::{Context, Result}; -use serde::de::DeserializeOwned; +use async_trait::async_trait; +use futures::StreamExt; +use serde::{ + de::{DeserializeOwned, Visitor}, + Deserialize, +}; +use tokio::{process::Command, select}; +use tokio_util::codec::{BytesCodec, FramedRead, LinesCodec}; +use tracing::{info, warn}; +#[async_trait] pub trait CommandExt { - fn run(&mut self) -> Result<()>; - fn run_json(&mut self) -> Result; - fn run_string(&mut self) -> Result; + async fn run_nix(&mut self) -> Result<()>; + async fn run_nix_json(&mut self) -> Result; + async fn run_nix_string(&mut self) -> Result; + async fn run(&mut self) -> Result<()>; + async fn run_json(&mut self) -> Result; + async fn run_string(&mut self) -> Result; fn inherit_stdio(&mut self) -> &mut Self; fn ssh_on(host: impl AsRef, command: impl AsRef) -> Self; } +#[derive(Debug)] +enum LogField { + String(String), + Num(u64), +} + +impl<'de> Deserialize<'de> for LogField { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct StringOrNum; + impl<'de> Visitor<'de> for StringOrNum { + type Value = LogField; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "string or unsigned") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(LogField::String(v.to_owned())) + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(LogField::Num(v)) + } + } + + deserializer.deserialize_any(StringOrNum) + } +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase", tag = "action")] +enum NixLog { + Msg { + level: u32, + msg: String, + raw_msg: Option, + }, + Start { + id: u64, + level: u32, + #[serde(default)] + fields: Vec, + text: String, + #[serde(rename = "type")] + typ: u32, + }, + Stop { + id: u64, + }, + Result { + id: u64, + #[serde(rename = "type")] + typ: u32, + }, +} + +#[async_trait] impl CommandExt for Command { + async fn run_nix(&mut self) -> Result<()> { + self.run_nix_string().await.map(|_| ()) + } + async fn run_nix_json(&mut self) -> Result { + let str = self.run_nix_string().await?; + serde_json::from_str(&str).with_context(|| format!("{:?}", str)) + } + + async fn run_nix_string(&mut self) -> Result { + self.arg("--log-format").arg("internal-json"); + self.stderr(Stdio::piped()); + self.stdout(Stdio::piped()); + let mut child = self.spawn()?; + let mut stderr = child.stderr.take().unwrap(); + let mut stdout = child.stdout.take().unwrap(); + let mut err = FramedRead::new(&mut stderr, LinesCodec::new()); + let mut out = FramedRead::new(&mut stdout, BytesCodec::new()); + + // while let Some(line) = read.next().await? {} + + let mut out_buf = vec![]; + loop { + select! { + e = err.next() => { + if let Some(e) = e { + let e = e?; + if let Some(e) = e.strip_prefix("@nix ") { + + let log: NixLog = match serde_json::from_str(e) { + Ok(l) => l, + Err(err) => { + warn!("failed to parse nix log line {:?}: {}", e, err); + continue; + }, + }; + match log { + NixLog::Msg { msg, raw_msg, .. } => { + if !(msg.ends_with(" is dirty") && msg.contains("warning:") && msg.contains(" Git tree ")) { + info!(target: "nix", "{}", raw_msg.unwrap_or(msg)) + } + }, + NixLog::Start { ref fields, typ, .. } if typ == 105 && fields.len() >= 1 => { + if let [LogField::String(drv), ..] = &fields[..] { + info!(target: "nix","building {}", drv) + } else { + warn!("bad build log: {:?}", log) + } + }, + NixLog::Start { ref fields, typ, .. } if typ == 100 && fields.len() >= 3 => { + if let [LogField::String(drv), LogField::String(from), LogField::String(to), ..] = &fields[..] { + info!(target: "nix","copying {} {} -> {}", drv, from, to) + } else { + warn!("bad copy log: {:?}", log) + } + }, + NixLog::Start { text, typ, .. } if typ == 0 || typ == 102 || typ == 103 || typ == 104 => { + if !text.is_empty() && text != "querying info about missing paths" && text != "copying 0 paths" { + info!(target: "nix", "{}", text) + } + }, + NixLog::Stop { .. } => {}, + NixLog::Result { .. } => {}, + _ => warn!("unknown log: {:?}", log) + }; + } else { + warn!(target="nix","unknown: {}", e) + } + } + }, + o = out.next() => { + if let Some(o) = o { + out_buf.extend_from_slice(&o?); + } + }, + code = child.wait() => { + let code = code?; + if !code.success() { + anyhow::bail!("command ({:?}) failed with status {}", self, code); + } + break; + } + } + } + + Ok(String::from_utf8(out_buf)?) + } + fn inherit_stdio(&mut self) -> &mut Self { self.stderr(Stdio::inherit()); self } - fn run(&mut self) -> Result<()> { + async fn run(&mut self) -> Result<()> { self.inherit_stdio(); - let out = self.output()?; + let out = self.output().await?; if !out.status.success() { anyhow::bail!("command ({:?}) failed with status {}", self, out.status); } Ok(()) } - fn run_json(&mut self) -> Result { - let str = self.run_string()?; + async fn run_json(&mut self) -> Result { + let str = self.run_string().await?; serde_json::from_str(&str).with_context(|| format!("{:?}", str)) } - fn run_string(&mut self) -> Result { + async fn run_string(&mut self) -> Result { self.inherit_stdio(); - let out = self.output()?; + let out = self.output().await?; if !out.status.success() { anyhow::bail!("command ({:?}) failed with status {}", self, out.status); } --- a/cmds/fleet/src/host.rs +++ b/cmds/fleet/src/host.rs @@ -4,7 +4,6 @@ ffi::{OsStr, OsString}, ops::Deref, path::PathBuf, - process::Command, sync::Arc, }; @@ -12,6 +11,7 @@ use serde::de::DeserializeOwned; use structopt::clap::ArgGroup; use structopt::StructOpt; +use tokio::process::Command; use crate::{command::CommandExt, fleetdata::FleetData}; @@ -73,15 +73,15 @@ str } - pub fn list_hosts(&self) -> Result> { + pub async fn list_hosts(&self) -> Result> { Command::new("nix") .arg("eval") .arg(self.full_attr_name("fleetConfigurations.default.configuredHosts")) .args(&["--apply", "builtins.attrNames", "--json", "--show-trace"]) - .inherit_stdio() - .run_json() + .run_nix_json() + .await } - pub fn config_attr(&self, host: &str, attr: &str) -> Result { + pub async fn config_attr(&self, host: &str, attr: &str) -> Result { Command::new("nix") .arg("eval") .arg(self.full_attr_name(&format!( @@ -89,8 +89,8 @@ host, attr ))) .args(&["--json", "--show-trace"]) - .inherit_stdio() - .run_json() + .run_nix_json() + .await } pub fn data(&self) -> Ref { --- a/cmds/fleet/src/keys.rs +++ b/cmds/fleet/src/keys.rs @@ -2,7 +2,7 @@ use crate::{command::CommandExt, host::Config}; use anyhow::{anyhow, Result}; -use log::warn; +use tracing::warn; impl Config { pub fn cached_key(&self, host: &str) -> Option { @@ -21,7 +21,7 @@ host.encryption_key = key.trim().to_string(); } - pub fn key(&self, host: &str) -> anyhow::Result { + pub async fn key(&self, host: &str) -> anyhow::Result { if let Some(key) = self.cached_key(host) { Ok(key) } else { @@ -29,19 +29,20 @@ let key = self .command_on(host, "cat", false) .arg("/etc/ssh/ssh_host_ed25519_key.pub") - .run_string()?; + .run_string() + .await?; self.update_key(host, key.clone()); Ok(key) } } - pub fn recipient(&self, host: &str) -> anyhow::Result { - let key = self.key(host)?; + pub async fn recipient(&self, host: &str) -> anyhow::Result { + let key = self.key(host).await?; age::ssh::Recipient::from_str(&key).map_err(|e| anyhow!("parse recipient error: {:?}", e)) } - pub fn orphaned_data(&self) -> Result> { + pub async fn orphaned_data(&self) -> Result> { let mut out = Vec::new(); - let host_names = self.list_hosts()?; + let host_names = self.list_hosts().await?; for hostname in self .data() .hosts --- a/cmds/fleet/src/main.rs +++ b/cmds/fleet/src/main.rs @@ -1,3 +1,5 @@ +#![feature(try_blocks)] + pub mod command; pub mod host; pub mod keys; @@ -7,12 +9,16 @@ mod fleetdata; -use anyhow::Result; +use std::io; + +use anyhow::{anyhow, Result}; use structopt::clap::AppSettings::*; use structopt::StructOpt; use cmds::{build_systems::BuildSystems, info::Info, secrets::Secrets}; use host::{Config, FleetOpts}; +use tracing::{info, metadata::LevelFilter}; +use tracing_subscriber::EnvFilter; #[derive(StructOpt)] enum Opts { @@ -38,23 +44,34 @@ command: Opts, } -fn run_command(config: &Config, command: Opts) -> Result<()> { +async fn run_command(config: &Config, command: Opts) -> Result<()> { match command { - Opts::BuildSystems(c) => c.run(config)?, - Opts::Secrets(s) => s.run(config)?, - Opts::Info(i) => i.run(config)?, + Opts::BuildSystems(c) => c.run(config).await?, + Opts::Secrets(s) => s.run(config).await?, + Opts::Info(i) => i.run(config).await?, }; Ok(()) } -fn main() -> Result<()> { - env_logger::Builder::new() - .filter_level(log::LevelFilter::Info) - .init(); +#[tokio::main] +async fn main() -> Result<()> { + let filter = EnvFilter::from_default_env().add_directive(LevelFilter::INFO.into()); + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(filter) + .without_time() + .with_target(false) + .with_writer(|| { + // eprintln!("Line"); + io::stderr() + }) + .try_init() + .map_err(|e| anyhow!("Failed to initialize logger: {}", e))?; + + info!("Starting"); let opts = RootOpts::from_args(); let config = opts.fleet_opts.build()?; - match run_command(&config, opts.command) { + match run_command(&config, opts.command).await { Ok(()) => { config.save()?; Ok(()) --- a/modules/fleet/meta.nix +++ b/modules/fleet/meta.nix @@ -45,6 +45,6 @@ hosts = fleet.hostsToAttrs (host: { modules = config.globalModules; }); - globalModules = import ../nixos/_modules.nix; + globalModules = import ../../nixos/modules/module-list.nix; }; } --- a/modules/nixos/_modules.nix +++ /dev/null @@ -1,5 +0,0 @@ -[ - ./fleetPkgs.nix - ./meta.nix - ./secrets.nix -] --- a/modules/nixos/fleetPkgs.nix +++ /dev/null @@ -1,3 +0,0 @@ -{ ... }: { - nixpkgs.overlays = [ (import ../../pkgs) ]; -} --- a/modules/nixos/meta.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ lib, ... }: -with lib; -{ - options = with types; { - tags = mkOption { - type = listOf str; - description = "Host tags"; - default = [ ]; - }; - network = mkOption { - type = submodule { - options = { - internalIps = mkOption { - type = listOf str; - description = "Internal ips"; - default = [ ]; - }; - externalIps = mkOption { - type = listOf str; - description = "External ips"; - default = [ ]; - }; - }; - }; - description = "Network definition of host"; - }; - }; - config = { - tags = [ "all" ]; - network = { }; - }; -} --- a/modules/nixos/secrets.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ lib, config, pkgs, ... }: - -with lib; - -let - sysConfig = config; - secretType = types.submodule ({ config, ... }: { - config = { - path = mkOptionDefault "/run/secrets/${config._module.args.name}"; - publicPath = mkOptionDefault (pkgs.writeText "pub-${config._module.args.name}" config.public); - }; - options = { - public = mkOption { - type = types.nullOr types.str; - description = "Secret public data"; - default = null; - }; - secret = mkOption { - type = types.nullOr types.str; - description = "Encrypted secret data"; - default = null; - }; - mode = mkOption { - type = types.str; - description = "Secret mode"; - default = "0440"; - }; - owner = mkOption { - type = types.str; - description = "Owner of the secret"; - default = "root"; - }; - group = mkOption { - type = types.str; - description = "Group of the secret"; - default = sysConfig.users.users.${config.owner}.group; - }; - - path = mkOption { - type = types.str; - description = "Path to the decrypted secret"; - }; - publicPath = mkOption { - type = types.package; - description = "Path to the public part of secret"; - }; - }; - }); - secretsFile = pkgs.writeTextFile { - name = "secrets.json"; - text = builtins.toJSON config.secrets; - }; -in -{ - options = { - secrets = mkOption { - type = types.attrsOf secretType; - default = { }; - description = "Host-local secrets"; - }; - }; - config = { - system.activationScripts.decryptSecrets = stringAfter [ "users" "groups" "specialfs" ] '' - 1>&2 echo "setting up secrets" - ${pkgs.fleet-install-secrets}/bin/fleet-install-secrets ${secretsFile} - ''; - }; -} --- /dev/null +++ b/nixos/fleetPkgs.nix @@ -0,0 +1,3 @@ +{ ... }: { + nixpkgs.overlays = [ (import ../pkgs) ]; +} --- /dev/null +++ b/nixos/meta.nix @@ -0,0 +1,32 @@ +{ lib, ... }: +with lib; +{ + options = with types; { + tags = mkOption { + type = listOf str; + description = "Host tags"; + default = [ ]; + }; + network = mkOption { + type = submodule { + options = { + internalIps = mkOption { + type = listOf str; + description = "Internal ips"; + default = [ ]; + }; + externalIps = mkOption { + type = listOf str; + description = "External ips"; + default = [ ]; + }; + }; + }; + description = "Network definition of host"; + }; + }; + config = { + tags = [ "all" ]; + network = { }; + }; +} --- /dev/null +++ b/nixos/modules/module-list.nix @@ -0,0 +1,5 @@ +[ + ../fleetPkgs.nix + ../meta.nix + ../secrets.nix +] --- /dev/null +++ b/nixos/secrets.nix @@ -0,0 +1,68 @@ +{ lib, config, pkgs, ... }: + +with lib; + +let + sysConfig = config; + secretType = types.submodule ({ config, ... }: { + config = { + path = mkOptionDefault "/run/secrets/${config._module.args.name}"; + publicPath = mkOptionDefault (pkgs.writeText "pub-${config._module.args.name}" config.public); + }; + options = { + public = mkOption { + type = types.nullOr types.str; + description = "Secret public data"; + default = null; + }; + secret = mkOption { + type = types.nullOr types.str; + description = "Encrypted secret data"; + default = null; + }; + mode = mkOption { + type = types.str; + description = "Secret mode"; + default = "0440"; + }; + owner = mkOption { + type = types.str; + description = "Owner of the secret"; + default = "root"; + }; + group = mkOption { + type = types.str; + description = "Group of the secret"; + default = sysConfig.users.users.${config.owner}.group; + }; + + path = mkOption { + type = types.str; + description = "Path to the decrypted secret"; + }; + publicPath = mkOption { + type = types.package; + description = "Path to the public part of secret"; + }; + }; + }); + secretsFile = pkgs.writeTextFile { + name = "secrets.json"; + text = builtins.toJSON config.secrets; + }; +in +{ + options = { + secrets = mkOption { + type = types.attrsOf secretType; + default = { }; + description = "Host-local secrets"; + }; + }; + config = { + system.activationScripts.decryptSecrets = stringAfter [ "users" "groups" "specialfs" ] '' + 1>&2 echo "setting up secrets" + ${pkgs.fleet-install-secrets}/bin/fleet-install-secrets ${secretsFile} + ''; + }; +} -- gitstuff