From 4469d5531eeaeb7bd958a7a7db621dc325fc115e Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Fri, 10 Mar 2023 14:39:24 +0800 Subject: [PATCH 1/5] feat: automatic retrying of downloads --- src/Cargo.lock | 458 +++++++++++++++++++++++++++++++++++++++- src/assets/Cargo.toml | 2 + src/assets/src/fetch.rs | 20 +- 3 files changed, 465 insertions(+), 15 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 6621f5f..fc1af73 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -32,6 +32,21 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" + [[package]] name = "assets" version = "0.1.0" @@ -39,6 +54,8 @@ dependencies = [ "env_logger", "log", "reqwest", + "reqwest-middleware", + "reqwest-retry", "tokio", ] @@ -56,6 +73,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-trait" +version = "0.1.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -119,6 +147,28 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -144,6 +194,50 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cxx" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "encoding_rs" version = "0.8.32" @@ -236,6 +330,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.26" @@ -243,6 +352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -251,6 +361,34 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +[[package]] +name = "futures-executor" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" + +[[package]] +name = "futures-macro" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.26" @@ -269,10 +407,29 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", ] [[package]] @@ -392,6 +549,30 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "idna" version = "0.3.0" @@ -419,13 +600,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "cfa919a82ea574332e2de6e74b4c36e74d41982b335080fa59d4ef31be20fdf3" dependencies = [ "libc", "windows-sys 0.45.0", @@ -472,9 +656,18 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] [[package]] name = "linux-raw-sys" @@ -482,6 +675,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -503,6 +706,16 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.6.2" @@ -542,6 +755,25 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -603,6 +835,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -627,6 +884,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.51" @@ -645,6 +908,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -692,6 +985,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", @@ -710,6 +1004,55 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c50db2c7ccd815f976473dd7d0bde296f8c3b77c383acf4fc021cdcf10852b" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + +[[package]] +name = "reqwest-retry" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d0fd6ef4c6d23790399fe15efc8d12cd9f3d4133958f9bd7801ee5cbaec6c4" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "futures", + "getrandom", + "http", + "hyper", + "parking_lot", + "reqwest", + "reqwest-middleware", + "retry-policies", + "task-local-extensions", + "tokio", + "tracing", + "wasm-timer", +] + +[[package]] +name = "retry-policies" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e09bbcb5003282bcb688f0bae741b278e9c7e8f378f561522c9806c58e075d9b" +dependencies = [ + "anyhow", + "chrono", + "rand", +] + [[package]] name = "rustix" version = "0.36.9" @@ -739,6 +1082,18 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + [[package]] name = "security-framework" version = "2.8.2" @@ -764,9 +1119,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "8cdd151213925e7f1ab45a9bbfb129316bd00799784b174b7cc7bcd16961c49e" [[package]] name = "serde_json" @@ -822,6 +1177,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + [[package]] name = "socket2" version = "0.4.9" @@ -843,6 +1204,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "task-local-extensions" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba323866e5d033818e3240feeb9f7db2c4296674e4d9e16b97b7bf8f490434e8" +dependencies = [ + "pin-utils", +] + [[package]] name = "tempfile" version = "3.4.0" @@ -865,6 +1235,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -990,9 +1380,21 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.30" @@ -1008,11 +1410,20 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "524b68aca1d05e03fdf03fcdce2c6c94b6daf6d16861ddaa7e4f2b6638a9052c" [[package]] name = "unicode-ident" @@ -1029,6 +1440,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "unsafe-libyaml" version = "0.2.7" @@ -1052,6 +1469,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.0" @@ -1134,6 +1557,21 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.61" @@ -1258,9 +1696,9 @@ checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winnow" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95fb4ff192527911dd18eb138ac30908e7165b8944e528b6af93aa4c842d345" +checksum = "ee7b2c67f962bf5042bfd8b6a916178df33a26eec343ae064cb8e069f638fa6f" dependencies = [ "memchr", ] diff --git a/src/assets/Cargo.toml b/src/assets/Cargo.toml index 761f136..839a81f 100644 --- a/src/assets/Cargo.toml +++ b/src/assets/Cargo.toml @@ -12,4 +12,6 @@ crate-type = ["staticlib"] env_logger = "0.10.0" log = "0.4.17" reqwest = { version = "0.11.14", features = ["deflate", "gzip", "brotli"] } +reqwest-middleware = "0.2.1" +reqwest-retry = "0.2.2" tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } diff --git a/src/assets/src/fetch.rs b/src/assets/src/fetch.rs index ca399ff..50bda38 100644 --- a/src/assets/src/fetch.rs +++ b/src/assets/src/fetch.rs @@ -1,9 +1,11 @@ use std::fs::File; use std::io::Read; -use reqwest::Client; use std::time::Duration; use log::{debug, info, warn}; use std::collections::HashSet; +use reqwest_middleware::ClientBuilder; +use reqwest_retry::RetryTransientMiddleware; +use reqwest_retry::policies::ExponentialBackoff; /// Http download timeout limit const TIMEOUT: u64 = 120; @@ -30,11 +32,19 @@ fn remove_dup(data: &Vec) -> Vec { /// Download the specified text file and organize it into a String array. async fn http_fetch(url: &str, timeout: u64) -> Result, String> { - let client = Client::builder() - .timeout(Duration::from_secs(timeout)) - .build().unwrap(); + let retry_policy = ExponentialBackoff::builder() + .backoff_exponent(2) // [2, 4, 8, 16, ...] + .retry_bounds( + Duration::from_secs(5), // min retry interval -> 1s + Duration::from_secs(60)) // max retry interval -> 60s + .build_with_max_retries(2); // total request 3 times + let client = ClientBuilder::new(reqwest::Client::new()) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) + .build(); debug!("Start downloading `{}`", url); - match client.get(url).send().await { + match client.get(url) + .timeout(Duration::from_secs(timeout)) + .send().await { Ok(response) => { match response.text().await { Ok(text) => { From 337706c49902dfaf761ba222f326001a1cd61539 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Fri, 10 Mar 2023 15:53:03 +0800 Subject: [PATCH 2/5] feat: new log format of rust modules --- include/utils/assets.h | 2 +- src/Cargo.lock | 25 +++++++++++++++++++++++-- src/assets/Cargo.toml | 1 + src/assets/src/fetch.rs | 6 +++--- src/assets/src/ffi.rs | 23 +++++++++++++++++++++-- src/cleardns.c | 4 ++-- 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/include/utils/assets.h b/include/utils/assets.h index 51279c1..4355f40 100644 --- a/include/utils/assets.h +++ b/include/utils/assets.h @@ -23,7 +23,7 @@ uint32_t assets_size(asset **asset_list); void assets_append(asset ***asset_list, asset *res); /// Rust assets interface -void assets_log_init(uint8_t verbose); +void assets_log_init(uint8_t verbose, const char *prefix); uint8_t asset_update(const char *file, char *const *sources, const char *assets_dir); #endif diff --git a/src/Cargo.lock b/src/Cargo.lock index fc1af73..e1937fd 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -51,6 +51,7 @@ checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" name = "assets" version = "0.1.0" dependencies = [ + "chrono", "env_logger", "log", "reqwest", @@ -154,8 +155,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", + "js-sys", "num-integer", "num-traits", + "time", + "wasm-bindgen", "winapi", ] @@ -428,7 +432,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -733,7 +737,7 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.45.0", ] @@ -1255,6 +1259,17 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1485,6 +1500,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/src/assets/Cargo.toml b/src/assets/Cargo.toml index 839a81f..c7b9a6e 100644 --- a/src/assets/Cargo.toml +++ b/src/assets/Cargo.toml @@ -9,6 +9,7 @@ crate-type = ["staticlib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +chrono = "0.4.23" env_logger = "0.10.0" log = "0.4.17" reqwest = { version = "0.11.14", features = ["deflate", "gzip", "brotli"] } diff --git a/src/assets/src/fetch.rs b/src/assets/src/fetch.rs index 50bda38..ba319a9 100644 --- a/src/assets/src/fetch.rs +++ b/src/assets/src/fetch.rs @@ -41,14 +41,14 @@ async fn http_fetch(url: &str, timeout: u64) -> Result, String> { let client = ClientBuilder::new(reqwest::Client::new()) .with(RetryTransientMiddleware::new_with_policy(retry_policy)) .build(); - debug!("Start downloading `{}`", url); + info!("Start downloading `{}`", url); match client.get(url) .timeout(Duration::from_secs(timeout)) .send().await { Ok(response) => { match response.text().await { Ok(text) => { - info!("Remote file `{}` download success", url); + debug!("Remote file `{}` download success", url); Ok(asset_tidy(&text)) }, Err(err) => Err(format!("http content error: {}", err)) @@ -66,7 +66,7 @@ async fn local_fetch(path: &str) -> Result, String> { if let Err(err) = file.read_to_string(&mut text) { return Err(format!("file `{}` read failed: {}", path, err)); }; - info!("Local file `{}` read success", path); + debug!("Local file `{}` read success", path); Ok(asset_tidy(&text)) }, Err(err) => Err(format!("file `{}` open failed: {}", path, err)), diff --git a/src/assets/src/ffi.rs b/src/assets/src/ffi.rs index 6cb3663..ac41a44 100644 --- a/src/assets/src/ffi.rs +++ b/src/assets/src/ffi.rs @@ -31,13 +31,32 @@ unsafe fn load_c_string_list(ptr: *const *const c_char) -> Vec { /// Initialize the rust module log, enable trace level log when verbose is not `0`. #[no_mangle] -pub unsafe extern "C" fn assets_log_init(verbose: u8) { +pub unsafe extern "C" fn assets_log_init(verbose: u8, prefix: *const c_char) { if verbose == FALSE { // bool value `FALSE` set_var("RUST_LOG", "info"); } else { set_var("RUST_LOG", "trace"); } - env_logger::init(); + let log_prefix = load_c_string(prefix); + env_logger::builder() + .target(env_logger::Target::Stderr) + .format(move |buf, record| { + let prefix = format!( + "\x1b[36m[{}]\x1b[0m \x1b[90m{}\x1b[0m \x1b[35m[{}]\x1b[0m", + log_prefix, + chrono::Local::now().format("%Y-%m-%d %H:%M:%S%.3f").to_string(), + record.module_path().unwrap() + ); + let level = format!("{}[{}]\x1b[0m", match record.level() { + log::Level::Trace => "\x1b[39m", + log::Level::Debug => "\x1b[36m", + log::Level::Info => "\x1b[32m", + log::Level::Warn => "\x1b[33m", + log::Level::Error => "\x1b[31m", + }, record.level()); + write!(buf, "{} {} {:?}\n", prefix, level, record.args()) + }) + .init(); } /// Update the specified resource file, return `0` on failure. diff --git a/src/cleardns.c b/src/cleardns.c index 492c8fc..75b1b93 100644 --- a/src/cleardns.c +++ b/src/cleardns.c @@ -66,9 +66,9 @@ void init(int argc, char *argv[]) { // return config file void cleardns() { // cleardns service if (settings.verbose || settings.debug) { LOG_LEVEL = LOG_DEBUG; // enable debug log level - assets_log_init(TRUE); + assets_log_init(TRUE, LOG_PREFIX); } else { - assets_log_init(FALSE); + assets_log_init(FALSE, LOG_PREFIX); } create_folder(EXPOSE_DIR); create_folder(WORK_DIR); From 40b90612765fe266129b4e45cfe731a3be5b4ac7 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Fri, 10 Mar 2023 17:24:31 +0800 Subject: [PATCH 3/5] perf: optimize cidr merge speed --- assets/china-ip.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/assets/china-ip.py b/assets/china-ip.py index 94f06dc..f964bd2 100755 --- a/assets/china-ip.py +++ b/assets/china-ip.py @@ -2,7 +2,9 @@ # -*- coding: utf-8 -*- import os -from IPy import IP, IPSet +from netaddr import IPSet +from netaddr import IPAddress +from netaddr import IPNetwork operators = ['china', 'cmcc', 'chinanet', 'unicom', 'tietong', 'cernet', 'cstnet', 'drpeng', 'googlecn'] operators += ['%s6' % x for x in operators] # add `...6` suffix @@ -12,20 +14,20 @@ source = [ 'curl -sL https://github.com/17mon/china_ip_list/raw/master/china_ip_list.txt', ] + ['curl -sL https://gaoyifan.github.io/china-operator-ip/%s.txt' % x for x in operators] -ipv4 = IPSet() -ipv6 = IPSet() ipAddrs = set() for script in source: # traverse fetch commands raw = os.popen(script).read().split('\n') ipAddrs.update(filter(None, raw)) -for ipAddr in ipAddrs: + +ipv4 = IPSet() +ipv6 = IPSet() +for ipAddr in ipAddrs: # load all IP data try: - ip = IP(ipAddr) # ip format check - (ipv4 if ip.version() == 4 else ipv6).add(ip) + ip = IPNetwork(ipAddr) if '/' in ipAddr else IPAddress(ipAddr) + ipv4.add(ip) if ip.version == 4 else ipv6.add(ip) except: pass -release = [('%s' if '/' in str(ip) else '%s/32') % str(ip) for ip in ipv4] # format into CIDR -release += [('%s' if '/' in str(ip) else '%s/128') % str(ip) for ip in ipv6] -with open('china-ip.txt', 'w') as fileObj: - fileObj.write('\n'.join(release) + '\n') +with open('china-ip.txt', 'w') as fileObj: # save to file + fileObj.write('\n'.join([str(ip) for ip in ipv4.iter_cidrs()]) + '\n') # format into CIDR + fileObj.write('\n'.join([str(ip) for ip in ipv6.iter_cidrs()]) + '\n') From 89c0086727d9a9e67a4b757007a6eff2b0c12b62 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Fri, 10 Mar 2023 18:26:26 +0800 Subject: [PATCH 4/5] build: Alpine already supports multi-architecture upx --- Dockerfile | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ca2166..3ea449d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,23 +2,12 @@ ARG ALPINE="alpine:3.17" ARG RUST="rust:1.67-alpine3.17" ARG GOLANG="golang:1.19-alpine3.17" -FROM ${ALPINE} AS upx -RUN apk add build-base cmake -ENV UPX="4.0.2" -RUN wget https://github.com/upx/upx/releases/download/v${UPX}/upx-${UPX}-src.tar.xz && tar xf upx-${UPX}-src.tar.xz -WORKDIR ./upx-${UPX}-src/ -RUN make UPX_CMAKE_CONFIG_FLAGS=-DCMAKE_EXE_LINKER_FLAGS=-static -WORKDIR ./build/release/ -RUN strip upx && mv upx /tmp/ - FROM ${GOLANG} AS dnsproxy ENV DNSPROXY="0.48.0" RUN wget https://github.com/AdguardTeam/dnsproxy/archive/refs/tags/v${DNSPROXY}.tar.gz && tar xf v${DNSPROXY}.tar.gz WORKDIR ./dnsproxy-${DNSPROXY}/ RUN go get RUN env CGO_ENABLED=0 go build -v -trimpath -ldflags "-X main.VersionString=${DNSPROXY} -s -w" && mv dnsproxy /tmp/ -COPY --from=upx /tmp/upx /usr/bin/ -RUN upx -9 /tmp/dnsproxy FROM ${GOLANG} AS overture ENV OVERTURE="1.8" @@ -26,8 +15,6 @@ RUN wget https://github.com/shawn1m/overture/archive/refs/tags/v${OVERTURE}.tar. WORKDIR ./overture-${OVERTURE}/main/ RUN go get RUN env CGO_ENABLED=0 go build -v -trimpath -ldflags "-X main.version=v${OVERTURE} -s -w" && mv main /tmp/overture -COPY --from=upx /tmp/upx /usr/bin/ -RUN upx -9 /tmp/overture FROM ${ALPINE} AS adguard-src RUN apk add git @@ -50,8 +37,6 @@ WORKDIR /AdGuardHome/ RUN go get COPY --from=adguard-web /tmp/static/ ./build/static/ RUN make CHANNEL="release" VERBOSE=1 go-build && mv AdGuardHome /tmp/ -COPY --from=upx /tmp/upx /usr/bin/ -RUN upx -9 /tmp/AdGuardHome FROM ${RUST} AS rust-mods RUN apk add libc-dev openssl-dev @@ -66,12 +51,10 @@ COPY ./ /cleardns/ COPY --from=rust-mods /tmp/libassets.a /cleardns/src/target/release/ COPY --from=rust-mods /tmp/libto_json.a /cleardns/src/target/release/ WORKDIR /cleardns/bin/ -RUN cmake -DCMAKE_EXE_LINKER_FLAGS=-static .. && make && mv cleardns /tmp/ -COPY --from=upx /tmp/upx /usr/bin/ -RUN strip /tmp/cleardns && upx -9 /tmp/cleardns +RUN cmake -DCMAKE_EXE_LINKER_FLAGS=-static .. && make && strip cleardns && mv cleardns /tmp/ FROM ${ALPINE} AS build -RUN apk add xz +RUN apk add upx xz WORKDIR /release/ RUN wget https://res.dnomd343.top/Share/cleardns/gfwlist.txt.xz && \ wget https://res.dnomd343.top/Share/cleardns/china-ip.txt.xz && \ @@ -81,6 +64,8 @@ COPY --from=cleardns /tmp/cleardns /release/usr/bin/ COPY --from=dnsproxy /tmp/dnsproxy /release/usr/bin/ COPY --from=overture /tmp/overture /release/usr/bin/ COPY --from=adguard /tmp/AdGuardHome /release/usr/bin/ +WORKDIR /release/usr/bin/ +RUN ls | xargs -n1 -P0 upx -9 FROM ${ALPINE} COPY --from=build /release/ / From 4751def29f40c5892b6fce5acc67ad081450eb64 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Fri, 10 Mar 2023 18:29:06 +0800 Subject: [PATCH 5/5] build: AdGuardHome version `0.107.26` --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3ea449d..23475c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN env CGO_ENABLED=0 go build -v -trimpath -ldflags "-X main.version=v${OVERTUR FROM ${ALPINE} AS adguard-src RUN apk add git -ENV ADGUARD="0.107.25" +ENV ADGUARD="0.107.26" RUN git clone https://github.com/AdguardTeam/AdGuardHome.git -b v${ADGUARD} --depth=1 FROM ${ALPINE} AS adguard-web