From 972982425b9f418aa9dc0717f1ed94c532675124 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Mon, 6 Mar 2023 17:39:19 +0800 Subject: [PATCH] feat: combine asset data from multiple sources, ref #3 --- include/utils/assets.h | 16 +++++----- src/CMakeLists.txt | 2 +- src/assets/src/fetch.rs | 43 ++++++++++--------------- src/assets/src/ffi.rs | 71 ++++++++++++++++++++++++++++++++++++----- src/assets/src/lib.rs | 31 +----------------- src/cleardns.c | 4 ++- src/utils/assets.c | 25 ++++++++------- 7 files changed, 105 insertions(+), 87 deletions(-) diff --git a/include/utils/assets.h b/include/utils/assets.h index 3d6da0d..a0fb8b7 100644 --- a/include/utils/assets.h +++ b/include/utils/assets.h @@ -1,25 +1,25 @@ #ifndef ASSETS_H_ #define ASSETS_H_ +#include + typedef struct { char *file; // string char **sources; // string list } asset; -asset* asset_init(const char *name); +void assets_extract(); +void assets_load(asset **info); asset** assets_init(); +asset* asset_init(const char *name); void assets_dump(asset **asset_list); void assets_free(asset **asset_list); uint32_t assets_size(asset **asset_list); void assets_append(asset ***asset_list, asset *res); -void assets_load(asset **info); - -void assets_extract(); - -//void assets_log_init(uint8_t verbose); -// -//uint8_t rust_assets_update(const char *file, char *const *sources, const char *assets_dir); +/// Rust assets interface +void assets_log_init(uint8_t verbose); +uint8_t asset_update(const char *file, char *const *sources, const char *assets_dir); #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6db7fa3..f6a05c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,4 +22,4 @@ add_executable(cleardns cleardns.c) target_link_libraries(cleardns utils applet bcrypt common loader) # TODO: just for test -target_link_libraries(cleardns assets) +target_link_libraries(cleardns assets ssl crypto) diff --git a/src/assets/src/fetch.rs b/src/assets/src/fetch.rs index a015399..b5a6e5e 100644 --- a/src/assets/src/fetch.rs +++ b/src/assets/src/fetch.rs @@ -1,18 +1,12 @@ use std::fs::File; use std::io::Read; use reqwest::Client; -use std::path::PathBuf; use std::time::Duration; use log::{debug, info, warn}; use std::collections::HashSet; -#[derive(Debug)] -pub(crate) struct Asset { - pub(crate) name: String, - pub(crate) timeout: u64, - pub(crate) workdir: String, // assets folder - pub(crate) sources: Vec, // http url or file path -} +/// Http download timeout limit +const TIMEOUT: u64 = 120; /// Cut text line by line and remove invisible characters on both sides. fn asset_tidy(data: &str) -> Vec { @@ -39,12 +33,12 @@ async fn http_fetch(url: &str, timeout: u64) -> Result, String> { let client = Client::builder() .timeout(Duration::from_secs(timeout)) .build().unwrap(); - info!("Start downloading `{}`", url); + debug!("Start downloading `{}`", url); match client.get(url).send().await { Ok(response) => { match response.text().await { Ok(text) => { - info!("Asset `{}` download success", url); + info!("Remote file `{}` download success", url); Ok(asset_tidy(&text)) }, Err(err) => Err(format!("http content error: {}", err)) @@ -55,36 +49,31 @@ async fn http_fetch(url: &str, timeout: u64) -> Result, String> { } /// Read the specified text file and organize it into a String array. -async fn local_fetch(path: &str, workdir: &str) -> Result, String> { - let mut path = String::from(path); - if !path.starts_with("/") { // relative path - let file_path = PathBuf::from(workdir).join(path); - path = String::from(file_path.to_str().unwrap()); - } - match File::open(&path) { +async fn local_fetch(path: &str) -> Result, String> { + match File::open(path) { Ok(mut file) => { let mut text = String::new(); if let Err(err) = file.read_to_string(&mut text) { - return Err(format!("File `{}` read failed: {}", path, err)); + return Err(format!("file `{}` read failed: {}", path, err)); }; - info!("Asset `{}` read success", path); + info!("Local file `{}` read success", path); Ok(asset_tidy(&text)) }, - Err(err) => Err(format!("File `{}` open failed: {}", path, err)), + Err(err) => Err(format!("file `{}` open failed: {}", path, err)), } } /// Get multiple resource data and merge them. -pub(crate) async fn asset_fetch(info: &Asset) -> Vec { +pub(crate) async fn asset_fetch(name: &str, sources: &Vec) -> Option> { let is_remote = |src: &str| { src.starts_with("http://") || src.starts_with("https://") }; let mut contents: Vec> = vec![]; - for source in &info.sources { + for source in sources { contents.push(match if is_remote(&source) { - http_fetch(source.trim(), info.timeout).await + http_fetch(source.trim(), TIMEOUT).await // from remote text file } else { - local_fetch(source.trim(), &info.workdir).await + local_fetch(source.trim()).await // from local text file } { Ok(data) => { debug!("Asset source `{}` fetch success with {} items", source.trim(), data.len()); @@ -92,7 +81,7 @@ pub(crate) async fn asset_fetch(info: &Asset) -> Vec { }, Err(err) => { warn!("Asset source `{}` fetch failed: {}", source.trim(), err); - break; + return None; // stop fetch process } }); } @@ -100,6 +89,6 @@ pub(crate) async fn asset_fetch(info: &Asset) -> Vec { .into_iter() .flatten() .collect::>()); - info!("Asset `{}` fetch complete with {} items", info.name, contents.len()); - contents + info!("Asset `{}` fetch complete with {} items", name, contents.len()); + Some(contents) } diff --git a/src/assets/src/ffi.rs b/src/assets/src/ffi.rs index 4dfa13b..513b289 100644 --- a/src/assets/src/ffi.rs +++ b/src/assets/src/ffi.rs @@ -1,7 +1,15 @@ -use log::debug; +use std::io::Write; use std::env::set_var; +use std::path::PathBuf; +use std::fs::OpenOptions; use std::os::raw::c_char; +use log::{debug, trace, warn}; use std::ffi::{CStr, CString}; +use crate::fetch::asset_fetch; + +/// Compatible with C89 bool value. +const TRUE: u8 = 1; +const FALSE: u8 = 0; /// Load c-style string from `char *` pointer. unsafe fn load_c_string(ptr: *const c_char) -> String { @@ -21,9 +29,10 @@ unsafe fn load_c_string_list(ptr: *const *const c_char) -> Vec { string_list } +/// Initialize the rust module log, enable trace level log when verbose is not `0`. #[no_mangle] pub unsafe extern "C" fn assets_log_init(verbose: u8) { - if verbose == 0 { // bool value `FALSE` + if verbose == FALSE { // bool value `FALSE` set_var("RUST_LOG", "info"); } else { set_var("RUST_LOG", "trace"); @@ -31,17 +40,63 @@ pub unsafe extern "C" fn assets_log_init(verbose: u8) { env_logger::init(); } +/// Update the specified resource file, return `0` on failure. #[no_mangle] -pub unsafe extern "C" fn rust_assets_update( +#[tokio::main] +pub async unsafe extern "C" fn asset_update( file: *const c_char, sources: *const *const c_char, assets_dir: *const c_char) -> u8 { - let file = load_c_string(file); // import c-style string + let name = load_c_string(file); // import c-style string let sources = load_c_string_list(sources); let assets_dir = load_c_string(assets_dir); + trace!("Working folder is `{}`", assets_dir); + trace!("Updating `{}` from {:?}", name, sources); - debug!("file: {}", file); - debug!("source: {:?}", sources); - debug!("assets dir: {}", assets_dir); + let assets_dir = PathBuf::from(&assets_dir); + let is_remote = |src: &str| { + src.starts_with("http://") || src.starts_with("https://") + }; + let sources = sources.iter() + .map(|src| { + if !is_remote(&src) && !src.starts_with("/") { // local relative path + let file_path = assets_dir.join(src); + String::from(file_path.to_str().unwrap()) + } else { + src.clone() + } + }) + .collect::>(); + let file = String::from(assets_dir.join(&name).to_str().unwrap()); + debug!("Asset sources -> {:?}", sources); + debug!("Asset target -> `{}`", file); - 0 + match asset_fetch(&name, &sources).await { + Some(data) => { + let mut content = String::new(); + let _ = data.iter().map(|item| { + content.push_str(item); + content.push('\n'); + }).collect::>(); + match OpenOptions::new() + .write(true) + .create(true) + .open(&file) { // open target file + Ok(mut fp) => { + match fp.write_all(content.trim().as_ref()) { + Err(err) => { + warn!("File `{}` save error: {}", file, err); + } + _ => { + debug!("File `{}` save success", file); + }, + } + }, + Err(err) => { + warn!("File `{}` open failed: {}", file, err); + }, + }; + TRUE // asset fetch success + }, + None => FALSE, // asset fetch failed + } } diff --git a/src/assets/src/lib.rs b/src/assets/src/lib.rs index 7b003b3..937b6d5 100644 --- a/src/assets/src/lib.rs +++ b/src/assets/src/lib.rs @@ -1,31 +1,2 @@ mod ffi; - -// use std::ffi::{c_char, CStr}; - -// const TIMEOUT: u64 = 60; -// -// const ASSETS_DIR: &str = "/cleardns/assets/"; - - -// #[tokio::main] -// async fn main() { -// -// set_var("RUST_LOG", "trace"); -// env_logger::init(); -// -// let d = vec![ -// String::from("https://res.343.re/Share/cleardns/gfwlist.txt"), -// String::from("/tmp/gfwlist.txt"), -// ]; -// let info = Asset { -// name: String::from("demo"), -// timeout: TIMEOUT, -// workdir: String::from(ASSETS_DIR), -// sources: d, -// }; -// fetch::asset_fetch(&info).await; -// -// -// println!("end demo"); -// -// } +mod fetch; diff --git a/src/cleardns.c b/src/cleardns.c index 6df1daf..fdfda2a 100644 --- a/src/cleardns.c +++ b/src/cleardns.c @@ -66,8 +66,10 @@ void init(int argc, char *argv[]) { // return config file void cleardns() { // cleardns service if (settings.verbose || settings.debug) { - // TODO: rust log module settings LOG_LEVEL = LOG_DEBUG; // enable debug log level + assets_log_init(TRUE); + } else { + assets_log_init(FALSE); } create_folder(EXPOSE_DIR); create_folder(WORK_DIR); diff --git a/src/utils/assets.c b/src/utils/assets.c index e40086b..73977da 100644 --- a/src/utils/assets.c +++ b/src/utils/assets.c @@ -72,18 +72,19 @@ void assets_update_entry() { // receive SIGALRM for update all assets log_info("Skip update assets"); return; } - log_info("Start assets update"); - - // TODO: call rust `assets_update` function -// for (char **file = update.update_file; *file != NULL; ++file) { -// char **url = file - update.update_file + update.update_url; -// char *asset_file = string_join(ASSETS_DIR, *file); -// log_info("Update asset `%s` -> %s", asset_file, *url); -// download_file(asset_file, *url); // download asset from url -// free(asset_file); -// } - - log_info("Restart overture"); + log_info("Start updating assets"); + for (asset **res = update_info; *res != NULL; ++res) { + char *content = string_list_dump((*res)->sources); + log_debug("Updating `%s` -> %s", (*res)->file, content); + if (asset_update((*res)->file, (*res)->sources, ASSETS_DIR)) { + log_debug("Asset `%s` update success", (*res)->file); + } else { + log_warn("Asset `%s` update failed", (*res)->file); + } + free(content); + } + // TODO: refresh `/etc/cleardns/*.txt` + log_info("Restart overture to apply new assets"); run_command("pgrep overture | xargs kill"); // restart overture log_info("Assets update complete"); }