feat: combine asset data from multiple sources, ref #3

include/utils/assets.h

@@ -1,25 +1,25 @@
 #ifndef ASSETS_H_
 #define ASSETS_H_
 
+#include <stdint.h>
+
 typedef struct {
     char *file; // string
     char **sources; // string list
 } asset;
 
-asset* asset_init(const char *name);
+void assets_extract();
+void assets_load(asset **info);
 
 asset** assets_init();
+asset* asset_init(const char *name);
 void assets_dump(asset **asset_list);
 void assets_free(asset **asset_list);
 uint32_t assets_size(asset **asset_list);
 void assets_append(asset ***asset_list, asset *res);
 
-void assets_load(asset **info);
+/// Rust assets interface
-
+void assets_log_init(uint8_t verbose);
-void assets_extract();
+uint8_t asset_update(const char *file, char *const *sources, const char *assets_dir);
-
-//void assets_log_init(uint8_t verbose);
-//
-//uint8_t rust_assets_update(const char *file, char *const *sources, const char *assets_dir);
 
 #endif

src/CMakeLists.txt

@@ -22,4 +22,4 @@ add_executable(cleardns cleardns.c)
 target_link_libraries(cleardns utils applet bcrypt common loader)
 
 # TODO: just for test
-target_link_libraries(cleardns assets)
+target_link_libraries(cleardns assets ssl crypto)

src/assets/src/fetch.rs

@@ -1,18 +1,12 @@
 use std::fs::File;
 use std::io::Read;
 use reqwest::Client;
-use std::path::PathBuf;
 use std::time::Duration;
 use log::{debug, info, warn};
 use std::collections::HashSet;
 
-#[derive(Debug)]
+/// Http download timeout limit
-pub(crate) struct Asset {
+const TIMEOUT: u64 = 120;
-    pub(crate) name: String,
-    pub(crate) timeout: u64,
-    pub(crate) workdir: String, // assets folder
-    pub(crate) sources: Vec<String>, // http url or file path
-}
 
 /// Cut text line by line and remove invisible characters on both sides.
 fn asset_tidy(data: &str) -> Vec<String> {
@@ -39,12 +33,12 @@ async fn http_fetch(url: &str, timeout: u64) -> Result<Vec<String>, String> {
     let client = Client::builder()
         .timeout(Duration::from_secs(timeout))
         .build().unwrap();
-    info!("Start downloading `{}`", url);
+    debug!("Start downloading `{}`", url);
     match client.get(url).send().await {
         Ok(response) => {
             match response.text().await {
                 Ok(text) => {
-                    info!("Asset `{}` download success", url);
+                    info!("Remote file `{}` download success", url);
                     Ok(asset_tidy(&text))
                 },
                 Err(err) => Err(format!("http content error: {}", err))
@@ -55,36 +49,31 @@ async fn http_fetch(url: &str, timeout: u64) -> Result<Vec<String>, String> {
 }
 
 /// Read the specified text file and organize it into a String array.
-async fn local_fetch(path: &str, workdir: &str) -> Result<Vec<String>, String> {
+async fn local_fetch(path: &str) -> Result<Vec<String>, String> {
-    let mut path = String::from(path);
+    match File::open(path) {
-    if !path.starts_with("/") { // relative path
-        let file_path = PathBuf::from(workdir).join(path);
-        path = String::from(file_path.to_str().unwrap());
-    }
-    match File::open(&path) {
         Ok(mut file) => {
             let mut text = String::new();
             if let Err(err) = file.read_to_string(&mut text) {
-                return Err(format!("File `{}` read failed: {}", path, err));
+                return Err(format!("file `{}` read failed: {}", path, err));
             };
-            info!("Asset `{}` read success", path);
+            info!("Local file `{}` read success", path);
             Ok(asset_tidy(&text))
         },
-        Err(err) => Err(format!("File `{}` open failed: {}", path, err)),
+        Err(err) => Err(format!("file `{}` open failed: {}", path, err)),
     }
 }
 
 /// Get multiple resource data and merge them.
-pub(crate) async fn asset_fetch(info: &Asset) -> Vec<String> {
+pub(crate) async fn asset_fetch(name: &str, sources: &Vec<String>) -> Option<Vec<String>> {
     let is_remote = |src: &str| {
         src.starts_with("http://") || src.starts_with("https://")
     };
     let mut contents: Vec<Vec<String>> = vec![];
-    for source in &info.sources {
+    for source in sources {
         contents.push(match if is_remote(&source) {
-            http_fetch(source.trim(), info.timeout).await
+            http_fetch(source.trim(), TIMEOUT).await // from remote text file
         } else {
-            local_fetch(source.trim(), &info.workdir).await
+            local_fetch(source.trim()).await // from local text file
         } {
             Ok(data) => {
                 debug!("Asset source `{}` fetch success with {} items", source.trim(), data.len());
@@ -92,7 +81,7 @@ pub(crate) async fn asset_fetch(info: &Asset) -> Vec<String> {
             },
             Err(err) => {
                 warn!("Asset source `{}` fetch failed: {}", source.trim(), err);
-                break;
+                return None; // stop fetch process
             }
         });
     }
@@ -100,6 +89,6 @@ pub(crate) async fn asset_fetch(info: &Asset) -> Vec<String> {
         .into_iter()
         .flatten()
         .collect::<Vec<String>>());
-    info!("Asset `{}` fetch complete with {} items", info.name, contents.len());
+    info!("Asset `{}` fetch complete with {} items", name, contents.len());
-    contents
+    Some(contents)
 }

src/assets/src/ffi.rs

@@ -1,7 +1,15 @@
-use log::debug;
+use std::io::Write;
 use std::env::set_var;
+use std::path::PathBuf;
+use std::fs::OpenOptions;
 use std::os::raw::c_char;
+use log::{debug, trace, warn};
 use std::ffi::{CStr, CString};
+use crate::fetch::asset_fetch;
+
+/// Compatible with C89 bool value.
+const TRUE: u8 = 1;
+const FALSE: u8 = 0;
 
 /// Load c-style string from `char *` pointer.
 unsafe fn load_c_string(ptr: *const c_char) -> String {
@@ -21,9 +29,10 @@ unsafe fn load_c_string_list(ptr: *const *const c_char) -> Vec<String> {
     string_list
 }
 
+/// Initialize the rust module log, enable trace level log when verbose is not `0`.
 #[no_mangle]
 pub unsafe extern "C" fn assets_log_init(verbose: u8) {
-    if verbose == 0 { // bool value `FALSE`
+    if verbose == FALSE { // bool value `FALSE`
         set_var("RUST_LOG", "info");
     } else {
         set_var("RUST_LOG", "trace");
@@ -31,17 +40,63 @@ pub unsafe extern "C" fn assets_log_init(verbose: u8) {
     env_logger::init();
 }
 
+/// Update the specified resource file, return `0` on failure.
 #[no_mangle]
-pub unsafe extern "C" fn rust_assets_update(
+#[tokio::main]
+pub async unsafe extern "C" fn asset_update(
     file: *const c_char, sources: *const *const c_char, assets_dir: *const c_char) -> u8 {
 
-    let file = load_c_string(file); // import c-style string
+    let name = load_c_string(file); // import c-style string
     let sources = load_c_string_list(sources);
     let assets_dir = load_c_string(assets_dir);
+    trace!("Working folder is `{}`", assets_dir);
+    trace!("Updating `{}` from {:?}", name, sources);
 
-    debug!("file: {}", file);
+    let assets_dir = PathBuf::from(&assets_dir);
-    debug!("source: {:?}", sources);
+    let is_remote = |src: &str| {
-    debug!("assets dir: {}", assets_dir);
+        src.starts_with("http://") || src.starts_with("https://")
+    };
+    let sources = sources.iter()
+        .map(|src| {
+            if !is_remote(&src) && !src.starts_with("/") { // local relative path
+                let file_path = assets_dir.join(src);
+                String::from(file_path.to_str().unwrap())
+            } else {
+                src.clone()
+            }
+        })
+        .collect::<Vec<String>>();
+    let file = String::from(assets_dir.join(&name).to_str().unwrap());
+    debug!("Asset sources -> {:?}", sources);
+    debug!("Asset target -> `{}`", file);
 
-    0
+    match asset_fetch(&name, &sources).await {
+        Some(data) => {
+            let mut content = String::new();
+            let _ = data.iter().map(|item| {
+                content.push_str(item);
+                content.push('\n');
+            }).collect::<Vec<()>>();
+            match OpenOptions::new()
+                .write(true)
+                .create(true)
+                .open(&file) { // open target file
+                Ok(mut fp) => {
+                    match fp.write_all(content.trim().as_ref()) {
+                        Err(err) => {
+                            warn!("File `{}` save error: {}", file, err);
+                        }
+                        _ => {
+                            debug!("File `{}` save success", file);
+                        },
+                    }
+                },
+                Err(err) => {
+                    warn!("File `{}` open failed: {}", file, err);
+                },
+            };
+            TRUE // asset fetch success
+        },
+        None => FALSE, // asset fetch failed
+    }
 }

src/assets/src/lib.rs

@@ -1,31 +1,2 @@
 mod ffi;
-
+mod fetch;
-// use std::ffi::{c_char, CStr};
-
-// const TIMEOUT: u64 = 60;
-//
-// const ASSETS_DIR: &str = "/cleardns/assets/";
-
-
-// #[tokio::main]
-// async fn main() {
-//
-//     set_var("RUST_LOG", "trace");
-//     env_logger::init();
-//
-//     let d = vec![
-//         String::from("https://res.343.re/Share/cleardns/gfwlist.txt"),
-//         String::from("/tmp/gfwlist.txt"),
-//     ];
-//     let info = Asset {
-//         name: String::from("demo"),
-//         timeout: TIMEOUT,
-//         workdir: String::from(ASSETS_DIR),
-//         sources: d,
-//     };
-//     fetch::asset_fetch(&info).await;
-//
-//
-//     println!("end demo");
-//
-// }

src/cleardns.c

@@ -66,8 +66,10 @@ void init(int argc, char *argv[]) { // return config file
 
 void cleardns() { // cleardns service
     if (settings.verbose || settings.debug) {
-        // TODO: rust log module settings
         LOG_LEVEL = LOG_DEBUG; // enable debug log level
+        assets_log_init(TRUE);
+    } else {
+        assets_log_init(FALSE);
     }
     create_folder(EXPOSE_DIR);
     create_folder(WORK_DIR);

src/utils/assets.c

@@ -72,18 +72,19 @@ void assets_update_entry() { // receive SIGALRM for update all assets
         log_info("Skip update assets");
         return;
     }
-    log_info("Start assets update");
+    log_info("Start updating assets");
-
+    for (asset **res = update_info; *res != NULL; ++res) {
-    // TODO: call rust `assets_update` function
+        char *content = string_list_dump((*res)->sources);
-//    for (char **file = update.update_file; *file != NULL; ++file) {
+        log_debug("Updating `%s` -> %s", (*res)->file, content);
-//        char **url = file - update.update_file + update.update_url;
+        if (asset_update((*res)->file, (*res)->sources, ASSETS_DIR)) {
-//        char *asset_file = string_join(ASSETS_DIR, *file);
+            log_debug("Asset `%s` update success", (*res)->file);
-//        log_info("Update asset `%s` -> %s", asset_file, *url);
+        } else {
-//        download_file(asset_file, *url); // download asset from url
+            log_warn("Asset `%s` update failed", (*res)->file);
-//        free(asset_file);
+        }
-//    }
+        free(content);
-
+    }
-    log_info("Restart overture");
+    // TODO: refresh `/etc/cleardns/*.txt`
+    log_info("Restart overture to apply new assets");
     run_command("pgrep overture | xargs kill"); // restart overture
     log_info("Assets update complete");
 }