feat: rust assets fetch

src/assets/src/main.rs

@@ -1,18 +1,50 @@
+use std::collections::{HashMap, HashSet};
 use reqwest::Client;
 use std::env::set_var;
 use std::time::Duration;
 
+use std::fs::File;
+use std::io::Read;
+use std::path::PathBuf;
+
+use log::{debug, info, warn};
+
 const TIMEOUT: u64 = 60;
 
-async fn asset_fetch(url: &str) -> Result<String, String> {
+const ASSETS_DIR: &str = "/cleardns/assets/";
+
+/// Cut text line by line and remove invisible characters on both sides.
+fn asset_tidy(data: &str) -> Vec<String> {
+    data.lines()
+        .map(|x| String::from(x.trim()))
+        .filter(|x| !x.is_empty())
+        .collect()
+}
+
+/// Remove duplicate elements from an array.
+fn remove_dup(data: &Vec<String>) -> Vec<String> {
+    let mut result: Vec<String> = vec![];
+    let mut tmp: HashSet<String> = HashSet::new();
+    for val in data {
+        if tmp.insert(val.clone()) { // value already exist
+            result.push(val.clone());
+        }
+    }
+    result
+}
+
+/// Download the specified text file and organize it into a String array.
+async fn http_fetch(url: &str, timeout: u64) -> Result<Vec<String>, String> {
     let client = Client::builder()
-        .timeout(Duration::from_secs(TIMEOUT))
+        .timeout(Duration::from_secs(timeout))
         .build().unwrap();
+    info!("Start downloading `{}`", url);
     match client.get(url).send().await {
         Ok(response) => {
             match response.text().await {
                 Ok(text) => {
-                    Ok(text)
+                    info!("Asset `{}` download success", url);
+                    Ok(asset_tidy(&text))
                 },
                 Err(err) => Err(format!("http content error: {}", err))
             }
@@ -21,17 +53,73 @@ async fn asset_fetch(url: &str) -> Result<String, String> {
     }
 }
 
+/// Read the specified text file and organize it into a String array.
+async fn local_fetch(path: &str) -> Result<Vec<String>, String> {
+    let mut path = String::from(path);
+    if !path.starts_with("/") { // relative path
+        let file_path = PathBuf::from(ASSETS_DIR).join(path);
+        path = String::from(file_path.to_str().unwrap());
+    }
+    match File::open(&path) {
+        Ok(mut file) => {
+            let mut text = String::new();
+            if let Err(err) = file.read_to_string(&mut text) {
+                return Err(format!("File `{}` read failed: {}", path, err));
+            };
+            info!("Asset `{}` read success", path);
+            Ok(asset_tidy(&text))
+        },
+        Err(err) => Err(format!("File `{}` open failed: {}", path, err)),
+    }
+}
+
+/// Get multiple resource data and merge them.
+async fn asset_fetch(name: &str, sources: Vec<&str>) -> Vec<String> {
+    let is_remote = |src: &str| {
+        src.starts_with("http://") || src.starts_with("https://")
+    };
+    let mut contents: Vec<Vec<String>> = vec![];
+    for source in sources {
+        contents.push(match if is_remote(source) {
+            http_fetch(source.trim(), TIMEOUT).await
+        } else {
+            local_fetch(source.trim()).await
+        } {
+            Ok(data) => {
+                debug!("Asset source `{}` fetch success with {} items", source.trim(), data.len());
+                data
+            },
+            Err(err) => {
+                warn!("Asset source `{}` fetch failed: {}", source.trim(), err);
+                break;
+            }
+        });
+    }
+    let contents = remove_dup(&contents
+        .into_iter()
+        .flatten()
+        .collect::<Vec<String>>());
+    info!("Asset `{}` fetch complete with {} items", name, contents.len());
+    contents
+}
+
 async fn demo() {
 
     println!("demo function start");
 
-    match asset_fetch("https://res.343.re/Share/cleardns/gfwlist.txt").await {
+    // match asset_fetch("https://res.343.re/Share/cleardns/gfwlist.txt", TIMEOUT).await {
+    //     Ok(data) => {
+    //         println!("{:?}", data);
+    //     },
+    //     Err(err) => println!("error -> {}", err)
+    // }
+
+    match local_fetch("../../tmp/gfwlist.txt").await {
         Ok(data) => {
-            // println!("{}", data);
+            // println!("{:?}", data);
         },
-        Err(err) => println!("error -> {}", err)
+        Err(err) => println!("error -> `{}`", err)
-    }
+    };
-
 
     println!("demo function exit");
 
@@ -40,11 +128,32 @@ async fn demo() {
 #[tokio::main]
 async fn main() {
 
-    // set_var("RUST_LOG", "debug");
     set_var("RUST_LOG", "trace");
     env_logger::init();
 
-    demo().await;
+    let d = vec![
+        "https://res.343.re/Share/cleardns/gfwlist.txt",
+        "/tmp/gfwlist.txt",
+    ];
+    asset_fetch("test", d).await;
+
+    // let demo = vec![
+    //     "baidu.com",
+    //     "ip.343.re",
+    //     "qq.com",
+    //     "google.com",
+    //     "res.343.re",
+    //     "ip.343.re",
+    //     "343.re",
+    // ];
+    // let demo = demo.into_iter()
+    //     .map(|x| String::from(x))
+    //     .collect();
+    // remove_dup(&demo);
+
+    // let data: &str = "dnomd343\n linjucong\n\nfuck\t\n7700\n";
+    // println!("{}", data);
+    // tidy(data);
 
     println!("end demo");