diff --git a/src/assets/src/main.rs b/src/assets/src/main.rs index 24f84fe..73381ba 100644 --- a/src/assets/src/main.rs +++ b/src/assets/src/main.rs @@ -1,18 +1,50 @@ +use std::collections::{HashMap, HashSet}; use reqwest::Client; use std::env::set_var; use std::time::Duration; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; + +use log::{debug, info, warn}; + const TIMEOUT: u64 = 60; -async fn asset_fetch(url: &str) -> Result { +const ASSETS_DIR: &str = "/cleardns/assets/"; + +/// Cut text line by line and remove invisible characters on both sides. +fn asset_tidy(data: &str) -> Vec { + data.lines() + .map(|x| String::from(x.trim())) + .filter(|x| !x.is_empty()) + .collect() +} + +/// Remove duplicate elements from an array. +fn remove_dup(data: &Vec) -> Vec { + let mut result: Vec = vec![]; + let mut tmp: HashSet = HashSet::new(); + for val in data { + if tmp.insert(val.clone()) { // value already exist + result.push(val.clone()); + } + } + result +} + +/// Download the specified text file and organize it into a String array. +async fn http_fetch(url: &str, timeout: u64) -> Result, String> { let client = Client::builder() - .timeout(Duration::from_secs(TIMEOUT)) + .timeout(Duration::from_secs(timeout)) .build().unwrap(); + info!("Start downloading `{}`", url); match client.get(url).send().await { Ok(response) => { match response.text().await { Ok(text) => { - Ok(text) + info!("Asset `{}` download success", url); + Ok(asset_tidy(&text)) }, Err(err) => Err(format!("http content error: {}", err)) } @@ -21,17 +53,73 @@ async fn asset_fetch(url: &str) -> Result { } } +/// Read the specified text file and organize it into a String array. +async fn local_fetch(path: &str) -> Result, String> { + let mut path = String::from(path); + if !path.starts_with("/") { // relative path + let file_path = PathBuf::from(ASSETS_DIR).join(path); + path = String::from(file_path.to_str().unwrap()); + } + match File::open(&path) { + Ok(mut file) => { + let mut text = String::new(); + if let Err(err) = file.read_to_string(&mut text) { + return Err(format!("File `{}` read failed: {}", path, err)); + }; + info!("Asset `{}` read success", path); + Ok(asset_tidy(&text)) + }, + Err(err) => Err(format!("File `{}` open failed: {}", path, err)), + } +} + +/// Get multiple resource data and merge them. +async fn asset_fetch(name: &str, sources: Vec<&str>) -> Vec { + let is_remote = |src: &str| { + src.starts_with("http://") || src.starts_with("https://") + }; + let mut contents: Vec> = vec![]; + for source in sources { + contents.push(match if is_remote(source) { + http_fetch(source.trim(), TIMEOUT).await + } else { + local_fetch(source.trim()).await + } { + Ok(data) => { + debug!("Asset source `{}` fetch success with {} items", source.trim(), data.len()); + data + }, + Err(err) => { + warn!("Asset source `{}` fetch failed: {}", source.trim(), err); + break; + } + }); + } + let contents = remove_dup(&contents + .into_iter() + .flatten() + .collect::>()); + info!("Asset `{}` fetch complete with {} items", name, contents.len()); + contents +} + async fn demo() { println!("demo function start"); - match asset_fetch("https://res.343.re/Share/cleardns/gfwlist.txt").await { + // match asset_fetch("https://res.343.re/Share/cleardns/gfwlist.txt", TIMEOUT).await { + // Ok(data) => { + // println!("{:?}", data); + // }, + // Err(err) => println!("error -> {}", err) + // } + + match local_fetch("../../tmp/gfwlist.txt").await { Ok(data) => { - // println!("{}", data); + // println!("{:?}", data); }, - Err(err) => println!("error -> {}", err) - } - + Err(err) => println!("error -> `{}`", err) + }; println!("demo function exit"); @@ -40,11 +128,32 @@ async fn demo() { #[tokio::main] async fn main() { - // set_var("RUST_LOG", "debug"); set_var("RUST_LOG", "trace"); env_logger::init(); - demo().await; + let d = vec![ + "https://res.343.re/Share/cleardns/gfwlist.txt", + "/tmp/gfwlist.txt", + ]; + asset_fetch("test", d).await; + + // let demo = vec![ + // "baidu.com", + // "ip.343.re", + // "qq.com", + // "google.com", + // "res.343.re", + // "ip.343.re", + // "343.re", + // ]; + // let demo = demo.into_iter() + // .map(|x| String::from(x)) + // .collect(); + // remove_dup(&demo); + + // let data: &str = "dnomd343\n linjucong\n\nfuck\t\n7700\n"; + // println!("{}", data); + // tidy(data); println!("end demo");