Browse Source

feat: rust assets fetch

dev
Dnomd343 2 years ago
parent
commit
70d0a8b028
  1. 129
      src/assets/src/main.rs

129
src/assets/src/main.rs

@ -1,18 +1,50 @@
use std::collections::{HashMap, HashSet};
use reqwest::Client;
use std::env::set_var;
use std::time::Duration;
use std::fs::File;
use std::io::Read;
use std::path::PathBuf;
use log::{debug, info, warn};
const TIMEOUT: u64 = 60;
async fn asset_fetch(url: &str) -> Result<String, String> {
const ASSETS_DIR: &str = "/cleardns/assets/";
/// Cut text line by line and remove invisible characters on both sides.
fn asset_tidy(data: &str) -> Vec<String> {
data.lines()
.map(|x| String::from(x.trim()))
.filter(|x| !x.is_empty())
.collect()
}
/// Remove duplicate elements from an array.
fn remove_dup(data: &Vec<String>) -> Vec<String> {
let mut result: Vec<String> = vec![];
let mut tmp: HashSet<String> = HashSet::new();
for val in data {
if tmp.insert(val.clone()) { // value already exist
result.push(val.clone());
}
}
result
}
/// Download the specified text file and organize it into a String array.
async fn http_fetch(url: &str, timeout: u64) -> Result<Vec<String>, String> {
let client = Client::builder()
.timeout(Duration::from_secs(TIMEOUT))
.timeout(Duration::from_secs(timeout))
.build().unwrap();
info!("Start downloading `{}`", url);
match client.get(url).send().await {
Ok(response) => {
match response.text().await {
Ok(text) => {
Ok(text)
info!("Asset `{}` download success", url);
Ok(asset_tidy(&text))
},
Err(err) => Err(format!("http content error: {}", err))
}
@ -21,17 +53,73 @@ async fn asset_fetch(url: &str) -> Result<String, String> {
}
}
/// Read the specified text file and organize it into a String array.
async fn local_fetch(path: &str) -> Result<Vec<String>, String> {
let mut path = String::from(path);
if !path.starts_with("/") { // relative path
let file_path = PathBuf::from(ASSETS_DIR).join(path);
path = String::from(file_path.to_str().unwrap());
}
match File::open(&path) {
Ok(mut file) => {
let mut text = String::new();
if let Err(err) = file.read_to_string(&mut text) {
return Err(format!("File `{}` read failed: {}", path, err));
};
info!("Asset `{}` read success", path);
Ok(asset_tidy(&text))
},
Err(err) => Err(format!("File `{}` open failed: {}", path, err)),
}
}
/// Get multiple resource data and merge them.
async fn asset_fetch(name: &str, sources: Vec<&str>) -> Vec<String> {
let is_remote = |src: &str| {
src.starts_with("http://") || src.starts_with("https://")
};
let mut contents: Vec<Vec<String>> = vec![];
for source in sources {
contents.push(match if is_remote(source) {
http_fetch(source.trim(), TIMEOUT).await
} else {
local_fetch(source.trim()).await
} {
Ok(data) => {
debug!("Asset source `{}` fetch success with {} items", source.trim(), data.len());
data
},
Err(err) => {
warn!("Asset source `{}` fetch failed: {}", source.trim(), err);
break;
}
});
}
let contents = remove_dup(&contents
.into_iter()
.flatten()
.collect::<Vec<String>>());
info!("Asset `{}` fetch complete with {} items", name, contents.len());
contents
}
async fn demo() {
println!("demo function start");
match asset_fetch("https://res.343.re/Share/cleardns/gfwlist.txt").await {
// match asset_fetch("https://res.343.re/Share/cleardns/gfwlist.txt", TIMEOUT).await {
// Ok(data) => {
// println!("{:?}", data);
// },
// Err(err) => println!("error -> {}", err)
// }
match local_fetch("../../tmp/gfwlist.txt").await {
Ok(data) => {
// println!("{}", data);
// println!("{:?}", data);
},
Err(err) => println!("error -> {}", err)
}
Err(err) => println!("error -> `{}`", err)
};
println!("demo function exit");
@ -40,11 +128,32 @@ async fn demo() {
#[tokio::main]
async fn main() {
// set_var("RUST_LOG", "debug");
set_var("RUST_LOG", "trace");
env_logger::init();
demo().await;
let d = vec![
"https://res.343.re/Share/cleardns/gfwlist.txt",
"/tmp/gfwlist.txt",
];
asset_fetch("test", d).await;
// let demo = vec![
// "baidu.com",
// "ip.343.re",
// "qq.com",
// "google.com",
// "res.343.re",
// "ip.343.re",
// "343.re",
// ];
// let demo = demo.into_iter()
// .map(|x| String::from(x))
// .collect();
// remove_dup(&demo);
// let data: &str = "dnomd343\n linjucong\n\nfuck\t\n7700\n";
// println!("{}", data);
// tidy(data);
println!("end demo");

Loading…
Cancel
Save