From 82e279b73fe14f017d5b61f7de5458340265e6b6 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Thu, 8 Dec 2022 16:42:39 +0800 Subject: [PATCH 1/3] perf: china-ip fetch script --- assets/china-ip.py | 52 +++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/assets/china-ip.py b/assets/china-ip.py index f4b0ecf..7a2f8ef 100755 --- a/assets/china-ip.py +++ b/assets/china-ip.py @@ -4,49 +4,27 @@ import os from IPy import IP, IPSet +operators = ['china', 'cmcc', 'chinanet', 'unicom', 'tietong', 'cernet', 'cstnet', 'drpeng', 'googlecn'] +operators += ['%s6' % x for x in operators] # add `...6` suffix source = [ 'curl -sL https://github.com/misakaio/chnroutes2/raw/master/chnroutes.txt | sed \'/^#/d\'', 'curl -sL https://github.com/metowolf/iplist/raw/master/data/special/china.txt', 'curl -sL https://github.com/17mon/china_ip_list/raw/master/china_ip_list.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/cernet.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/china.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/chinanet.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/cmcc.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/cstnet.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/drpeng.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/googlecn.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/tietong.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/unicom.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/cernet6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/china6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/chinanet6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/cmcc6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/cstnet6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/drpeng6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/googlecn6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/tietong6.txt', - 'curl -sL https://gaoyifan.github.io/china-operator-ip/unicom6.txt', -] +] + ['curl -sL https://gaoyifan.github.io/china-operator-ip/%s.txt' % x for x in operators] -ips = set() -for script in source: +ipv4 = IPSet() +ipv6 = IPSet() +ipAddrs = set() +for script in source: # traverse fetch commands raw = os.popen(script).read().split('\n') - ips.update(filter(None, raw)) - -v4 = IPSet() -v6 = IPSet() -for ip in ips: + ipAddrs.update(filter(None, raw)) +for ipAddr in ipAddrs: try: - ipAddr = IP(ip) - (v4 if ipAddr.version() == 4 else v6).add(ipAddr) - except: - pass - -ips = [] -for ip in v4: - ips.append(str(ip) + ('' if '/' in str(ip) else '/32')) -for ip in v6: - ips.append(str(ip) + ('' if '/' in str(ip) else '/128')) + ip = IP(ipAddr) # ip format check + (ipv4 if ip.version() == 4 else ipv6).add(ip) + except: pass +release = [('%s' if '/' in str(ip) else '%s/32') % str(ip) for ip in ipv4] # format into CIDR +release += [('%s' if '/' in str(ip) else '%s/128') % str(ip) for ip in ipv6] with open('china-ip.txt', 'w') as fileObj: - fileObj.write('\n'.join(ips) + '\n') + fileObj.write('\n'.join(release) + '\n') From 09e037675d6e5a04df57782e3e9e82a301e6233e Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Thu, 8 Dec 2022 16:49:47 +0800 Subject: [PATCH 2/3] perf: domain list fetch script --- assets/chinalist.py | 5 ++--- assets/gfwlist.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/assets/chinalist.py b/assets/chinalist.py index 72d926e..82f13d0 100755 --- a/assets/chinalist.py +++ b/assets/chinalist.py @@ -14,11 +14,10 @@ source = [ ] domains = set() -for script in source: +for script in source: # traverse fetch commands raw = os.popen(script).read().split('\n') domains.update(filter(None, raw)) regex = r'^(?=^.{3,255}$)[a-zA-Z0-9][a-zA-Z0-9\-]{0,62}(.[a-zA-Z0-9][a-zA-Z0-9\-]{0,62})+$' -domains = {x for x in domains if re.search(regex, str(x)) is not None} - +domains = {x for x in domains if re.search(regex, str(x)) is not None} # filter invalid domains with open('chinalist.txt', 'w') as fileObj: fileObj.write('\n'.join(sorted(domains)) + '\n') diff --git a/assets/gfwlist.py b/assets/gfwlist.py index 9c7bcaa..727d6c6 100755 --- a/assets/gfwlist.py +++ b/assets/gfwlist.py @@ -19,11 +19,10 @@ source = [ ] domains = set() -for script in source: +for script in source: # traverse fetch commands raw = os.popen(script).read().split('\n') domains.update(filter(None, raw)) regex = r'^(?=^.{3,255}$)[a-zA-Z0-9][a-zA-Z0-9\-]{0,62}(.[a-zA-Z0-9][a-zA-Z0-9\-]{0,62})+$' -domains = {x for x in domains if re.search(regex, str(x)) is not None} - +domains = {x for x in domains if re.search(regex, str(x)) is not None} # filter invalid domains with open('gfwlist.txt', 'w') as fileObj: fileObj.write('\n'.join(sorted(domains)) + '\n') From 3661bcfce792ee935d14014996df334c0c812cc7 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Thu, 8 Dec 2022 16:55:16 +0800 Subject: [PATCH 3/3] update: to_json file structure --- src/to-json/src/ffi.rs | 2 +- src/to-json/src/{convert.rs => json.rs} | 9 +++++---- src/to-json/src/lib.rs | 2 +- src/to-json/src/tests.rs | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) rename src/to-json/src/{convert.rs => json.rs} (63%) diff --git a/src/to-json/src/ffi.rs b/src/to-json/src/ffi.rs index dcaf66d..f728598 100644 --- a/src/to-json/src/ffi.rs +++ b/src/to-json/src/ffi.rs @@ -1,4 +1,4 @@ -use crate::convert::to_json; +use crate::json::to_json; use std::ffi::{c_char, CStr, CString}; fn to_c_string(string: String) -> *const c_char { // fetch c-style ptr of string diff --git a/src/to-json/src/convert.rs b/src/to-json/src/json.rs similarity index 63% rename from src/to-json/src/convert.rs rename to src/to-json/src/json.rs index 9393fad..8f79d44 100644 --- a/src/to-json/src/convert.rs +++ b/src/to-json/src/json.rs @@ -1,10 +1,11 @@ +use serde_json as json; use crate::parser::{parser, Value}; fn json_convert(content: &str) -> Result { // convert to JSON format let data = match parser(content)? { - Value::JSON(json) => serde_json::to_string(&json), - Value::YAML(yaml) => serde_json::to_string(&yaml), - Value::TOML(toml) => serde_json::to_string(&toml), + Value::JSON(_json) => json::to_string(&_json), + Value::YAML(_yaml) => json::to_string(&_yaml), + Value::TOML(_toml) => json::to_string(&_toml), }; match data { Ok(data) => Ok(data), @@ -15,6 +16,6 @@ fn json_convert(content: &str) -> Result { // convert to JSON fo pub fn to_json(content: &str) -> Option { // to JSON string match json_convert(content) { Ok(data) => Some(data), - Err(_) => None, + Err(_) => None, // convert failed } } diff --git a/src/to-json/src/lib.rs b/src/to-json/src/lib.rs index f04b9c3..d5af57d 100644 --- a/src/to-json/src/lib.rs +++ b/src/to-json/src/lib.rs @@ -1,4 +1,4 @@ mod ffi; +mod json; mod tests; mod parser; -mod convert; diff --git a/src/to-json/src/tests.rs b/src/to-json/src/tests.rs index ee74871..3c832c9 100644 --- a/src/to-json/src/tests.rs +++ b/src/to-json/src/tests.rs @@ -1,4 +1,4 @@ -use crate::convert::to_json; +use crate::json::to_json; #[allow(dead_code)] const JSON_TEST_CONTENT: &str = r#"