Merge branch 'dev'

assets/china-ip.py

@@ -4,49 +4,27 @@
 import os
 from IPy import IP, IPSet
 
+operators = ['china', 'cmcc', 'chinanet', 'unicom', 'tietong', 'cernet', 'cstnet', 'drpeng', 'googlecn']
+operators += ['%s6' % x for x in operators]  # add `...6` suffix
 source = [
     'curl -sL https://github.com/misakaio/chnroutes2/raw/master/chnroutes.txt | sed \'/^#/d\'',
     'curl -sL https://github.com/metowolf/iplist/raw/master/data/special/china.txt',
     'curl -sL https://github.com/17mon/china_ip_list/raw/master/china_ip_list.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/cernet.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/china.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/chinanet.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/cmcc.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/cstnet.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/drpeng.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/googlecn.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/tietong.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/unicom.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/cernet6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/china6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/chinanet6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/cmcc6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/cstnet6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/drpeng6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/googlecn6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/tietong6.txt',
-    'curl -sL https://gaoyifan.github.io/china-operator-ip/unicom6.txt',
-]
+] + ['curl -sL https://gaoyifan.github.io/china-operator-ip/%s.txt' % x for x in operators]
 
-ips = set()
-for script in source:
+ipv4 = IPSet()
+ipv6 = IPSet()
+ipAddrs = set()
+for script in source:  # traverse fetch commands
     raw = os.popen(script).read().split('\n')
-    ips.update(filter(None, raw))
-
-v4 = IPSet()
-v6 = IPSet()
-for ip in ips:
+    ipAddrs.update(filter(None, raw))
+for ipAddr in ipAddrs:
     try:
-        ipAddr = IP(ip)
-        (v4 if ipAddr.version() == 4 else v6).add(ipAddr)
-    except:
-        pass
-
-ips = []
-for ip in v4:
-    ips.append(str(ip) + ('' if '/' in str(ip) else '/32'))
-for ip in v6:
-    ips.append(str(ip) + ('' if '/' in str(ip) else '/128'))
+        ip = IP(ipAddr)  # ip format check
+        (ipv4 if ip.version() == 4 else ipv6).add(ip)
+    except: pass
 
+release = [('%s' if '/' in str(ip) else '%s/32') % str(ip) for ip in ipv4]  # format into CIDR
+release += [('%s' if '/' in str(ip) else '%s/128') % str(ip) for ip in ipv6]
 with open('china-ip.txt', 'w') as fileObj:
-    fileObj.write('\n'.join(ips) + '\n')
+    fileObj.write('\n'.join(release) + '\n')

assets/chinalist.py

@@ -14,11 +14,10 @@ source = [
 ]
 
 domains = set()
-for script in source:
+for script in source:  # traverse fetch commands
     raw = os.popen(script).read().split('\n')
     domains.update(filter(None, raw))
 regex = r'^(?=^.{3,255}$)[a-zA-Z0-9][a-zA-Z0-9\-]{0,62}(.[a-zA-Z0-9][a-zA-Z0-9\-]{0,62})+$'
-domains = {x for x in domains if re.search(regex, str(x)) is not None}
-
+domains = {x for x in domains if re.search(regex, str(x)) is not None}  # filter invalid domains
 with open('chinalist.txt', 'w') as fileObj:
     fileObj.write('\n'.join(sorted(domains)) + '\n')

assets/gfwlist.py

@@ -19,11 +19,10 @@ source = [
 ]
 
 domains = set()
-for script in source:
+for script in source:  # traverse fetch commands
     raw = os.popen(script).read().split('\n')
     domains.update(filter(None, raw))
 regex = r'^(?=^.{3,255}$)[a-zA-Z0-9][a-zA-Z0-9\-]{0,62}(.[a-zA-Z0-9][a-zA-Z0-9\-]{0,62})+$'
-domains = {x for x in domains if re.search(regex, str(x)) is not None}
-
+domains = {x for x in domains if re.search(regex, str(x)) is not None}  # filter invalid domains
 with open('gfwlist.txt', 'w') as fileObj:
     fileObj.write('\n'.join(sorted(domains)) + '\n')

src/to-json/src/ffi.rs

@@ -1,4 +1,4 @@
-use crate::convert::to_json;
+use crate::json::to_json;
 use std::ffi::{c_char, CStr, CString};
 
 fn to_c_string(string: String) -> *const c_char { // fetch c-style ptr of string

src/to-json/src/convert.rs → src/to-json/src/json.rs

@@ -1,10 +1,11 @@
+use serde_json as json;
 use crate::parser::{parser, Value};
 
 fn json_convert(content: &str) -> Result<String, String> { // convert to JSON format
     let data = match parser(content)? {
-        Value::JSON(json) => serde_json::to_string(&json),
-        Value::YAML(yaml) => serde_json::to_string(&yaml),
-        Value::TOML(toml) => serde_json::to_string(&toml),
+        Value::JSON(_json) => json::to_string(&_json),
+        Value::YAML(_yaml) => json::to_string(&_yaml),
+        Value::TOML(_toml) => json::to_string(&_toml),
     };
     match data {
         Ok(data) => Ok(data),
@@ -15,6 +16,6 @@ fn json_convert(content: &str) -> Result<String, String> { // convert to JSON fo
 pub fn to_json(content: &str) -> Option<String> { // to JSON string
     match json_convert(content) {
         Ok(data) => Some(data),
-        Err(_) => None,
+        Err(_) => None, // convert failed
     }
 }

src/to-json/src/lib.rs

@@ -1,4 +1,4 @@
 mod ffi;
+mod json;
 mod tests;
 mod parser;
-mod convert;

src/to-json/src/tests.rs

@@ -1,4 +1,4 @@
-use crate::convert::to_json;
+use crate::json::to_json;
 
 #[allow(dead_code)]
 const JSON_TEST_CONTENT: &str = r#"