From bc238330ec98d4fc88a9df1942c9d617439ad0d6 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Fri, 25 Mar 2022 13:06:21 +0800 Subject: [PATCH] feat: asset release script --- .gitignore | 6 +++++ asset/chinalist/build/build.py | 35 ++++++++++++++++++++++++++++++ asset/chinalist/build/source-ip.sh | 20 +++++++++++++++++ asset/chinalist/build/source.sh | 2 ++ asset/gfwlist/build/build.py | 13 +++++++++++ asset/gfwlist/build/source.sh | 7 ++++++ 6 files changed, 83 insertions(+) create mode 100755 asset/chinalist/build/build.py create mode 100644 asset/chinalist/build/source-ip.sh create mode 100644 asset/chinalist/build/source.sh create mode 100755 asset/gfwlist/build/build.py create mode 100644 asset/gfwlist/build/source.sh diff --git a/.gitignore b/.gitignore index 2234789..4630219 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ /bin/ /build/ /cmake-build-debug/ +/asset/chinalist/china-ip.txt +/asset/chinalist/china-ip.txt.gz +/asset/chinalist/chinalist.txt +/asset/chinalist/chinalist.txt.gz +/asset/gfwlist/gfwlist.txt +/asset/gfwlist/gfwlist.txt.gz diff --git a/asset/chinalist/build/build.py b/asset/chinalist/build/build.py new file mode 100755 index 0000000..c225f79 --- /dev/null +++ b/asset/chinalist/build/build.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- + +import os, re +workDir = os.path.split(os.path.realpath(__file__))[0] +domainList = os.popen('sh ' + workDir + '/source.sh | sort -u').read().split('\n') +regex = r'^(?=^.{3,255}$)[a-zA-Z0-9][a-zA-Z0-9\-]{0,62}(.[a-zA-Z0-9][a-zA-Z0-9\-]{0,62})+$' +for domain in domainList: + if re.search(regex, domain) is None: + domainList.remove(domain) +with open (workDir + '/../chinalist.txt', 'w') as fileObj: + fileObj.write('\n'.join(domainList) + '\n') +os.popen('gzip -cf9 ' + workDir + '/../chinalist.txt > ' + workDir + '/../chinalist.txt.gz') + +from IPy import IP, IPSet +ipv4List = IPSet() +ipv6List = IPSet() +ipList = os.popen('sh ' + workDir + '/source-ip.sh | sort -u').read().split('\n') +for ipStr in ipList: + try: + ipAddr = IP(ipStr) + if ipAddr.version() == 4: + ipv4List.add(ipAddr) + else: + ipv6List.add(ipAddr) + except: + pass +ipList = [] +for ipAddr in ipv4List: + ipList.append(str(ipAddr)) +for ipAddr in ipv6List: + ipList.append(str(ipAddr)) +with open (workDir + '/../china-ip.txt', 'w') as fileObj: + fileObj.write('\n'.join(ipList) + '\n') +os.popen('gzip -cf9 ' + workDir + '/../china-ip.txt > ' + workDir + '/../china-ip.txt.gz') diff --git a/asset/chinalist/build/source-ip.sh b/asset/chinalist/build/source-ip.sh new file mode 100644 index 0000000..f29120f --- /dev/null +++ b/asset/chinalist/build/source-ip.sh @@ -0,0 +1,20 @@ +curl -sL https://github.com/17mon/china_ip_list/raw/master/china_ip_list.txt +curl -sL https://github.com/misakaio/chnroutes2/raw/master/chnroutes.txt | sed '/^#/d' +curl -sL https://gaoyifan.github.io/china-operator-ip/cernet.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/china.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/chinanet.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/cmcc.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/cstnet.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/drpeng.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/googlecn.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/tietong.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/unicom.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/cernet6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/china6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/chinanet6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/cmcc6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/cstnet6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/drpeng6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/googlecn6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/tietong6.txt +curl -sL https://gaoyifan.github.io/china-operator-ip/unicom6.txt diff --git a/asset/chinalist/build/source.sh b/asset/chinalist/build/source.sh new file mode 100644 index 0000000..3e40fed --- /dev/null +++ b/asset/chinalist/build/source.sh @@ -0,0 +1,2 @@ +curl -sL https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf | sed 's/server=\///g;s/\/114.114.114.114//g' | sort -u +curl -sL https://raw.githubusercontent.com/hq450/fancyss/master/rules/WhiteList_new.txt | sed 's/Server=\///g;s/\///g' | sort -u diff --git a/asset/gfwlist/build/build.py b/asset/gfwlist/build/build.py new file mode 100755 index 0000000..da10dea --- /dev/null +++ b/asset/gfwlist/build/build.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- + +import os, re +workDir = os.path.split(os.path.realpath(__file__))[0] +domainList = os.popen('sh ' + workDir + '/source.sh | sort -u').read().split('\n') +regex = r'^(?=^.{3,255}$)[a-zA-Z0-9][a-zA-Z0-9\-]{0,62}(.[a-zA-Z0-9][a-zA-Z0-9\-]{0,62})+$' +for domain in domainList: + if re.search(regex, domain) is None: + domainList.remove(domain) +with open (workDir + '/../gfwlist.txt', 'w') as fileObj: + fileObj.write('\n'.join(domainList) + '\n') +os.popen('gzip -cf9 ' + workDir + '/../gfwlist.txt > ' + workDir + '/../gfwlist.txt.gz') diff --git a/asset/gfwlist/build/source.sh b/asset/gfwlist/build/source.sh new file mode 100644 index 0000000..8c835a5 --- /dev/null +++ b/asset/gfwlist/build/source.sh @@ -0,0 +1,7 @@ +curl -sL https://raw.githubusercontent.com/Loukky/gfwlist-by-loukky/master/gfwlist.txt | base64 -d | sort -u | sed '/^$\|@@/d'| sed 's#!.\+##; s#|##g; s#@##g; s#http:\/\/##; s#https:\/\/##;' | sed '/\*/d; /apple\.com/d; /sina\.cn/d; /sina\.com\.cn/d; /baidu\.com/d; /qq\.com/d' | sed '/^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$/d' | grep '^[0-9a-zA-Z\.-]\+$' | grep '\.' | sed 's#^\.\+##' | sort -u + +curl -sL https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt | base64 -d | sort -u | sed '/^$\|@@/d'| sed 's#!.\+##; s#|##g; s#@##g; s#http:\/\/##; s#https:\/\/##;' | sed '/\*/d; /apple\.com/d; /sina\.cn/d; /sina\.com\.cn/d; /baidu\.com/d; /qq\.com/d' | sed '/^[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+$/d' | grep '^[0-9a-zA-Z\.-]\+$' | grep '\.' | sed 's#^\.\+##' | sort -u + +curl -sL https://github.com/Loyalsoldier/v2ray-rules-dat/releases/download/$(curl -sL https://api.github.com/repos/Loyalsoldier/v2ray-rules-dat/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')/gfw.txt | sort -u + +curl -sL https://raw.githubusercontent.com/hq450/fancyss/master/rules/gfwlist.conf | sed 's/ipset=\/\.//g; s/\/gfwlist//g; /^server/d' | sort -u