From 40b90612765fe266129b4e45cfe731a3be5b4ac7 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Fri, 10 Mar 2023 17:24:31 +0800 Subject: [PATCH] perf: optimize cidr merge speed --- assets/china-ip.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/assets/china-ip.py b/assets/china-ip.py index 94f06dc..f964bd2 100755 --- a/assets/china-ip.py +++ b/assets/china-ip.py @@ -2,7 +2,9 @@ # -*- coding: utf-8 -*- import os -from IPy import IP, IPSet +from netaddr import IPSet +from netaddr import IPAddress +from netaddr import IPNetwork operators = ['china', 'cmcc', 'chinanet', 'unicom', 'tietong', 'cernet', 'cstnet', 'drpeng', 'googlecn'] operators += ['%s6' % x for x in operators] # add `...6` suffix @@ -12,20 +14,20 @@ source = [ 'curl -sL https://github.com/17mon/china_ip_list/raw/master/china_ip_list.txt', ] + ['curl -sL https://gaoyifan.github.io/china-operator-ip/%s.txt' % x for x in operators] -ipv4 = IPSet() -ipv6 = IPSet() ipAddrs = set() for script in source: # traverse fetch commands raw = os.popen(script).read().split('\n') ipAddrs.update(filter(None, raw)) -for ipAddr in ipAddrs: + +ipv4 = IPSet() +ipv6 = IPSet() +for ipAddr in ipAddrs: # load all IP data try: - ip = IP(ipAddr) # ip format check - (ipv4 if ip.version() == 4 else ipv6).add(ip) + ip = IPNetwork(ipAddr) if '/' in ipAddr else IPAddress(ipAddr) + ipv4.add(ip) if ip.version == 4 else ipv6.add(ip) except: pass -release = [('%s' if '/' in str(ip) else '%s/32') % str(ip) for ip in ipv4] # format into CIDR -release += [('%s' if '/' in str(ip) else '%s/128') % str(ip) for ip in ipv6] -with open('china-ip.txt', 'w') as fileObj: - fileObj.write('\n'.join(release) + '\n') +with open('china-ip.txt', 'w') as fileObj: # save to file + fileObj.write('\n'.join([str(ip) for ip in ipv4.iter_cidrs()]) + '\n') # format into CIDR + fileObj.write('\n'.join([str(ip) for ip in ipv6.iter_cidrs()]) + '\n')