From 8783e0e9aea644f3839bfbcf9763090a1c39d0f1 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sat, 31 Jan 2015 11:56:17 +0800 Subject: [PATCH 1/9] Move is_ip from asyncdns to common Since implement CIDR forbidden need this function, move it to common file seems to be a better choice. --- shadowsocks/asyncdns.py | 18 +++--------------- shadowsocks/common.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 6f60dc9..6fee6b9 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -233,18 +233,6 @@ def parse_response(data): return None -def is_ip(address): - for family in (socket.AF_INET, socket.AF_INET6): - try: - if type(address) != str: - address = address.decode('utf8') - socket.inet_pton(family, address) - return family - except (TypeError, ValueError, OSError, IOError): - pass - return False - - def is_valid_hostname(hostname): if len(hostname) > 255: return False @@ -296,7 +284,7 @@ class DNSResolver(object): parts = line.split() if len(parts) >= 2: server = parts[1] - if is_ip(server) == socket.AF_INET: + if common.is_ip(server) == socket.AF_INET: if type(server) != str: server = server.decode('utf8') self._servers.append(server) @@ -316,7 +304,7 @@ class DNSResolver(object): parts = line.split() if len(parts) >= 2: ip = parts[0] - if is_ip(ip): + if common.is_ip(ip): for i in range(1, len(parts)): hostname = parts[i] if hostname: @@ -423,7 +411,7 @@ class DNSResolver(object): hostname = hostname.encode('utf8') if not hostname: callback(None, Exception('empty hostname')) - elif is_ip(hostname): + elif common.is_ip(hostname): callback((hostname, hostname), None) elif hostname in self._hosts: logging.debug('hit hosts: %s', hostname) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index e4f698c..2be17e5 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -101,6 +101,18 @@ def inet_pton(family, addr): raise RuntimeError("What family?") +def is_ip(address): + for family in (socket.AF_INET, socket.AF_INET6): + try: + if type(address) != str: + address = address.decode('utf8') + inet_pton(family, address) + return family + except (TypeError, ValueError, OSError, IOError): + pass + return False + + def patch_socket(): if not hasattr(socket, 'inet_pton'): socket.inet_pton = inet_pton From 100ebcf064f4a8e5840012bdccef106794208569 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sat, 31 Jan 2015 19:50:10 +0800 Subject: [PATCH 2/9] Add IPNetwork class to support CIDR calculation Usage: Use IPNetwork(str|list) to create an IPNetwork object. Use operator 'in' to determine whether the specified IP address is in the IP network or not, like: >>> '192.168.1.1' in IPNetwork('192.168.1.0/24') True Both IPv4 and IPv6 address are supported. Note: When using string to initialize the IPNetwork, a comma seperated IP network list should be provided. Currently, IPNetwork just support standard CIDR like: x.x.x.x/y eg. 192.168.1.0/24 ::x/y eg. ::1/10 If pure IP address was provided, it will be treated as implicit IP network, like 192.168.0.0 will be treated as 192.168.0.0/16 and 192.168.1.1 will be treated as 192.168.1.1/32 This implicit translate may cause some unexpected behavior, like user provide 192.168.2.0 and expect it will be treated as 192.168.2.0/24 but actually it will be translated to 192.168.2.0/23 because there are 9 continuous 0 from right. In order to avoid confusion, a warning message will be displayed when pure IP address was provided. Other variants of CIDR are not supported yet. --- shadowsocks/common.py | 67 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 2be17e5..d582923 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -184,6 +184,57 @@ def parse_header(data): return addrtype, to_bytes(dest_addr), dest_port, header_length +class IPNetwork(object): + ADDRLENGTH = {socket.AF_INET: 32, socket.AF_INET6: 128} + + def __init__(self, addrs): + self._network_list_v4 = [] + self._network_list_v6 = [] + if type(addrs) == str: + addrs = addrs.split(',') + map(self.add_network, addrs) + + def add_network(self, addr): + block = addr.split('/') + addr_family = is_ip(block[0]) + if addr_family is socket.AF_INET: + ip, = struct.unpack("!I", socket.inet_aton(block[0])) + elif addr_family is socket.AF_INET6: + hi, lo = struct.unpack("!QQ", inet_pton(addr_family, block[0])) + ip = (hi << 64) | lo + else: + raise SyntaxError("Not a valid CIDR notation: %s" % addr) + if len(block) is 1: + prefix_size = 0 + while ((ip & 1) == 0): + ip >>= 1 + prefix_size += 1 + logging.warn("You did't specify CIDR routing prefix size for %s, " + "implicit treated as %s/%d" % (addr, addr, + IPNetwork.ADDRLENGTH[addr_family] - prefix_size)) + elif block[1].isdigit() and int(block[1]) <= IPNetwork.ADDRLENGTH[addr_family]: + prefix_size = IPNetwork.ADDRLENGTH[addr_family] - int(block[1]) + ip >>= prefix_size + else: + raise SyntaxError("Not a valid CIDR notation: %s" % addr) + if addr_family is socket.AF_INET: + self._network_list_v4.append((ip, prefix_size)) + else: + self._network_list_v6.append((ip, prefix_size)) + + def __contains__(self, addr): + addr_family = is_ip(addr) + if addr_family is socket.AF_INET: + ip, = struct.unpack("!I", socket.inet_aton(addr)) + return any(map(lambda (naddr, ps): naddr == ip >> ps, self._network_list_v4)) + elif addr_family is socket.AF_INET6: + hi, lo = struct.unpack("!QQ", inet_pton(addr_family, addr)) + ip = (hi << 64) | lo + return any(map(lambda (naddr, ps): naddr == ip >> ps, self._network_list_v6)) + else: + return False + + def test_inet_conv(): ipv4 = b'8.8.4.4' b = inet_pton(socket.AF_INET, ipv4) @@ -210,7 +261,23 @@ def test_pack_header(): assert pack_addr(b'www.google.com') == b'\x03\x0ewww.google.com' +def test_ip_network(): + ip_network = IPNetwork('127.0.0.0/24,::ff:1/112,::1,192.168.1.1,192.168.2.0') + assert '127.0.0.1' in ip_network + assert '127.0.1.1' not in ip_network + assert ':ff:ffff' in ip_network + assert '::ffff:1' not in ip_network + assert '::1' in ip_network + assert '::2' not in ip_network + assert '192.168.1.1' in ip_network + assert '192.168.1.2' not in ip_network + assert '192.168.2.1' in ip_network + assert '192.168.3.1' in ip_network # 192.168.2.0 is treated as 192.168.2.0/23 + assert 'www.google.com' not in ip_network + + if __name__ == '__main__': test_inet_conv() test_parse_header() test_pack_header() + test_ip_network() From 8af359ae05d6fa60b9f60e6f732c6e02bb4b6b93 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sat, 31 Jan 2015 20:32:09 +0800 Subject: [PATCH 3/9] Use IPNetwork supporting forbidden ip feature in config utils This commit also make "forbidden_ip" field available in config file. If no forbidden ip specified in command line and config file, default to "127.0.0.0/8,::1". --- shadowsocks/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index a51c965..0c77125 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -29,7 +29,7 @@ import json import sys import getopt import logging -from shadowsocks.common import to_bytes, to_str +from shadowsocks.common import to_bytes, to_str, IPNetwork VERBOSE_LEVEL = 5 @@ -186,6 +186,7 @@ def get_config(is_local): config['verbose'] = config.get('verbose', False) config['local_address'] = config.get('local_address', '127.0.0.1') config['local_port'] = config.get('local_port', 1080) + config['forbidden_ip'] = IPNetwork(config.get('forbidden_ip', '127.0.0.0/8,::1')) if is_local: if config.get('server', None) is None: logging.error('server addr not specified') From aa28796524eb7ff9ccb9a73afa4353079d2a9f71 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 1 Feb 2015 00:17:03 +0800 Subject: [PATCH 4/9] Make common fit PEP8 --- shadowsocks/common.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index d582923..0f94e07 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -185,7 +185,7 @@ def parse_header(data): class IPNetwork(object): - ADDRLENGTH = {socket.AF_INET: 32, socket.AF_INET6: 128} + ADDRLENGTH = {socket.AF_INET: 32, socket.AF_INET6: 128, False: 0} def __init__(self, addrs): self._network_list_v4 = [] @@ -197,6 +197,7 @@ class IPNetwork(object): def add_network(self, addr): block = addr.split('/') addr_family = is_ip(block[0]) + addr_len = IPNetwork.ADDRLENGTH[addr_family] if addr_family is socket.AF_INET: ip, = struct.unpack("!I", socket.inet_aton(block[0])) elif addr_family is socket.AF_INET6: @@ -210,10 +211,9 @@ class IPNetwork(object): ip >>= 1 prefix_size += 1 logging.warn("You did't specify CIDR routing prefix size for %s, " - "implicit treated as %s/%d" % (addr, addr, - IPNetwork.ADDRLENGTH[addr_family] - prefix_size)) - elif block[1].isdigit() and int(block[1]) <= IPNetwork.ADDRLENGTH[addr_family]: - prefix_size = IPNetwork.ADDRLENGTH[addr_family] - int(block[1]) + "implicit treated as %s/%d" % (addr, addr, addr_len)) + elif block[1].isdigit() and int(block[1]) <= addr_len: + prefix_size = addr_len - int(block[1]) ip >>= prefix_size else: raise SyntaxError("Not a valid CIDR notation: %s" % addr) @@ -226,11 +226,13 @@ class IPNetwork(object): addr_family = is_ip(addr) if addr_family is socket.AF_INET: ip, = struct.unpack("!I", socket.inet_aton(addr)) - return any(map(lambda (naddr, ps): naddr == ip >> ps, self._network_list_v4)) + return any(map(lambda (n, ps): n == ip >> ps, + self._network_list_v4)) elif addr_family is socket.AF_INET6: hi, lo = struct.unpack("!QQ", inet_pton(addr_family, addr)) ip = (hi << 64) | lo - return any(map(lambda (naddr, ps): naddr == ip >> ps, self._network_list_v6)) + return any(map(lambda (n, ps): n == ip >> ps, + self._network_list_v6)) else: return False @@ -262,7 +264,7 @@ def test_pack_header(): def test_ip_network(): - ip_network = IPNetwork('127.0.0.0/24,::ff:1/112,::1,192.168.1.1,192.168.2.0') + ip_network = IPNetwork('127.0.0.0/24,::ff:1/112,::1,192.168.1.1,192.0.2.0') assert '127.0.0.1' in ip_network assert '127.0.1.1' not in ip_network assert ':ff:ffff' in ip_network @@ -271,8 +273,8 @@ def test_ip_network(): assert '::2' not in ip_network assert '192.168.1.1' in ip_network assert '192.168.1.2' not in ip_network - assert '192.168.2.1' in ip_network - assert '192.168.3.1' in ip_network # 192.168.2.0 is treated as 192.168.2.0/23 + assert '192.0.2.1' in ip_network + assert '192.0.3.1' in ip_network # 192.0.2.0 is treated as 192.0.2.0/23 assert 'www.google.com' not in ip_network From b11d8489862765a2aa13d1594c855c4e652f8ccf Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 1 Feb 2015 00:42:12 +0800 Subject: [PATCH 5/9] Fix for Python3 lambda behavior change In Python3, lambda no longer support use tuple as syntax. So, ugly changes is inevitable. --- shadowsocks/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 0f94e07..68ca025 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -226,12 +226,12 @@ class IPNetwork(object): addr_family = is_ip(addr) if addr_family is socket.AF_INET: ip, = struct.unpack("!I", socket.inet_aton(addr)) - return any(map(lambda (n, ps): n == ip >> ps, + return any(map(lambda n_ps: n_ps[0] == ip >> n_ps[1], self._network_list_v4)) elif addr_family is socket.AF_INET6: hi, lo = struct.unpack("!QQ", inet_pton(addr_family, addr)) ip = (hi << 64) | lo - return any(map(lambda (n, ps): n == ip >> ps, + return any(map(lambda n_ps: n_ps[0] == ip >> n_ps[1], self._network_list_v6)) else: return False From 070108f78bb81f4dc6391fd575d5268f51481cea Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 1 Feb 2015 00:48:15 +0800 Subject: [PATCH 6/9] Disable forbidden ip feature for local shadowsocks Since forbidden ip is server-side only, disable it for local-side. This commit also supress warning about IPv6 loopback because I can confirm ::1/128 is the only loopback address, not like IPv4. --- shadowsocks/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 0c77125..6ea3daa 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -186,7 +186,6 @@ def get_config(is_local): config['verbose'] = config.get('verbose', False) config['local_address'] = config.get('local_address', '127.0.0.1') config['local_port'] = config.get('local_port', 1080) - config['forbidden_ip'] = IPNetwork(config.get('forbidden_ip', '127.0.0.0/8,::1')) if is_local: if config.get('server', None) is None: logging.error('server addr not specified') @@ -194,6 +193,8 @@ def get_config(is_local): sys.exit(2) else: config['server'] = config.get('server', '0.0.0.0') + config['forbidden_ip'] = \ + IPNetwork(config.get('forbidden_ip', '127.0.0.0/8,::1/128')) config['server_port'] = config.get('server_port', 8388) if is_local and not config.get('password', None): From 79b9b53dbea59627d43ec7ee4eb2dd54777460be Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 1 Feb 2015 01:57:17 +0800 Subject: [PATCH 7/9] Never process empty string and prevent infinite loop If user provide an empty string as network range, inet_pton will treate it as an IPv6 unspecified address, it seems a bug but I can't confirm. Then empty string will be converted to 0, 0 & 1 always be zero, so it caused dead loop. --- shadowsocks/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 68ca025..d4a9a67 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -195,6 +195,8 @@ class IPNetwork(object): map(self.add_network, addrs) def add_network(self, addr): + if addr is "": + return block = addr.split('/') addr_family = is_ip(block[0]) addr_len = IPNetwork.ADDRLENGTH[addr_family] @@ -207,7 +209,7 @@ class IPNetwork(object): raise SyntaxError("Not a valid CIDR notation: %s" % addr) if len(block) is 1: prefix_size = 0 - while ((ip & 1) == 0): + while (ip & 1) == 0 and ip is not 0: ip >>= 1 prefix_size += 1 logging.warn("You did't specify CIDR routing prefix size for %s, " From a0aa9173a8001b5c20e270d7b5dcec28172cfa3f Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 1 Feb 2015 02:37:30 +0800 Subject: [PATCH 8/9] Fix for Python3 map changed behavior In Python3, map returns an iterator instead of list in Python2, which cause map "lazier" than before, wrap with list() force it running. --- shadowsocks/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index d4a9a67..0c4e278 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -192,7 +192,7 @@ class IPNetwork(object): self._network_list_v6 = [] if type(addrs) == str: addrs = addrs.split(',') - map(self.add_network, addrs) + list(map(self.add_network, addrs)) def add_network(self, addr): if addr is "": From 17624d0b99731a5947a2df9090039ad531883cc7 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 1 Feb 2015 02:40:48 +0800 Subject: [PATCH 9/9] Fix large file test since forbidden ip list is default to localhost Use an empty forbidden ip list to override default list should work. --- tests/test_large_file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh index e8acd79..33bcb59 100755 --- a/tests/test_large_file.sh +++ b/tests/test_large_file.sh @@ -8,7 +8,7 @@ mkdir -p tmp $PYTHON shadowsocks/local.py -c tests/aes.json & LOCAL=$! -$PYTHON shadowsocks/server.py -c tests/aes.json & +$PYTHON shadowsocks/server.py -c tests/aes.json --forbidden-ip "" & SERVER=$! sleep 3