From 62f38df8fb8fab9fe9de9aef419b42238419e6d0 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 17:04:18 +0800 Subject: [PATCH 01/18] perf: Tester module --- Basis/Constant.py | 9 +++ Basis/Logger.py | 5 +- Basis/Test.py | 159 +++++++++++++++++++++++++++++++++++++++++ Builder/Shadowsocks.py | 10 +-- Tester/Brook.py | 5 +- Tester/Hysteria.py | 5 +- Tester/Plugin.py | 10 +-- Tester/Settings.py | 14 ---- Tester/Shadowsocks.py | 58 +++++++-------- Tester/ShadowsocksR.py | 5 +- Tester/Trojan.py | 5 +- Tester/TrojanGo.py | 14 ++-- Tester/V2ray.py | 2 +- Tester/VLESS.py | 5 +- Tester/VMess.py | 5 +- Tester/Xray.py | 2 +- Tester/__init__.py | 128 +-------------------------------- test.py | 13 ++-- 18 files changed, 237 insertions(+), 217 deletions(-) create mode 100644 Basis/Test.py delete mode 100644 Tester/Settings.py diff --git a/Basis/Constant.py b/Basis/Constant.py index 2f44b3f..15d3f0a 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -6,6 +6,15 @@ WorkDir = '/tmp/ProxyC' PathEnv = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' # Shadowsocks Info +mbedtlsMethods = [ + 'aes-128-cfb128', + 'aes-192-cfb128', + 'aes-256-cfb128', + 'camellia-128-cfb128', + 'camellia-192-cfb128', + 'camellia-256-cfb128', +] + ssMethods = { # methods support of different Shadowsocks project 'ss-rust': [ # table method removed refer to https://github.com/shadowsocks/shadowsocks-rust/issues/887 'none', 'plain', 'rc4', 'rc4-md5', diff --git a/Basis/Logger.py b/Basis/Logger.py index 383cee2..0069306 100644 --- a/Basis/Logger.py +++ b/Basis/Logger.py @@ -6,8 +6,9 @@ import logging from colorlog import ColoredFormatter logFile = 'runtime.log' -logLevel = logging.DEBUG -# logLevel = logging.WARNING +# logLevel = logging.INFO +# logLevel = logging.DEBUG +logLevel = logging.WARNING dateFormat = '%Y-%m-%d %H:%M:%S' logFormat = '[%(asctime)s] [%(levelname)s] %(message)s (%(module)s.%(funcName)s)' logging.basicConfig( diff --git a/Basis/Test.py b/Basis/Test.py new file mode 100644 index 0000000..e23c8ef --- /dev/null +++ b/Basis/Test.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import time +import requests +from threading import Thread +from Basis.Logger import logging +from Basis.Constant import WorkDir +from Basis.Functions import md5Sum, genFlag, hostFormat, checkPortStatus + +Settings = { + 'workDir': WorkDir, + 'site': 'www.bing.com', + 'serverBind': '', + 'clientBind': '', + 'host': '', + 'cert': '', + 'key': '', +} + + +def loadBind(serverV6: bool = False, clientV6: bool = False) -> None: + Settings['serverBind'] = '::1' if serverV6 else '127.0.0.1' + Settings['clientBind'] = '::1' if clientV6 else '127.0.0.1' + + +def waitPort(port: int, times: int = 50, delay: int = 100) -> bool: # wait until port occupied + for i in range(times): + if not checkPortStatus(port): # port occupied + return True + time.sleep(delay / 1000) # default wait 100ms * 50 => 5s + return False # timeout + + +def genCert(host: str, certInfo: dict, remark: str = 'ProxyC') -> None: # generate self-signed certificate + certOrgInfo = ['--organization', remark, '--organizationUnit', remark] # organization info + logging.critical('Load self-signed certificate') + os.system('mkdir -p %s' % Settings['workDir']) # make sure that work directory exist + + # create CA data at first (by mad) + logging.critical('Creating CA certificate and key...') + os.system(' '.join( # generate CA certificate and privkey + ['mad', 'ca', '--ca', certInfo['caCert'], '--key', certInfo['caKey'], '--commonName', remark] + certOrgInfo + )) + + # generate private key and sign certificate + logging.critical('Signing certificate...') + os.system(' '.join(['mad', 'cert', '--domain', host] + [ # generate certificate and privkey, then signed by CA + '--ca', certInfo['caCert'], '--ca_key', certInfo['caKey'], + '--cert', certInfo['cert'], '--key', certInfo['key'], + ] + certOrgInfo)) + + # install CA certificate and record self-signed cert info + logging.critical('Installing CA certificate...') + os.system('cat %s >> /etc/ssl/certs/ca-certificates.crt' % certInfo['caCert']) # add into system's trust list + + +def loadCert(host: str = 'proxyc.net', certId: str = '') -> None: # load certificate + newCert = (certId == '') + certId = genFlag(length = 8) if certId == '' else certId + certInfo = { + 'caCert': os.path.join(Settings['workDir'], 'proxyc_%s_ca.pem' % certId), + 'caKey': os.path.join(Settings['workDir'], 'proxyc_%s_ca_key.pem' % certId), + 'cert': os.path.join(Settings['workDir'], 'proxyc_%s_cert.pem' % certId), + 'key': os.path.join(Settings['workDir'], 'proxyc_%s_cert_key.pem' % certId), + } + if newCert: + genCert(host, certInfo) # generate new certificate + Settings['host'] = host + Settings['cert'] = certInfo['cert'] + Settings['key'] = certInfo['key'] + logging.warning('Certificate load complete -> ID = %s' % certId) + + +def httpCheck(socksInfo: dict, url: str, testId: str, timeout: int = 10) -> None: + socksProxy = 'socks5://%s:%i' % (hostFormat(socksInfo['addr'], v6Bracket = True), socksInfo['port']) + try: + logging.debug('[%s] Http request via %s' % (testId, socksProxy)) + request = requests.get(url, timeout = timeout, proxies = { # http request via socks5 + 'http': socksProxy, + 'https': socksProxy, + }) + request.raise_for_status() # throw error when server return 4xx or 5xx (don't actually need) + logging.info('[%s] %s -> ok' % (testId, socksProxy)) + except Exception as exp: + logging.error('[%s] %s -> error\n%s' % (testId, socksProxy, exp)) # show detail of error reason + raise RuntimeError('Http request via socks5 failed') + + +def runTest(testInfo: dict, testUrl: str, testSelect: set or None, delay: int = 1) -> None: + testId = md5Sum(testInfo['caption'])[:12] # generate test ID + if testSelect is not None: # testSelect is None -> run all test + if testId not in testSelect: # skip unselected task + return + logging.warning('[%s] %s' % (testId, testInfo['caption'])) # show caption + logging.debug('[%s] Server ID -> %s | Client ID -> %s' % ( + testId, testInfo['server'].id, testInfo['client'].id + )) + testInfo['server'].id = testId + '-server' + testInfo['client'].id = testId + '-client' + + # build server and client and wait them start + testInfo['server'].start() # start test server + if waitPort(testInfo['interface']['port']): # wait for server + logging.debug('[%s] Test server start complete' % testId) + testInfo['client'].start() # start test client + if waitPort(testInfo['socks']['port']): # wait for client + logging.debug('[%s] Test client start complete' % testId) + + # start test process + try: + logging.debug('[%s] Test process start' % testId) + time.sleep(delay) # delay a short time before check + httpCheck(testInfo['socks'], testUrl, testId) # run http request test + testInfo['client'].quit() # clean up client + testInfo['server'].quit() # clean up server + except: + testInfo['client'].quit() + testInfo['server'].quit() + logging.warning('[%s] Client info' % testId) + logging.error('[%(id)s-server]\n▲ CMD => %(cmd)s\n▲ ENV => %(env)s\n▲ FILE => %(file)s\n%(output)s' % { + 'id': testId, + 'cmd': testInfo['client'].cmd, + 'env': testInfo['client'].env, + 'file': testInfo['client'].file, + 'output': '-' * 96 + '\n' + testInfo['client'].output + '-' * 96, + }) + logging.warning('[%s] Server info' % testId) + logging.error('[%(id)s-client]\n▲ CMD => %(cmd)s\n▲ ENV => %(env)s\n▲ FILE => %(file)s\n%(output)s' % { + 'id': testId, + 'cmd': testInfo['server'].cmd, + 'env': testInfo['server'].env, + 'file': testInfo['server'].file, + 'output': '-' * 96 + '\n' + testInfo['server'].output + '-' * 96, + }) + + +def Test(testIter: iter, threadNum: int, testUrl: str, testFilter: set or None = None) -> None: + threads = [] + while True: # infinite loop + try: + for thread in threads: + if thread.is_alive(): # skip running thread + continue + threads.remove(thread) # remove dead thread + if len(threads) < threadNum: + for i in range(threadNum - len(threads)): # start threads within limit + thread = Thread( # create new thread + target = runTest, + args = (next(testIter), testUrl, testFilter) + ) + thread.start() + threads.append(thread) # record thread info + time.sleep(0.1) + except StopIteration: # traverse completed + break + for thread in threads: # wait until all threads exit + thread.join() diff --git a/Builder/Shadowsocks.py b/Builder/Shadowsocks.py index c985a60..7f326b5 100644 --- a/Builder/Shadowsocks.py +++ b/Builder/Shadowsocks.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import json -from Basis.Constant import ssMethods, ssAllMethods +from Basis.Constant import ssMethods, ssAllMethods, mbedtlsMethods def loadConfig(proxyInfo: dict, socksInfo: dict) -> dict: # load basic config option @@ -45,14 +45,6 @@ def ssLibev(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list]: def ssPython(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list]: config = loadConfig(proxyInfo, socksInfo) - mbedtlsMethods = [ - 'aes-128-cfb128', - 'aes-192-cfb128', - 'aes-256-cfb128', - 'camellia-128-cfb128', - 'camellia-192-cfb128', - 'camellia-256-cfb128', - ] if config['method'] in mbedtlsMethods: # mbedtls methods should use prefix `mbedtls:` config['method'] = 'mbedtls:' + config['method'] if config['method'] in ['idea-cfb', 'seed-cfb']: # only older versions of openssl are supported diff --git a/Tester/Brook.py b/Tester/Brook.py index b8aab33..45a1685 100644 --- a/Tester/Brook.py +++ b/Tester/Brook.py @@ -4,9 +4,9 @@ import copy import itertools from Builder import Brook +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process -from Tester.Settings import Settings from Basis.Functions import hostFormat, genFlag, getAvailablePort @@ -78,7 +78,7 @@ def loadTest(stream: dict) -> dict: 'port': proxyInfo['port'], } } - logging.debug('New brook test -> %s' % testInfo) + logging.debug('New Brook test -> %s' % testInfo) return testInfo @@ -91,3 +91,4 @@ def load(): addStream(wsStream(isRaw, isSecure)) # websocket stream test for stream in streams: yield loadTest(stream) + logging.info('Brook test yield complete') diff --git a/Tester/Hysteria.py b/Tester/Hysteria.py index a625e6d..6e36b5f 100644 --- a/Tester/Hysteria.py +++ b/Tester/Hysteria.py @@ -5,9 +5,9 @@ import os import json import itertools from Builder import Hysteria +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process -from Tester.Settings import Settings from Basis.Constant import hysteriaProtocols from Basis.Functions import hostFormat, genFlag, getAvailablePort @@ -78,10 +78,11 @@ def loadTest(protocol: str, isObfs: bool, isAuth: bool) -> dict: 'port': proxyInfo['port'], } } - logging.debug('New hysteria test -> %s' % testInfo) + logging.debug('New Hysteria test -> %s' % testInfo) return testInfo def load(): for protocol, isObfs, isAuth in itertools.product(hysteriaProtocols, [False, True], [False, True]): yield loadTest(protocol, isObfs, isAuth) + logging.info('Hysteria test yield complete') diff --git a/Tester/Plugin.py b/Tester/Plugin.py index 55f3517..d873271 100644 --- a/Tester/Plugin.py +++ b/Tester/Plugin.py @@ -4,10 +4,10 @@ import os import re import json +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process from Basis.Constant import Plugins -from Tester.Settings import Settings from Basis.Functions import genFlag, hostFormat, getAvailablePort pluginParams = {} @@ -310,7 +310,7 @@ def paramFill(param: str) -> str: return param -def load(proxyType: str): +def load(proxyType: str) -> list: if proxyType not in ['ss', 'trojan-go']: raise RuntimeError('Unknown proxy type for sip003 plugin') pluginParams.update({ @@ -321,13 +321,14 @@ def load(proxyType: str): 'PASSWD': genFlag(length = 8), # random password for test 'PATH': '/' + genFlag(length = 6), # random uri path for test }) + result = [] cloakLoad() # init cloak config kcptunLoad() # init kcptun config for pluginType in pluginConfig: for pluginTest, pluginTestInfo in pluginConfig[pluginType].items(): # traverse all plugin test item pluginParams['RANDOM'] = genFlag(length = 8) # refresh RANDOM field pluginParams['RABBIT_PORT'] = str(getAvailablePort()) # allocate port before rabbit plugin start - yield { + result.append({ 'type': pluginType, 'caption': pluginTest, 'server': { # plugin info for server @@ -339,4 +340,5 @@ def load(proxyType: str): 'param': paramFill(pluginTestInfo[1]), }, 'inject': ssInject if proxyType == 'ss' else trojanInject # for some special plugins - } + }) + return result diff --git a/Tester/Settings.py b/Tester/Settings.py deleted file mode 100644 index 5acdee0..0000000 --- a/Tester/Settings.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from Basis.Constant import WorkDir - -Settings = { - 'workDir': WorkDir, - 'site': 'www.bing.com', - 'serverBind': '', - 'clientBind': '', - 'host': '', - 'cert': '', - 'key': '', -} diff --git a/Tester/Shadowsocks.py b/Tester/Shadowsocks.py index 26fc199..a193965 100644 --- a/Tester/Shadowsocks.py +++ b/Tester/Shadowsocks.py @@ -7,11 +7,11 @@ import base64 import itertools from Tester import Plugin from Builder import Shadowsocks +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process -from Tester.Settings import Settings -from Basis.Constant import ssMethods, ssAllMethods from Basis.Functions import md5Sum, genFlag, getAvailablePort +from Basis.Constant import ssMethods, ssAllMethods, mbedtlsMethods def loadConfig(proxyInfo: dict) -> dict: # load basic config option @@ -43,14 +43,6 @@ def ssLibev(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: def ssPython(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: config = loadConfig(proxyInfo) - mbedtlsMethods = [ - 'aes-128-cfb128', - 'aes-192-cfb128', - 'aes-256-cfb128', - 'camellia-128-cfb128', - 'camellia-192-cfb128', - 'camellia-256-cfb128', - ] if config['method'] in mbedtlsMethods: # mbedtls methods should use prefix `mbedtls:` config['method'] = 'mbedtls:' + config['method'] if config['method'] in ['idea-cfb', 'seed-cfb']: # only older versions of openssl are supported @@ -135,33 +127,37 @@ def loadTest(serverType: str, clientType: str, method: str, plugin: dict or None } if plugin is not None: testInfo['server'] = plugin['inject'](testInfo['server'], plugin) - logging.debug('New shadowsocks test -> %s' % testInfo) + logging.debug('New Shadowsocks test -> %s' % testInfo) return testInfo -def load(isExtra: bool = False): - pluginTest = [] - pluginIter = Plugin.load('ss') - while True: - try: - pluginTest.append(next(pluginIter)) # export data of plugin generator - except StopIteration: - break - if not isExtra: # just test basic connection - for method in ssAllMethods: # test every method for once - for ssType in ssMethods: # found the client which support this method - if method not in ssMethods[ssType]: continue - yield loadTest(ssType, ssType, method) # ssType <-- method --> ssType - break # don't need other client - for ssType in ssMethods: # test plugin for every shadowsocks project - yield loadTest(ssType, ssType, ssMethods[ssType][0], pluginTest[0]) - ssType = list(ssMethods.keys())[0] # choose the first one - for plugin in pluginTest[1:]: # test every plugin (except the first one that has been checked) - yield loadTest(ssType, ssType, ssMethods[ssType][0], plugin) - return +def loadCommon(pluginTest: list): # shadowsocks basic test + for method in ssAllMethods: # test every method for once + for ssType in ssMethods: # found the client which support this method + if method not in ssMethods[ssType]: continue + yield loadTest(ssType, ssType, method) # ssType <-- method --> ssType + break # don't need other client + for ssType in ssMethods: # test plugin for every shadowsocks project + yield loadTest(ssType, ssType, ssMethods[ssType][0], pluginTest[0]) + ssType = list(ssMethods.keys())[0] # choose the first one + for plugin in pluginTest[1:]: # test every plugin (except the first one that has been checked) + yield loadTest(ssType, ssType, ssMethods[ssType][0], plugin) + + +def loadExtra(pluginTest: list): for ssServer in ssMethods: # traverse all shadowsocks type as server for method, ssClient in itertools.product(ssMethods[ssServer], ssMethods): # supported methods and clients if method not in ssMethods[ssClient]: continue yield loadTest(ssServer, ssClient, method) # ssServer <-- method --> ssClient for ssType, plugin in itertools.product(ssMethods, pluginTest): # test every plugin with different ss project yield loadTest(ssType, ssType, ssMethods[ssType][0], plugin) + + +def load(isExtra: bool = False): + ssIter = (loadExtra if isExtra else loadCommon)(Plugin.load('ss')) + while True: + try: + yield next(ssIter) + except StopIteration: + break + logging.info('Shadowsocks test yield complete') diff --git a/Tester/ShadowsocksR.py b/Tester/ShadowsocksR.py index 484bcc4..439d114 100644 --- a/Tester/ShadowsocksR.py +++ b/Tester/ShadowsocksR.py @@ -3,10 +3,10 @@ import os import json +from Basis.Test import Settings from Builder import ShadowsocksR from Basis.Logger import logging from Basis.Process import Process -from Tester.Settings import Settings from Basis.Functions import genFlag, getAvailablePort from Basis.Constant import ssrMethods, ssrProtocols, ssrObfuscations @@ -64,7 +64,7 @@ def loadTest(method: str, protocol: str, obfs: str) -> dict: 'port': proxyInfo['port'], } } - logging.debug('New shadowsocksr test -> %s' % testInfo) + logging.debug('New ShadowsocksR test -> %s' % testInfo) return testInfo @@ -75,3 +75,4 @@ def load(): yield loadTest('aes-128-ctr', protocol, 'plain') for obfs in ssrObfuscations: yield loadTest('aes-128-ctr', 'origin', obfs) + logging.info('ShadowsocksR test yield complete') diff --git a/Tester/Trojan.py b/Tester/Trojan.py index 79a07e5..abc33a5 100644 --- a/Tester/Trojan.py +++ b/Tester/Trojan.py @@ -5,10 +5,10 @@ import os import json from Tester import Xray from Builder import Trojan +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process from Basis.Constant import xtlsFlows -from Tester.Settings import Settings from Basis.Functions import md5Sum, genFlag, getAvailablePort @@ -110,7 +110,7 @@ def loadTest(stream: dict) -> dict: 'port': proxyInfo['port'], } } - logging.debug('New trojan test -> %s' % testInfo) + logging.debug('New Trojan test -> %s' % testInfo) return testInfo @@ -119,3 +119,4 @@ def load(): yield loadBasicTest(streams[1]) # Trojan basic test -> TCP stream with TLS for stream in streams: # test all stream cases yield loadTest(stream) + logging.info('Trojan test yield complete') diff --git a/Tester/TrojanGo.py b/Tester/TrojanGo.py index 0e1684d..93d9521 100644 --- a/Tester/TrojanGo.py +++ b/Tester/TrojanGo.py @@ -5,9 +5,9 @@ import os import json from Tester import Plugin from Builder import TrojanGo +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process -from Tester.Settings import Settings from Basis.Constant import trojanGoMethods from Basis.Functions import md5Sum, genFlag, getAvailablePort @@ -85,18 +85,11 @@ def loadTest(wsObject: dict or None, ssObject: dict or None, plugin: dict or Non } if plugin is not None: testInfo['server'] = plugin['inject'](testInfo['server'], plugin) - logging.debug('New trojan-go test -> %s' % testInfo) + logging.debug('New Trojan-Go test -> %s' % testInfo) return testInfo def load(): - pluginTest = [] - pluginIter = Plugin.load('trojan-go') - while True: - try: - pluginTest.append(next(pluginIter)) # export data of plugin generator - except StopIteration: - break wsObject = { 'host': Settings['host'], 'path': '/' + genFlag(length = 6), @@ -108,5 +101,6 @@ def load(): 'passwd': genFlag(length = 8) } yield loadTest(wsObject, None if ssObject['method'] == '' else ssObject, None) - for plugin in pluginTest: # different plugin for trojan-go + for plugin in Plugin.load('trojan-go'): # different plugin for trojan-go yield loadTest(None, None, plugin) + logging.info('Trojan-Go test yield complete') diff --git a/Tester/V2ray.py b/Tester/V2ray.py index b651ff7..f278a70 100644 --- a/Tester/V2ray.py +++ b/Tester/V2ray.py @@ -3,8 +3,8 @@ import copy import itertools +from Basis.Test import Settings from Basis.Functions import genFlag -from Tester.Settings import Settings from Basis.Constant import quicMethods, udpObfuscations httpConfig = { diff --git a/Tester/VLESS.py b/Tester/VLESS.py index ed22e7c..78a92e1 100644 --- a/Tester/VLESS.py +++ b/Tester/VLESS.py @@ -5,10 +5,10 @@ import os import json from Tester import Xray from Builder import VLESS +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process from Basis.Constant import xtlsFlows -from Tester.Settings import Settings from Basis.Functions import md5Sum, genUUID, getAvailablePort @@ -71,7 +71,7 @@ def loadTest(stream: dict) -> dict: 'port': proxyInfo['port'], } } - logging.debug('New vless test -> %s' % testInfo) + logging.debug('New VLESS test -> %s' % testInfo) return testInfo @@ -79,3 +79,4 @@ def load(): streams = Xray.loadStream() # load xray-core stream list for stream in streams: # test all stream cases yield loadTest(stream) + logging.info('VLESS test yield complete') diff --git a/Tester/VMess.py b/Tester/VMess.py index daffe63..bc8dce7 100644 --- a/Tester/VMess.py +++ b/Tester/VMess.py @@ -6,9 +6,9 @@ import json import itertools from Tester import V2ray from Builder import VMess +from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process -from Tester.Settings import Settings from Basis.Constant import PathEnv, vmessMethods from Basis.Functions import md5Sum, genUUID, getAvailablePort @@ -69,7 +69,7 @@ def loadTest(method: str, aid: int, stream: dict) -> dict: 'port': proxyInfo['port'], } } - logging.debug('New vmess test -> %s' % testInfo) + logging.debug('New VMess test -> %s' % testInfo) return testInfo @@ -79,3 +79,4 @@ def load(): yield loadTest(method, aid, streams[0]) for stream in streams[1:]: # skip first stream that has benn checked yield loadTest('auto', 0, stream) # aead with auto security + logging.info('VMess test yield complete') diff --git a/Tester/Xray.py b/Tester/Xray.py index 2e61c3f..2e5bb53 100644 --- a/Tester/Xray.py +++ b/Tester/Xray.py @@ -4,8 +4,8 @@ import copy import itertools from Tester import V2ray +from Basis.Test import Settings from Basis.Functions import genFlag -from Tester.Settings import Settings from Basis.Constant import xtlsFlows, quicMethods, udpObfuscations loadConfig = V2ray.loadConfig diff --git a/Tester/__init__.py b/Tester/__init__.py index dfd464b..09563df 100644 --- a/Tester/__init__.py +++ b/Tester/__init__.py @@ -1,14 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import os -import time -import requests -from threading import Thread -from Basis.Logger import logging -from Tester.Settings import Settings -from Basis.Functions import md5Sum, genFlag, hostFormat, checkPortStatus - from Tester import Brook from Tester import VMess from Tester import VLESS @@ -18,7 +10,7 @@ from Tester import Hysteria from Tester import Shadowsocks from Tester import ShadowsocksR -entry = { +testEntry = { 'ss': Shadowsocks.load(), 'ss-all': Shadowsocks.load(isExtra = True), 'ssr': ShadowsocksR.load(), @@ -29,121 +21,3 @@ entry = { 'brook': Brook.load(), 'hysteria': Hysteria.load(), } - - -def waitPort(port: int, times: int = 100, delay: int = 100) -> bool: # wait until port occupied - for i in range(times): - if not checkPortStatus(port): # port occupied - return True - time.sleep(delay / 1000) # default wait 100ms - return False # timeout - - -def httpCheck(socksInfo: dict, url: str, testId: str, timeout: int = 10) -> None: - socksProxy = 'socks5://%s:%i' % (hostFormat(socksInfo['addr'], v6Bracket = True), socksInfo['port']) - try: - proxy = { - 'http': socksProxy, - 'https': socksProxy, - } - request = requests.get(url, timeout = timeout, proxies = proxy) - request.raise_for_status() - logging.info('[%s] %s -> ok' % (testId, socksProxy)) - except Exception as exp: - logging.error('[%s] %s -> error' % (testId, socksProxy)) - logging.error('requests exception\n' + str(exp)) - raise RuntimeError('socks5 test failed') - - -def runTest(testInfo: dict, testUrl: str, testFilter: set or None, delay: int = 1) -> None: - testInfo['hash'] = md5Sum(testInfo['caption'])[:12] - if testFilter is not None and testInfo['hash'] not in testFilter: return - logging.warning('[%s] %s' % (testInfo['hash'], testInfo['caption'])) - testInfo['server'].start() # start test server - if waitPort(testInfo['interface']['port']): # wait for server - logging.debug('server start complete') - testInfo['client'].start() # start test client - if waitPort(testInfo['socks']['port']): # wait for client - logging.debug('client start complete') - try: - logging.debug('start test process') - time.sleep(delay) - httpCheck(testInfo['socks'], testUrl, testInfo['hash']) - testInfo['client'].quit() - testInfo['server'].quit() - except: - # client debug info - testInfo['client'].quit() - logging.warning('[%s] client info' % testInfo['hash']) - logging.error('command -> %s' % testInfo['client'].cmd) - logging.error('envVar -> %s' % testInfo['client'].env) - logging.error('file -> %s' % testInfo['client'].file) - logging.warning('[%s] client capture output' % testInfo['hash']) - logging.error('\n%s' % testInfo['client'].output) - # server debug info - testInfo['server'].quit() - logging.warning('[%s] server info' % testInfo['hash']) - logging.error('command -> %s' % testInfo['server'].cmd) - logging.error('envVar -> %s' % testInfo['server'].env) - logging.error('file -> %s' % testInfo['server'].file) - logging.warning('[%s] server capture output' % testInfo['hash']) - logging.error('\n%s' % testInfo['server'].output) - - -def test(testIter: iter, threadNum: int, testUrl: str, testFilter: set or None = None) -> None: - threads = [] - while True: # infinite loop - try: - for thread in threads: - if thread.is_alive(): continue - threads.remove(thread) # remove dead thread - if len(threads) < threadNum: - for i in range(threadNum - len(threads)): # start threads within limit - thread = Thread( # create new thread - target = runTest, - args = (next(testIter), testUrl, testFilter) - ) - thread.start() - threads.append(thread) # record thread info - time.sleep(0.1) - except StopIteration: # traverse completed - break - for thread in threads: # wait until all threads exit - thread.join() - - -def loadBind(serverV6: bool = False, clientV6: bool = False) -> None: - Settings['serverBind'] = '::1' if serverV6 else '127.0.0.1' - Settings['clientBind'] = '::1' if clientV6 else '127.0.0.1' - - -def loadCert(host: str = 'proxyc.net', remark: str = 'ProxyC') -> None: - loadPath = lambda x: os.path.join(Settings['workDir'], x) - certFlag = genFlag(length = 8) - caCert = loadPath('proxyc_%s_ca.pem' % certFlag) - caKey = loadPath('proxyc_%s_ca_key.pem' % certFlag) - cert = loadPath('proxyc_%s_cert.pem' % certFlag) - key = loadPath('proxyc_%s_cert_key.pem' % certFlag) - logging.critical('Load self-signed certificate') - os.system('mkdir -p %s' % Settings['workDir']) # create work directory - logging.critical('Creating CA certificate and key...') - os.system(' '.join(['mad', 'ca'] + [ # generate CA certificate and privkey - '--ca', caCert, '--key', caKey, - '--commonName', remark, - '--organization', remark, - '--organizationUnit', remark, - ])) - logging.critical('Signing certificate...') - os.system(' '.join(['mad', 'cert'] + [ # generate certificate and privkey, then signed by CA - '--ca', caCert, '--ca_key', caKey, - '--cert', cert, '--key', key, - '--domain', host, - '--organization', remark, - '--organizationUnit', remark, - ])) - logging.critical('Installing CA certificate...') - os.system('cat %s >> /etc/ssl/certs/ca-certificates.crt' % caCert) # add into system's trust list - Settings['host'] = host - Settings['cert'] = cert - Settings['key'] = key - logging.warning('Certificate loading complete') diff --git a/test.py b/test.py index 7f142d1..a6d507d 100755 --- a/test.py +++ b/test.py @@ -2,8 +2,9 @@ # -*- coding: utf-8 -*- import sys -import Tester +from Tester import testEntry from Basis.Logger import logging +from Basis.Test import Test, loadBind, loadCert threadNum = 16 testItem = None @@ -45,8 +46,8 @@ if getArg('--filter') is not None: testFilter = set(getArg('--filter').split(',')) isV6 = '--ipv6' in sys.argv -Tester.loadBind(serverV6 = isV6, clientV6 = isV6) # ipv4 / ipv6 (127.0.0.1 / ::1) -Tester.loadCert('proxyc.net', 'ProxyC') # default cert config +loadBind(serverV6 = isV6, clientV6 = isV6) # ipv4 / ipv6 (127.0.0.1 / ::1) +loadCert('proxyc.net') # default cert config logging.critical('TEST ITEM: ' + ('all' if testItem is None else testItem)) logging.critical('FILTER: %s' % testFilter) logging.critical('URL: ' + testUrl) @@ -56,11 +57,11 @@ logging.critical('-------------------------------- TEST START ------------------ if testItem is not None: if testItem == 'ss' and '--all' in sys.argv: testItem = 'ss-all' - Tester.test(Tester.entry[testItem], threadNum, testUrl, testFilter) + Test(testEntry[testItem], threadNum, testUrl, testFilter) else: - for item in Tester.entry: + for item in testEntry: if item == ('ss' if '--all' in sys.argv else 'ss-all'): # skip ss / ss-all continue logging.critical('TEST ITEM -> ' + item) - Tester.test(Tester.entry[item], threadNum, testUrl, testFilter) + Test(testEntry[item], threadNum, testUrl, testFilter) logging.critical('-------------------------------- TEST COMPLETE --------------------------------') From 720dbdfd3c6b01e90e29b78d4896acf9ef5316b3 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 18:42:18 +0800 Subject: [PATCH 02/18] update: global options --- Basis/Compile.py | 11 ----------- Basis/Constant.py | 7 +++++++ Basis/Logger.py | 27 ++++++++++++--------------- Basis/Manager.py | 2 +- Basis/Test.py | 4 ++-- main.py | 33 ++++++++++++++++++--------------- 6 files changed, 40 insertions(+), 44 deletions(-) delete mode 100644 Basis/Compile.py diff --git a/Basis/Compile.py b/Basis/Compile.py deleted file mode 100644 index ad90c81..0000000 --- a/Basis/Compile.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import compileall -from Basis.Logger import logging - - -def startCompile(dirRange: str = '/') -> None: - for optimize in [-1, 1, 2]: - compileall.compile_dir(dirRange, quiet = 1, maxlevels = 256, optimize = optimize) - logging.warning('Python optimize compile -> %s (level = %i)' % (dirRange, optimize)) diff --git a/Basis/Constant.py b/Basis/Constant.py index 15d3f0a..b81e32e 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -2,7 +2,14 @@ # -*- coding: utf-8 -*- Version = 'dev' +ApiToken = '' +# ApiToken = 'dnomd343' +# LogLevel = 'DEBUG' +LogLevel = 'WARNING' +LogFile = 'runtime.log' WorkDir = '/tmp/ProxyC' +ObfsSite = 'www.bing.com' +DnsServer = ['223.5.5.5', '119.29.29.29'] PathEnv = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' # Shadowsocks Info diff --git a/Basis/Logger.py b/Basis/Logger.py index 0069306..cf8f9b8 100644 --- a/Basis/Logger.py +++ b/Basis/Logger.py @@ -4,18 +4,23 @@ import sys import logging from colorlog import ColoredFormatter +from Basis.Constant import LogLevel, LogFile + +logLevel = { # log level + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL +}[LogLevel.lower()] +dateFormat = '%Y-%m-%d %H:%M:%S' # log date format +logFormat = '[%(asctime)s] [%(levelname)s] %(message)s (%(module)s.%(funcName)s)' # log format -logFile = 'runtime.log' -# logLevel = logging.INFO -# logLevel = logging.DEBUG -logLevel = logging.WARNING -dateFormat = '%Y-%m-%d %H:%M:%S' -logFormat = '[%(asctime)s] [%(levelname)s] %(message)s (%(module)s.%(funcName)s)' logging.basicConfig( level = logLevel, format = logFormat, datefmt = dateFormat, - filename = logFile, + filename = LogFile, ) logHandler = logging.StreamHandler(stream = sys.stdout) logHandler.setFormatter(ColoredFormatter( @@ -30,11 +35,3 @@ logHandler.setFormatter(ColoredFormatter( } )) logging.getLogger().addHandler(logHandler) - - -if __name__ == '__main__': - logging.debug('debug') - logging.info('info') - logging.warning('warn') - logging.error('error') - logging.critical('critical') diff --git a/Basis/Manager.py b/Basis/Manager.py index 691631f..3d272be 100644 --- a/Basis/Manager.py +++ b/Basis/Manager.py @@ -78,7 +78,7 @@ class Task(object): task['status'] = self.__TASK_RUNNING # set task status as running logging.info('Manager pop task [%s] -> %s' % (taskId, task['data'])) return taskId, copy.deepcopy(task['data']) - logging.debug('Manager has no more loaded tasks') + logging.debug('Manager has no more task') raise RuntimeError('No more tasks') def finishTask(self, taskId: str, taskData: dict) -> None: # update task data when completed diff --git a/Basis/Test.py b/Basis/Test.py index e23c8ef..83f4523 100644 --- a/Basis/Test.py +++ b/Basis/Test.py @@ -6,12 +6,12 @@ import time import requests from threading import Thread from Basis.Logger import logging -from Basis.Constant import WorkDir +from Basis.Constant import WorkDir, ObfsSite from Basis.Functions import md5Sum, genFlag, hostFormat, checkPortStatus Settings = { 'workDir': WorkDir, - 'site': 'www.bing.com', + 'site': ObfsSite, 'serverBind': '', 'clientBind': '', 'host': '', diff --git a/main.py b/main.py index 6bdada4..36b3622 100755 --- a/main.py +++ b/main.py @@ -1,41 +1,44 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- + import time import _thread -from Basis import DnsProxy +import compileall +from Basis import Constant from Basis.Check import Check +from Basis import Api, DnsProxy from Basis.Logger import logging from Basis.Manager import Manager -from Basis.Api import startServer -from Basis.Constant import Version -from Basis.Compile import startCompile from concurrent.futures import ThreadPoolExecutor -# dnsServers = None -dnsServers = ['223.5.5.5', '119.28.28.28'] + +def pythonCompile(dirRange: str = '/') -> None: # python optimize compile + for optimize in [-1, 1, 2]: + compileall.compile_dir(dirRange, quiet = 1, optimize = optimize) + logging.warning('Python optimize compile -> %s (level = %i)' % (dirRange, optimize)) def runCheck(taskId: str, taskInfo: dict) -> None: - checkResult = Check(taskId, taskInfo) + checkResult = Check(taskId, taskInfo) # check by task info logging.warning('[%s] Task finish' % taskId) - Manager.finishTask(taskId, checkResult) + Manager.finishTask(taskId, checkResult) # commit check result def loopCheck(threadNum: int = 16) -> None: - threadPool = ThreadPoolExecutor(max_workers = threadNum) + threadPool = ThreadPoolExecutor(max_workers = threadNum) # init thread pool while True: try: - taskId, taskInfo = Manager.popTask() + taskId, taskInfo = Manager.popTask() # pop a task logging.warning('[%s] Load new task' % taskId) except: # no more task time.sleep(2) continue - threadPool.submit(runCheck, taskId, taskInfo) + threadPool.submit(runCheck, taskId, taskInfo) # submit into thread pool -logging.warning('ProxyC starts running (%s)' % Version) -_thread.start_new_thread(startCompile, ('/usr', )) # python compile (generate .pyc file) -_thread.start_new_thread(DnsProxy.start, (dnsServers, 53)) # start dns server +logging.warning('ProxyC starts running (%s)' % Constant.Version) +_thread.start_new_thread(pythonCompile, ('/usr', )) # python compile (generate .pyc file) +_thread.start_new_thread(DnsProxy.start, (Constant.DnsServer, 53)) # start dns server _thread.start_new_thread(loopCheck, ()) # start loop check -startServer(apiToken = '') # start api server +Api.startServer(apiToken = Constant.ApiToken) # start api server From 7f357604ebb6764fbcf3469180096183a19b0050 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 20:34:13 +0800 Subject: [PATCH 03/18] feat: argparse for input args --- Basis/Api.py | 29 +++++++++++------------- Basis/Constant.py | 12 +++++++--- main.py | 57 ++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/Basis/Api.py b/Basis/Api.py index 506c5c3..8cb5b36 100644 --- a/Basis/Api.py +++ b/Basis/Api.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os import json from gevent import pywsgi from Checker import formatCheck from Basis.Logger import logging from Basis.Manager import Manager -from Basis.Constant import Version from flask import Flask, Response, request +from Basis.Constant import ApiPort, ApiPath, ApiToken, Version -token = '' -webPath = '/' # root of api server webApi = Flask(__name__) # init flask server @@ -43,16 +42,16 @@ def genError(message: str) -> Response: def tokenCheck() -> bool: - if token == '': return True # without token check + if ApiToken == '': return True # without token check if request.method == 'GET': - return request.args.get('token') == token + return request.args.get('token') == ApiToken elif request.method == 'POST': - return request.json.get('token') == token + return request.json.get('token') == ApiToken else: return False # polyfill -@webApi.route('/task', methods = ['GET']) +@webApi.route(os.path.join(ApiPath, 'task'), methods = ['GET']) def getTaskList() -> Response: if not tokenCheck(): # token check return genError('Invalid token') @@ -64,7 +63,7 @@ def getTaskList() -> Response: }) -@webApi.route('/task', methods = ['POST']) +@webApi.route(os.path.join(ApiPath, 'task'), methods = ['POST']) def createTask() -> Response: if not tokenCheck(): # token check return genError('Invalid token') @@ -93,7 +92,7 @@ def createTask() -> Response: }) -@webApi.route('/task/', methods = ['GET']) +@webApi.route(os.path.join(ApiPath, 'task/'), methods = ['GET']) def getTaskInfo(taskId: str) -> Response: if not tokenCheck(): # token check return genError('Invalid token') @@ -106,7 +105,7 @@ def getTaskInfo(taskId: str) -> Response: }) -@webApi.route('/version', methods = ['GET']) +@webApi.route(os.path.join(ApiPath, 'version'), methods = ['GET']) def getVersion() -> Response: logging.debug('API get version -> %s' + Version) return jsonResponse({ @@ -115,9 +114,7 @@ def getVersion() -> Response: }) -def startServer(apiToken: str = '', apiPort: int = 7839) -> None: - global token - token = apiToken # api token (default empty) - logging.warning('API server at http://:%i/' % apiPort) - logging.warning('API ' + ('without token' if apiToken == '' else 'token -> %s' % apiToken)) - pywsgi.WSGIServer(('0.0.0.0', apiPort), webApi).serve_forever() # powered by gevent +def startServer() -> None: + logging.warning('API server at http://:%i/' % ApiPort) + logging.warning('API ' + ('without token' if ApiToken == '' else 'token -> %s' % ApiToken)) + pywsgi.WSGIServer(('0.0.0.0', ApiPort), webApi).serve_forever() # powered by gevent diff --git a/Basis/Constant.py b/Basis/Constant.py index b81e32e..971509a 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -1,15 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# Global Options Version = 'dev' + +ApiPath = '/' +ApiPort = 7839 ApiToken = '' -# ApiToken = 'dnomd343' + # LogLevel = 'DEBUG' -LogLevel = 'WARNING' +# LogLevel = 'WARNING' +LogLevel = 'INFO' LogFile = 'runtime.log' + +DnsServer = [] WorkDir = '/tmp/ProxyC' ObfsSite = 'www.bing.com' -DnsServer = ['223.5.5.5', '119.29.29.29'] PathEnv = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' # Shadowsocks Info diff --git a/main.py b/main.py index 36b3622..b652388 100755 --- a/main.py +++ b/main.py @@ -1,11 +1,58 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- - +import sys +import copy import time import _thread +import argparse import compileall from Basis import Constant + + +def mainArgParse(rawArgs: list) -> argparse.Namespace: + mainParser = argparse.ArgumentParser(description = 'Start running API server') + mainParser.add_argument('--log', type = str, default = 'debug', help = 'output log level') + mainParser.add_argument('--port', type = int, default = 7839, help = 'port for running') + mainParser.add_argument('--token', type = str, default = '', help = 'token for api server') + mainParser.add_argument('-v', '--version', help = 'show version', action = 'store_true') + # TODO: api path / dns server + return mainParser.parse_args(rawArgs) + + +def testArgParse(rawArgs: list) -> argparse.Namespace: + testParser = argparse.ArgumentParser(description = 'Test that each function is working properly') + testParser.add_argument('PROTOCOL', type = str, help = 'test protocol name') + testParser.add_argument('-a', '--all', help = 'test extra shadowsocks items', action = 'store_true') + testParser.add_argument('-6', '--ipv6', help = 'test on ipv6 network', action = 'store_true') + testParser.add_argument('--url', type = str, default = 'http://baidu.com', help = 'http request url') + testParser.add_argument('--cert', type = str, help = 'specify the certificate id') + testParser.add_argument('--thread', type = int, default = 16, help = 'thread number in check process') + testParser.add_argument('--select', type = str, nargs = '+', help = 'select id list for test') + return testParser.parse_args(rawArgs) + + +inputArgs = copy.copy(sys.argv) +if len(inputArgs) >= 0: # remove first arg (normally file name) + inputArgs.pop(0) +if len(inputArgs) != 0 and inputArgs[0].lower() == 'test': # test mode + inputArgs.pop(0) # remove `test` + if len(inputArgs) == 0 or inputArgs[0].startswith('-'): # no protocol is specified + inputArgs = ['all'] + inputArgs + testArgs = testArgParse(inputArgs) + print(testArgs) + sys.exit(1) + # TODO: start test process +else: + mainArgs = mainArgParse(inputArgs) + if mainArgs.version: # output version and exit + print('ProxyC version -> %s' % Constant.Version) + sys.exit(0) + Constant.LogLevel = mainArgs.log # overwrite global options + Constant.ApiPort = mainArgs.port + Constant.ApiToken = mainArgs.token + + from Basis.Check import Check from Basis import Api, DnsProxy from Basis.Logger import logging @@ -25,7 +72,7 @@ def runCheck(taskId: str, taskInfo: dict) -> None: Manager.finishTask(taskId, checkResult) # commit check result -def loopCheck(threadNum: int = 16) -> None: +def loop(threadNum: int = 16) -> None: threadPool = ThreadPoolExecutor(max_workers = threadNum) # init thread pool while True: try: @@ -38,7 +85,7 @@ def loopCheck(threadNum: int = 16) -> None: logging.warning('ProxyC starts running (%s)' % Constant.Version) -_thread.start_new_thread(pythonCompile, ('/usr', )) # python compile (generate .pyc file) +_thread.start_new_thread(pythonCompile, ('/usr',)) # python compile (generate .pyc file) _thread.start_new_thread(DnsProxy.start, (Constant.DnsServer, 53)) # start dns server -_thread.start_new_thread(loopCheck, ()) # start loop check -Api.startServer(apiToken = Constant.ApiToken) # start api server +_thread.start_new_thread(loop, ()) # start check loop +Api.startServer() # start api server From 1d5618e264ab848d6bef2273dd82926c48284e19 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 21:12:15 +0800 Subject: [PATCH 04/18] update: finish test sub command --- Basis/Constant.py | 3 ++- Basis/Test.py | 6 +++--- main.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/Basis/Constant.py b/Basis/Constant.py index 971509a..4175aa0 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -15,7 +15,8 @@ LogFile = 'runtime.log' DnsServer = [] WorkDir = '/tmp/ProxyC' -ObfsSite = 'www.bing.com' +TestHost = 'proxyc.net' +TestSite = 'www.bing.com' PathEnv = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' # Shadowsocks Info diff --git a/Basis/Test.py b/Basis/Test.py index 83f4523..2cb96cb 100644 --- a/Basis/Test.py +++ b/Basis/Test.py @@ -6,12 +6,12 @@ import time import requests from threading import Thread from Basis.Logger import logging -from Basis.Constant import WorkDir, ObfsSite +from Basis.Constant import WorkDir, TestHost, TestSite from Basis.Functions import md5Sum, genFlag, hostFormat, checkPortStatus Settings = { 'workDir': WorkDir, - 'site': ObfsSite, + 'site': TestSite, 'serverBind': '', 'clientBind': '', 'host': '', @@ -56,7 +56,7 @@ def genCert(host: str, certInfo: dict, remark: str = 'ProxyC') -> None: # gener os.system('cat %s >> /etc/ssl/certs/ca-certificates.crt' % certInfo['caCert']) # add into system's trust list -def loadCert(host: str = 'proxyc.net', certId: str = '') -> None: # load certificate +def loadCert(host: str = TestHost, certId: str = '') -> None: # load certificate newCert = (certId == '') certId = genFlag(length = 8) if certId == '' else certId certInfo = { diff --git a/main.py b/main.py index b652388..2729378 100755 --- a/main.py +++ b/main.py @@ -25,13 +25,16 @@ def testArgParse(rawArgs: list) -> argparse.Namespace: testParser.add_argument('PROTOCOL', type = str, help = 'test protocol name') testParser.add_argument('-a', '--all', help = 'test extra shadowsocks items', action = 'store_true') testParser.add_argument('-6', '--ipv6', help = 'test on ipv6 network', action = 'store_true') + testParser.add_argument('--debug', help = 'enable debug log level', action = 'store_true') testParser.add_argument('--url', type = str, default = 'http://baidu.com', help = 'http request url') - testParser.add_argument('--cert', type = str, help = 'specify the certificate id') + testParser.add_argument('--cert', type = str, default = '', help = 'specify the certificate id') testParser.add_argument('--thread', type = int, default = 16, help = 'thread number in check process') testParser.add_argument('--select', type = str, nargs = '+', help = 'select id list for test') return testParser.parse_args(rawArgs) +testArgs = None +testMode = False inputArgs = copy.copy(sys.argv) if len(inputArgs) >= 0: # remove first arg (normally file name) inputArgs.pop(0) @@ -40,9 +43,8 @@ if len(inputArgs) != 0 and inputArgs[0].lower() == 'test': # test mode if len(inputArgs) == 0 or inputArgs[0].startswith('-'): # no protocol is specified inputArgs = ['all'] + inputArgs testArgs = testArgParse(inputArgs) - print(testArgs) - sys.exit(1) - # TODO: start test process + Constant.LogLevel = 'debug' if testArgs.debug else 'warning' + testMode = True else: mainArgs = mainArgParse(inputArgs) if mainArgs.version: # output version and exit @@ -53,10 +55,12 @@ else: Constant.ApiToken = mainArgs.token +from Tester import testEntry from Basis.Check import Check from Basis import Api, DnsProxy from Basis.Logger import logging from Basis.Manager import Manager +from Basis.Test import Test, loadBind, loadCert from concurrent.futures import ThreadPoolExecutor @@ -84,6 +88,28 @@ def loop(threadNum: int = 16) -> None: threadPool.submit(runCheck, taskId, taskInfo) # submit into thread pool +if testMode: # test mode + loadBind(serverV6 = testArgs.ipv6, clientV6 = testArgs.ipv6) # ipv4 / ipv6 (127.0.0.1 / ::1) + loadCert(certId = testArgs.cert) # cert config + logging.critical('TEST ITEM: %s' % testArgs.PROTOCOL) + logging.critical('SELECT: ' + str(testArgs.select)) + logging.critical('URL: %s' % testArgs.url) + logging.critical('THREAD NUMBER: %i' % testArgs.thread) + logging.critical('-' * 32 + ' TEST START ' + '-' * 32) + if testArgs.PROTOCOL == 'all': # run all test items + for item in testEntry: + if item == ('ss' if testArgs.all else 'ss-all'): # skip ss / ss-all + continue + logging.critical('TEST ITEM -> ' + item) + Test(testEntry[item], testArgs.thread, testArgs.url, testArgs.select) + else: # run single item + if testArgs.PROTOCOL == 'ss' and testArgs.all: # test shadowsocks extra items + testItem = 'ss-all' + Test(testEntry[testArgs.PROTOCOL], testArgs.thread, testArgs.url, testArgs.select) + logging.critical('-' * 32 + ' TEST COMPLETE ' + '-' * 32) + sys.exit(0) # test complete + + logging.warning('ProxyC starts running (%s)' % Constant.Version) _thread.start_new_thread(pythonCompile, ('/usr',)) # python compile (generate .pyc file) _thread.start_new_thread(DnsProxy.start, (Constant.DnsServer, 53)) # start dns server From 700f2d68db625fc527c08fdb198585a039bbe859 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 21:25:31 +0800 Subject: [PATCH 05/18] update: add dns and api path options --- Basis/Api.py | 2 +- Basis/Constant.py | 4 +--- main.py | 11 +++++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Basis/Api.py b/Basis/Api.py index 8cb5b36..3a37bda 100644 --- a/Basis/Api.py +++ b/Basis/Api.py @@ -115,6 +115,6 @@ def getVersion() -> Response: def startServer() -> None: - logging.warning('API server at http://:%i/' % ApiPort) + logging.warning('API server at http://:%i%s' % (ApiPort, ApiPath)) logging.warning('API ' + ('without token' if ApiToken == '' else 'token -> %s' % ApiToken)) pywsgi.WSGIServer(('0.0.0.0', ApiPort), webApi).serve_forever() # powered by gevent diff --git a/Basis/Constant.py b/Basis/Constant.py index 4175aa0..72c3b55 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -8,12 +8,10 @@ ApiPath = '/' ApiPort = 7839 ApiToken = '' -# LogLevel = 'DEBUG' -# LogLevel = 'WARNING' LogLevel = 'INFO' LogFile = 'runtime.log' -DnsServer = [] +DnsServer = None WorkDir = '/tmp/ProxyC' TestHost = 'proxyc.net' TestSite = 'www.bing.com' diff --git a/main.py b/main.py index 2729378..4d5384e 100755 --- a/main.py +++ b/main.py @@ -12,11 +12,12 @@ from Basis import Constant def mainArgParse(rawArgs: list) -> argparse.Namespace: mainParser = argparse.ArgumentParser(description = 'Start running API server') - mainParser.add_argument('--log', type = str, default = 'debug', help = 'output log level') - mainParser.add_argument('--port', type = int, default = 7839, help = 'port for running') - mainParser.add_argument('--token', type = str, default = '', help = 'token for api server') + mainParser.add_argument('--log', type = str, default = Constant.LogLevel, help = 'output log level') + mainParser.add_argument('--dns', type = str, nargs = '+', help = 'specify dns server') + mainParser.add_argument('--port', type = int, default = Constant.ApiPort, help = 'port for running') + mainParser.add_argument('--path', type = str, default = Constant.ApiPath, help = 'root path for api server') + mainParser.add_argument('--token', type = str, default = Constant.ApiToken, help = 'token for api server') mainParser.add_argument('-v', '--version', help = 'show version', action = 'store_true') - # TODO: api path / dns server return mainParser.parse_args(rawArgs) @@ -51,7 +52,9 @@ else: print('ProxyC version -> %s' % Constant.Version) sys.exit(0) Constant.LogLevel = mainArgs.log # overwrite global options + Constant.DnsServer = mainArgs.dns Constant.ApiPort = mainArgs.port + Constant.ApiPath = mainArgs.path Constant.ApiToken = mainArgs.token From c5b72f7678ee4e32405e68c2a19f497904dabb29 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 21:26:47 +0800 Subject: [PATCH 06/18] update: remove legacy test script --- test.py | 67 --------------------------------------------------------- 1 file changed, 67 deletions(-) delete mode 100755 test.py diff --git a/test.py b/test.py deleted file mode 100755 index a6d507d..0000000 --- a/test.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import sys -from Tester import testEntry -from Basis.Logger import logging -from Basis.Test import Test, loadBind, loadCert - -threadNum = 16 -testItem = None -testFilter = None -testUrl = 'http://baidu.com' -helpMsg = ''' - ./test.py [ITEM] [OPTIONS] - - [ITEM]: ss / ssr / vmess / vless / trojan / trojan-go / brook / hysteria - - [OPTIONS]: - --thread NUM thread number - --url URL http check url - --filter ID1[,ID2...] test the specified id - --all test extra shadowsocks items - --ipv6 test on ipv6 network - --help show this message -''' - - -def getArg(field: str) -> str or None: - try: - index = sys.argv.index(field) - return sys.argv[index + 1] - except: - return None - - -if '--help' in sys.argv: - print(helpMsg) - sys.exit(0) -if len(sys.argv) > 1 and not sys.argv[1].startswith('--'): - testItem = sys.argv[1] -if getArg('--url') is not None: - testUrl = getArg('--url') -if getArg('--thread') is not None: - threadNum = int(getArg('--thread')) -if getArg('--filter') is not None: - testFilter = set(getArg('--filter').split(',')) - -isV6 = '--ipv6' in sys.argv -loadBind(serverV6 = isV6, clientV6 = isV6) # ipv4 / ipv6 (127.0.0.1 / ::1) -loadCert('proxyc.net') # default cert config -logging.critical('TEST ITEM: ' + ('all' if testItem is None else testItem)) -logging.critical('FILTER: %s' % testFilter) -logging.critical('URL: ' + testUrl) -logging.critical('THREAD NUMBER: %i' % threadNum) - -logging.critical('-------------------------------- TEST START --------------------------------') -if testItem is not None: - if testItem == 'ss' and '--all' in sys.argv: - testItem = 'ss-all' - Test(testEntry[testItem], threadNum, testUrl, testFilter) -else: - for item in testEntry: - if item == ('ss' if '--all' in sys.argv else 'ss-all'): # skip ss / ss-all - continue - logging.critical('TEST ITEM -> ' + item) - Test(testEntry[item], threadNum, testUrl, testFilter) -logging.critical('-------------------------------- TEST COMPLETE --------------------------------') From ea92ff82c98948ceea0755debdd0daa169e630bc Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 21:53:18 +0800 Subject: [PATCH 07/18] feat: local env configure --- Basis/Constant.py | 23 +++++++++++++++++++++++ Dockerfile | 2 +- env.yaml | 10 ++++++++++ main.py | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 env.yaml diff --git a/Basis/Constant.py b/Basis/Constant.py index 72c3b55..9ffbc7f 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os +import yaml + # Global Options Version = 'dev' @@ -17,6 +20,26 @@ TestHost = 'proxyc.net' TestSite = 'www.bing.com' PathEnv = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' +# Load Env Options +yamlFile = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../env.yaml') +yamlContent = open(yamlFile, 'r', encoding = 'utf-8').read() +envOptions = yaml.load(yamlContent, Loader = yaml.FullLoader) +if 'version' in envOptions: + Version = envOptions['version'] +if 'loglevel' in envOptions: + LogLevel = envOptions['loglevel'] +if 'dir' in envOptions: + WorkDir = envOptions['dir'] +if 'dns' in envOptions: + DnsServer = envOptions['dns'] +if 'api' in envOptions: + if 'port' in envOptions['api']: + ApiPort = envOptions['api']['port'] + if 'path' in envOptions['api']: + ApiPath = envOptions['api']['path'] + if 'token' in envOptions['api']: + ApiToken = envOptions['api']['token'] + # Shadowsocks Info mbedtlsMethods = [ 'aes-128-cfb128', diff --git a/Dockerfile b/Dockerfile index 699f252..e3aa413 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ FROM ${PYTHON_IMG} AS wheels WORKDIR /wheels/ RUN apk add linux-headers COPY --from=build-base /apk/ /apk/ -RUN /apk/build-base && pip wheel colorlog flask IPy psutil pysocks requests salsa20 +RUN /apk/build-base && pip wheel colorlog flask IPy psutil pysocks pyyaml requests salsa20 COPY --from=gevent /wheels/*.whl /wheels/ COPY --from=numpy /wheels/*.whl /wheels/ diff --git a/env.yaml b/env.yaml new file mode 100644 index 0000000..7432cd0 --- /dev/null +++ b/env.yaml @@ -0,0 +1,10 @@ +version: 'v0.1' +loglevel: 'INFO' +dir: '/tmp/ProxyC' +dns: + - 223.5.5.5 + - 119.29.29.29 +api: + port: 17839 + path: '/v1' + token: 'dnomd343' diff --git a/main.py b/main.py index 4d5384e..80fc9fd 100755 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ from Basis import Constant def mainArgParse(rawArgs: list) -> argparse.Namespace: mainParser = argparse.ArgumentParser(description = 'Start running API server') mainParser.add_argument('--log', type = str, default = Constant.LogLevel, help = 'output log level') - mainParser.add_argument('--dns', type = str, nargs = '+', help = 'specify dns server') + mainParser.add_argument('--dns', type = str, default = Constant.DnsServer, nargs = '+', help = 'specify dns server') mainParser.add_argument('--port', type = int, default = Constant.ApiPort, help = 'port for running') mainParser.add_argument('--path', type = str, default = Constant.ApiPath, help = 'root path for api server') mainParser.add_argument('--token', type = str, default = Constant.ApiToken, help = 'token for api server') From ea86a6610bbf5df208dba5a85b4d7bff8510ee14 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Sun, 7 Aug 2022 22:02:54 +0800 Subject: [PATCH 08/18] fix: filter dnsproxy input --- Basis/DnsProxy.py | 8 ++++++-- env.yaml | 10 ++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Basis/DnsProxy.py b/Basis/DnsProxy.py index 3b738f9..276c3a4 100644 --- a/Basis/DnsProxy.py +++ b/Basis/DnsProxy.py @@ -15,12 +15,16 @@ def daemon(process: subprocess.Popen, command: list, gap: int = 2) -> None: # d while True: # start daemon time.sleep(gap) # check time gap if process.poll() is not None: # unexpected exit - logging.warning('dnsproxy unexpected exit') - logging.debug('output of dnsproxy\n%s' % process.stdout.read().decode('utf-8')) + logging.error('DnsProxy unexpected exit\n%s\n%s%s' % ( + '-' * 96, process.stdout.read().decode('utf-8'), '-' * 96) + ) process = run(command) def start(servers: list or None, port: int = 53, cache: int = 4194304) -> None: # default cache size -> 4MiB + if servers is not None and type(servers) != list: # invalid server content + logging.error('Invalid DNS server -> %s' % servers) + return if servers is None or len(servers) == 0: # use origin dns server logging.info('Skip dnsproxy process') return diff --git a/env.yaml b/env.yaml index 7432cd0..83407f9 100644 --- a/env.yaml +++ b/env.yaml @@ -1,10 +1,8 @@ version: 'v0.1' loglevel: 'INFO' dir: '/tmp/ProxyC' -dns: - - 223.5.5.5 - - 119.29.29.29 +dns: null api: - port: 17839 - path: '/v1' - token: 'dnomd343' + port: 7839 + path: '/' + token: '' From d47b9162b88056b0b7a4f81ee7e26bc09c0e5525 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 01:16:47 +0800 Subject: [PATCH 09/18] build: update naiveproxy --- Dockerfile | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3aa413..e56f6d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,16 +359,17 @@ COPY --from=upx /upx/ /usr/ RUN upx -9 /tmp/clash # Download naiveproxy -FROM ${ALPINE_IMG} AS naiveproxy -ENV NAIVE_VERSION="v103.0.5060.53-3" +FROM ${ALPINE_IMG} AS naive +ENV NAIVE_VERSION="v104.0.5112.79-2" RUN apk add curl libgcc jq RUN curl -sL https://api.github.com/repos/klzgrad/naiveproxy/releases/tags/${NAIVE_VERSION} \ | jq .assets | jq .[].name | grep naiveproxy-${NAIVE_VERSION}-openwrt-$(uname -m) \ | cut -b 2- | rev | cut -b 2- | rev | tac > list.dat -RUN echo -e "while read FILE_NAME;do\nwget https://github.com/klzgrad/naiveproxy/releases/download/\${NAIVE_VERSION}/\${FILE_NAME}\n \ - tar xf \${FILE_NAME} && ldd ./\$(echo \$FILE_NAME | rev | cut -b 8- | rev)/naive\n \ - [ \$? -eq 0 ] && cp ./\$(echo \$FILE_NAME | rev | cut -b 8- | rev)/naive /tmp/ && break\ndone < list.dat" > naiveproxy.sh && \ - sh naiveproxy.sh +RUN echo "while read FILE_NAME; do" >> naive.sh && \ + echo "wget https://github.com/klzgrad/naiveproxy/releases/download/\${NAIVE_VERSION}/\${FILE_NAME}" >> naive.sh && \ + echo "tar xf \${FILE_NAME} && ldd ./\$(echo \$FILE_NAME | rev | cut -b 8- | rev)/naive" >> naive.sh && \ + echo "[ \$? -eq 0 ] && cp ./\$(echo \$FILE_NAME | rev | cut -b 8- | rev)/naive /tmp/ && break" >> naive.sh && \ + echo "done < list.dat" >> naive.sh && sh naive.sh COPY --from=build-base /apk/ /apk/ RUN /apk/build-base && strip /tmp/naive @@ -475,9 +476,9 @@ COPY --from=trojan /tmp/trojan* /asset/usr/bin/ COPY --from=gost /tmp/gost* /asset/usr/bin/ COPY --from=brook /tmp/brook /asset/usr/bin/ COPY --from=clash /tmp/clash /asset/usr/bin/ +COPY --from=naive /tmp/naive /asset/usr/bin/ COPY --from=snell /tmp/snell-* /asset/usr/bin/ COPY --from=hysteria /tmp/hysteria /asset/usr/bin/ -COPY --from=naiveproxy /tmp/naive /asset/usr/bin/ COPY --from=relaybaton /tmp/relaybaton /asset/usr/bin/ COPY --from=pingtunnel /tmp/pingtunnel /asset/usr/bin/ COPY --from=wireproxy /tmp/wireproxy /asset/usr/bin/ @@ -492,4 +493,4 @@ RUN apk add --no-cache boost-program_options c-ares \ ln -s /usr/local/share/ProxyC/main.py /usr/bin/proxyc COPY --from=asset /asset / EXPOSE 7839 -CMD ["proxyc"] +ENTRYPOINT ["proxyc"] From 412750b75c7d9257068b5c0d06d44d1323eed133 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 02:30:41 +0800 Subject: [PATCH 10/18] update: trace env for ss-rust --- Builder/Shadowsocks.py | 20 ++++++++++---------- Dockerfile | 4 ++-- Tester/Shadowsocks.py | 33 ++++++++++++++++++++------------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/Builder/Shadowsocks.py b/Builder/Shadowsocks.py index 7f326b5..5a5b627 100644 --- a/Builder/Shadowsocks.py +++ b/Builder/Shadowsocks.py @@ -29,21 +29,21 @@ def pluginUdp(plugin: str, pluginParam: str) -> bool: # whether the plugin uses return True # UDP is assumed by default -def ssRust(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssRust(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) if isUdp: # proxy UDP traffic config['mode'] = 'tcp_and_udp' - return config, ['ss-rust-local', '-v'] + return config, ['ss-rust-local', '-v'], {'RUST_BACKTRACE': 'full'} -def ssLibev(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssLibev(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) if isUdp: # proxy UDP traffic config['mode'] = 'tcp_and_udp' - return config, ['ss-libev-local', '-v'] + return config, ['ss-libev-local', '-v'], {} -def ssPython(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssPython(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) if config['method'] in mbedtlsMethods: # mbedtls methods should use prefix `mbedtls:` config['method'] = 'mbedtls:' + config['method'] @@ -52,15 +52,15 @@ def ssPython(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list] if not isUdp: config['no_udp'] = True # UDP traffic is not proxied config['shadowsocks'] = 'ss-python-local' - return config, ['ss-bootstrap-local', '--debug', '-vv'] + return config, ['ss-bootstrap-local', '--debug', '-vv'], {} -def ssPythonLegacy(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssPythonLegacy(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) if not isUdp: config['no_udp'] = True # UDP traffic is not proxied config['shadowsocks'] = 'ss-python-legacy-local' - return config, ['ss-bootstrap-local', '--debug', '-vv'] + return config, ['ss-bootstrap-local', '--debug', '-vv'], {} def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, dict]: @@ -72,10 +72,10 @@ def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, for client in ssMethods: # traverse all shadowsocks client if proxyInfo['method'] not in ssMethods[client]: continue - ssConfig, ssClient = { + ssConfig, ssClient, ssEnv = { 'ss-rust': ssRust, 'ss-libev': ssLibev, 'ss-python': ssPython, 'ss-python-legacy': ssPythonLegacy }[client](proxyInfo, socksInfo, isUdp) # generate config file - return ssClient + ['-c', configFile], json.dumps(ssConfig), {} # command, fileContent, envVar + return ssClient + ['-c', configFile], json.dumps(ssConfig), ssEnv # command, fileContent, envVar diff --git a/Dockerfile b/Dockerfile index e56f6d0..070d583 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ COPY --from=build-base /apk/ /apk/ RUN wget https://github.com/shadowsocks/shadowsocks-rust/archive/refs/tags/v${SS_RUST}.tar.gz && \ tar xf v${SS_RUST}.tar.gz && /apk/build-base WORKDIR ./shadowsocks-rust-${SS_RUST}/ -RUN cargo update +RUN cargo fetch RUN cargo build --target-dir ./ --release --bin sslocal --bin ssserver \ --features "stream-cipher aead-cipher-extra aead-cipher-2022 aead-cipher-2022-extra" && \ mv ./release/sslocal /tmp/ss-rust-local && mv ./release/ssserver /tmp/ss-rust-server && \ @@ -169,7 +169,7 @@ RUN git submodule update --init --recursive && \ mv ./src/obfs-local ./src/obfs-server /plugins/ # Compile qtun WORKDIR ../qtun/ -RUN cargo update +RUN cargo fetch RUN cargo build --target-dir ./ --release && \ mv ./release/qtun-client ./release/qtun-server /plugins/ && \ strip /plugins/* diff --git a/Tester/Shadowsocks.py b/Tester/Shadowsocks.py index a193965..d101c3f 100644 --- a/Tester/Shadowsocks.py +++ b/Tester/Shadowsocks.py @@ -11,7 +11,7 @@ from Basis.Test import Settings from Basis.Logger import logging from Basis.Process import Process from Basis.Functions import md5Sum, genFlag, getAvailablePort -from Basis.Constant import ssMethods, ssAllMethods, mbedtlsMethods +from Basis.Constant import PathEnv, ssMethods, ssAllMethods, mbedtlsMethods def loadConfig(proxyInfo: dict) -> dict: # load basic config option @@ -27,21 +27,28 @@ def loadConfig(proxyInfo: dict) -> dict: # load basic config option return config -def ssRust(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: +def addPathEnv(env: dict) -> dict: + return { + **env, + 'PATH': PathEnv # add PATH env + } + + +def ssRust(proxyInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo) if isUdp: # proxy UDP traffic config['mode'] = 'tcp_and_udp' - return config, ['ss-rust-server', '-v'] + return config, ['ss-rust-server', '-v'], {'RUST_BACKTRACE': 'full'} -def ssLibev(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssLibev(proxyInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo) if isUdp: # proxy UDP traffic config['mode'] = 'tcp_and_udp' - return config, ['ss-libev-server', '-v'] + return config, ['ss-libev-server', '-v'], {} -def ssPython(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssPython(proxyInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo) if config['method'] in mbedtlsMethods: # mbedtls methods should use prefix `mbedtls:` config['method'] = 'mbedtls:' + config['method'] @@ -50,15 +57,15 @@ def ssPython(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: if not isUdp: config['no_udp'] = True # UDP traffic is not proxied config['shadowsocks'] = 'ss-python-server' - return config, ['ss-bootstrap-server', '--debug', '-vv'] + return config, ['ss-bootstrap-server', '--debug', '-vv'], {} -def ssPythonLegacy(proxyInfo: dict, isUdp: bool) -> tuple[dict, list]: +def ssPythonLegacy(proxyInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo) if not isUdp: config['no_udp'] = True # UDP traffic is not proxied config['shadowsocks'] = 'ss-python-legacy-server' - return config, ['ss-bootstrap-server', '--debug', '-vv'] + return config, ['ss-bootstrap-server', '--debug', '-vv'], {} def loadPassword(method: str) -> str: @@ -71,7 +78,7 @@ def loadPassword(method: str) -> str: def loadClient(ssType: str, configFile: str, proxyInfo: dict, socksInfo: dict) -> Process: - ssConfig, ssClient = { # generate client start command and its config file + ssConfig, ssClient, ssEnv = { # generate client start command and its config file 'ss-rust': Shadowsocks.ssRust, 'ss-libev': Shadowsocks.ssLibev, 'ss-python': Shadowsocks.ssPython, @@ -81,11 +88,11 @@ def loadClient(ssType: str, configFile: str, proxyInfo: dict, socksInfo: dict) - return Process(Settings['workDir'], cmd = ssClient + ['-c', clientFile], file = { # load client process 'path': clientFile, 'content': json.dumps(ssConfig) - }, isStart = False) + }, env = addPathEnv(ssEnv), isStart = False) def loadServer(ssType: str, configFile: str, proxyInfo: dict) -> Process: - ssConfig, ssServer = { # generate server start command and its config file + ssConfig, ssServer, ssEnv = { # generate server start command and its config file 'ss-rust': ssRust, 'ss-libev': ssLibev, 'ss-python': ssPython, @@ -95,7 +102,7 @@ def loadServer(ssType: str, configFile: str, proxyInfo: dict) -> Process: return Process(Settings['workDir'], cmd = ssServer + ['-c', serverFile], file = { # load server process 'path': serverFile, 'content': json.dumps(ssConfig) - }, isStart = False) + }, env = addPathEnv(ssEnv), isStart = False) def loadTest(serverType: str, clientType: str, method: str, plugin: dict or None = None) -> dict: From 7efbf8708353fb9fbda13338270edb1095a30ecc Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 03:14:40 +0800 Subject: [PATCH 11/18] perf: qtun build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 070d583..29d9474 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,6 +168,7 @@ RUN git submodule update --init --recursive && \ ./autogen.sh && ./configure --disable-documentation && make && \ mv ./src/obfs-local ./src/obfs-server /plugins/ # Compile qtun +COPY --from=ss-rust /usr/local/cargo/registry/ /usr/local/cargo/registry/ WORKDIR ../qtun/ RUN cargo fetch RUN cargo build --target-dir ./ --release && \ From a966c8cf5d42b8664fb2beeb5b1c7cd0b676033d Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 14:48:45 +0800 Subject: [PATCH 12/18] feat: add build exception --- Basis/Constant.py | 6 ++++++ Basis/Exception.py | 6 ++++++ Basis/Functions.py | 4 ++++ Builder/Brook.py | 26 ++++++++++++-------------- Builder/Hysteria.py | 6 +++--- Builder/Shadowsocks.py | 25 +++++++++++++------------ Builder/ShadowsocksR.py | 7 ++++--- Builder/Trojan.py | 2 +- Builder/V2ray.py | 7 ++++--- Builder/VLESS.py | 5 ++++- Builder/VMess.py | 3 ++- Builder/Xray.py | 7 ++++--- Builder/__init__.py | 12 +++++++----- 13 files changed, 70 insertions(+), 46 deletions(-) create mode 100644 Basis/Exception.py diff --git a/Basis/Constant.py b/Basis/Constant.py index 9ffbc7f..c17a187 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -40,6 +40,12 @@ if 'api' in envOptions: if 'token' in envOptions['api']: ApiToken = envOptions['api']['token'] +# WorkDir Create +try: + os.makedirs(WorkDir) # just like `mkdir -p ...` +except: + pass # folder exist or target is another thing + # Shadowsocks Info mbedtlsMethods = [ 'aes-128-cfb128', diff --git a/Basis/Exception.py b/Basis/Exception.py new file mode 100644 index 0000000..685b394 --- /dev/null +++ b/Basis/Exception.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +class buildException(Exception): # for build error + def __init__(self, reason): + self.reason = reason diff --git a/Basis/Functions.py b/Basis/Functions.py index daefe4f..7297183 100644 --- a/Basis/Functions.py +++ b/Basis/Functions.py @@ -55,6 +55,10 @@ def hostFormat(host: str, v6Bracket: bool = False) -> str: return host +def v6AddBracket(host: str) -> str: # add bracket for ipv6 + return hostFormat(host, v6Bracket = True) + + def genFlag(length: int = 12) -> str: # generate random task flag flag = '' for i in range(0, length): diff --git a/Builder/Brook.py b/Builder/Brook.py index ab2940b..99a4776 100644 --- a/Builder/Brook.py +++ b/Builder/Brook.py @@ -1,30 +1,29 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from Basis.Functions import hostFormat - +from Basis.Functions import v6AddBracket def loadOrigin(proxyInfo: dict) -> list: # origin stream return ['client'] + [ - '--server', '%s:%i' % (hostFormat(proxyInfo['server'], v6Bracket = True), proxyInfo['port']), + '--server', '%s:%i' % (v6AddBracket(proxyInfo['server']), proxyInfo['port']), '--password', proxyInfo['passwd'], - ] + (['--udpovertcp'] if proxyInfo['stream']['uot'] else []) + ] + (['--udpovertcp'] if proxyInfo['stream']['uot'] else []) # add uot option -def loadWebsocket(proxyInfo: dict) -> list: - isTls = proxyInfo['stream']['secure'] is not None - wsAddress = (('wss' if isTls else 'ws') + '://%s:%i%s') % ( - hostFormat(proxyInfo['stream']['host'], v6Bracket = True), proxyInfo['port'], proxyInfo['stream']['path'] +def loadWebsocket(proxyInfo: dict) -> list: # websocket stream + isTls = proxyInfo['stream']['secure'] is not None # ws or wss + wsAddress = (('wss' if isTls else 'ws') + '://%s:%i%s') % ( # websocket address + v6AddBracket(proxyInfo['stream']['host']), proxyInfo['port'], proxyInfo['stream']['path'] ) brookCommand = [ 'wssclient' if isTls else 'wsclient', - '--address', '%s:%i' % (hostFormat(proxyInfo['server'], v6Bracket = True), proxyInfo['port']), + '--address', '%s:%i' % (v6AddBracket(proxyInfo['server']), proxyInfo['port']), # real address '--password', proxyInfo['passwd'], - ] + (['--withoutBrookProtocol'] if proxyInfo['stream']['raw'] else []) + ] + (['--withoutBrookProtocol'] if proxyInfo['stream']['raw'] else []) # raw transmission on ws or wss if not isTls: return brookCommand + ['--wsserver', wsAddress] return brookCommand + ['--wssserver', wsAddress] + ( - [] if proxyInfo['stream']['secure']['verify'] else ['--insecure'] + [] if proxyInfo['stream']['secure']['verify'] else ['--insecure'] # add tls options ) @@ -32,7 +31,6 @@ def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, brookCommand = ['brook', '--debug', '--listen', ':'] + { # debug module listen on random port 'origin': loadOrigin, 'ws': loadWebsocket, - }[proxyInfo['stream']['type']](proxyInfo) + [ - '--socks5', '%s:%i' % (hostFormat(socksInfo['addr'], v6Bracket = True), socksInfo['port']) - ] + }[proxyInfo['stream']['type']](proxyInfo) # choose origin or websocket stream + brookCommand += ['--socks5', '%s:%i' % (v6AddBracket(socksInfo['addr']), socksInfo['port'])] return brookCommand, 'Config file %s no need' % configFile, {} # command, fileContent, envVar diff --git a/Builder/Hysteria.py b/Builder/Hysteria.py index 4f4be18..2afdc4e 100644 --- a/Builder/Hysteria.py +++ b/Builder/Hysteria.py @@ -2,19 +2,19 @@ # -*- coding: utf-8 -*- import json -from Basis.Functions import hostFormat +from Basis.Functions import v6AddBracket def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, dict]: hysteriaConfig = { - 'server': '%s:%i' % (hostFormat(proxyInfo['server'], v6Bracket = True), proxyInfo['port']), + 'server': '%s:%i' % (v6AddBracket(proxyInfo['server']), proxyInfo['port']), 'protocol': proxyInfo['protocol'], 'up_mbps': proxyInfo['up'], 'down_mbps': proxyInfo['down'], 'retry_interval': 2, 'retry': 3, 'socks5': { - 'listen': '%s:%i' % (hostFormat(socksInfo['addr'], v6Bracket = True), socksInfo['port']) + 'listen': '%s:%i' % (v6AddBracket(socksInfo['addr']), socksInfo['port']) }, **({} if proxyInfo['obfs'] is None else { 'obfs': proxyInfo['obfs'] diff --git a/Builder/Shadowsocks.py b/Builder/Shadowsocks.py index 5a5b627..da6fe63 100644 --- a/Builder/Shadowsocks.py +++ b/Builder/Shadowsocks.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import json +from Basis.Exception import buildException from Basis.Constant import ssMethods, ssAllMethods, mbedtlsMethods @@ -20,25 +21,25 @@ def loadConfig(proxyInfo: dict, socksInfo: dict) -> dict: # load basic config o return config -def pluginUdp(plugin: str, pluginParam: str) -> bool: # whether the plugin uses UDP +def pluginUdp(plugin: str, pluginParam: str) -> bool: # whether the plugin uses udp if plugin in ['obfs-local', 'simple-tls', 'ck-client', 'gq-client', 'mtt-client', 'rabbit-plugin']: return False # UDP is not used if plugin in ['v2ray-plugin', 'xray-plugin', 'gost-plugin']: - if 'mode=quic' not in pluginParam.split(';'): # non-quic mode does not use UDP + if 'mode=quic' not in pluginParam.split(';'): # non-quic mode does not use udp return False - return True # UDP is assumed by default + return True # udp is assumed by default def ssRust(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) - if isUdp: # proxy UDP traffic + if isUdp: # proxy udp traffic config['mode'] = 'tcp_and_udp' - return config, ['ss-rust-local', '-v'], {'RUST_BACKTRACE': 'full'} + return config, ['ss-rust-local', '-v'], {'RUST_BACKTRACE': 'full'} # enable rust trace def ssLibev(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) - if isUdp: # proxy UDP traffic + if isUdp: # proxy udp traffic config['mode'] = 'tcp_and_udp' return config, ['ss-libev-local', '-v'], {} @@ -50,7 +51,7 @@ def ssPython(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, if config['method'] in ['idea-cfb', 'seed-cfb']: # only older versions of openssl are supported config['extra_opts'] = '--libopenssl=libcrypto.so.1.0.0' if not isUdp: - config['no_udp'] = True # UDP traffic is not proxied + config['no_udp'] = True # udp traffic is not proxied config['shadowsocks'] = 'ss-python-local' return config, ['ss-bootstrap-local', '--debug', '-vv'], {} @@ -58,21 +59,21 @@ def ssPython(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, def ssPythonLegacy(proxyInfo: dict, socksInfo: dict, isUdp: bool) -> tuple[dict, list, dict]: config = loadConfig(proxyInfo, socksInfo) if not isUdp: - config['no_udp'] = True # UDP traffic is not proxied + config['no_udp'] = True # udp traffic is not proxied config['shadowsocks'] = 'ss-python-legacy-local' return config, ['ss-bootstrap-local', '--debug', '-vv'], {} def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, dict]: - isUdp = True if proxyInfo['plugin'] is None else ( # UDP enabled when server without plugin - not pluginUdp(proxyInfo['plugin']['type'], proxyInfo['plugin']['param']) # UDP conflict status of plugins + isUdp = True if proxyInfo['plugin'] is None else ( # udp enabled when server without plugin + not pluginUdp(proxyInfo['plugin']['type'], proxyInfo['plugin']['param']) # udp conflict status of plugins ) if proxyInfo['method'] not in ssAllMethods: # unknown shadowsocks method - raise RuntimeError('Unknown shadowsocks method') + raise buildException('Unknown shadowsocks method') for client in ssMethods: # traverse all shadowsocks client if proxyInfo['method'] not in ssMethods[client]: continue - ssConfig, ssClient, ssEnv = { + ssConfig, ssClient, ssEnv = { # found appropriate client 'ss-rust': ssRust, 'ss-libev': ssLibev, 'ss-python': ssPython, diff --git a/Builder/ShadowsocksR.py b/Builder/ShadowsocksR.py index 8155da6..70549ca 100644 --- a/Builder/ShadowsocksR.py +++ b/Builder/ShadowsocksR.py @@ -2,16 +2,17 @@ # -*- coding: utf-8 -*- import json +from Basis.Exception import buildException from Basis.Constant import ssrMethods, ssrProtocols, ssrObfuscations def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, dict]: if proxyInfo['method'] not in ssrMethods: - raise RuntimeError('Unknown shadowsocksr method') + raise buildException('Unknown shadowsocksr method') if proxyInfo['protocol'] not in ssrProtocols: - raise RuntimeError('Unknown shadowsocksr protocol') + raise buildException('Unknown shadowsocksr protocol') if proxyInfo['obfs'] not in ssrObfuscations: - raise RuntimeError('Unknown shadowsocksr obfuscation') + raise buildException('Unknown shadowsocksr obfuscation') ssrConfig = { 'server': proxyInfo['server'], 'server_port': proxyInfo['port'], # type -> int diff --git a/Builder/Trojan.py b/Builder/Trojan.py index b6aaa5e..29b3680 100644 --- a/Builder/Trojan.py +++ b/Builder/Trojan.py @@ -13,7 +13,7 @@ def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, 'address': proxyInfo['server'], 'port': proxyInfo['port'], 'password': proxyInfo['passwd'], - **Xray.xtlsFlow(proxyInfo['stream']) + **Xray.xtlsFlow(proxyInfo['stream']) # add xtls flow option }] }, 'streamSettings': Xray.loadStream(proxyInfo['stream']) diff --git a/Builder/V2ray.py b/Builder/V2ray.py index 47bc8fb..a66709d 100644 --- a/Builder/V2ray.py +++ b/Builder/V2ray.py @@ -2,8 +2,9 @@ # -*- coding: utf-8 -*- import copy +from Basis.Exception import buildException -httpConfig = { +httpConfig = { # http obfs configure in default 'type': 'http', 'request': { 'version': '1.1', @@ -24,7 +25,7 @@ httpConfig = { } } -kcpConfig = { +kcpConfig = { # kcp options in default 'mtu': 1350, 'tti': 50, 'uplinkCapacity': 12, @@ -139,7 +140,7 @@ def loadStream(streamInfo: dict) -> dict: 'grpc': grpcStream, } if streamInfo['type'] not in streamEntry: - raise RuntimeError('Unknown stream type') + raise buildException('Unknown v2ray stream type') streamObject = streamEntry[streamInfo['type']](streamInfo) return { **streamObject, diff --git a/Builder/VLESS.py b/Builder/VLESS.py index 3180801..a9b6d8f 100644 --- a/Builder/VLESS.py +++ b/Builder/VLESS.py @@ -3,9 +3,12 @@ import json from Builder import Xray +from Basis.Exception import buildException def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, dict]: + if proxyInfo['method'] != 'none': + raise buildException('Unknown VLESS method') outboundConfig = { 'protocol': 'vless', 'settings': { @@ -15,7 +18,7 @@ def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, 'users': [{ 'id': proxyInfo['id'], 'encryption': proxyInfo['method'], - **Xray.xtlsFlow(proxyInfo['stream']) + **Xray.xtlsFlow(proxyInfo['stream']) # add xtls flow option }] }] }, diff --git a/Builder/VMess.py b/Builder/VMess.py index 76076ad..eebf5db 100644 --- a/Builder/VMess.py +++ b/Builder/VMess.py @@ -4,11 +4,12 @@ import json from Builder import V2ray from Basis.Constant import vmessMethods +from Basis.Exception import buildException def load(proxyInfo: dict, socksInfo: dict, configFile: str) -> tuple[list, str, dict]: if proxyInfo['method'] not in vmessMethods: - raise RuntimeError('Unknown vmess method') + raise buildException('Unknown VMess method') outboundConfig = { 'protocol': 'vmess', 'settings': { diff --git a/Builder/Xray.py b/Builder/Xray.py index bde216c..5726ae0 100644 --- a/Builder/Xray.py +++ b/Builder/Xray.py @@ -3,6 +3,7 @@ from Builder import V2ray from Basis.Constant import xtlsFlows +from Basis.Exception import buildException loadConfig = V2ray.loadConfig @@ -11,7 +12,7 @@ def loadSecure(secureInfo: dict or None) -> dict: # TLS / XTLS encrypt config if secureInfo is None: return {'security': 'none'} # without TLS / XTLS options if secureInfo['type'] not in ['tls', 'xtls']: - raise RuntimeError('Unknown secure type') + raise buildException('Unknown xray secure type') secureObject = { 'allowInsecure': not secureInfo['verify'] # whether verify server's certificate } @@ -53,7 +54,7 @@ def loadStream(streamInfo: dict) -> dict: 'grpc': V2ray.grpcStream, } if streamInfo['type'] not in streamEntry: - raise RuntimeError('Unknown stream type') + raise buildException('Unknown xray stream type') streamObject = streamEntry[streamInfo['type']](streamInfo) return { **streamObject, @@ -67,7 +68,7 @@ def xtlsFlow(streamInfo: dict or None) -> dict: if streamInfo['secure']['type'] != 'xtls': # not XTLS secure type return {} if streamInfo['secure']['flow'] not in xtlsFlows: - raise RuntimeError('Unknown xtls flow') + raise buildException('Unknown xtls flow') return { 'flow': xtlsFlows[streamInfo['secure']['flow']] + ( # xtls-rprx-xxx '-udp443' if streamInfo['secure']['udp443'] else '' # whether block udp/443 (disable http/3) diff --git a/Builder/__init__.py b/Builder/__init__.py index dd1a60e..5bbe0dc 100644 --- a/Builder/__init__.py +++ b/Builder/__init__.py @@ -5,8 +5,10 @@ import os import copy from Basis.Logger import logging from Basis.Process import Process +from Basis.Functions import v6AddBracket +from Basis.Exception import buildException from Basis.Constant import WorkDir, PathEnv -from Basis.Functions import hostFormat, genFlag, getAvailablePort +from Basis.Functions import genFlag, getAvailablePort from Builder import Brook from Builder import VMess @@ -44,19 +46,19 @@ class Builder(object): Attributes: id, proxyType, proxyInfo, socksAddr, socksPort, output """ - output = None + output = None # output capture of proxy client (after process exit) def __loadClient(self): logging.info('[%s] Builder load %s client at %s -> %s' % ( self.id, self.proxyType, - 'socks5://%s:%i' % (hostFormat(self.socksAddr, v6Bracket = True), self.socksPort), self.proxyInfo + 'socks5://%s:%i' % (v6AddBracket(self.socksAddr), self.socksPort), self.proxyInfo )) configFile = os.path.join( # config file path WorkDir, self.id + clientEntry[self.proxyType][1] # workDir + taskId + file suffix ) logging.debug('[%s] Builder config file -> %s' % (self.id, configFile)) command, fileContent, envVar = clientEntry[self.proxyType][0](self.proxyInfo, { # load client boot info - 'addr': self.socksAddr, + 'addr': self.socksAddr, # specify socks5 info 'port': self.socksPort, }, configFile) envVar['PATH'] = PathEnv # add PATH env (some programs need it) @@ -69,7 +71,7 @@ class Builder(object): self.id = genFlag(length = 12) if taskId == '' else taskId # load task ID if proxyType not in clientEntry: logging.error('[%s] Builder receive unknown proxy type %s' % (self.id, proxyType)) - raise RuntimeError('Unknown proxy type') + raise buildException('Unknown proxy type') self.proxyType = proxyType # proxy type -> ss / ssr / vmess ... self.proxyInfo = copy.copy(proxyInfo) # connection info self.socksAddr = bindAddr From a12e00dcafa8b86a1e1f1f6fa4867d31837569c0 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 15:13:11 +0800 Subject: [PATCH 13/18] feat: filter error --- Basis/Exception.py | 5 +++++ Basis/Filter.py | 30 +++++++++++++++++------------- Builder/__init__.py | 4 +++- Filter/__init__.py | 10 ++++++++-- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/Basis/Exception.py b/Basis/Exception.py index 685b394..8192bea 100644 --- a/Basis/Exception.py +++ b/Basis/Exception.py @@ -4,3 +4,8 @@ class buildException(Exception): # for build error def __init__(self, reason): self.reason = reason + + +class filterException(Exception): # for filter error + def __init__(self, reason): + self.reason = reason diff --git a/Basis/Filter.py b/Basis/Filter.py index fdc81a5..341c851 100644 --- a/Basis/Filter.py +++ b/Basis/Filter.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import copy +from Basis.Exception import filterException filterObject = { 'optional': { @@ -67,7 +68,7 @@ for field in filterObject: def Filter(raw: dict, rules: dict) -> dict: if type(raw) != dict: - raise RuntimeError('Invalid input for filter') + raise filterException('Invalid input for filter') data = {} raw = copy.deepcopy(raw) rules = copy.deepcopy(rules) @@ -75,19 +76,19 @@ def Filter(raw: dict, rules: dict) -> dict: # pretreatment process (raw --[copy / default value]--> data) if key not in raw: # key not exist if not rule['optional']: # force require key not exist - raise RuntimeError('Missing `%s` field' % key) + raise filterException('Missing `%s` field' % key) data[key] = rule['default'] # set default value else: # key exist data[key] = raw[key] # format process (data --[format]--> data) if data[key] is None: # key content is None if not rule['allowNone']: # key is not allow None - raise RuntimeError('Field `%s` shouldn\'t be None' % key) + raise filterException('Field `%s` shouldn\'t be None' % key) continue # skip following process try: data[key] = rule['format'](data[key]) # run format except: - raise RuntimeError(rule['errMsg']) # format error + raise filterException(rule['errMsg']) # format error # filter process (data --[type check (& filter check)]--> pass / non-pass) if type(rule['type']) == type: # str / int / bool / ... rule['type'] = [rule['type']] # str -> [str] / int -> [int] / ... @@ -95,35 +96,38 @@ def Filter(raw: dict, rules: dict) -> dict: if data[key] == any and any in rule['type']: # special case -> skip type filter pass elif type(data[key]) not in rule['type']: # type not in allow list - raise RuntimeError('Invalid `%s` field' % key) + raise filterException('Invalid `%s` field' % key) elif type(rule['type']) == dict: # check subObject if type(data[key]) != dict: - raise RuntimeError('Invalid sub object in `%s`' % key) # subObject content should be dict + raise filterException('Invalid sub object in `%s`' % key) # subObject content should be dict if not rule['multiSub']: # single subObject subRules = rule['type'] else: # multi subObject if rule['indexKey'] not in data[key]: # confirm index key exist - raise RuntimeError('Index key `%s` not found in `%s`' % (rule['indexKey'], key)) + raise filterException('Index key `%s` not found in `%s`' % (rule['indexKey'], key)) subType = data[key][rule['indexKey']].lower() if subType not in rule['type']: # confirm subObject rule exist - raise RuntimeError('Unknown index `%s` in key `%s`' % (subType, key)) + raise filterException('Unknown index `%s` in key `%s`' % (subType, key)) subRules = rule['type'][subType] try: data[key] = Filter(data[key], subRules) - except RuntimeError as exp: - raise RuntimeError('%s (in `%s`)' % (exp, key)) # add located info + except filterException as exp: + raise filterException('%s (in `%s`)' % (exp, key)) # add located info continue elif rule['type'] != any: # type == any -> skip type filter - raise RuntimeError('Unknown `type` in rules') + raise filterException('Unknown `type` in rules') if not rule['filter'](data[key]): # run filter - raise RuntimeError(rule['errMsg']) + raise filterException(rule['errMsg']) return data def rulesFilter(rules: dict) -> dict: result = {} for key, rule in rules.items(): # filter by basic rules - result[key] = Filter(rule, filterObject) + try: + result[key] = Filter(rule, filterObject) + except filterException as exp: + raise filterException('%s (`%s` in rules)' % (exp, key)) # rules error return result diff --git a/Builder/__init__.py b/Builder/__init__.py index 5bbe0dc..63aea63 100644 --- a/Builder/__init__.py +++ b/Builder/__init__.py @@ -3,12 +3,13 @@ import os import copy +from Filter import Filter from Basis.Logger import logging from Basis.Process import Process from Basis.Functions import v6AddBracket -from Basis.Exception import buildException from Basis.Constant import WorkDir, PathEnv from Basis.Functions import genFlag, getAvailablePort +from Basis.Exception import buildException, filterException from Builder import Brook from Builder import VMess @@ -73,6 +74,7 @@ class Builder(object): logging.error('[%s] Builder receive unknown proxy type %s' % (self.id, proxyType)) raise buildException('Unknown proxy type') self.proxyType = proxyType # proxy type -> ss / ssr / vmess ... + self.proxyInfo = Filter(proxyType, proxyInfo) # filter input proxy info self.proxyInfo = copy.copy(proxyInfo) # connection info self.socksAddr = bindAddr self.socksPort = getAvailablePort() # random port for socks5 exposed diff --git a/Filter/__init__.py b/Filter/__init__.py index 4b571b5..25d071e 100644 --- a/Filter/__init__.py +++ b/Filter/__init__.py @@ -9,6 +9,7 @@ from Filter.Shadowsocks import ssFilter from Filter.ShadowsocksR import ssrFilter from Filter.TrojanGo import trojanGoFilter from Filter.Hysteria import hysteriaFilter +from Basis.Exception import filterException filterEntry = { 'ss': ssFilter, @@ -23,5 +24,10 @@ filterEntry = { def Filter(proxyType: str, proxyInfo: dict) -> dict: if proxyType not in filterEntry: - raise RuntimeError('Unknown proxy type') - return filterEntry[proxyType](proxyInfo) + raise filterException('Unknown proxy type') + try: + return filterEntry[proxyType](proxyInfo) + except filterException as exp: + raise filterException(exp) + except: + raise filterException('Unknown filter error') From 8a3ad7dc834f5d0750fa41c5724daf05934453fe Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 15:25:57 +0800 Subject: [PATCH 14/18] feat: add more Exception --- Basis/Exception.py | 15 +++++++++++++++ Basis/Manager.py | 8 ++++---- Basis/Process.py | 23 ++++++++++++++--------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/Basis/Exception.py b/Basis/Exception.py index 8192bea..aa841e5 100644 --- a/Basis/Exception.py +++ b/Basis/Exception.py @@ -9,3 +9,18 @@ class buildException(Exception): # for build error class filterException(Exception): # for filter error def __init__(self, reason): self.reason = reason + + +class processException(Exception): # for process error + def __init__(self, reason): + self.reason = reason + + +class managerException(Exception): # for manager error + def __init__(self, reason): + self.reason = reason + + +class checkException(Exception): # for check error + def __init__(self, reason): + self.reason = reason diff --git a/Basis/Manager.py b/Basis/Manager.py index 3d272be..25d25b2 100644 --- a/Basis/Manager.py +++ b/Basis/Manager.py @@ -4,7 +4,7 @@ import copy from Basis.Logger import logging from Basis.Functions import genFlag - +from Basis.Exception import managerException class Task(object): """ Manage global check task. @@ -47,7 +47,7 @@ class Task(object): def getUnion(self, unionId: str) -> dict: # get union status (remove tasks when all completed) if unionId not in self.__unions: logging.error('Manager union [%s] not found' % unionId) - raise RuntimeError('Union id not found') + raise managerException('Union id not found') tasks = self.__unions[unionId]['items'] finishNum = 0 for taskId in tasks: @@ -79,12 +79,12 @@ class Task(object): logging.info('Manager pop task [%s] -> %s' % (taskId, task['data'])) return taskId, copy.deepcopy(task['data']) logging.debug('Manager has no more task') - raise RuntimeError('No more tasks') + raise managerException('No more tasks') def finishTask(self, taskId: str, taskData: dict) -> None: # update task data when completed if taskId not in self.__tasks: logging.error('Manager task [%s] not found' % taskId) - raise RuntimeError('Task id not found') + raise managerException('Task id not found') self.__tasks[taskId]['data'] = copy.deepcopy(taskData) self.__tasks[taskId]['status'] = self.__TASK_FINISH # set task status as completed diff --git a/Basis/Process.py b/Basis/Process.py index 3e3b2a0..5b942e7 100644 --- a/Basis/Process.py +++ b/Basis/Process.py @@ -8,6 +8,7 @@ import ctypes import signal from Basis.Logger import logging from Basis.Functions import genFlag +from Basis.Exception import processException from subprocess import Popen, STDOUT, DEVNULL libcPaths = [ @@ -73,7 +74,7 @@ class Process(object): logging.error('[%s] %s already exist but not folder' % (self.id, self.workDir)) else: logging.error('[%s] Unable to create new folder -> %s' % (self.id, self.workDir)) - raise RuntimeError('Working directory error') # fatal error + raise processException('Working directory error') # fatal error def __killProcess(self, killSignal: int) -> None: try: @@ -134,10 +135,10 @@ class Process(object): )) if self.cmd is None: # ERROR CASE logging.error('[%s] Process miss start command' % self.id) - raise RuntimeError('Miss start command') + raise processException('Miss start command') if self.__process is not None and self.__process.poll() is None: # ERROR CASE - logging.error('[%s] Process is still running' % self.id) - raise RuntimeError('Process is still running') + logging.error('[%s] Process try to start but it is running' % self.id) + raise processException('Process is still running') if self.env is not None and 'PATH' not in self.env and '/' not in self.cmd[0]: # WARNING CASE logging.warning('[%s] Executable file in relative path but miss PATH in environ' % self.id) if self.file is not None: # create and write file contents @@ -153,11 +154,15 @@ class Process(object): else: # discard all the output of sub process stdout = DEVNULL stderr = DEVNULL - self.__process = Popen( - self.cmd, env = self.env, - stdout = stdout, stderr = stderr, - preexec_fn = None if libcPath is None else Process.__preExec - ) + try: + self.__process = Popen( + self.cmd, env = self.env, + stdout = stdout, stderr = stderr, + preexec_fn = None if libcPath is None else Process.__preExec + ) + except Exception as exp: + logging.error('[%s] Process unable to start -> %s' % (self.id, exp)) + raise processException('Unable to start process') logging.info('[%s] Process running -> PID = %i' % (self.id, self.__process.pid)) def signal(self, signalNum: int) -> None: # send specified signal to sub process From 29742f2c78220860d27159294f8a5bc1a2440dbc Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 16:13:57 +0800 Subject: [PATCH 15/18] update: enhance api logical --- Basis/Api.py | 30 ++++++++++++++++++++++++++---- Basis/Check.py | 18 ++++++++++++------ Basis/Constant.py | 26 ++++++++++++++++++-------- Basis/Manager.py | 25 +++++++++++++++++-------- main.py | 23 ++++++++++++++++++++--- 5 files changed, 93 insertions(+), 29 deletions(-) diff --git a/Basis/Api.py b/Basis/Api.py index 3a37bda..5efc514 100644 --- a/Basis/Api.py +++ b/Basis/Api.py @@ -8,6 +8,7 @@ from Checker import formatCheck from Basis.Logger import logging from Basis.Manager import Manager from flask import Flask, Response, request +from Basis.Exception import managerException from Basis.Constant import ApiPort, ApiPath, ApiToken, Version webApi = Flask(__name__) # init flask server @@ -68,12 +69,17 @@ def createTask() -> Response: if not tokenCheck(): # token check return genError('Invalid token') - # TODO: format check and proxy list - checkList = formatCheck(request.json.get('check')) + try: + # TODO: format check and proxy list + checkList = formatCheck(request.json.get('check')) + except: + return genError('Some error in check options') proxyList = [] for proxy in request.json.get('proxy'): - proxyList.append(formatProxy(proxy)) - logging.critical(proxyList) + try: + proxyList.append(formatProxy(proxy)) + except Exception as exp: + return genError('Proxy error in %s -> %s' % (proxy, exp)) logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList)) tasks = [] @@ -105,6 +111,22 @@ def getTaskInfo(taskId: str) -> Response: }) +@webApi.route(os.path.join(ApiPath, 'task/'), methods = ['DELETE']) +def deleteTask(taskId: str) -> Response: + if not tokenCheck(): # token check + return genError('Invalid token') + logging.debug('API get task -> %s' % taskId) + if not Manager.isUnion(taskId): + return genError('Task not found') + try: + Manager.delUnion(taskId) + return jsonResponse({ + 'success': True + }) + except managerException as exp: + return genError(str(exp)) + + @webApi.route(os.path.join(ApiPath, 'version'), methods = ['GET']) def getVersion() -> Response: logging.debug('API get version -> %s' + Version) diff --git a/Basis/Check.py b/Basis/Check.py index b067b43..597aa6f 100644 --- a/Basis/Check.py +++ b/Basis/Check.py @@ -6,6 +6,8 @@ import time from Checker import Checker from Basis.Logger import logging from Builder import Builder, clientEntry +from Basis.Exception import checkException +from Basis.Functions import checkPortStatus def buildClient(taskId: str, taskInfo: dict) -> Builder: @@ -18,24 +20,27 @@ def buildClient(taskId: str, taskInfo: dict) -> Builder: ) except Exception as reason: logging.error('[%s] Client build error -> %s' % (taskId, reason)) - raise RuntimeError('Client build error') + raise checkException('Client build error') -def waitClient(taskId: str, client: Builder): - # TODO: wait port occupied (client.socksPort) - time.sleep(1) # TODO: simple delay for now +def waitClient(taskId: str, client: Builder, times: int = 150, delay: int = 100): # wait until client port occupied + for i in range(times): + if not checkPortStatus(client.socksPort): # port occupied + break + time.sleep(delay / 1000) # wait in default: 100ms * 150 => 15s + time.sleep(1) # wait a short time before check process if not client.status(): # client unexpected exit logging.warning('[%s] Client unexpected exit' % taskId) client.destroy() # remove file and kill sub process logging.debug('[%s] Client output\n%s', (taskId, client.output)) - raise RuntimeError('Client unexpected exit') + raise checkException('Client unexpected exit') def Check(taskId: str, taskInfo: dict) -> dict: logging.info('[%s] Start checking process -> %s' % (taskId, taskInfo)) if taskInfo['type'] not in clientEntry: # unknown proxy type logging.error('[%s] Unknown proxy type %s' % (taskId, taskInfo['type'])) - raise RuntimeError('Unknown proxy type') + raise checkException('Unknown proxy type') client = buildClient(taskId, taskInfo) # build proxy client logging.info('[%s] Client loaded successfully' % taskId) waitClient(taskId, client) # wait for the client to start @@ -49,5 +54,6 @@ def Check(taskId: str, taskInfo: dict) -> dict: taskInfo.pop('check') # remove check items return { **taskInfo, + 'success': True, 'result': checkResult, # add check result } diff --git a/Basis/Constant.py b/Basis/Constant.py index c17a187..b0673b9 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -20,10 +20,15 @@ TestHost = 'proxyc.net' TestSite = 'www.bing.com' PathEnv = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' + # Load Env Options -yamlFile = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../env.yaml') -yamlContent = open(yamlFile, 'r', encoding = 'utf-8').read() -envOptions = yaml.load(yamlContent, Loader = yaml.FullLoader) +envOptions = {} +try: + yamlFile = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../env.yaml') + yamlContent = open(yamlFile, 'r', encoding = 'utf-8').read() + envOptions = yaml.load(yamlContent, Loader = yaml.FullLoader) +except: # something error in env.yaml + pass if 'version' in envOptions: Version = envOptions['version'] if 'loglevel' in envOptions: @@ -40,12 +45,14 @@ if 'api' in envOptions: if 'token' in envOptions['api']: ApiToken = envOptions['api']['token'] + # WorkDir Create try: os.makedirs(WorkDir) # just like `mkdir -p ...` except: pass # folder exist or target is another thing + # Shadowsocks Info mbedtlsMethods = [ 'aes-128-cfb128', @@ -120,6 +127,7 @@ ssAllMethods = set() [ssAllMethods.update(ssMethods[x]) for x in ssMethods] ssAllMethods = sorted(list(ssAllMethods)) # methods of Shadowsocks + # Plugin Info Plugins = { 'simple-obfs': ['obfs-local', 'obfs-server'], @@ -140,6 +148,7 @@ Plugins = {x: [Plugins[x][0], Plugins[x][1 if len(Plugins[x]) == 2 else 0]] for Plugins = {x: {'client': Plugins[x][0], 'server': Plugins[x][1]} for x in Plugins} # format plugins info pluginClients = [Plugins[x]['client'] for x in Plugins] # plugin client list -> obfs-local / simple-tls / ... + # ShadowsocksR Info ssrMethods = [ # methods of ShadowsocksR 'aes-128-ctr', 'aes-192-ctr', 'aes-256-ctr', @@ -168,19 +177,20 @@ ssrObfuscations = [ # obfuscations of ShadowsocksR (obfs) 'tls_simple', 'tls1.2_ticket_auth', 'tls1.2_ticket_fastauth', ] -# VMess Info + +# V2ray / Xray Info vmessMethods = ['aes-128-gcm', 'chacha20-poly1305', 'auto', 'none', 'zero'] -# XTLS Info +quicMethods = ['none', 'aes-128-gcm', 'chacha20-poly1305'] +udpObfuscations = ['none', 'srtp', 'utp', 'wechat-video', 'dtls', 'wireguard'] + xtlsFlows = ['xtls-origin', 'xtls-direct', 'xtls-splice'] xtlsFlows = {x: x.replace('-', '-rprx-') for x in xtlsFlows} -# v2ray / Xray Info -quicMethods = ['none', 'aes-128-gcm', 'chacha20-poly1305'] -udpObfuscations = ['none', 'srtp', 'utp', 'wechat-video', 'dtls', 'wireguard'] # Trojan-Go Info trojanGoMethods = ['aes-128-gcm', 'aes-256-gcm', 'chacha20-ietf-poly1305'] + # Hysteria Info hysteriaProtocols = ['udp', 'wechat-video', 'faketcp'] diff --git a/Basis/Manager.py b/Basis/Manager.py index 25d25b2..5ed3502 100644 --- a/Basis/Manager.py +++ b/Basis/Manager.py @@ -39,15 +39,26 @@ class Task(object): logging.info('Manager add task [%s] -> %s' % (taskId, task)) self.__tasks.update(tasks) # load into task list self.__unions[unionId] = { + 'finish': False, 'items': taskIds # record task items } logging.info('Manager add union [%s] -> %s' % (unionId, taskIds)) return unionId + def delUnion(self, unionId) -> None: # remove union + if unionId not in self.__unions: + logging.error('Manager union [%s] not found' % unionId) + raise managerException('Union id not found') + if not self.__unions[unionId]['finish']: # some tasks are still running + raise managerException('Couldn\'t remove working union') + self.__unions.pop(unionId) + def getUnion(self, unionId: str) -> dict: # get union status (remove tasks when all completed) if unionId not in self.__unions: logging.error('Manager union [%s] not found' % unionId) raise managerException('Union id not found') + if self.__unions[unionId]['finish']: # all tasks are finished + return self.__unions[unionId] tasks = self.__unions[unionId]['items'] finishNum = 0 for taskId in tasks: @@ -60,17 +71,15 @@ class Task(object): 'finish': False, 'percent': round(finishNum / len(tasks), 2) } - self.__unions.pop(unionId) # remove from union list - unionResult = [] # temporary storage + self.__unions[unionId]['result'] = [] for taskId in tasks: task = self.__tasks[taskId] self.__tasks.pop(taskId) # remove from task list - unionResult.append(task['data']) - logging.info('Manager release union [%s] -> %s' % (unionId, unionResult)) - return { - 'finish': True, - 'result': unionResult - } + self.__unions[unionId]['result'].append(task['data']) + self.__unions[unionId]['finish'] = True + self.__unions[unionId].pop('items') + logging.info('Manager release union [%s] -> %s' % (unionId, self.__unions[unionId]['result'])) + return self.__unions[unionId] def popTask(self) -> tuple[str or None, any]: # fetch a loaded task for taskId, task in self.__tasks.items(): diff --git a/main.py b/main.py index 80fc9fd..89f3c34 100755 --- a/main.py +++ b/main.py @@ -8,6 +8,7 @@ import _thread import argparse import compileall from Basis import Constant +from Basis.Exception import checkException def mainArgParse(rawArgs: list) -> argparse.Namespace: @@ -74,9 +75,25 @@ def pythonCompile(dirRange: str = '/') -> None: # python optimize compile def runCheck(taskId: str, taskInfo: dict) -> None: - checkResult = Check(taskId, taskInfo) # check by task info - logging.warning('[%s] Task finish' % taskId) - Manager.finishTask(taskId, checkResult) # commit check result + success = True + checkResult = {} + try: + checkResult = Check(taskId, taskInfo) # check by task info + logging.warning('[%s] Task finish' % taskId) + except checkException as exp: + success = False + logging.error('[%s] Task error -> %s' % (taskId, exp)) + except: + success = False + logging.error('[%s] Task error -> Unknown error' % taskId) + finally: + if not success: # got some error in check process + taskInfo.pop('check') + checkResult = { + **taskInfo, + 'success': False, + } + Manager.finishTask(taskId, checkResult) # commit check result def loop(threadNum: int = 16) -> None: From c65e4911b32ca114eed9ba701d260c09adce2540 Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 16:18:23 +0800 Subject: [PATCH 16/18] feat: add check thread option --- Basis/Constant.py | 2 ++ main.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Basis/Constant.py b/Basis/Constant.py index b0673b9..d30ef82 100644 --- a/Basis/Constant.py +++ b/Basis/Constant.py @@ -11,6 +11,8 @@ ApiPath = '/' ApiPort = 7839 ApiToken = '' +CheckThread = 64 + LogLevel = 'INFO' LogFile = 'runtime.log' diff --git a/main.py b/main.py index 89f3c34..fa104fb 100755 --- a/main.py +++ b/main.py @@ -18,6 +18,7 @@ def mainArgParse(rawArgs: list) -> argparse.Namespace: mainParser.add_argument('--port', type = int, default = Constant.ApiPort, help = 'port for running') mainParser.add_argument('--path', type = str, default = Constant.ApiPath, help = 'root path for api server') mainParser.add_argument('--token', type = str, default = Constant.ApiToken, help = 'token for api server') + mainParser.add_argument('--thread', type = int, default = Constant.CheckThread, help = 'number of check thread') mainParser.add_argument('-v', '--version', help = 'show version', action = 'store_true') return mainParser.parse_args(rawArgs) @@ -96,7 +97,8 @@ def runCheck(taskId: str, taskInfo: dict) -> None: Manager.finishTask(taskId, checkResult) # commit check result -def loop(threadNum: int = 16) -> None: +def loop(threadNum: int) -> None: + logging.warning('Loop check start -> %i threads' % threadNum) threadPool = ThreadPoolExecutor(max_workers = threadNum) # init thread pool while True: try: @@ -133,5 +135,5 @@ if testMode: # test mode logging.warning('ProxyC starts running (%s)' % Constant.Version) _thread.start_new_thread(pythonCompile, ('/usr',)) # python compile (generate .pyc file) _thread.start_new_thread(DnsProxy.start, (Constant.DnsServer, 53)) # start dns server -_thread.start_new_thread(loop, ()) # start check loop +_thread.start_new_thread(loop, (Constant.CheckThread, )) # start check loop Api.startServer() # start api server From 8937d51a1f5530eb5dab2dcadcd83e5b82546bcc Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 17:05:20 +0800 Subject: [PATCH 17/18] build: remove the reuse of cargo cache --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 29d9474..070d583 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,7 +168,6 @@ RUN git submodule update --init --recursive && \ ./autogen.sh && ./configure --disable-documentation && make && \ mv ./src/obfs-local ./src/obfs-server /plugins/ # Compile qtun -COPY --from=ss-rust /usr/local/cargo/registry/ /usr/local/cargo/registry/ WORKDIR ../qtun/ RUN cargo fetch RUN cargo build --target-dir ./ --release && \ From a6f4a6d8ea218dbeed0d7f9402f6227688df4a8d Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Mon, 8 Aug 2022 17:52:28 +0800 Subject: [PATCH 18/18] fix: load of thread option --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index fa104fb..8c16917 100755 --- a/main.py +++ b/main.py @@ -58,6 +58,7 @@ else: Constant.ApiPort = mainArgs.port Constant.ApiPath = mainArgs.path Constant.ApiToken = mainArgs.token + Constant.CheckThread = mainArgs.thread from Tester import testEntry