From 428d2d414d18e5400d5e63c6bb62b71783a59c8a Mon Sep 17 00:00:00 2001 From: dnomd343 Date: Wed, 7 Sep 2022 18:47:52 +0800 Subject: [PATCH] update: proxyc init process --- Dockerfile | 2 +- Main.py | 97 +++++++++++++++++++++++++++ Test.py | 65 ++++++++++++++++++ {Basis => Utils}/Api.py | 24 +++---- env.yml | 6 +- main.py | 141 ---------------------------------------- 6 files changed, 178 insertions(+), 157 deletions(-) create mode 100755 Main.py create mode 100644 Test.py rename {Basis => Utils}/Api.py (83%) delete mode 100755 main.py diff --git a/Dockerfile b/Dockerfile index 2d9391a..9cf87d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -448,7 +448,7 @@ COPY . /asset/usr/local/share/ProxyC/ # Release docker image FROM ${PYTHON_IMG} RUN apk add --no-cache boost-program_options c-ares ca-certificates libev libsodium libstdc++ mbedtls pcre && \ - ln -s /usr/local/share/ProxyC/main.py /usr/bin/proxyc + ln -s /usr/local/share/ProxyC/Main.py /usr/bin/proxyc COPY --from=asset /asset / EXPOSE 7839 ENTRYPOINT ["proxyc"] diff --git a/Main.py b/Main.py new file mode 100755 index 0000000..664a341 --- /dev/null +++ b/Main.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import time +import _thread +import argparse +import compileall +from Utils import Constant +from Utils.Exception import checkException + + +def parseArgs(rawArgs: list) -> argparse.Namespace: + mainParser = argparse.ArgumentParser(description = 'Start running API server') + mainParser.add_argument('--log', type = str, default = Constant.LogLevel, help = 'output log level') + mainParser.add_argument('--dns', type = str, default = Constant.DnsServer, nargs = '+', help = 'specify dns server') + mainParser.add_argument('--port', type = int, default = Constant.ApiPort, help = 'port for running') + mainParser.add_argument('--path', type = str, default = Constant.ApiPath, help = 'root path for api server') + mainParser.add_argument('--token', type = str, default = Constant.ApiToken, help = 'token for api server') + mainParser.add_argument('--thread', type = int, default = Constant.CheckThread, help = 'number of check thread') + mainParser.add_argument('-v', '--version', help = 'show version', action = 'store_true') + return mainParser.parse_args(rawArgs) + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1].lower() == 'test': # test mode + from Test import runTest + runTest(sys.argv[2:]) + sys.exit(0) + + mainArgs = parseArgs(sys.argv[1:]) + if mainArgs.version: # output version and exit + print('ProxyC version %s' % Constant.Version) + sys.exit(0) + Constant.LogLevel = mainArgs.log # overwrite global options + Constant.DnsServer = mainArgs.dns + Constant.ApiPort = mainArgs.port + Constant.ApiPath = mainArgs.path + Constant.ApiToken = mainArgs.token + Constant.CheckThread = mainArgs.thread + + +if __name__ == '__main__': + from Utils.Check import Check + from Utils import Api, DnsProxy + from Utils.Logger import logger + from Utils.Manager import Manager + from concurrent.futures import ThreadPoolExecutor + + +def pythonCompile(dirRange: str = '/') -> None: # python optimize compile + for optimize in [-1, 1, 2]: + compileall.compile_dir(dirRange, quiet = 1, optimize = optimize) + logger.info('Python optimize compile -> %s (level = %i)' % (dirRange, optimize)) + + +def runCheck(taskId: str, taskInfo: dict) -> None: + success = True + checkResult = {} + try: + checkResult = Check(taskId, taskInfo) # check by task info + logger.warning('[%s] Task finish' % taskId) + except checkException as exp: + success = False + logger.error('[%s] Task error -> %s' % (taskId, exp)) + except: + success = False + logger.error('[%s] Task error -> Unknown error' % taskId) + finally: + if not success: # got some error in check process + taskInfo.pop('check') + checkResult = { + **taskInfo, + 'success': False, + } + Manager.finishTask(taskId, checkResult) # commit check result + + +def loop(threadNum: int) -> None: + logger.warning('Loop check start -> %i threads' % threadNum) + threadPool = ThreadPoolExecutor(max_workers = threadNum) # init thread pool + while True: + try: + taskId, taskInfo = Manager.popTask() # pop a task + logger.warning('[%s] Load new task' % taskId) + except: # no more task + time.sleep(2) + continue + threadPool.submit(runCheck, taskId, taskInfo) # submit into thread pool + + +if __name__ == '__main__': + logger.warning('ProxyC starts running (%s)' % Constant.Version) + _thread.start_new_thread(pythonCompile, ('/usr',)) # python compile (generate .pyc file) + _thread.start_new_thread(DnsProxy.start, (Constant.DnsServer, 53)) # start dns server + _thread.start_new_thread(loop, (Constant.CheckThread, )) # start check loop + Api.startServer() # start api server diff --git a/Test.py b/Test.py new file mode 100644 index 0000000..8c1015a --- /dev/null +++ b/Test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import argparse +from Utils import Constant + + +def parseArgs(rawArgs: list) -> argparse.Namespace: + testParser = argparse.ArgumentParser(description = 'Test that each function is working properly') + testParser.add_argument('PROTOCOL', type = str, help = 'test protocol name') + testParser.add_argument('-a', '--all', help = 'test extra shadowsocks items', action = 'store_true') + testParser.add_argument('-6', '--ipv6', help = 'test on ipv6 network', action = 'store_true') + testParser.add_argument('--debug', help = 'enable debug log level', action = 'store_true') + testParser.add_argument('--url', type = str, default = 'http://google.com', help = 'http request url') + testParser.add_argument('--cert', type = str, default = '', help = 'specify the certificate id') + testParser.add_argument('--thread', type = int, default = 16, help = 'thread number in check process') + testParser.add_argument('--select', type = str, nargs = '+', help = 'select id list for test') + return testParser.parse_args(rawArgs) + + +def initTest(args: list) -> argparse.Namespace: + if len(args) == 0 or args[0].startswith('-'): # no protocol is specified + args = ['all'] + args # test all items + testArgs = parseArgs(args) + Constant.LogLevel = 'debug' if testArgs.debug else 'warning' + + from Tester import testEntry + from Utils.Logger import logger # load logger after setting up LogLevel + from Utils.Tester import loadBind, loadCert + if testArgs.PROTOCOL != 'all' and testArgs.PROTOCOL not in testEntry: + logger.error('Unknown protocol -> %s' % testArgs.PROTOCOL) + sys.exit(1) + + loadBind(serverV6 = testArgs.ipv6, clientV6 = testArgs.ipv6) # ipv4 / ipv6 (127.0.0.1 / ::1) + loadCert(certId = testArgs.cert) # certificate config + logger.critical('TEST ITEM: %s' % testArgs.PROTOCOL) + logger.critical('SELECT: ' + str(testArgs.select)) + logger.critical('URL: %s' % testArgs.url) + logger.critical('THREAD NUMBER: %i' % testArgs.thread) + return testArgs + + +def runTest(args: list) -> None: # run test process + testArgs = initTest(args) + from Tester import testEntry + from Utils.Tester import Test + from Utils.Logger import logger + + logger.critical('-' * 32 + ' TEST START ' + '-' * 32) # test start + if testArgs.PROTOCOL == 'all': # run all test items + for item in testEntry: + if item == ('ss' if testArgs.all else 'ss-all'): # skip ss / ss-all + continue + logger.critical('TEST ITEM -> ' + item) + Test(testEntry[item], testArgs.thread, testArgs.url, testArgs.select) + else: # run single item + if testArgs.PROTOCOL == 'ss' and testArgs.all: # test shadowsocks extra items + testArgs.PROTOCOL = 'ss-all' + Test(testEntry[testArgs.PROTOCOL], testArgs.thread, testArgs.url, testArgs.select) + logger.critical('-' * 32 + ' TEST COMPLETE ' + '-' * 32) # test complete + + +if __name__ == '__main__': + runTest(sys.argv[1:]) diff --git a/Basis/Api.py b/Utils/Api.py similarity index 83% rename from Basis/Api.py rename to Utils/Api.py index 5efc514..4bf0505 100644 --- a/Basis/Api.py +++ b/Utils/Api.py @@ -5,11 +5,11 @@ import os import json from gevent import pywsgi from Checker import formatCheck -from Basis.Logger import logging -from Basis.Manager import Manager +from Utils.Logger import logger +from Utils.Manager import Manager from flask import Flask, Response, request -from Basis.Exception import managerException -from Basis.Constant import ApiPort, ApiPath, ApiToken, Version +from Utils.Exception import managerException +from Utils.Constant import ApiPort, ApiPath, ApiToken, Version webApi = Flask(__name__) # init flask server @@ -57,7 +57,7 @@ def getTaskList() -> Response: if not tokenCheck(): # token check return genError('Invalid token') taskList = Manager.listUnion() - logging.debug('API get task list -> %s' % taskList) + logger.debug('API get task list -> %s' % taskList) return jsonResponse({ 'success': True, 'task': taskList, @@ -81,7 +81,7 @@ def createTask() -> Response: except Exception as exp: return genError('Proxy error in %s -> %s' % (proxy, exp)) - logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList)) + logger.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList)) tasks = [] for proxy in proxyList: tasks.append({ @@ -89,7 +89,7 @@ def createTask() -> Response: 'check': checkList # load check items }) checkId = Manager.addUnion(tasks) # add into manager -> get id - logging.debug('API return task id -> %s' % checkId) + logger.debug('API return task id -> %s' % checkId) return jsonResponse({ 'success': True, 'id': checkId, @@ -102,7 +102,7 @@ def createTask() -> Response: def getTaskInfo(taskId: str) -> Response: if not tokenCheck(): # token check return genError('Invalid token') - logging.debug('API get task -> %s' % taskId) + logger.debug('API get task -> %s' % taskId) if not Manager.isUnion(taskId): return genError('Task not found') return jsonResponse({ @@ -115,7 +115,7 @@ def getTaskInfo(taskId: str) -> Response: def deleteTask(taskId: str) -> Response: if not tokenCheck(): # token check return genError('Invalid token') - logging.debug('API get task -> %s' % taskId) + logger.debug('API get task -> %s' % taskId) if not Manager.isUnion(taskId): return genError('Task not found') try: @@ -129,7 +129,7 @@ def deleteTask(taskId: str) -> Response: @webApi.route(os.path.join(ApiPath, 'version'), methods = ['GET']) def getVersion() -> Response: - logging.debug('API get version -> %s' + Version) + logger.debug('API get version -> %s' + Version) return jsonResponse({ 'success': True, 'version': Version, @@ -137,6 +137,6 @@ def getVersion() -> Response: def startServer() -> None: - logging.warning('API server at http://:%i%s' % (ApiPort, ApiPath)) - logging.warning('API ' + ('without token' if ApiToken == '' else 'token -> %s' % ApiToken)) + logger.warning('API server at http://:%i%s' % (ApiPort, ApiPath)) + logger.warning('API ' + ('without token' if ApiToken == '' else 'token -> %s' % ApiToken)) pywsgi.WSGIServer(('0.0.0.0', ApiPort), webApi).serve_forever() # powered by gevent diff --git a/env.yml b/env.yml index a276699..dbe8fb6 100644 --- a/env.yml +++ b/env.yml @@ -1,8 +1,8 @@ -version: '0.1' +version: '0.9.0' loglevel: 'INFO' -dir: '/tmp/ProxyC-test' +dir: '/tmp/ProxyC' dns: null api: port: 7839 -# path: '/' + path: '/' token: '' diff --git a/main.py b/main.py deleted file mode 100755 index 61ab223..0000000 --- a/main.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import sys -import copy -import time -import _thread -import argparse -import compileall -from Utils import Constant -from Utils.Exception import checkException - - -def mainArgParse(rawArgs: list) -> argparse.Namespace: - mainParser = argparse.ArgumentParser(description = 'Start running API server') - mainParser.add_argument('--log', type = str, default = Constant.LogLevel, help = 'output log level') - mainParser.add_argument('--dns', type = str, default = Constant.DnsServer, nargs = '+', help = 'specify dns server') - mainParser.add_argument('--port', type = int, default = Constant.ApiPort, help = 'port for running') - mainParser.add_argument('--path', type = str, default = Constant.ApiPath, help = 'root path for api server') - mainParser.add_argument('--token', type = str, default = Constant.ApiToken, help = 'token for api server') - mainParser.add_argument('--thread', type = int, default = Constant.CheckThread, help = 'number of check thread') - mainParser.add_argument('-v', '--version', help = 'show version', action = 'store_true') - return mainParser.parse_args(rawArgs) - - -def testArgParse(rawArgs: list) -> argparse.Namespace: - testParser = argparse.ArgumentParser(description = 'Test that each function is working properly') - testParser.add_argument('PROTOCOL', type = str, help = 'test protocol name') - testParser.add_argument('-a', '--all', help = 'test extra shadowsocks items', action = 'store_true') - testParser.add_argument('-6', '--ipv6', help = 'test on ipv6 network', action = 'store_true') - testParser.add_argument('--debug', help = 'enable debug log level', action = 'store_true') - testParser.add_argument('--url', type = str, default = 'http://baidu.com', help = 'http request url') - testParser.add_argument('--cert', type = str, default = '', help = 'specify the certificate id') - testParser.add_argument('--thread', type = int, default = 16, help = 'thread number in check process') - testParser.add_argument('--select', type = str, nargs = '+', help = 'select id list for test') - return testParser.parse_args(rawArgs) - - -testArgs = None -testMode = False -inputArgs = copy.copy(sys.argv) -if len(inputArgs) >= 0: # remove first arg (normally file name) - inputArgs.pop(0) -if len(inputArgs) != 0 and inputArgs[0].lower() == 'test': # test mode - inputArgs.pop(0) # remove `test` - if len(inputArgs) == 0 or inputArgs[0].startswith('-'): # no protocol is specified - inputArgs = ['all'] + inputArgs - testArgs = testArgParse(inputArgs) - Constant.LogLevel = 'debug' if testArgs.debug else 'warning' - testMode = True -else: - mainArgs = mainArgParse(inputArgs) - if mainArgs.version: # output version and exit - print('ProxyC version %s' % Constant.Version) - sys.exit(0) - Constant.LogLevel = mainArgs.log # overwrite global options - Constant.DnsServer = mainArgs.dns - Constant.ApiPort = mainArgs.port - Constant.ApiPath = mainArgs.path - Constant.ApiToken = mainArgs.token - Constant.CheckThread = mainArgs.thread - - -from Tester import testEntry -from Utils.Check import Check -# from Utils import Api, DnsProxy -from Utils import DnsProxy -from Utils.Logger import logger -from Utils.Manager import Manager -from Utils.Tester import Test, loadBind, loadCert -from concurrent.futures import ThreadPoolExecutor - - -def pythonCompile(dirRange: str = '/') -> None: # python optimize compile - for optimize in [-1, 1, 2]: - compileall.compile_dir(dirRange, quiet = 1, optimize = optimize) - logger.warning('Python optimize compile -> %s (level = %i)' % (dirRange, optimize)) - - -def runCheck(taskId: str, taskInfo: dict) -> None: - success = True - checkResult = {} - try: - checkResult = Check(taskId, taskInfo) # check by task info - logger.warning('[%s] Task finish' % taskId) - except checkException as exp: - success = False - logger.error('[%s] Task error -> %s' % (taskId, exp)) - except: - success = False - logger.error('[%s] Task error -> Unknown error' % taskId) - finally: - if not success: # got some error in check process - taskInfo.pop('check') - checkResult = { - **taskInfo, - 'success': False, - } - Manager.finishTask(taskId, checkResult) # commit check result - - -def loop(threadNum: int) -> None: - logger.warning('Loop check start -> %i threads' % threadNum) - threadPool = ThreadPoolExecutor(max_workers = threadNum) # init thread pool - while True: - try: - taskId, taskInfo = Manager.popTask() # pop a task - logger.warning('[%s] Load new task' % taskId) - except: # no more task - time.sleep(2) - continue - threadPool.submit(runCheck, taskId, taskInfo) # submit into thread pool - - -if testMode: # test mode - loadBind(serverV6 = testArgs.ipv6, clientV6 = testArgs.ipv6) # ipv4 / ipv6 (127.0.0.1 / ::1) - loadCert(certId = testArgs.cert) # cert config - logger.critical('TEST ITEM: %s' % testArgs.PROTOCOL) - logger.critical('SELECT: ' + str(testArgs.select)) - logger.critical('URL: %s' % testArgs.url) - logger.critical('THREAD NUMBER: %i' % testArgs.thread) - logger.critical('-' * 32 + ' TEST START ' + '-' * 32) - if testArgs.PROTOCOL == 'all': # run all test items - for item in testEntry: - if item == ('ss' if testArgs.all else 'ss-all'): # skip ss / ss-all - continue - logger.critical('TEST ITEM -> ' + item) - Test(testEntry[item], testArgs.thread, testArgs.url, testArgs.select) - else: # run single item - if testArgs.PROTOCOL == 'ss' and testArgs.all: # test shadowsocks extra items - testItem = 'ss-all' - Test(testEntry[testArgs.PROTOCOL], testArgs.thread, testArgs.url, testArgs.select) - logger.critical('-' * 32 + ' TEST COMPLETE ' + '-' * 32) - sys.exit(0) # test complete - - -logger.warning('ProxyC starts running (%s)' % Constant.Version) -_thread.start_new_thread(pythonCompile, ('/usr',)) # python compile (generate .pyc file) -_thread.start_new_thread(DnsProxy.start, (Constant.DnsServer, 53)) # start dns server -_thread.start_new_thread(loop, (Constant.CheckThread, )) # start check loop -# Api.startServer() # start api server