diff --git a/Basis/Api.py b/Basis/Api.py index 2a25dfb..9cc84d9 100644 --- a/Basis/Api.py +++ b/Basis/Api.py @@ -14,6 +14,20 @@ webPath = '/' # root of api server webApi = Flask(__name__) # init flask server +def formatProxy(raw: str or dict) -> dict: + from ProxyFilter import filte + from ProxyDecoder import decode + if type(raw) == str: + raw = decode(raw) + if raw is None: + raise RuntimeError('decode error') + print(raw) + status, raw = filte(raw, isExtra = True) + if not status: + raise RuntimeError('filter error') + return raw + + def jsonResponse(data: dict) -> Response: # return json mime return Response(json.dumps(data), mimetype = 'application/json') @@ -54,7 +68,10 @@ def createTask() -> Response: # TODO: format check and proxy list checkList = formatCheck(request.json.get('check')) - proxyList = request.json.get('proxy') + proxyList = [] + for proxy in request.json.get('proxy'): + proxyList.append(formatProxy(proxy)) + logging.critical(proxyList) logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList)) tasks = [] @@ -79,7 +96,7 @@ def getTaskInfo(taskId: str) -> Response: return genError('Invalid token') logging.debug('API get task -> %s' % taskId) if not Manager.isUnion(taskId): - return genError('Task id not found') + return genError('Task not found') return jsonResponse({ 'success': True, **Manager.getUnion(taskId) diff --git a/Basis/Manager.py b/Basis/Manager.py index e9154d3..691631f 100644 --- a/Basis/Manager.py +++ b/Basis/Manager.py @@ -58,7 +58,7 @@ class Task(object): logging.debug('Manager union [%s] still working' % unionId) return { 'finish': False, - 'percent': '%f' % (round(finishNum / len(tasks), 2)) + 'percent': round(finishNum / len(tasks), 2) } self.__unions.pop(unionId) # remove from union list unionResult = [] # temporary storage diff --git a/Checker/__init__.py b/Checker/__init__.py index d2501d4..264ebee 100644 --- a/Checker/__init__.py +++ b/Checker/__init__.py @@ -14,7 +14,7 @@ def formatCheck(rawInfo: list) -> dict: return { 'http': { 'times': 3, - 'url': 'http://baidu.com', + 'url': 'http://gstatic.com/generate_204', 'timeout': 20, } } diff --git a/main.py b/main.py index d6990db..6bdada4 100755 --- a/main.py +++ b/main.py @@ -10,26 +10,28 @@ from Basis.Manager import Manager from Basis.Api import startServer from Basis.Constant import Version from Basis.Compile import startCompile +from concurrent.futures import ThreadPoolExecutor # dnsServers = None dnsServers = ['223.5.5.5', '119.28.28.28'] -def runCheck() -> None: # try to pop a task and check - try: - taskId, taskInfo = Manager.popTask() - logging.warning('[%s] Load new task' % taskId) - except: - return # no more task +def runCheck(taskId: str, taskInfo: dict) -> None: checkResult = Check(taskId, taskInfo) logging.warning('[%s] Task finish' % taskId) Manager.finishTask(taskId, checkResult) -def loopCheck() -> None: - while True: # TODO: thread pool working - runCheck() - time.sleep(2) +def loopCheck(threadNum: int = 16) -> None: + threadPool = ThreadPoolExecutor(max_workers = threadNum) + while True: + try: + taskId, taskInfo = Manager.popTask() + logging.warning('[%s] Load new task' % taskId) + except: # no more task + time.sleep(2) + continue + threadPool.submit(runCheck, taskId, taskInfo) logging.warning('ProxyC starts running (%s)' % Version)