feat: proxy decode and multi threading

Basis/Api.py

@@ -14,6 +14,20 @@ webPath = '/'  # root of api server
 webApi = Flask(__name__)  # init flask server
 
 
+def formatProxy(raw: str or dict) -> dict:
+    from ProxyFilter import filte
+    from ProxyDecoder import decode
+    if type(raw) == str:
+        raw = decode(raw)
+        if raw is None:
+            raise RuntimeError('decode error')
+        print(raw)
+    status, raw = filte(raw, isExtra = True)
+    if not status:
+        raise RuntimeError('filter error')
+    return raw
+
+
 def jsonResponse(data: dict) -> Response:  # return json mime
     return Response(json.dumps(data), mimetype = 'application/json')
 
@@ -54,7 +68,10 @@ def createTask() -> Response:
 
     # TODO: format check and proxy list
     checkList = formatCheck(request.json.get('check'))
-    proxyList = request.json.get('proxy')
+    proxyList = []
+    for proxy in request.json.get('proxy'):
+        proxyList.append(formatProxy(proxy))
+    logging.critical(proxyList)
 
     logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList))
     tasks = []
@@ -79,7 +96,7 @@ def getTaskInfo(taskId: str) -> Response:
         return genError('Invalid token')
     logging.debug('API get task -> %s' % taskId)
     if not Manager.isUnion(taskId):
-        return genError('Task id not found')
+        return genError('Task not found')
     return jsonResponse({
         'success': True,
         **Manager.getUnion(taskId)

Basis/Manager.py

@@ -58,7 +58,7 @@ class Task(object):
             logging.debug('Manager union [%s] still working' % unionId)
             return {
                 'finish': False,
-                'percent': '%f' % (round(finishNum / len(tasks), 2))
+                'percent': round(finishNum / len(tasks), 2)
             }
         self.__unions.pop(unionId)  # remove from union list
         unionResult = []  # temporary storage

Checker/__init__.py

@@ -14,7 +14,7 @@ def formatCheck(rawInfo: list) -> dict:
     return {
         'http': {
             'times': 3,
-            'url': 'http://baidu.com',
+            'url': 'http://gstatic.com/generate_204',
             'timeout': 20,
         }
     }

main.py

@@ -10,26 +10,28 @@ from Basis.Manager import Manager
 from Basis.Api import startServer
 from Basis.Constant import Version
 from Basis.Compile import startCompile
+from concurrent.futures import ThreadPoolExecutor
 
 # dnsServers = None
 dnsServers = ['223.5.5.5', '119.28.28.28']
 
 
-def runCheck() -> None:  # try to pop a task and check
-    try:
-        taskId, taskInfo = Manager.popTask()
-        logging.warning('[%s] Load new task' % taskId)
-    except:
-        return  # no more task
+def runCheck(taskId: str, taskInfo: dict) -> None:
     checkResult = Check(taskId, taskInfo)
     logging.warning('[%s] Task finish' % taskId)
     Manager.finishTask(taskId, checkResult)
 
 
-def loopCheck() -> None:
-    while True:  # TODO: thread pool working
-        runCheck()
+def loopCheck(threadNum: int = 16) -> None:
+    threadPool = ThreadPoolExecutor(max_workers = threadNum)
+    while True:
+        try:
+            taskId, taskInfo = Manager.popTask()
+            logging.warning('[%s] Load new task' % taskId)
+        except:  # no more task
             time.sleep(2)
+            continue
+        threadPool.submit(runCheck, taskId, taskInfo)
 
 
 logging.warning('ProxyC starts running (%s)' % Version)