Browse Source

feat: proxy decode and multi threading

master
dnomd343 2 years ago
parent
commit
176567a7d9
  1. 21
      Basis/Api.py
  2. 2
      Basis/Manager.py
  3. 2
      Checker/__init__.py
  4. 20
      main.py

21
Basis/Api.py

@ -14,6 +14,20 @@ webPath = '/' # root of api server
webApi = Flask(__name__) # init flask server
def formatProxy(raw: str or dict) -> dict:
from ProxyFilter import filte
from ProxyDecoder import decode
if type(raw) == str:
raw = decode(raw)
if raw is None:
raise RuntimeError('decode error')
print(raw)
status, raw = filte(raw, isExtra = True)
if not status:
raise RuntimeError('filter error')
return raw
def jsonResponse(data: dict) -> Response: # return json mime
return Response(json.dumps(data), mimetype = 'application/json')
@ -54,7 +68,10 @@ def createTask() -> Response:
# TODO: format check and proxy list
checkList = formatCheck(request.json.get('check'))
proxyList = request.json.get('proxy')
proxyList = []
for proxy in request.json.get('proxy'):
proxyList.append(formatProxy(proxy))
logging.critical(proxyList)
logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList))
tasks = []
@ -79,7 +96,7 @@ def getTaskInfo(taskId: str) -> Response:
return genError('Invalid token')
logging.debug('API get task -> %s' % taskId)
if not Manager.isUnion(taskId):
return genError('Task id not found')
return genError('Task not found')
return jsonResponse({
'success': True,
**Manager.getUnion(taskId)

2
Basis/Manager.py

@ -58,7 +58,7 @@ class Task(object):
logging.debug('Manager union [%s] still working' % unionId)
return {
'finish': False,
'percent': '%f' % (round(finishNum / len(tasks), 2))
'percent': round(finishNum / len(tasks), 2)
}
self.__unions.pop(unionId) # remove from union list
unionResult = [] # temporary storage

2
Checker/__init__.py

@ -14,7 +14,7 @@ def formatCheck(rawInfo: list) -> dict:
return {
'http': {
'times': 3,
'url': 'http://baidu.com',
'url': 'http://gstatic.com/generate_204',
'timeout': 20,
}
}

20
main.py

@ -10,26 +10,28 @@ from Basis.Manager import Manager
from Basis.Api import startServer
from Basis.Constant import Version
from Basis.Compile import startCompile
from concurrent.futures import ThreadPoolExecutor
# dnsServers = None
dnsServers = ['223.5.5.5', '119.28.28.28']
def runCheck() -> None: # try to pop a task and check
try:
taskId, taskInfo = Manager.popTask()
logging.warning('[%s] Load new task' % taskId)
except:
return # no more task
def runCheck(taskId: str, taskInfo: dict) -> None:
checkResult = Check(taskId, taskInfo)
logging.warning('[%s] Task finish' % taskId)
Manager.finishTask(taskId, checkResult)
def loopCheck() -> None:
while True: # TODO: thread pool working
runCheck()
def loopCheck(threadNum: int = 16) -> None:
threadPool = ThreadPoolExecutor(max_workers = threadNum)
while True:
try:
taskId, taskInfo = Manager.popTask()
logging.warning('[%s] Load new task' % taskId)
except: # no more task
time.sleep(2)
continue
threadPool.submit(runCheck, taskId, taskInfo)
logging.warning('ProxyC starts running (%s)' % Version)

Loading…
Cancel
Save