Browse Source

feat: proxy decode and multi threading

master
dnomd343 2 years ago
parent
commit
176567a7d9
  1. 21
      Basis/Api.py
  2. 2
      Basis/Manager.py
  3. 2
      Checker/__init__.py
  4. 20
      main.py

21
Basis/Api.py

@ -14,6 +14,20 @@ webPath = '/' # root of api server
webApi = Flask(__name__) # init flask server webApi = Flask(__name__) # init flask server
def formatProxy(raw: str or dict) -> dict:
from ProxyFilter import filte
from ProxyDecoder import decode
if type(raw) == str:
raw = decode(raw)
if raw is None:
raise RuntimeError('decode error')
print(raw)
status, raw = filte(raw, isExtra = True)
if not status:
raise RuntimeError('filter error')
return raw
def jsonResponse(data: dict) -> Response: # return json mime def jsonResponse(data: dict) -> Response: # return json mime
return Response(json.dumps(data), mimetype = 'application/json') return Response(json.dumps(data), mimetype = 'application/json')
@ -54,7 +68,10 @@ def createTask() -> Response:
# TODO: format check and proxy list # TODO: format check and proxy list
checkList = formatCheck(request.json.get('check')) checkList = formatCheck(request.json.get('check'))
proxyList = request.json.get('proxy') proxyList = []
for proxy in request.json.get('proxy'):
proxyList.append(formatProxy(proxy))
logging.critical(proxyList)
logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList)) logging.debug('API create task -> check = %s | proxy = %s' % (checkList, proxyList))
tasks = [] tasks = []
@ -79,7 +96,7 @@ def getTaskInfo(taskId: str) -> Response:
return genError('Invalid token') return genError('Invalid token')
logging.debug('API get task -> %s' % taskId) logging.debug('API get task -> %s' % taskId)
if not Manager.isUnion(taskId): if not Manager.isUnion(taskId):
return genError('Task id not found') return genError('Task not found')
return jsonResponse({ return jsonResponse({
'success': True, 'success': True,
**Manager.getUnion(taskId) **Manager.getUnion(taskId)

2
Basis/Manager.py

@ -58,7 +58,7 @@ class Task(object):
logging.debug('Manager union [%s] still working' % unionId) logging.debug('Manager union [%s] still working' % unionId)
return { return {
'finish': False, 'finish': False,
'percent': '%f' % (round(finishNum / len(tasks), 2)) 'percent': round(finishNum / len(tasks), 2)
} }
self.__unions.pop(unionId) # remove from union list self.__unions.pop(unionId) # remove from union list
unionResult = [] # temporary storage unionResult = [] # temporary storage

2
Checker/__init__.py

@ -14,7 +14,7 @@ def formatCheck(rawInfo: list) -> dict:
return { return {
'http': { 'http': {
'times': 3, 'times': 3,
'url': 'http://baidu.com', 'url': 'http://gstatic.com/generate_204',
'timeout': 20, 'timeout': 20,
} }
} }

20
main.py

@ -10,26 +10,28 @@ from Basis.Manager import Manager
from Basis.Api import startServer from Basis.Api import startServer
from Basis.Constant import Version from Basis.Constant import Version
from Basis.Compile import startCompile from Basis.Compile import startCompile
from concurrent.futures import ThreadPoolExecutor
# dnsServers = None # dnsServers = None
dnsServers = ['223.5.5.5', '119.28.28.28'] dnsServers = ['223.5.5.5', '119.28.28.28']
def runCheck() -> None: # try to pop a task and check def runCheck(taskId: str, taskInfo: dict) -> None:
try:
taskId, taskInfo = Manager.popTask()
logging.warning('[%s] Load new task' % taskId)
except:
return # no more task
checkResult = Check(taskId, taskInfo) checkResult = Check(taskId, taskInfo)
logging.warning('[%s] Task finish' % taskId) logging.warning('[%s] Task finish' % taskId)
Manager.finishTask(taskId, checkResult) Manager.finishTask(taskId, checkResult)
def loopCheck() -> None: def loopCheck(threadNum: int = 16) -> None:
while True: # TODO: thread pool working threadPool = ThreadPoolExecutor(max_workers = threadNum)
runCheck() while True:
try:
taskId, taskInfo = Manager.popTask()
logging.warning('[%s] Load new task' % taskId)
except: # no more task
time.sleep(2) time.sleep(2)
continue
threadPool.submit(runCheck, taskId, taskInfo)
logging.warning('ProxyC starts running (%s)' % Version) logging.warning('ProxyC starts running (%s)' % Version)

Loading…
Cancel
Save