mirror of https://github.com/dnomd343/ProxyC
12 changed files with 215 additions and 211 deletions
@ -1,86 +0,0 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
import copy |
|||
from Basis.Logger import logging |
|||
from Basis.Functions import genFlag |
|||
|
|||
|
|||
class Task(object): |
|||
""" Manage global check task. |
|||
""" |
|||
__tasks = {} |
|||
__subTasks = {} # sub task status -> loaded / running / complete |
|||
|
|||
def __init__(self): |
|||
logging.info('task manager start') |
|||
|
|||
def addTask(self, tasks: list) -> str: # add task to manager (multi sub tasks) |
|||
subTasks = {} |
|||
subTaskIds = [] |
|||
for subTask in tasks: |
|||
subTaskId = genFlag(length = 24) # generate sub task id (24 bytes) |
|||
subTaskIds.append(subTaskId) |
|||
subTasks[subTaskId] = { |
|||
'status': 'loaded', |
|||
'data': copy.deepcopy(subTask) |
|||
} |
|||
logging.info('add sub task %s -> %s' % (subTaskId, subTasks[subTaskId]['data'])) |
|||
taskId = genFlag(length = 16) # generate task id (16 bytes) |
|||
self.__tasks[taskId] = { # load task |
|||
'sub': subTaskIds |
|||
} |
|||
self.__subTasks.update(subTasks) # load sub tasks |
|||
logging.info('task %s loaded' % taskId) |
|||
return taskId |
|||
|
|||
def isTask(self, taskId: str) -> bool: # check if the task id exist |
|||
return taskId in self.__tasks |
|||
|
|||
def getTask(self, taskId: str) -> dict: # get task status (remove sub tasks when all completed) |
|||
if taskId not in self.__tasks: |
|||
logging.error('task id %s not found' % taskId) |
|||
raise RuntimeError('task id not found') |
|||
subList = self.__tasks[taskId]['sub'] |
|||
completed = 0 |
|||
for subTaskId in subList: |
|||
if self.__subTasks[subTaskId]['status'] == 'complete': # get number of completed sub task |
|||
completed += 1 |
|||
logging.debug('[%s] statistics sub task status -> %i/%i' % (taskId, completed, len(subList))) |
|||
if completed < len(subList): # some sub tasks are not completed |
|||
logging.debug('[%s] task still running' % taskId) |
|||
return { |
|||
'finish': False, |
|||
'percent': '%i%%' % (completed / len(subList)) |
|||
} |
|||
logging.debug('[%s] task work complete' % taskId) # all sub tasks completed |
|||
result = [] |
|||
for subTaskId in subList: |
|||
subTask = self.__subTasks[subTaskId] |
|||
self.__subTasks.pop(subTaskId) |
|||
result.append(subTask['data']) |
|||
logging.debug('release sub tasks -> %s' % result) |
|||
return { |
|||
'finish': True, |
|||
'result': result |
|||
} |
|||
|
|||
def listTask(self) -> list: # get all task ids |
|||
return [x for x in self.__tasks] |
|||
|
|||
def popSubTask(self) -> tuple[str or None, any]: # fetch a loaded sub task |
|||
for subTaskId, subTask in self.__subTasks.items(): |
|||
if subTask['status'] != 'loaded': continue # only get loaded sub task |
|||
subTask['status'] = 'running' # set status as running |
|||
return subTaskId, copy.deepcopy(subTask['data']) |
|||
return None, None |
|||
|
|||
def updateSubTask(self, subTaskId: str, subTaskData: dict) -> None: # update sub task data when completed |
|||
if subTaskId not in self.__subTasks: |
|||
logging.error('sub task id %s not found' % subTaskId) |
|||
raise RuntimeError('sub task id not found') |
|||
self.__subTasks[subTaskId]['data'] = copy.deepcopy(subTaskData) |
|||
self.__subTasks[subTaskId]['status'] = 'complete' |
|||
|
|||
|
|||
Manage = Task() # global task manager |
|||
@ -0,0 +1,92 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
import copy |
|||
from Basis.Logger import logging |
|||
from Basis.Functions import genFlag |
|||
|
|||
|
|||
class Task(object): |
|||
""" Manage global check task. |
|||
""" |
|||
__tasks = {} # task status -> loaded / running / complete |
|||
__unions = {} # one union include multi tasks |
|||
__TASK_LOADED = -1 |
|||
__TASK_RUNNING = 0 |
|||
__TASK_FINISH = 1 |
|||
|
|||
def __init__(self): |
|||
logging.info('Manager start') |
|||
|
|||
def listUnion(self) -> list: # get all union ids |
|||
return [x for x in self.__unions] |
|||
|
|||
def isUnion(self, unionId: str) -> bool: # check if the union id exist |
|||
return unionId in self.__unions |
|||
|
|||
def addUnion(self, union: list) -> str: # add union to manager (include multi tasks) |
|||
tasks = {} # temporary task storage |
|||
taskIds = [] # task id list for manage union |
|||
unionId = genFlag(length = 16) # generate union id (16 bytes) |
|||
logging.debug('Manager start to load union [%s]' % unionId) |
|||
for task in union: |
|||
taskId = genFlag(length = 24) # generate task id (24 bytes) |
|||
taskIds.append(taskId) |
|||
tasks[taskId] = { |
|||
'status': self.__TASK_LOADED, # task status -> loaded |
|||
'data': copy.deepcopy(task) # save task info |
|||
} |
|||
logging.info('Manager add task [%s] -> %s' % (taskId, task)) |
|||
self.__tasks.update(tasks) # load into task list |
|||
self.__unions[unionId] = { |
|||
'items': taskIds # record task items |
|||
} |
|||
logging.info('Manager add union [%s] -> %s' % (unionId, taskIds)) |
|||
return unionId |
|||
|
|||
def getUnion(self, unionId: str) -> dict: # get union status (remove tasks when all completed) |
|||
if unionId not in self.__unions: |
|||
logging.error('Manager union [%s] not found' % unionId) |
|||
raise RuntimeError('Union id not found') |
|||
tasks = self.__unions[unionId]['items'] |
|||
finishNum = 0 |
|||
for taskId in tasks: |
|||
if self.__tasks[taskId]['status'] == self.__TASK_FINISH: # get number of completed task |
|||
finishNum += 1 |
|||
logging.info('Manager statistics union [%s] -> %i/%i' % (unionId, finishNum, len(tasks))) |
|||
if finishNum < len(tasks): # some tasks are not yet completed |
|||
logging.debug('Manager union [%s] still working' % unionId) |
|||
return { |
|||
'finish': False, |
|||
'percent': '%f' % (round(finishNum / len(tasks), 2)) |
|||
} |
|||
self.__unions.pop(unionId) # remove from union list |
|||
unionResult = [] # temporary storage |
|||
for taskId in tasks: |
|||
task = self.__tasks[taskId] |
|||
self.__tasks.pop(taskId) # remove from task list |
|||
unionResult.append(task['data']) |
|||
logging.info('Manager release union [%s] -> %s' % (unionId, unionResult)) |
|||
return { |
|||
'finish': True, |
|||
'result': unionResult |
|||
} |
|||
|
|||
def popTask(self) -> tuple[str or None, any]: # fetch a loaded task |
|||
for taskId, task in self.__tasks.items(): |
|||
if task['status'] != self.__TASK_LOADED: continue # only get loaded task |
|||
task['status'] = self.__TASK_RUNNING # set task status as running |
|||
logging.info('Manager pop task [%s] -> %s' % (taskId, task['data'])) |
|||
return taskId, copy.deepcopy(task['data']) |
|||
logging.debug('Manager has no more loaded tasks') |
|||
raise RuntimeError('No more tasks') |
|||
|
|||
def finishTask(self, taskId: str, taskData: dict) -> None: # update task data when completed |
|||
if taskId not in self.__tasks: |
|||
logging.error('Manager task [%s] not found' % taskId) |
|||
raise RuntimeError('Task id not found') |
|||
self.__tasks[taskId]['data'] = copy.deepcopy(taskData) |
|||
self.__tasks[taskId]['status'] = self.__TASK_FINISH # set task status as completed |
|||
|
|||
|
|||
Manager = Task() # global task manager |
|||
@ -0,0 +1,34 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
import time |
|||
import requests |
|||
from Basis.Logger import logging |
|||
from Basis.Functions import hostFormat |
|||
|
|||
|
|||
def httpPing(taskId: str, url: str, socksAddr: str, socksPort: int, timeout: int) -> float: |
|||
try: |
|||
startTime = time.time_ns() # request start time |
|||
socksProxy = 'socks5://%s:%i' % (hostFormat(socksAddr, v6Bracket = True), socksPort) |
|||
logging.debug('[%s] Http ping -> request %s via %s' % (taskId, url, socksProxy)) |
|||
httpRequest = requests.get(url, proxies = { # send http request by socks5 proxy |
|||
'http': socksProxy, |
|||
'https': socksProxy, |
|||
}, timeout = timeout) |
|||
except: # something error on request process (timeout or proxy not working) |
|||
logging.debug('[%s] Http ping -> request error' % taskId) |
|||
return -1 |
|||
if httpRequest.status_code not in range(200, 300): # http code not 2xx |
|||
logging.debug('[%s] Http ping -> status code %i not expected' % (taskId, httpRequest.status_code)) |
|||
return -1 |
|||
delay = (time.time_ns() - startTime) / (10 ** 6) # ns -> ms |
|||
logging.debug('[%s] Http ping -> delay %f ms' % (taskId, delay)) |
|||
return round(delay, 2) # two decimal places |
|||
|
|||
|
|||
def check(taskId: str, socksInfo: dict, options: dict) -> dict: |
|||
# TODO: multi check |
|||
return { # TODO: just demo |
|||
'delay': httpPing(taskId, options['url'], socksInfo['addr'], socksInfo['port'], options['timeout']) |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
from Checker import Http |
|||
|
|||
def Checker(taskId: str, checkInfo: dict, socksInfo: dict) -> dict: |
|||
|
|||
# TODO: ignore checkInfo for now |
|||
|
|||
httpRet = Http.check(taskId, socksInfo, { |
|||
'url': 'http://baidu.com/', |
|||
'timeout': 20, |
|||
}) |
|||
return { |
|||
'http': httpRet # TODO: just check http delay for now |
|||
} |
|||
|
|||
# TODO: return check result |
|||
@ -1,56 +0,0 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*- coding: utf-8 -*- |
|||
|
|||
from Basis.Manage import Manage |
|||
from Basis.Logger import logging |
|||
|
|||
|
|||
taskId_1 = Manage.addTask([ |
|||
{'test': 1}, |
|||
{'test': 2}, |
|||
{'test': 3}, |
|||
]) |
|||
logging.critical('task id 1 -> %s' % taskId_1) |
|||
|
|||
taskId_2 = Manage.addTask([ |
|||
{'demo': 1}, |
|||
{'demo': 2}, |
|||
{'demo': 3}, |
|||
]) |
|||
logging.critical('task id 2 -> %s' % taskId_2) |
|||
|
|||
logging.critical('list task -> %s' % Manage.listTask()) |
|||
|
|||
logging.critical('is task 1234 -> %s' % Manage.isTask('1234')) |
|||
logging.critical('is task %s -> %s' % (taskId_1, Manage.isTask(taskId_1))) |
|||
|
|||
logging.critical('get task %s -> %s' % (taskId_1, Manage.getTask(taskId_1))) |
|||
|
|||
subTaskId_1, subTask_1 = Manage.popSubTask() |
|||
logging.critical('pop sub task %s -> %s' % (subTaskId_1, subTask_1)) |
|||
subTaskId_2, subTask_2 = Manage.popSubTask() |
|||
logging.critical('pop sub task %s -> %s' % (subTaskId_2, subTask_2)) |
|||
subTaskId_3, subTask_3 = Manage.popSubTask() |
|||
logging.critical('pop sub task %s -> %s' % (subTaskId_3, subTask_3)) |
|||
|
|||
Manage.updateSubTask(subTaskId_3, {'test': 33}) |
|||
logging.critical('update sub task %s' % subTaskId_3) |
|||
Manage.updateSubTask(subTaskId_2, {'test': 22}) |
|||
logging.critical('update sub task %s' % subTaskId_2) |
|||
Manage.updateSubTask(subTaskId_1, {'test': 11}) |
|||
logging.critical('update sub task %s' % subTaskId_1) |
|||
|
|||
logging.critical('get task %s -> %s' % (taskId_1, Manage.getTask(taskId_1))) |
|||
|
|||
subTaskId_4, subTask_4 = Manage.popSubTask() |
|||
logging.critical('pop sub task %s -> %s' % (subTaskId_4, subTask_4)) |
|||
|
|||
Manage.updateSubTask(subTaskId_4, {'demo': 2333}) |
|||
logging.critical('update sub task %s' % subTaskId_4) |
|||
|
|||
logging.critical('get task %s -> %s' % (taskId_2, Manage.getTask(taskId_2))) |
|||
|
|||
logging.critical('sub task pop -> ' + str(Manage.popSubTask())) |
|||
logging.critical('sub task pop -> ' + str(Manage.popSubTask())) |
|||
logging.critical('sub task pop -> ' + str(Manage.popSubTask())) |
|||
|
|||
Loading…
Reference in new issue