diff --git a/src/wxsy.net/catalog.py b/src/wxsy.net/catalog.py index 869bfd5..1fab35f 100644 --- a/src/wxsy.net/catalog.py +++ b/src/wxsy.net/catalog.py @@ -2,7 +2,9 @@ # -*- coding: utf-8 -*- """ -Fetch catalog and output as JSON format +Fetch catalog and output as JSON format. + + USAGE: python3 catalog.py """ import re diff --git a/src/wxsy.net/fetch.py b/src/wxsy.net/fetch.py new file mode 100644 index 0000000..9d4c825 --- /dev/null +++ b/src/wxsy.net/fetch.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Download raw html content as `.html` files. + + USAGE: python3 fetch.py [CATALOG] [OUTPUT_DIR] +""" + +import sys +import json +import time +import requests +from logger import logger + +basicUrl = 'https://m.wxsy.net/novel/57104' + +userAgent = ( # default user-agent + 'Mozilla/5.0 (Linux; Android 10; moto g(7) play) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/100.0.4896.79 Mobile Safari/537.36' +) + + +def httpRequest(url: str, fileName: str) -> bool: # save html content + try: + logger.debug('Http request `%s` -> %s' % (url, fileName)) + request = requests.get(url, timeout = 30, # timeout -> 30s + headers = { + 'user-agent': userAgent, # with fake user-agent + } + ) + if request.status_code not in range(200, 300): # http status code 2xx + logger.warning('Http request failed -> %s' % url) + return False + logger.debug('Http request success -> %s' % url) + with open(fileName, 'w') as fileObj: # save html content + fileObj.write(request.text) + logger.debug('File save success -> %s' % fileName) + except: + return False + return True + + +catalog = json.loads(open(sys.argv[1]).read()) # load catalog + +for _, pageId in catalog.items(): # traverse all chapters + for subPage in [1, 2]: # two sub pages in one chapter + pageUrl = '%s/read_%s/%d.html' % (basicUrl, pageId, subPage) + pageFile = '%s/%s-%d.html' % (sys.argv[2], pageId, subPage) + if httpRequest(pageUrl, pageFile): # save html content + logger.info('Page request success -> %s' % pageUrl) + else: + logger.error('Page request failed -> %s' % pageUrl) + time.sleep(1) # avoid being blocked by the server diff --git a/src/wxsy.net/logger.py b/src/wxsy.net/logger.py new file mode 100644 index 0000000..b9f9a06 --- /dev/null +++ b/src/wxsy.net/logger.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import logging +import colorlog + +logColor = { # log color + 'DEBUG': 'white', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'bold_red', +} + +dateFormat = '%Y-%m-%d %H:%M:%S' +timeFormat = '%(asctime)s.%(msecs)03d' +logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)' + +# load fileHandler -> log file +fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8') +fileHandler.setFormatter(logging.Formatter( + '[' + timeFormat + '] ' + logFormat, + datefmt = dateFormat +)) +fileHandler.setLevel(logging.DEBUG) # debug level for log file + +# load stdHandler -> stderr +stdHandler = colorlog.StreamHandler() +stdHandler.setFormatter(colorlog.ColoredFormatter( + '%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat, + datefmt = dateFormat, + log_colors = logColor, + stream = sys.stderr +)) +stdHandler.setLevel(logging.INFO) # info level for stderr + +logger = logging.getLogger() +logger.addHandler(stdHandler) +logger.addHandler(fileHandler) +logger.setLevel(logging.DEBUG) # set log level in handler