Browse Source

feat: fetch script for `ixsw.la`

master
Dnomd343 2 years ago
parent
commit
af84b5d968
  1. 53
      src/ixsw.la/fetch.py
  2. 47
      src/ixsw.la/logger.py

53
src/ixsw.la/fetch.py

@ -0,0 +1,53 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Download raw html content as `.html` files.
USAGE: python3 fetch.py [CATALOG] [OUTPUT_DIR]
"""
import os
import sys
import json
import time
import requests
from logger import logger
userAgent = ( # default user agent
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.47'
)
def httpRequest(fileUrl: str, fileName: str) -> bool: # save html content
try:
logger.debug('Http request `%s` -> `%s`' % (fileUrl, fileName))
request = requests.get(fileUrl, timeout = 30, # timeout -> 30s
headers = {
'user-agent': userAgent, # with fake user-agent
}
)
if request.status_code not in range(200, 300): # http status code 2xx
logger.warning('Http request failed -> `%s`' % fileUrl)
return False
logger.debug('Http request success -> `%s`' % fileUrl)
with open(fileName, 'wb') as fileObj: # save html content
fileObj.write(request.content)
logger.debug('File save success -> `%s`' % fileName)
except:
return False
return True
catalog = json.loads(open(sys.argv[1]).read()) # load catalog
for _, chapterId in catalog.items(): # traverse all chapters
pageUrl = 'https://www.ixsw.la/ks82668/%s.html' % chapterId
pageFile = os.path.join(sys.argv[2], '%s.html' % chapterId)
if httpRequest(pageUrl, pageFile): # save html content
logger.info('Page request success -> %s' % pageUrl)
else:
logger.error('Page request failed -> %s' % pageUrl)
time.sleep(1) # avoid being blocked by the server

47
src/ixsw.la/logger.py

@ -0,0 +1,47 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Logger module:
INFO level -> stderr
DEBUG level -> runtime.log
"""
import sys
import logging
import colorlog
logColor = { # log color
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
}
dateFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%(asctime)s.%(msecs)03d'
logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
# load fileHandler -> log file
fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
fileHandler.setFormatter(logging.Formatter(
'[' + timeFormat + '] ' + logFormat,
datefmt = dateFormat
))
fileHandler.setLevel(logging.DEBUG) # debug level for log file
# load stdHandler -> stderr
stdHandler = colorlog.StreamHandler()
stdHandler.setFormatter(colorlog.ColoredFormatter(
'%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
datefmt = dateFormat,
log_colors = logColor,
stream = sys.stderr
))
stdHandler.setLevel(logging.INFO) # info level for stderr
logger = logging.getLogger()
logger.addHandler(stdHandler)
logger.addHandler(fileHandler)
logger.setLevel(logging.DEBUG) # set log level in handler
Loading…
Cancel
Save