diff --git a/src/108shu.com/catalog.py b/src/108shu.com/catalog.py new file mode 100644 index 0000000..1e7c0cf --- /dev/null +++ b/src/108shu.com/catalog.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Fetch catalog and output as JSON format. + + USAGE: python3 catalog.py +""" + +import re +import json +import requests +from logger import logger +from bs4 import BeautifulSoup + +userAgent = ( # default user agent + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.47' +) + + +def httpRequest(url: str) -> str: # fetch raw html content + request = requests.get(url, headers = { + 'user-agent': userAgent, # with fake user-agent + 'accept-encoding': 'gzip, deflate', # allow content compress + }) + if request.status_code not in range(200, 300): # http status code 2xx + raise RuntimeError('Http request failed') + return request.text + + +def extractCatalog(rawHtml: str) -> dict: # extract catalog from html content + catalog = {} + html = BeautifulSoup(rawHtml, 'lxml') + items = html.select('div[class="section-box"]')[1] + for item in items.select('a'): + name = re.search(r'^(第\d+章)(.*)$', item.text) + pageId = item.attrs['href'].replace('/book/54247/', '').replace('.html', '') + catalog['%s %s' % (name[1], name[2].strip())] = pageId + return catalog + + +def fetchCatalog(pageNum: int) -> dict: # fetch all catalog + catalog = {} + for pageId in range(1, pageNum + 1): # traverse all pages + pageUrl = 'http://www.108shu.com/book/54247/index_%d.html' % pageId + logger.info('Page: %d -> `%s`' % (pageId, pageUrl)) + catalog.update(extractCatalog(httpRequest(pageUrl))) + return catalog + + +print(json.dumps(fetchCatalog(45))) diff --git a/src/108shu.com/logger.py b/src/108shu.com/logger.py new file mode 100644 index 0000000..36cfaf8 --- /dev/null +++ b/src/108shu.com/logger.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Logger module: + INFO level -> stderr + DEBUG level -> runtime.log +""" + +import sys +import logging +import colorlog + +logColor = { # log color + 'DEBUG': 'white', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'bold_red', +} + +dateFormat = '%Y-%m-%d %H:%M:%S' +timeFormat = '%(asctime)s.%(msecs)03d' +logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)' + +# load fileHandler -> log file +fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8') +fileHandler.setFormatter(logging.Formatter( + '[' + timeFormat + '] ' + logFormat, + datefmt = dateFormat +)) +fileHandler.setLevel(logging.DEBUG) # debug level for log file + +# load stdHandler -> stderr +stdHandler = colorlog.StreamHandler() +stdHandler.setFormatter(colorlog.ColoredFormatter( + '%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat, + datefmt = dateFormat, + log_colors = logColor, + stream = sys.stderr +)) +stdHandler.setLevel(logging.INFO) # info level for stderr + +logger = logging.getLogger() +logger.addHandler(stdHandler) +logger.addHandler(fileHandler) +logger.setLevel(logging.DEBUG) # set log level in handler