From 4e611cc7fb5854640f8728d08dfe7b32a662db16 Mon Sep 17 00:00:00 2001
From: Dnomd343 <i@343.re>
Date: Sat, 15 Oct 2022 02:27:22 +0800
Subject: [PATCH] update: html fetch script

---
 src/wxsy.net/catalog.py |  4 ++-
 src/wxsy.net/fetch.py   | 55 +++++++++++++++++++++++++++++++++++++++++
 src/wxsy.net/logger.py  | 41 ++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 src/wxsy.net/fetch.py
 create mode 100644 src/wxsy.net/logger.py

diff --git a/src/wxsy.net/catalog.py b/src/wxsy.net/catalog.py
index 869bfd5..1fab35f 100644
--- a/src/wxsy.net/catalog.py
+++ b/src/wxsy.net/catalog.py
@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 
 """
-Fetch catalog and output as JSON format
+Fetch catalog and output as JSON format.
+
+    USAGE: python3 catalog.py
 """
 
 import re
diff --git a/src/wxsy.net/fetch.py b/src/wxsy.net/fetch.py
new file mode 100644
index 0000000..9d4c825
--- /dev/null
+++ b/src/wxsy.net/fetch.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Download raw html content as `.html` files.
+
+    USAGE: python3 fetch.py [CATALOG] [OUTPUT_DIR]
+"""
+
+import sys
+import json
+import time
+import requests
+from logger import logger
+
+basicUrl = 'https://m.wxsy.net/novel/57104'
+
+userAgent = (  # default user-agent
+    'Mozilla/5.0 (Linux; Android 10; moto g(7) play) '
+    'AppleWebKit/537.36 (KHTML, like Gecko) '
+    'Chrome/100.0.4896.79 Mobile Safari/537.36'
+)
+
+
+def httpRequest(url: str, fileName: str) -> bool:  # save html content
+    try:
+        logger.debug('Http request `%s` -> %s' % (url, fileName))
+        request = requests.get(url, timeout = 30,  # timeout -> 30s
+            headers = {
+                'user-agent': userAgent,  # with fake user-agent
+            }
+        )
+        if request.status_code not in range(200, 300):  # http status code 2xx
+            logger.warning('Http request failed -> %s' % url)
+            return False
+        logger.debug('Http request success -> %s' % url)
+        with open(fileName, 'w') as fileObj:  # save html content
+            fileObj.write(request.text)
+        logger.debug('File save success -> %s' % fileName)
+    except:
+        return False
+    return True
+
+
+catalog = json.loads(open(sys.argv[1]).read())  # load catalog
+
+for _, pageId in catalog.items():  # traverse all chapters
+    for subPage in [1, 2]:  # two sub pages in one chapter
+        pageUrl = '%s/read_%s/%d.html' % (basicUrl, pageId, subPage)
+        pageFile = '%s/%s-%d.html' % (sys.argv[2], pageId, subPage)
+        if httpRequest(pageUrl, pageFile):  # save html content
+            logger.info('Page request success -> %s' % pageUrl)
+        else:
+            logger.error('Page request failed -> %s' % pageUrl)
+        time.sleep(1)  # avoid being blocked by the server
diff --git a/src/wxsy.net/logger.py b/src/wxsy.net/logger.py
new file mode 100644
index 0000000..b9f9a06
--- /dev/null
+++ b/src/wxsy.net/logger.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+import logging
+import colorlog
+
+logColor = {  # log color
+    'DEBUG': 'white',
+    'INFO': 'green',
+    'WARNING': 'yellow',
+    'ERROR': 'red',
+    'CRITICAL': 'bold_red',
+}
+
+dateFormat = '%Y-%m-%d %H:%M:%S'
+timeFormat = '%(asctime)s.%(msecs)03d'
+logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
+
+# load fileHandler -> log file
+fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
+fileHandler.setFormatter(logging.Formatter(
+    '[' + timeFormat + '] ' + logFormat,
+    datefmt = dateFormat
+))
+fileHandler.setLevel(logging.DEBUG)  # debug level for log file
+
+# load stdHandler -> stderr
+stdHandler = colorlog.StreamHandler()
+stdHandler.setFormatter(colorlog.ColoredFormatter(
+    '%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
+    datefmt = dateFormat,
+    log_colors = logColor,
+    stream = sys.stderr
+))
+stdHandler.setLevel(logging.INFO)  # info level for stderr
+
+logger = logging.getLogger()
+logger.addHandler(stdHandler)
+logger.addHandler(fileHandler)
+logger.setLevel(logging.DEBUG)  # set log level in handler