From a1eb72fbc6c7b2e4273b03c16086d2413d3db31a Mon Sep 17 00:00:00 2001
From: Dnomd343 <i@343.re>
Date: Sat, 15 Oct 2022 20:17:30 +0800
Subject: [PATCH] feat: extract script for `108shu.com`

---
 src/108shu.com/extract.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 src/108shu.com/extract.py

diff --git a/src/108shu.com/extract.py b/src/108shu.com/extract.py
new file mode 100644
index 0000000..0b7eb18
--- /dev/null
+++ b/src/108shu.com/extract.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Extract data from raw html content.
+
+    USAGE: python3 extract.py [CATALOG] [HTML_DIR]
+"""
+
+import os
+import re
+import sys
+import json
+from logger import logger
+from bs4 import BeautifulSoup
+
+
+def splitHtml(rawHtml: str) -> dict:  # extract from raw html content
+    html = BeautifulSoup(rawHtml, 'lxml')
+    content = [x.text.strip() for x in html.select('div[class="content"]')[0].select('p')]
+    title = re.search(r'^(第\d+章)(.*)$', html.select('h1')[0].text)
+    return {
+        'title': '%s %s' % (title[1], title[2].strip()),
+        'content': [x.text.strip() for x in html.select('div[class="content"]')[0].select('p')]
+    }
+
+
+result = {}
+catalog = json.loads(open(sys.argv[1]).read())  # load catalog
+
+for chapterName, chapterId in catalog.items():  # traverse all chapters
+    logger.info('Analyse chapter `%s`' % chapterId)
+    htmlFile = os.path.join(sys.argv[2], '%s.html' % chapterId)
+    info = splitHtml(open(htmlFile).read())
+    if chapterName != info['title']:
+        logger.error('Title error -> %s' % info['title'])
+    result[chapterName] = info['content']
+
+print(json.dumps(result))