From 9883dfa8ec8c7cb21fdf9160d4298b0a9b89cec3 Mon Sep 17 00:00:00 2001
From: Dnomd343 <i@343.re>
Date: Sat, 15 Oct 2022 17:58:05 +0800
Subject: [PATCH] feat: fetch catalog from `aidusk.com`

---
 src/aidusk.com/catalog.py | 44 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 src/aidusk.com/catalog.py

diff --git a/src/aidusk.com/catalog.py b/src/aidusk.com/catalog.py
new file mode 100644
index 0000000..35c871e
--- /dev/null
+++ b/src/aidusk.com/catalog.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Fetch catalog and output as JSON format.
+
+    USAGE: python3 catalog.py
+"""
+
+import re
+import json
+import requests
+from bs4 import BeautifulSoup
+
+userAgent = (  # default user agent
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+    'AppleWebKit/537.36 (KHTML, like Gecko) '
+    'Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.47'
+)
+
+
+def httpRequest(url: str) -> str:  # fetch raw html content
+    request = requests.get(url, headers = {
+        'user-agent': userAgent,  # with fake user-agent
+        'accept-encoding': 'gzip, deflate',  # allow content compress
+    })
+    if request.status_code not in range(200, 300):  # http status code 2xx
+        raise RuntimeError('Http request failed')
+    return str(request.content, encoding = 'utf-8')
+
+
+def extractCatalog(rawHtml: str) -> dict:  # extract catalog from html content
+    catalog = {}
+    html = BeautifulSoup(rawHtml, 'lxml')
+    div = html.select('div[class="book_con_list"]')[1]
+    for item in div.select('a'):
+        name = re.search(r'^(第\d+章)(.*)$', item.text)
+        catalog['%s %s' % (name[1], name[2].strip())] = item.attrs['href'].replace('.html', '')
+    return catalog
+
+
+print(json.dumps(
+    extractCatalog(httpRequest('http://www.aidusk.com/t/134659/'))
+))