#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Fetch catalog and output as JSON format. USAGE: python3 catalog.py """ import re import sys import json sys.path.append('..') from utils import logger from utils import httpRequest from bs4 import BeautifulSoup def extractCatalog(rawHtml: bytes) -> dict: # extract catalog from html content catalog = {} html = BeautifulSoup(str(rawHtml, encoding = 'utf-8'), 'lxml') for item in [x.select('a')[0] for x in html.select('dd')]: title = re.search(r'^(第\d+章)(.*)', item.text.strip()) pageId = item.attrs['href'].replace('/book/56718/', '').replace('.html', '') catalog['%s %s' % (title[1], title[2].strip())] = pageId catalog = sorted(catalog.items(), key = lambda d: int( re.search(r'^第(\d+)章', d[0])[1] # sort by chapter )) return {x[0]: x[1] for x in catalog} # formatted output logger.warning('Fetch catalog of `xswang.com`') print(json.dumps( extractCatalog(httpRequest('https://www.xswang.com/book/56718/')) ))