From 2acbbadd13182eba3a9a2d8bdf8e5f60fabbb1d0 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Tue, 11 Oct 2022 14:25:37 +0800 Subject: [PATCH] feat: analyse html content --- analyse.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 analyse.py diff --git a/analyse.py b/analyse.py new file mode 100644 index 0000000..1232c4b --- /dev/null +++ b/analyse.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import re +import json +from logger import logger +from bs4 import BeautifulSoup + + +def splitPage(rawHtml: str) -> dict: + body = BeautifulSoup(rawHtml, 'lxml').body + script = body.select('script')[5].text + info = { + 'title': body.select('div[class="size18 w100 text-center lh100 pt30 pb15"]')[0].text.strip(), + 'contents': [x.text.strip() for x in body.select('p[class="content_detail"]')], + 'prePage': body.select('div[class="pt-prechapter"]')[0].a.attrs['href'], + 'nextPage': body.select('div[class="pt-nextchapter"]')[0].a.attrs['href'], + 'preId': re.search(r'window\.__PREVPAGE = "(\d*)"', script)[1], + 'nextId': re.search(r'window\.__NEXTPAGE = "(\d*)"', script)[1], + 'myId': re.search(r'window\.chapterNum = (\d+)', script)[1], + 'caption': re.search(r'window\.chapterName = \'(.+)\'', script)[1], + } + if not info['title'].startswith(info['caption']): + logger.error('Title error -> %s' % info['caption']) + info['index'] = info['title'].replace(info['caption'], '') + info.pop('title') + return info + + +def combinePage(id: str) -> dict: + page_1 = splitPage(open('./html/%s-1.html' % id).read()) + page_2 = splitPage(open('./html/%s-2.html' % id).read()) + + if not page_1['index'] == '[1/2页]' or not page_2['index'] == '[2/2页]': + logger.error('Sub page error -> `%s` <-> `%s`' % (page_1['index'], page_2['index'])) + if not page_1['caption'] == page_2['caption']: + logger.error('Caption error -> `%s` <-> `%s`' % (page_1['caption'], page_2['caption'])) + if not page_1['myId'] == page_2['myId']: + logger.error('Page ID error -> `%s` <-> `%s`' % (page_1['myId'], page_2['myId'])) + if not page_1['preId'] == page_2['preId']: + logger.error('Pre page ID error -> `%s` <-> `%s`' % (page_1['preId'], page_2['preId'])) + if not page_1['nextId'] == page_2['nextId']: + logger.error('Next page ID error -> `%s` <-> `%s`' % (page_1['nextId'], page_2['nextId'])) + + if not page_1['prePage'] == '/novel/57104/read_%s.html' % page_1['preId']: + logger.warning('Page-1 pre url -> `%s` (ID = %s)' % (page_1['prePage'], id)) + if not page_1['nextPage'] == '/novel/57104/read_%s/2.html' % page_1['myId']: + logger.warning('Page-1 next url -> `%s` (ID = %s)' % (page_1['nextPage'], id)) + if not page_2['prePage'] == '/novel/57104/read_%s.html' % page_2['myId']: + logger.warning('Page-2 pre url -> `%s` (ID = %s)' % (page_2['prePage'], id)) + if not page_2['nextPage'] == '/novel/57104/read_%s.html' % page_2['nextId']: + logger.warning('Page-2 next url -> `%s` (ID = %s)' % (page_2['nextPage'], id)) + + return { + 'title': page_1['caption'], + 'preId': page_1['preId'], + 'myId': page_1['myId'], + 'nextId': page_1['nextId'], + 'contents': page_1['contents'] + page_2['contents'] + } + + +catalog = json.loads(open('./catalog/catalog.json').read()) + +for _, pageId in catalog.items(): + logger.info('Analyse page `%s`' % pageId) + with open('./json/%s.json' % pageId, 'w') as fileObj: + fileObj.write(json.dumps(combinePage(pageId)))