feat: crawler script for `xswang.com`

2 years ago · 36607e0d26
1 changed files with 8 additions and 0 deletions
--- a/src/xswang.com/crawler.sh
+++ b/src/xswang.com/crawler.sh
@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+cd `dirname $0`
+mkdir -p ./data/html/
+
+python3 catalog.py > ./data/catalog.json
+python3 fetch.py ./data/catalog.json ./data/html/
+python3 extract.py ./data/catalog.json ./data/html/ > ./data/xxrs.json