diff --git a/src/aidusk.com/crawler.sh b/src/aidusk.com/crawler.sh index 4f53aca..4a5836b 100755 --- a/src/aidusk.com/crawler.sh +++ b/src/aidusk.com/crawler.sh @@ -6,12 +6,3 @@ mkdir -p ./data/html/ python3 catalog.py > ./data/catalog.json python3 fetch.py ./data/catalog.json ./data/html/ python3 extract.py ./data/catalog.json ./data/html/ > ./data/xxrs.json - -cd ./data/ -xz -k9 catalog.json -tar cJf html.tar.xz html/ -xz -k9 xxrs.json - -mkdir -p ../archive/ -mv *.xz ../archive/ -cd ../