Browse Source

update: file structure

master
Dnomd343 2 years ago
parent
commit
1148fe804f
  1. 47
      src/aidusk.com/logger.py
  2. 0
      src/crawler/108shu.com/catalog.py
  3. 0
      src/crawler/108shu.com/crawler.sh
  4. 0
      src/crawler/108shu.com/extract.py
  5. 0
      src/crawler/108shu.com/fetch.py
  6. 0
      src/crawler/aidusk.com/archive/catalog.json.xz
  7. 0
      src/crawler/aidusk.com/archive/html.tar.xz
  8. 0
      src/crawler/aidusk.com/archive/xxrs.json.xz
  9. 0
      src/crawler/aidusk.com/catalog.py
  10. 0
      src/crawler/aidusk.com/check.sh
  11. 0
      src/crawler/aidusk.com/crawler.sh
  12. 0
      src/crawler/aidusk.com/extract.py
  13. 0
      src/crawler/aidusk.com/fetch.py
  14. 0
      src/crawler/ixsw.la/archive/catalog.json.xz
  15. 0
      src/crawler/ixsw.la/archive/html.tar.xz
  16. 0
      src/crawler/ixsw.la/archive/xxrs.json.xz
  17. 0
      src/crawler/ixsw.la/catalog.py
  18. 0
      src/crawler/ixsw.la/check.sh
  19. 0
      src/crawler/ixsw.la/crawler.sh
  20. 0
      src/crawler/ixsw.la/extract.py
  21. 0
      src/crawler/ixsw.la/fetch.py
  22. 0
      src/crawler/logger.py
  23. 0
      src/crawler/m.wxsy.net/archive/catalog.json.xz
  24. 0
      src/crawler/m.wxsy.net/archive/html.tar.xz
  25. 0
      src/crawler/m.wxsy.net/archive/json.sha1sum
  26. 0
      src/crawler/m.wxsy.net/archive/json.tar.xz
  27. 0
      src/crawler/m.wxsy.net/archive/xxrs.json.xz
  28. 4
      src/crawler/m.wxsy.net/catalog.py
  29. 0
      src/crawler/m.wxsy.net/check.sh
  30. 0
      src/crawler/m.wxsy.net/crawler.sh
  31. 0
      src/crawler/m.wxsy.net/extract.py
  32. 0
      src/crawler/m.wxsy.net/fetch.py
  33. 0
      src/crawler/m.wxsy.net/release.py
  34. 0
      src/crawler/wxsy.net/archive/catalog.json.xz
  35. 0
      src/crawler/wxsy.net/archive/html.tar.xz
  36. 0
      src/crawler/wxsy.net/archive/json.sha1sum
  37. 0
      src/crawler/wxsy.net/archive/json.tar.xz
  38. 0
      src/crawler/wxsy.net/archive/xxrs.json.xz
  39. 0
      src/crawler/wxsy.net/catalog.py
  40. 0
      src/crawler/wxsy.net/check.sh
  41. 0
      src/crawler/wxsy.net/crawler.sh
  42. 0
      src/crawler/wxsy.net/extract.py
  43. 0
      src/crawler/wxsy.net/fetch.py
  44. 0
      src/crawler/wxsy.net/release.py
  45. 0
      src/crawler/xswang.com/catalog.py
  46. 0
      src/crawler/xswang.com/crawler.sh
  47. 0
      src/crawler/xswang.com/extract.py
  48. 0
      src/crawler/xswang.com/fetch.py
  49. 47
      src/ixsw.la/logger.py
  50. 47
      src/m.wxsy.net/logger.py
  51. 47
      src/wxsy.net/logger.py
  52. 47
      src/xswang.com/logger.py

47
src/aidusk.com/logger.py

@ -1,47 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Logger module:
INFO level -> stderr
DEBUG level -> runtime.log
"""
import sys
import logging
import colorlog
logColor = { # log color
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
}
dateFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%(asctime)s.%(msecs)03d'
logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
# load fileHandler -> log file
fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
fileHandler.setFormatter(logging.Formatter(
'[' + timeFormat + '] ' + logFormat,
datefmt = dateFormat
))
fileHandler.setLevel(logging.DEBUG) # debug level for log file
# load stdHandler -> stderr
stdHandler = colorlog.StreamHandler()
stdHandler.setFormatter(colorlog.ColoredFormatter(
'%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
datefmt = dateFormat,
log_colors = logColor,
stream = sys.stderr
))
stdHandler.setLevel(logging.INFO) # info level for stderr
logger = logging.getLogger()
logger.addHandler(stdHandler)
logger.addHandler(fileHandler)
logger.setLevel(logging.DEBUG) # set log level in handler

0
src/108shu.com/catalog.py → src/crawler/108shu.com/catalog.py

0
src/108shu.com/crawler.sh → src/crawler/108shu.com/crawler.sh

0
src/108shu.com/extract.py → src/crawler/108shu.com/extract.py

0
src/108shu.com/fetch.py → src/crawler/108shu.com/fetch.py

0
src/aidusk.com/archive/catalog.json.xz → src/crawler/aidusk.com/archive/catalog.json.xz

0
src/aidusk.com/archive/html.tar.xz → src/crawler/aidusk.com/archive/html.tar.xz

0
src/aidusk.com/archive/xxrs.json.xz → src/crawler/aidusk.com/archive/xxrs.json.xz

0
src/aidusk.com/catalog.py → src/crawler/aidusk.com/catalog.py

0
src/aidusk.com/check.sh → src/crawler/aidusk.com/check.sh

0
src/aidusk.com/crawler.sh → src/crawler/aidusk.com/crawler.sh

0
src/aidusk.com/extract.py → src/crawler/aidusk.com/extract.py

0
src/aidusk.com/fetch.py → src/crawler/aidusk.com/fetch.py

0
src/ixsw.la/archive/catalog.json.xz → src/crawler/ixsw.la/archive/catalog.json.xz

0
src/ixsw.la/archive/html.tar.xz → src/crawler/ixsw.la/archive/html.tar.xz

0
src/ixsw.la/archive/xxrs.json.xz → src/crawler/ixsw.la/archive/xxrs.json.xz

0
src/ixsw.la/catalog.py → src/crawler/ixsw.la/catalog.py

0
src/ixsw.la/check.sh → src/crawler/ixsw.la/check.sh

0
src/ixsw.la/crawler.sh → src/crawler/ixsw.la/crawler.sh

0
src/ixsw.la/extract.py → src/crawler/ixsw.la/extract.py

0
src/ixsw.la/fetch.py → src/crawler/ixsw.la/fetch.py

0
src/108shu.com/logger.py → src/crawler/logger.py

0
src/m.wxsy.net/archive/catalog.json.xz → src/crawler/m.wxsy.net/archive/catalog.json.xz

0
src/m.wxsy.net/archive/html.tar.xz → src/crawler/m.wxsy.net/archive/html.tar.xz

0
src/m.wxsy.net/archive/json.sha1sum → src/crawler/m.wxsy.net/archive/json.sha1sum

0
src/m.wxsy.net/archive/json.tar.xz → src/crawler/m.wxsy.net/archive/json.tar.xz

0
src/m.wxsy.net/archive/xxrs.json.xz → src/crawler/m.wxsy.net/archive/xxrs.json.xz

4
src/m.wxsy.net/catalog.py → src/crawler/m.wxsy.net/catalog.py

@ -7,8 +7,10 @@ Fetch catalog and output as JSON format.
USAGE: python3 catalog.py USAGE: python3 catalog.py
""" """
import re
import sys import sys
sys.path.append('..')
import re
import json import json
import time import time
import requests import requests

0
src/m.wxsy.net/check.sh → src/crawler/m.wxsy.net/check.sh

0
src/m.wxsy.net/crawler.sh → src/crawler/m.wxsy.net/crawler.sh

0
src/m.wxsy.net/extract.py → src/crawler/m.wxsy.net/extract.py

0
src/m.wxsy.net/fetch.py → src/crawler/m.wxsy.net/fetch.py

0
src/m.wxsy.net/release.py → src/crawler/m.wxsy.net/release.py

0
src/wxsy.net/archive/catalog.json.xz → src/crawler/wxsy.net/archive/catalog.json.xz

0
src/wxsy.net/archive/html.tar.xz → src/crawler/wxsy.net/archive/html.tar.xz

0
src/wxsy.net/archive/json.sha1sum → src/crawler/wxsy.net/archive/json.sha1sum

0
src/wxsy.net/archive/json.tar.xz → src/crawler/wxsy.net/archive/json.tar.xz

0
src/wxsy.net/archive/xxrs.json.xz → src/crawler/wxsy.net/archive/xxrs.json.xz

0
src/wxsy.net/catalog.py → src/crawler/wxsy.net/catalog.py

0
src/wxsy.net/check.sh → src/crawler/wxsy.net/check.sh

0
src/wxsy.net/crawler.sh → src/crawler/wxsy.net/crawler.sh

0
src/wxsy.net/extract.py → src/crawler/wxsy.net/extract.py

0
src/wxsy.net/fetch.py → src/crawler/wxsy.net/fetch.py

0
src/wxsy.net/release.py → src/crawler/wxsy.net/release.py

0
src/xswang.com/catalog.py → src/crawler/xswang.com/catalog.py

0
src/xswang.com/crawler.sh → src/crawler/xswang.com/crawler.sh

0
src/xswang.com/extract.py → src/crawler/xswang.com/extract.py

0
src/xswang.com/fetch.py → src/crawler/xswang.com/fetch.py

47
src/ixsw.la/logger.py

@ -1,47 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Logger module:
INFO level -> stderr
DEBUG level -> runtime.log
"""
import sys
import logging
import colorlog
logColor = { # log color
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
}
dateFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%(asctime)s.%(msecs)03d'
logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
# load fileHandler -> log file
fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
fileHandler.setFormatter(logging.Formatter(
'[' + timeFormat + '] ' + logFormat,
datefmt = dateFormat
))
fileHandler.setLevel(logging.DEBUG) # debug level for log file
# load stdHandler -> stderr
stdHandler = colorlog.StreamHandler()
stdHandler.setFormatter(colorlog.ColoredFormatter(
'%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
datefmt = dateFormat,
log_colors = logColor,
stream = sys.stderr
))
stdHandler.setLevel(logging.INFO) # info level for stderr
logger = logging.getLogger()
logger.addHandler(stdHandler)
logger.addHandler(fileHandler)
logger.setLevel(logging.DEBUG) # set log level in handler

47
src/m.wxsy.net/logger.py

@ -1,47 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Logger module:
INFO level -> stderr
DEBUG level -> runtime.log
"""
import sys
import logging
import colorlog
logColor = { # log color
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
}
dateFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%(asctime)s.%(msecs)03d'
logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
# load fileHandler -> log file
fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
fileHandler.setFormatter(logging.Formatter(
'[' + timeFormat + '] ' + logFormat,
datefmt = dateFormat
))
fileHandler.setLevel(logging.DEBUG) # debug level for log file
# load stdHandler -> stderr
stdHandler = colorlog.StreamHandler()
stdHandler.setFormatter(colorlog.ColoredFormatter(
'%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
datefmt = dateFormat,
log_colors = logColor,
stream = sys.stderr
))
stdHandler.setLevel(logging.INFO) # info level for stderr
logger = logging.getLogger()
logger.addHandler(stdHandler)
logger.addHandler(fileHandler)
logger.setLevel(logging.DEBUG) # set log level in handler

47
src/wxsy.net/logger.py

@ -1,47 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Logger module:
INFO level -> stderr
DEBUG level -> runtime.log
"""
import sys
import logging
import colorlog
logColor = { # log color
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
}
dateFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%(asctime)s.%(msecs)03d'
logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
# load fileHandler -> log file
fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
fileHandler.setFormatter(logging.Formatter(
'[' + timeFormat + '] ' + logFormat,
datefmt = dateFormat
))
fileHandler.setLevel(logging.DEBUG) # debug level for log file
# load stdHandler -> stderr
stdHandler = colorlog.StreamHandler()
stdHandler.setFormatter(colorlog.ColoredFormatter(
'%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
datefmt = dateFormat,
log_colors = logColor,
stream = sys.stderr
))
stdHandler.setLevel(logging.INFO) # info level for stderr
logger = logging.getLogger()
logger.addHandler(stdHandler)
logger.addHandler(fileHandler)
logger.setLevel(logging.DEBUG) # set log level in handler

47
src/xswang.com/logger.py

@ -1,47 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Logger module:
INFO level -> stderr
DEBUG level -> runtime.log
"""
import sys
import logging
import colorlog
logColor = { # log color
'DEBUG': 'white',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
}
dateFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%(asctime)s.%(msecs)03d'
logFormat = '[%(levelname)s] %(message)s (%(module)s.%(funcName)s:%(lineno)d)'
# load fileHandler -> log file
fileHandler = logging.FileHandler('runtime.log', encoding = 'utf-8')
fileHandler.setFormatter(logging.Formatter(
'[' + timeFormat + '] ' + logFormat,
datefmt = dateFormat
))
fileHandler.setLevel(logging.DEBUG) # debug level for log file
# load stdHandler -> stderr
stdHandler = colorlog.StreamHandler()
stdHandler.setFormatter(colorlog.ColoredFormatter(
'%(light_black)s' + timeFormat + '%(log_color)s ' + logFormat,
datefmt = dateFormat,
log_colors = logColor,
stream = sys.stderr
))
stdHandler.setLevel(logging.INFO) # info level for stderr
logger = logging.getLogger()
logger.addHandler(stdHandler)
logger.addHandler(fileHandler)
logger.setLevel(logging.DEBUG) # set log level in handler
Loading…
Cancel
Save