cobweb-launcher 1.3.5__py3-none-any.whl → 1.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {cobweb_launcher-1.3.5.dist-info → cobweb_launcher-1.3.7.dist-info}/METADATA +1 -1
  2. cobweb_launcher-1.3.7.dist-info/RECORD +40 -0
  3. cobweb_launcher-1.3.7.dist-info/top_level.txt +1 -0
  4. cobweb/base/decorators.py +0 -40
  5. cobweb/crawlers/base_crawler.py +0 -144
  6. cobweb/crawlers/file_crawler.py +0 -98
  7. cobweb/pipelines/base_pipeline.py +0 -54
  8. cobweb/pipelines/loghub_pipeline.py +0 -34
  9. cobweb/utils/dotting.py +0 -32
  10. cobweb_/__init__.py +0 -2
  11. cobweb_/base/__init__.py +0 -9
  12. cobweb_/base/common_queue.py +0 -30
  13. cobweb_/base/decorators.py +0 -40
  14. cobweb_/base/item.py +0 -46
  15. cobweb_/base/log.py +0 -94
  16. cobweb_/base/request.py +0 -82
  17. cobweb_/base/response.py +0 -23
  18. cobweb_/base/seed.py +0 -114
  19. cobweb_/constant.py +0 -94
  20. cobweb_/crawlers/__init__.py +0 -1
  21. cobweb_/crawlers/crawler.py +0 -184
  22. cobweb_/db/__init__.py +0 -2
  23. cobweb_/db/api_db.py +0 -82
  24. cobweb_/db/redis_db.py +0 -130
  25. cobweb_/exceptions/__init__.py +0 -1
  26. cobweb_/exceptions/oss_db_exception.py +0 -28
  27. cobweb_/launchers/__init__.py +0 -3
  28. cobweb_/launchers/launcher.py +0 -235
  29. cobweb_/launchers/launcher_air.py +0 -88
  30. cobweb_/launchers/launcher_api.py +0 -221
  31. cobweb_/launchers/launcher_pro.py +0 -222
  32. cobweb_/pipelines/__init__.py +0 -3
  33. cobweb_/pipelines/pipeline.py +0 -69
  34. cobweb_/pipelines/pipeline_console.py +0 -22
  35. cobweb_/pipelines/pipeline_loghub.py +0 -34
  36. cobweb_/setting.py +0 -74
  37. cobweb_/utils/__init__.py +0 -5
  38. cobweb_/utils/bloom.py +0 -58
  39. cobweb_/utils/dotting.py +0 -32
  40. cobweb_/utils/oss.py +0 -94
  41. cobweb_/utils/tools.py +0 -42
  42. cobweb_launcher-1.3.5.dist-info/RECORD +0 -111
  43. cobweb_launcher-1.3.5.dist-info/top_level.txt +0 -2
  44. cobweb_new/__init__.py +0 -2
  45. cobweb_new/base/__init__.py +0 -72
  46. cobweb_new/base/common_queue.py +0 -53
  47. cobweb_new/base/decorators.py +0 -72
  48. cobweb_new/base/item.py +0 -46
  49. cobweb_new/base/log.py +0 -94
  50. cobweb_new/base/request.py +0 -82
  51. cobweb_new/base/response.py +0 -23
  52. cobweb_new/base/seed.py +0 -118
  53. cobweb_new/constant.py +0 -105
  54. cobweb_new/crawlers/__init__.py +0 -1
  55. cobweb_new/crawlers/crawler-new.py +0 -85
  56. cobweb_new/crawlers/crawler.py +0 -170
  57. cobweb_new/db/__init__.py +0 -2
  58. cobweb_new/db/api_db.py +0 -82
  59. cobweb_new/db/redis_db.py +0 -158
  60. cobweb_new/exceptions/__init__.py +0 -1
  61. cobweb_new/exceptions/oss_db_exception.py +0 -28
  62. cobweb_new/launchers/__init__.py +0 -3
  63. cobweb_new/launchers/launcher.py +0 -237
  64. cobweb_new/launchers/launcher_air.py +0 -88
  65. cobweb_new/launchers/launcher_api.py +0 -161
  66. cobweb_new/launchers/launcher_pro.py +0 -96
  67. cobweb_new/launchers/tesss.py +0 -47
  68. cobweb_new/pipelines/__init__.py +0 -3
  69. cobweb_new/pipelines/pipeline.py +0 -68
  70. cobweb_new/pipelines/pipeline_console.py +0 -22
  71. cobweb_new/pipelines/pipeline_loghub.py +0 -34
  72. cobweb_new/setting.py +0 -95
  73. cobweb_new/utils/__init__.py +0 -5
  74. cobweb_new/utils/bloom.py +0 -58
  75. cobweb_new/utils/oss.py +0 -94
  76. cobweb_new/utils/tools.py +0 -42
  77. {cobweb_launcher-1.3.5.dist-info → cobweb_launcher-1.3.7.dist-info}/LICENSE +0 -0
  78. {cobweb_launcher-1.3.5.dist-info → cobweb_launcher-1.3.7.dist-info}/WHEEL +0 -0
cobweb_new/utils/tools.py DELETED
@@ -1,42 +0,0 @@
1
- import re
2
- import hashlib
3
- from typing import Union
4
- from importlib import import_module
5
-
6
-
7
- def md5(text: Union[str, bytes]) -> str:
8
- if isinstance(text, str):
9
- text = text.encode('utf-8')
10
- return hashlib.md5(text).hexdigest()
11
-
12
-
13
- def build_path(site, url, file_type):
14
- return f"{site}/{md5(url)}.{file_type}"
15
-
16
-
17
- def format_size(content_length: int) -> str:
18
- units = ["KB", "MB", "GB", "TB"]
19
- for i in range(4):
20
- num = content_length / (1024 ** (i + 1))
21
- if num < 1024:
22
- return f"{round(num, 2)} {units[i]}"
23
-
24
-
25
- def dynamic_load_class(model_info):
26
- if isinstance(model_info, str):
27
- if "import" in model_info:
28
- model_path, class_name = re.search(
29
- r"from (.*?) import (.*?)$", model_info
30
- ).groups()
31
- model = import_module(model_path)
32
- class_object = getattr(model, class_name)
33
- else:
34
- model_path, class_name = model_info.rsplit(".", 1)
35
- model = import_module(model_path)
36
- class_object = getattr(model, class_name)
37
- return class_object
38
- raise TypeError()
39
-
40
-
41
- # def download_log_info(item:dict) -> str:
42
- # return "\n".join([" " * 12 + f"{str(k).ljust(14)}: {str(v)}" for k, v in item.items()])