cobweb-launcher 1.2.6__tar.gz → 1.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (38) hide show
  1. {cobweb-launcher-1.2.6/cobweb_launcher.egg-info → cobweb-launcher-1.2.7}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/launchers/launcher.py +2 -1
  3. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/setting.py +1 -4
  4. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  5. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/setup.py +1 -1
  6. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/LICENSE +0 -0
  7. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/README.md +0 -0
  8. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/decorators.py +0 -0
  12. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/item.py +0 -0
  13. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/log.py +0 -0
  14. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/request.py +0 -0
  15. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/response.py +0 -0
  16. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/base/seed.py +0 -0
  17. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/constant.py +0 -0
  18. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/crawlers/__init__.py +0 -0
  19. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/crawlers/crawler.py +0 -0
  20. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/db/__init__.py +0 -0
  21. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/db/redis_db.py +0 -0
  22. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/exceptions/__init__.py +0 -0
  23. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/exceptions/oss_db_exception.py +0 -0
  24. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/launchers/__init__.py +0 -0
  25. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/launchers/launcher_air.py +0 -0
  26. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/launchers/launcher_pro.py +0 -0
  27. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/pipelines/__init__.py +0 -0
  28. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/pipelines/pipeline.py +0 -0
  29. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/pipelines/pipeline_console.py +0 -0
  30. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/pipelines/pipeline_loghub.py +0 -0
  31. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/utils/__init__.py +0 -0
  32. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/utils/oss.py +0 -0
  33. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/utils/tools.py +0 -0
  34. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  35. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  36. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb_launcher.egg-info/requires.txt +0 -0
  37. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb_launcher.egg-info/top_level.txt +0 -0
  38. {cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/setup.cfg +0 -0
{cobweb-launcher-1.2.6/cobweb_launcher.egg-info → cobweb-launcher-1.2.7}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.6
3
+ Version: 1.2.7
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/launchers/launcher.py RENAMED
@@ -4,7 +4,7 @@ import threading
4
4
  import importlib
5
5
 
6
6
  from cobweb import setting
7
- from cobweb.base import Seed, Queue
7
+ from cobweb.base import Seed, Queue, logger
8
8
  from cobweb.utils.tools import dynamic_load_class
9
9
 
10
10
 
@@ -138,6 +138,7 @@ class Launcher(threading.Thread):
138
138
  def _remove_doing_seeds(self, seeds):
139
139
  for seed in seeds:
140
140
  self.__DOING__.pop(seed, None)
141
+ logger.info("remove %s seeds from __DOING__" % len(seeds))
141
142
 
142
143
  def _execute(self):
143
144
  for func_name in self.__LAUNCHER_FUNC__:
{cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/cobweb/setting.py RENAMED
@@ -26,9 +26,6 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
26
26
  OSS_CHUNK_SIZE = 10 * 1024 ** 2
27
27
  OSS_MIN_UPLOAD_SIZE = 1024
28
28
 
29
- # message
30
- MESSAGE = ""
31
-
32
29
 
33
30
  # 采集器选择
34
31
  CRAWLER = "cobweb.crawlers.Crawler"
@@ -43,7 +40,7 @@ BEFORE_SCHEDULER_WAIT_SECONDS = 60 # 调度前等待时间,只作用于单次
43
40
  SCHEDULER_WAIT_SECONDS = 15 # 调度等待时间
44
41
  TODO_QUEUE_FULL_WAIT_SECONDS = 5 # todo队列已满时等待时间
45
42
  NEW_QUEUE_WAIT_SECONDS = 30 # new队列等待时间
46
- DONE_QUEUE_WAIT_SECONDS = 15 # done队列等待时间
43
+ DONE_QUEUE_WAIT_SECONDS = 5 # done队列等待时间
47
44
  UPLOAD_QUEUE_WAIT_SECONDS = 15 # upload队列等待时间
48
45
  SEED_RESET_SECONDS = 30 # 种子重制时间
49
46
 
{cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.6
3
+ Version: 1.2.7
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.6",
8
+ version="1.2.7",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",
{cobweb-launcher-1.2.6 → cobweb-launcher-1.2.7}/LICENSE RENAMED
File without changes