cobweb-launcher 1.2.53__tar.gz → 1.2.55__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (45) hide show
  1. {cobweb-launcher-1.2.53/cobweb_launcher.egg-info → cobweb-launcher-1.2.55}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/launcher.py +3 -0
  3. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/launcher_pro.py +8 -2
  4. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  5. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/setup.py +1 -1
  6. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/LICENSE +0 -0
  7. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/README.md +0 -0
  8. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/decorators.py +0 -0
  12. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/item.py +0 -0
  13. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/log.py +0 -0
  14. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/request.py +0 -0
  15. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/response.py +0 -0
  16. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/base/seed.py +0 -0
  17. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/constant.py +0 -0
  18. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/crawlers/__init__.py +0 -0
  19. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/crawlers/base_crawler.py +0 -0
  20. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/crawlers/crawler.py +0 -0
  21. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/crawlers/file_crawler.py +0 -0
  22. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/db/__init__.py +0 -0
  23. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/db/api_db.py +0 -0
  24. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/db/redis_db.py +0 -0
  25. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/exceptions/__init__.py +0 -0
  26. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/exceptions/oss_db_exception.py +0 -0
  27. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/__init__.py +0 -0
  28. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/launcher_air.py +0 -0
  29. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/launcher_api.py +0 -0
  30. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/pipelines/__init__.py +0 -0
  31. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/pipelines/pipeline.py +0 -0
  32. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/pipelines/pipeline_console.py +0 -0
  33. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/pipelines/pipeline_loghub.py +0 -0
  34. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/setting.py +0 -0
  35. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/utils/__init__.py +0 -0
  36. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/utils/bloom.py +0 -0
  37. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/utils/dotting.py +0 -0
  38. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/utils/oss.py +0 -0
  39. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/utils/tools.py +0 -0
  40. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  41. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  42. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb_launcher.egg-info/requires.txt +0 -0
  43. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb_launcher.egg-info/top_level.txt +0 -0
  44. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/setup.cfg +0 -0
  45. {cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/test/test.py +0 -0
{cobweb-launcher-1.2.53/cobweb_launcher.egg-info → cobweb-launcher-1.2.55}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.53
3
+ Version: 1.2.55
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/launcher.py RENAMED
@@ -188,6 +188,9 @@ class Launcher(threading.Thread):
188
188
 
189
189
  def _monitor(self):
190
190
  while True:
191
+ if not self.__WORKER_THREAD__:
192
+ time.sleep(15)
193
+ continue
191
194
  for func_name, worker_thread in self.__WORKER_THREAD__.items():
192
195
  if not worker_thread.is_alive():
193
196
  logger.info(f"{func_name} thread is dead. Restarting...")
{cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/cobweb/launchers/launcher_pro.py RENAMED
@@ -175,7 +175,7 @@ class LauncherPro(Launcher):
175
175
  logger.info("Done! ready to close thread...")
176
176
  self._stop.set()
177
177
 
178
- elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
178
+ elif self._db.zcard(self._todo_key):
179
179
  logger.info(f"Recovery {self.task} task run！")
180
180
  self._pause.clear()
181
181
  self._execute()
@@ -183,8 +183,14 @@ class LauncherPro(Launcher):
183
183
  logger.info("pause! waiting for resume...")
184
184
  elif check_emtpy_times > 2:
185
185
  self.__DOING__ = {}
186
- if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
186
+ seed_count = self._db.zcard(self._todo_key)
187
+ logger.info(f"队列剩余种子数:{seed_count}")
188
+ if not seed_count:
189
+ logger.info("Done! pause set...")
187
190
  self._pause.set()
191
+ else:
192
+ self._pause.clear()
193
+
188
194
  else:
189
195
  logger.info(
190
196
  "check whether the task is complete, "
{cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.53
3
+ Version: 1.2.55
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.53 → cobweb-launcher-1.2.55}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.53",
8
+ version="1.2.55",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",