cobweb-launcher 1.2.59__tar.gz → 1.2.60__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (46) hide show
  1. {cobweb-launcher-1.2.59/cobweb_launcher.egg-info → cobweb-launcher-1.2.60}/PKG-INFO +1 -1
  2. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/launcher_api.py +8 -1
  3. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/launcher_pro.py +2 -0
  4. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60/cobweb_launcher.egg-info}/PKG-INFO +1 -1
  5. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/setup.py +1 -1
  6. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/LICENSE +0 -0
  7. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/README.md +0 -0
  8. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/__init__.py +0 -0
  10. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/common_queue.py +0 -0
  11. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/decorators.py +0 -0
  12. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/item.py +0 -0
  13. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/log.py +0 -0
  14. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/request.py +0 -0
  15. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/response.py +0 -0
  16. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/base/seed.py +0 -0
  17. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/constant.py +0 -0
  18. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/crawlers/__init__.py +0 -0
  19. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/crawlers/base_crawler.py +0 -0
  20. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/crawlers/crawler.py +0 -0
  21. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/crawlers/file_crawler.py +0 -0
  22. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/db/__init__.py +0 -0
  23. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/db/api_db.py +0 -0
  24. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/db/redis_db.py +0 -0
  25. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/db/redis_db_new.py +0 -0
  26. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/exceptions/__init__.py +0 -0
  27. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/exceptions/oss_db_exception.py +0 -0
  28. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/__init__.py +0 -0
  29. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/launcher.py +0 -0
  30. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/launcher_air.py +0 -0
  31. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/pipelines/__init__.py +0 -0
  32. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/pipelines/pipeline.py +0 -0
  33. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/pipelines/pipeline_console.py +0 -0
  34. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/pipelines/pipeline_loghub.py +0 -0
  35. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/utils/dotting.py +0 -0
  39. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/utils/oss.py +0 -0
  40. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/utils/tools.py +0 -0
  41. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  42. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  43. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb_launcher.egg-info/requires.txt +0 -0
  44. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb_launcher.egg-info/top_level.txt +0 -0
  45. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/setup.cfg +0 -0
  46. {cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/test/test.py +0 -0
{cobweb-launcher-1.2.59/cobweb_launcher.egg-info → cobweb-launcher-1.2.60}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.59
3
+ Version: 1.2.60
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/launcher_api.py RENAMED
@@ -183,8 +183,13 @@ class LauncherApi(Launcher):
183
183
  logger.info("pause! waiting for resume...")
184
184
  elif check_emtpy_times > 2:
185
185
  self.__DOING__ = {}
186
- if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
186
+ seed_count = self._db.zcard(self._todo_key)
187
+ logger.info(f"队列剩余种子数:{seed_count}")
188
+ if not seed_count:
189
+ logger.info("Done! pause set...")
187
190
  self._pause.set()
191
+ else:
192
+ self._pause.clear()
188
193
  else:
189
194
  logger.info(
190
195
  "check whether the task is complete, "
@@ -192,6 +197,8 @@ class LauncherApi(Launcher):
192
197
  )
193
198
  check_emtpy_times += 1
194
199
  else:
200
+ if self._pause.is_set():
201
+ self._pause.clear()
195
202
  logger.info(LogTemplate.launcher_pro_polling.format(
196
203
  task=self.task,
197
204
  doing_len=len(self.__DOING__.keys()),
{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/cobweb/launchers/launcher_pro.py RENAMED
@@ -198,6 +198,8 @@ class LauncherPro(Launcher):
198
198
  )
199
199
  check_emtpy_times += 1
200
200
  else:
201
+ if self._pause.is_set():
202
+ self._pause.clear()
201
203
  logger.info(LogTemplate.launcher_pro_polling.format(
202
204
  task=self.task,
203
205
  doing_len=len(self.__DOING__.keys()),
{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60/cobweb_launcher.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.59
3
+ Version: 1.2.60
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.60}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="1.2.59",
8
+ version="1.2.60",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",