cobweb-launcher 0.0.22__tar.gz → 0.0.24__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/PKG-INFO +1 -1
  2. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/redis_db.py +1 -1
  3. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/distributed/launcher.py +6 -2
  4. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb_launcher.egg-info/PKG-INFO +1 -1
  5. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/setup.py +1 -1
  6. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/LICENSE +0 -0
  7. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/README.md +0 -0
  8. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/__init__.py +0 -0
  9. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/bbb.py +0 -0
  10. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/__init__.py +0 -0
  11. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/oss_db.py +0 -0
  12. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/scheduler/__init__.py +0 -0
  13. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/scheduler/default.py +0 -0
  14. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/scheduler/textfile.py +0 -0
  15. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/storer/__init__.py +0 -0
  16. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/storer/console.py +0 -0
  17. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/storer/loghub.py +0 -0
  18. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/storer/redis.py +0 -0
  19. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/storer/textfile.py +0 -0
  20. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/decorators.py +0 -0
  21. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/distributed/__init__.py +0 -0
  22. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/distributed/models.py +0 -0
  23. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/interface.py +0 -0
  24. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/log.py +0 -0
  25. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/setting.py +0 -0
  26. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/single/__init__.py +0 -0
  27. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/single/models.py +0 -0
  28. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/single/nest.py +0 -0
  29. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/task.py +0 -0
  30. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/utils.py +0 -0
  31. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  32. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  33. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb_launcher.egg-info/requires.txt +0 -0
  34. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb_launcher.egg-info/top_level.txt +0 -0
  35. {cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/setup.cfg +0 -0
{cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 0.0.22
3
+ Version: 0.0.24
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/db/redis_db.py RENAMED
@@ -158,7 +158,7 @@ class RedisDB:
158
158
  score = -int(time.time()) + self.rs_time if heartbeat else "-inf"
159
159
 
160
160
  keys = self.client.keys(self.storer_key % "*")
161
- if len(keys) == storer_num:
161
+ if len(keys) >= storer_num:
162
162
  intersection_key = self.storer_key % "intersection"
163
163
  self.client.delete(intersection_key)
164
164
  self.client.zinterstore(intersection_key, keys)
{cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb/distributed/launcher.py RENAMED
@@ -72,7 +72,7 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
72
72
  )
73
73
  if (
74
74
  scheduler.stop and
75
- not redis_ready_seed_length and
75
+ # not redis_ready_seed_length and
76
76
  not memory_seed_queue_length and
77
77
  not running_spider_thread_num
78
78
  ):
@@ -88,7 +88,11 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
88
88
  storer_upload_queue_list.append(
89
89
  f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
90
90
  )
91
- if storer_queue_empty and not redis_spider_seed_length:
91
+ if (
92
+ storer_queue_empty and
93
+ not redis_ready_seed_length and
94
+ not redis_spider_seed_length
95
+ ):
92
96
  if MODEL:
93
97
  log.info("waiting for push seeds...")
94
98
  status = "waiting"
{cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/cobweb_launcher.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 0.0.22
3
+ Version: 0.0.24
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb-launcher-0.0.22 → cobweb-launcher-0.0.24}/setup.py RENAMED
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="0.0.22",
8
+ version="0.0.24",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",