cobweb-launcher 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

cobweb/db/redis_db.py

@@ -158,7 +158,7 @@ class RedisDB:
                 score = -int(time.time()) + self.rs_time if heartbeat else "-inf"
 
                 keys = self.client.keys(self.storer_key % "*")
-                if len(keys) == storer_num:
+                if len(keys) >= storer_num:
                     intersection_key = self.storer_key % "intersection"
                     self.client.delete(intersection_key)
                     self.client.zinterstore(intersection_key, keys)

cobweb/distributed/launcher.py

@@ -72,7 +72,7 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
             )
         if (
                 scheduler.stop and
-                not redis_ready_seed_length and
+                # not redis_ready_seed_length and
                 not memory_seed_queue_length and
                 not running_spider_thread_num
         ):
@@ -88,7 +88,11 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
                 storer_upload_queue_list.append(
                     f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
                 )
-            if storer_queue_empty and not redis_spider_seed_length:
+            if (
+                    storer_queue_empty and
+                    not redis_ready_seed_length and
+                    not redis_spider_seed_length
+            ):
                 if MODEL:
                     log.info("waiting for push seeds...")
                     status = "waiting"

{cobweb_launcher-0.0.22.dist-info → cobweb_launcher-0.0.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.22
+Version: 0.0.24
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.0.22.dist-info → cobweb_launcher-0.0.24.dist-info}/RECORD

@@ -8,7 +8,7 @@ cobweb/task.py,sha256=77F5EaopSVlSX2TANv1lhuPHFI8ER8Jh4tSGrwDWAc0,1405
 cobweb/utils.py,sha256=3iQRn2s4fY-5ClrVo46RO9OhziRAyOn5hGJOQGjMMwQ,2176
 cobweb/db/__init__.py,sha256=4m9lqmxZCRbaih3Z3rl_BT0GugMd0dkOIgu_P9aeC84,63
 cobweb/db/oss_db.py,sha256=lFGNuH3tdIMsohVXQ_fTZPyBfS2oxYNmFNuQ-ZBQgm0,4221
-cobweb/db/redis_db.py,sha256=ldUCKryDzZuFyb_2tee0fhbjqX3Y8OHfyu7iQG8DM7s,8171
+cobweb/db/redis_db.py,sha256=kxTNRpnl0zPuIyPIWRVoG6udfz6_j_kiS7L0BP85K9A,8171
 cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/db/scheduler/default.py,sha256=OxmFX7OvMEhKEq-NF7A8I9cA4V4qWw5vayS-yIbng0A,114
 cobweb/db/scheduler/textfile.py,sha256=atRDeNT-e5toNvyGsCXAxL1FJi77uSYktdCzH_hXGo8,821
@@ -18,13 +18,13 @@ cobweb/db/storer/loghub.py,sha256=4VqZacXWhidzINHXQu2_-E0HOBRCcc86f6LkKfnXD5I,17
 cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
 cobweb/db/storer/textfile.py,sha256=3mDHMvF6Sh5fn3IHzWQxyTUd45V-zUoH8vY3EoRlMx0,415
 cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=6Xrsj5y87uKmJfNi3n_7cfPRoys25-60Oc97M62ei5Q,10252
+cobweb/distributed/launcher.py,sha256=e3obQJ1KQPjL2NZmD3-unpI7N7CLUwFO7brPjHwIESc,10362
 cobweb/distributed/models.py,sha256=5rRy16BHBq4EwWZxHcH_hJ6GJ0jEZ9hechSlZxzvdFQ,4513
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
 cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
-cobweb_launcher-0.0.22.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.0.22.dist-info/METADATA,sha256=vQxzP4fmsGXH-IPzVZBDdV6fiejqhXIMm3RCg5BvKgY,1226
-cobweb_launcher-0.0.22.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cobweb_launcher-0.0.22.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.0.22.dist-info/RECORD,,
+cobweb_launcher-0.0.24.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.0.24.dist-info/METADATA,sha256=gnQRFdNwlYUDERWmZhjRkmwob2Uu012sQJn__uLQU1g,1226
+cobweb_launcher-0.0.24.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cobweb_launcher-0.0.24.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.0.24.dist-info/RECORD,,