cobweb-launcher 3.1.16__py3-none-any.whl → 3.1.17__py3-none-any.whl

cobweb/schedulers/scheduler.py

@@ -65,15 +65,16 @@ class Scheduler(ABC, threading.Thread):
         with self.lock:
             return self.__WORKING_ITEMS__.copy()
 
-    def remove_working_items(self, items: list[str] = None):
+    def remove_working_items(self, items: list[str] = None) -> int:
         if not items:
-            return
-        for _ in range(5):
-            with self.lock:
-                for item in items:
-                    self.__WORKING_ITEMS__.pop(item, None)
-                return True
-            time.sleep(1)
+            return 0
+        with self.lock:
+            deleted_count = 0
+            for item in items:
+                if item in self.__WORKING_ITEMS__:
+                    del self.__WORKING_ITEMS__[item]
+                    deleted_count += 1
+            return deleted_count
 
     def get_working_items_count(self) -> int:
         with self.lock:

cobweb/schedulers/scheduler_with_redis.py

@@ -97,12 +97,12 @@ class RedisScheduler(Scheduler):
         """
         刷新doing种子过期时间，防止reset重新消费
         """
+        time.sleep(20)
         if item_info := self.get_working_items():
             refresh_time = int(time.time())
             seed_info = {k: -refresh_time - v / 1000 for k, v in item_info.items()}
             self.db.zadd(self.todo_key, seed_info, xx=True)
             self.set_working_items(seed_info)
-        time.sleep(20)
 
     @check_pause
     def delete(self):
@@ -114,7 +114,7 @@ class RedisScheduler(Scheduler):
 
         if self.remove_working_items(items):
             self.db.zrem(self.todo_key, *items)
-        else:
+        elif seeds:
             self.done.push(seeds)
 
         if self.done.length < self.done_queue_max_size:

{cobweb_launcher-3.1.16.dist-info → cobweb_launcher-3.1.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.16
+Version: 3.1.17
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.16.dist-info → cobweb_launcher-3.1.17.dist-info}/RECORD

@@ -24,16 +24,16 @@ cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,
 cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
 cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
-cobweb/schedulers/scheduler.py,sha256=v1GIAwdxBjSLNugwVsfrh_zn_V887x02BxhVr5O8Xyo,2870
-cobweb/schedulers/scheduler_with_redis.py,sha256=VW6umnyfFeExd7c_v5e--QZEQ-bWSIlHIxz7ySVkLGU,6352
+cobweb/schedulers/scheduler.py,sha256=lBN0ZgfZADq3EH_lPdLOxhrgmZ2GCxrZqzdQQT_bdR8,2937
+cobweb/schedulers/scheduler_with_redis.py,sha256=d8lG4QZEfS9m4V9QnOqia-I3hZ_2gwM4InOX7BOR3-M,6358
 cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
 cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
 cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.16.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.16.dist-info/METADATA,sha256=CxdyN7vkyoMCAhJpBuvEwPBBq4v7jg4Nw8p3mGnVEQs,5998
-cobweb_launcher-3.1.16.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.16.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.16.dist-info/RECORD,,
+cobweb_launcher-3.1.17.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.17.dist-info/METADATA,sha256=vz70vRxivUpOV_beizaVdV3EJQucKbJFdXz6-ntliDg,5998
+cobweb_launcher-3.1.17.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.17.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.17.dist-info/RECORD,,