cobweb-launcher 3.1.16__tar.gz → 3.1.18__tar.gz

{cobweb-launcher-3.1.16 → cobweb-launcher-3.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.16
+Version: 3.1.18
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.16 → cobweb-launcher-3.1.18}/cobweb/schedulers/scheduler.py

@@ -57,7 +57,7 @@ class Scheduler(ABC, threading.Thread):
 
     def set_working_items(self, item_info: dict = None):
         if not item_info:
-            return
+            return False
         with self.lock:
             self.__WORKING_ITEMS__.update(item_info)
 
@@ -65,15 +65,16 @@ class Scheduler(ABC, threading.Thread):
         with self.lock:
             return self.__WORKING_ITEMS__.copy()
 
-    def remove_working_items(self, items: list[str] = None):
+    def remove_working_items(self, items: list[str] = None) -> int:
         if not items:
-            return
-        for _ in range(5):
-            with self.lock:
-                for item in items:
-                    self.__WORKING_ITEMS__.pop(item, None)
-                return True
-            time.sleep(1)
+            return 0
+        with self.lock:
+            deleted_count = 0
+            for item in items:
+                if item in self.__WORKING_ITEMS__:
+                    del self.__WORKING_ITEMS__[item]
+                    deleted_count += 1
+            return deleted_count
 
     def get_working_items_count(self) -> int:
         with self.lock:

{cobweb-launcher-3.1.16 → cobweb-launcher-3.1.18}/cobweb/schedulers/scheduler_with_redis.py

@@ -101,7 +101,7 @@ class RedisScheduler(Scheduler):
             refresh_time = int(time.time())
             seed_info = {k: -refresh_time - v / 1000 for k, v in item_info.items()}
             self.db.zadd(self.todo_key, seed_info, xx=True)
-            self.set_working_items(seed_info)
+            # self.set_working_items(seed_info)
         time.sleep(20)
 
     @check_pause
@@ -114,7 +114,7 @@ class RedisScheduler(Scheduler):
 
         if self.remove_working_items(items):
             self.db.zrem(self.todo_key, *items)
-        else:
+        elif seeds:
             self.done.push(seeds)
 
         if self.done.length < self.done_queue_max_size:

{cobweb-launcher-3.1.16 → cobweb-launcher-3.1.18}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.16
+Version: 3.1.18
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.16 → cobweb-launcher-3.1.18}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="3.1.16",
+    version="3.1.18",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",