cobweb-launcher 1.2.62__py3-none-any.whl → 1.2.64__py3-none-any.whl

cobweb/launchers/launcher_api.py

@@ -49,8 +49,11 @@ class LauncherApi(Launcher):
         if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
                 not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
             expire_time = self._db.ttl(self._speed_control_key)
-            logger.info(f"Too fast! Please wait {expire_time} seconds...")
-            time.sleep(expire_time / 2)
+            if isinstance(expire_time, int) and expire_time <= -1:
+                self._db.delete(self._speed_control_key)
+            elif isinstance(expire_time, int):
+                logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                time.sleep(expire_time / 2)
             return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
         return seed
@@ -115,11 +118,11 @@ class LauncherApi(Launcher):
                 break
             seed, new_seed = seed_tuple
             new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed.to_string)
+            del_seeds.add(seed)
         if new_seeds:
             self._db.zadd(self._todo_key, new_seeds, nx=True)
         if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
+            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

cobweb/launchers/launcher_pro.py

@@ -47,7 +47,7 @@ class LauncherPro(Launcher):
             spider_speed = self._db.get(self._speed_control_key)
             if int(spider_speed or 0) > self._spider_max_count:
                 expire_time = self._db.ttl(self._speed_control_key)
-                if expire_time <= -1:
+                if isinstance(expire_time, int) and expire_time <= -1:
                     self._db.delete(self._speed_control_key)
                 elif isinstance(expire_time, int):
                     logger.info(f"Too fast! Please wait {expire_time} seconds...")
@@ -117,11 +117,11 @@ class LauncherPro(Launcher):
                 break
             seed, new_seed = seed_tuple
             new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed.to_string)
+            del_seeds.add(seed)
         if new_seeds:
             self._db.zadd(self._todo_key, new_seeds, nx=True)
         if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
+            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
         if status:
             time.sleep(self._new_queue_wait_seconds)
 

{cobweb_launcher-1.2.62.dist-info → cobweb_launcher-1.2.64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.62
+Version: 1.2.64
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.62.dist-info → cobweb_launcher-1.2.64.dist-info}/RECORD

@@ -24,8 +24,8 @@ cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BX
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=aR1cnQymjQQUJe-W_dRoQTd9t3Qav9QkLDzeWW7i0xE,7921
 cobweb/launchers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
-cobweb/launchers/launcher_api.py,sha256=RlWuGSb_izevqJBVlyi_D8mlUXMKReTSOHTHHMEn8FE,8453
-cobweb/launchers/launcher_pro.py,sha256=Jn5T89eWZafr4wGTllqrXB5IDyZ8G9hiuk7dW3Xl1VA,8395
+cobweb/launchers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
+cobweb/launchers/launcher_pro.py,sha256=xM2z3nCTb6gehuD1Ggyvli2knGAPjGczbjOhKWswVpo,8424
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -38,8 +38,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.62.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.62.dist-info/METADATA,sha256=fgQXgBFjxcRQdCLc4m_2qmM11V8QHleCbROVG2dPIFo,6510
-cobweb_launcher-1.2.62.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.62.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.62.dist-info/RECORD,,
+cobweb_launcher-1.2.64.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.64.dist-info/METADATA,sha256=NikLvEH211SWsw5jTUhaLsXROPJWvc2bkEUq3n4wlaM,6510
+cobweb_launcher-1.2.64.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.64.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.64.dist-info/RECORD,,