cobweb-launcher 1.2.59__py3-none-any.whl → 1.2.61__py3-none-any.whl

cobweb/launchers/launcher_api.py

@@ -183,8 +183,13 @@ class LauncherApi(Launcher):
                         logger.info("pause! waiting for resume...")
                 elif check_emtpy_times > 2:
                     self.__DOING__ = {}
-                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                    seed_count = self._db.zcard(self._todo_key)
+                    logger.info(f"队列剩余种子数:{seed_count}")
+                    if not seed_count:
+                        logger.info("Done! pause set...")
                         self._pause.set()
+                    else:
+                        self._pause.clear()
                 else:
                     logger.info(
                         "check whether the task is complete, "
@@ -192,6 +197,8 @@ class LauncherApi(Launcher):
                     )
                     check_emtpy_times += 1
             else:
+                if self._pause.is_set():
+                    self._pause.clear()
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),

cobweb/launchers/launcher_pro.py

@@ -198,6 +198,8 @@ class LauncherPro(Launcher):
                     )
                     check_emtpy_times += 1
             else:
+                if self._pause.is_set():
+                    self._pause.clear()
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),

cobweb/utils/oss.py

@@ -1,4 +1,4 @@
-from typing import List
+
 from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models, PartIterator
@@ -23,6 +23,9 @@ class OssUtil:
         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
 
+        self.failed_count = 0
+        self._kw = kwargs
+
         self._auth = Auth(
             access_key_id=access_key or setting.OSS_ACCESS_KEY,
             access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -31,11 +34,27 @@ class OssUtil:
             auth=self._auth,
             endpoint=self.endpoint,
             bucket_name=self.bucket,
-            **kwargs
+            **self._kw
         )
 
+    def failed(self):
+        self.failed_count += 1
+        if self.failed_count >= 5:
+            self._client = Bucket(
+                auth=self._auth,
+                endpoint=self.endpoint,
+                bucket_name=self.bucket,
+                **self._kw
+            )
+
     def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
+        try:
+            result = self._client.object_exists(key)
+            self.failed_count = 0
+            return result
+        except Exception as e:
+            self.failed()
+            raise e
 
     def head(self, key: str) -> models.HeadObjectResult:
         return self._client.head_object(key)

{cobweb_launcher-1.2.59.dist-info → cobweb_launcher-1.2.61.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.59
+Version: 1.2.61
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.59.dist-info → cobweb_launcher-1.2.61.dist-info}/RECORD

@@ -24,8 +24,8 @@ cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BX
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=y1a72sANV-oJVpy_BI7BwKy7fY4AnpHqJK3t6czoCts,7896
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
-cobweb/launchers/launcher_pro.py,sha256=TOGwlPq75K1pURR0aI0PhVXpG5S0Lod3QfJZsjIF5Cc,8066
+cobweb/launchers/launcher_api.py,sha256=ydx_xrOjGMhGTDdyN3Oxy5EHUvvbvetNj3mCX1GlNsw,8205
+cobweb/launchers/launcher_pro.py,sha256=0Z3E1HA_D-yy_7hD7Q-HkYdy4v4AHYwyrR4i6HErIBo,8147
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -36,10 +36,10 @@ cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2
 cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
-cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
+cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.59.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.59.dist-info/METADATA,sha256=QL6FY17qhB83qCW51T4gRRAM-FiB_xw97SeArPYogtE,6510
-cobweb_launcher-1.2.59.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.59.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.59.dist-info/RECORD,,
+cobweb_launcher-1.2.61.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.61.dist-info/METADATA,sha256=vtAEmhDSAlOHdtb9XfsPaMGCtXk4z-zXao5IifOxBhY,6510
+cobweb_launcher-1.2.61.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.61.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.61.dist-info/RECORD,,