cobweb-launcher 1.2.59__tar.gz → 1.2.61__tar.gz

{cobweb-launcher-1.2.59/cobweb_launcher.egg-info → cobweb-launcher-1.2.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.59
+Version: 1.2.61
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.61}/cobweb/launchers/launcher_api.py

@@ -183,8 +183,13 @@ class LauncherApi(Launcher):
                         logger.info("pause! waiting for resume...")
                 elif check_emtpy_times > 2:
                     self.__DOING__ = {}
-                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                    seed_count = self._db.zcard(self._todo_key)
+                    logger.info(f"队列剩余种子数:{seed_count}")
+                    if not seed_count:
+                        logger.info("Done! pause set...")
                         self._pause.set()
+                    else:
+                        self._pause.clear()
                 else:
                     logger.info(
                         "check whether the task is complete, "
@@ -192,6 +197,8 @@ class LauncherApi(Launcher):
                     )
                     check_emtpy_times += 1
             else:
+                if self._pause.is_set():
+                    self._pause.clear()
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),

{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.61}/cobweb/launchers/launcher_pro.py

@@ -198,6 +198,8 @@ class LauncherPro(Launcher):
                     )
                     check_emtpy_times += 1
             else:
+                if self._pause.is_set():
+                    self._pause.clear()
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=len(self.__DOING__.keys()),

{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.61}/cobweb/utils/oss.py

@@ -1,4 +1,4 @@
-from typing import List
+
 from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models, PartIterator
@@ -23,6 +23,9 @@ class OssUtil:
         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
 
+        self.failed_count = 0
+        self._kw = kwargs
+
         self._auth = Auth(
             access_key_id=access_key or setting.OSS_ACCESS_KEY,
             access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -31,11 +34,27 @@ class OssUtil:
             auth=self._auth,
             endpoint=self.endpoint,
             bucket_name=self.bucket,
-            **kwargs
+            **self._kw
         )
 
+    def failed(self):
+        self.failed_count += 1
+        if self.failed_count >= 5:
+            self._client = Bucket(
+                auth=self._auth,
+                endpoint=self.endpoint,
+                bucket_name=self.bucket,
+                **self._kw
+            )
+
     def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
+        try:
+            result = self._client.object_exists(key)
+            self.failed_count = 0
+            return result
+        except Exception as e:
+            self.failed()
+            raise e
 
     def head(self, key: str) -> models.HeadObjectResult:
         return self._client.head_object(key)

{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.61/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.59
+Version: 1.2.61
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.59 → cobweb-launcher-1.2.61}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.59",
+    version="1.2.61",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",