cobweb-launcher 1.2.9__py3-none-any.whl → 1.2.11__py3-none-any.whl

cobweb/launchers/launcher_pro.py

@@ -125,15 +125,18 @@ class LauncherPro(Launcher):
                 seeds.append(seed.to_string)
         if seeds:
             count = self._db.zrem(self._todo_key, *seeds)
-            self.statistics(self._statistics_done_key, count)
+            if count:
+                self.statistics(self._statistics_done_key, count)
             self._remove_doing_seeds(seeds)
         if s_seeds:
             count = self._db.done([self._todo_key, self._done_key], *s_seeds)
-            self.statistics(self._statistics_done_key, count)
+            if count:
+                self.statistics(self._statistics_done_key, count)
             self._remove_doing_seeds(s_seeds)
         if f_seeds:
             count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
-            self.statistics(self._statistics_fail_key, count)
+            if count:
+                self.statistics(self._statistics_fail_key, count)
             self._remove_doing_seeds(f_seeds)
 
         if status:

cobweb/pipelines/pipeline.py

@@ -34,26 +34,35 @@ class Pipeline(threading.Thread, ABC):
 
     def run(self):
         while not self._stop.is_set():
-            status = self._upload.length < self.upload_size
-            if status:
+            if not self._upload.length:
                 time.sleep(self.wait_seconds)
+                continue
+            if self._upload.length < self.upload_size:
+                time.sleep(self.wait_seconds)
+            status = True
             data_info, seeds = {}, []
-            for _ in range(self.upload_size):
-                item = self._upload.pop()
-                if not item:
-                    break
-                data = self.build(item)
-                seeds.append(item.seed)
-                data_info.setdefault(item.table, []).append(data)
-            for table, datas in data_info.items():
-                try:
-                    self.upload(table, datas)
-                    status = True
-                except Exception as e:
-                    logger.info(e)
-                    status = False
-                if status:
-                    self._done.push(seeds)
+            try:
+                for _ in range(self.upload_size):
+                    item = self._upload.pop()
+                    if not item:
+                        break
+                    seeds.append(item.seed)
+                    data = self.build(item)
+                    data_info.setdefault(item.table, []).append(data)
+                for table, datas in data_info.items():
+                    try:
+                        self.upload(table, datas)
+                    except Exception as e:
+                        logger.info(e)
+                        status = False
+            except Exception as e:
+                logger.info(e)
+                status = False
+            if not status:
+                for seed in seeds:
+                    seed.params.seed_status = "deal model: fail"
+            if seeds:
+                self._done.push(seeds)
 
         logger.info("upload pipeline close!")
 

{cobweb_launcher-1.2.9.dist-info → cobweb_launcher-1.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.9
+Version: 1.2.11
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.9.dist-info → cobweb_launcher-1.2.11.dist-info}/RECORD

@@ -20,18 +20,18 @@ cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BX
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
 cobweb/launchers/launcher.py,sha256=mBpq0CmxXXv-KdXQ2x7vwOkkAvaKECLkZLGhraafkQA,5953
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_pro.py,sha256=IpckN0Xl5NAPVjuq-pWbCx0RerDmYG9gZ182EDURixo,7599
+cobweb/launchers/launcher_pro.py,sha256=OmlC5o3BdcsTFwOVAKNJdnwdlTl-fUiXC2kViWEpcoU,7677
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
-cobweb/pipelines/pipeline.py,sha256=29O7CzASDv52NJl_1KWlKYxNd79HiaQAm0oO5K9PQzw,1650
+cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.9.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.9.dist-info/METADATA,sha256=gIYkBqrrJQ41kVqJN-9wxRAFh1lCKPvXd-a6IU4dj9A,6489
-cobweb_launcher-1.2.9.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.9.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.9.dist-info/RECORD,,
+cobweb_launcher-1.2.11.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.11.dist-info/METADATA,sha256=GSDZ3aWvattuW3Md-faK0cSBPyGDA8bJcVZRlSVp4ag,6490
+cobweb_launcher-1.2.11.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.11.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.11.dist-info/RECORD,,