cobweb-launcher 1.2.9__tar.gz → 1.2.11__tar.gz

{cobweb-launcher-1.2.9/cobweb_launcher.egg-info → cobweb-launcher-1.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.9
+Version: 1.2.11
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.9 → cobweb-launcher-1.2.11}/cobweb/launchers/launcher_pro.py

@@ -125,15 +125,18 @@ class LauncherPro(Launcher):
                 seeds.append(seed.to_string)
         if seeds:
             count = self._db.zrem(self._todo_key, *seeds)
-            self.statistics(self._statistics_done_key, count)
+            if count:
+                self.statistics(self._statistics_done_key, count)
             self._remove_doing_seeds(seeds)
         if s_seeds:
             count = self._db.done([self._todo_key, self._done_key], *s_seeds)
-            self.statistics(self._statistics_done_key, count)
+            if count:
+                self.statistics(self._statistics_done_key, count)
             self._remove_doing_seeds(s_seeds)
         if f_seeds:
             count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
-            self.statistics(self._statistics_fail_key, count)
+            if count:
+                self.statistics(self._statistics_fail_key, count)
             self._remove_doing_seeds(f_seeds)
 
         if status:

cobweb-launcher-1.2.11/cobweb/pipelines/pipeline.py

@@ -0,0 +1,69 @@
+import time
+import threading
+
+from abc import ABC, abstractmethod
+from cobweb.base import BaseItem, Queue, logger
+
+
+class Pipeline(threading.Thread, ABC):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            upload: Queue, done: Queue,
+            upload_size: int,
+            wait_seconds: int
+    ):
+        super().__init__()
+        self._stop = stop
+        self._pause = pause
+        self._upload = upload
+        self._done = done
+
+        self.upload_size = upload_size
+        self.wait_seconds = wait_seconds
+
+    @abstractmethod
+    def build(self, item: BaseItem) -> dict:
+        pass
+
+    @abstractmethod
+    def upload(self, table: str, data: list) -> bool:
+        pass
+
+    def run(self):
+        while not self._stop.is_set():
+            if not self._upload.length:
+                time.sleep(self.wait_seconds)
+                continue
+            if self._upload.length < self.upload_size:
+                time.sleep(self.wait_seconds)
+            status = True
+            data_info, seeds = {}, []
+            try:
+                for _ in range(self.upload_size):
+                    item = self._upload.pop()
+                    if not item:
+                        break
+                    seeds.append(item.seed)
+                    data = self.build(item)
+                    data_info.setdefault(item.table, []).append(data)
+                for table, datas in data_info.items():
+                    try:
+                        self.upload(table, datas)
+                    except Exception as e:
+                        logger.info(e)
+                        status = False
+            except Exception as e:
+                logger.info(e)
+                status = False
+            if not status:
+                for seed in seeds:
+                    seed.params.seed_status = "deal model: fail"
+            if seeds:
+                self._done.push(seeds)
+
+        logger.info("upload pipeline close!")
+
+

{cobweb-launcher-1.2.9 → cobweb-launcher-1.2.11/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.9
+Version: 1.2.11
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.9 → cobweb-launcher-1.2.11}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.9",
+    version="1.2.11",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-1.2.9/cobweb/pipelines/pipeline.py

@@ -1,60 +0,0 @@
-import time
-import threading
-
-from abc import ABC, abstractmethod
-from cobweb.base import BaseItem, Queue, logger
-
-
-class Pipeline(threading.Thread, ABC):
-
-    def __init__(
-            self,
-            stop: threading.Event,
-            pause: threading.Event,
-            upload: Queue, done: Queue,
-            upload_size: int,
-            wait_seconds: int
-    ):
-        super().__init__()
-        self._stop = stop
-        self._pause = pause
-        self._upload = upload
-        self._done = done
-
-        self.upload_size = upload_size
-        self.wait_seconds = wait_seconds
-
-    @abstractmethod
-    def build(self, item: BaseItem) -> dict:
-        pass
-
-    @abstractmethod
-    def upload(self, table: str, data: list) -> bool:
-        pass
-
-    def run(self):
-        while not self._stop.is_set():
-            status = self._upload.length < self.upload_size
-            if status:
-                time.sleep(self.wait_seconds)
-            data_info, seeds = {}, []
-            for _ in range(self.upload_size):
-                item = self._upload.pop()
-                if not item:
-                    break
-                data = self.build(item)
-                seeds.append(item.seed)
-                data_info.setdefault(item.table, []).append(data)
-            for table, datas in data_info.items():
-                try:
-                    self.upload(table, datas)
-                    status = True
-                except Exception as e:
-                    logger.info(e)
-                    status = False
-                if status:
-                    self._done.push(seeds)
-
-        logger.info("upload pipeline close!")
-
-