cobweb-launcher 3.1.23__tar.gz → 3.1.24__tar.gz

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.23
+Version: 3.1.24
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.24}/cobweb/launchers/uploader.py

@@ -39,45 +39,28 @@ class Uploader(threading.Thread):
 
     @check_pause
     def upload_data(self):
-        if task_list := self.task_queue.get_task_by_status(
-                status=Status.UPLOAD, limit=self.upload_size
-        ):
-            try:
-                data_info, task_ids = dict(), set()
+        try:
+            data_info, task_ids = dict(), set()
+            if task_list := self.task_queue.get_task_by_status(
+                    status=Status.UPLOAD, limit=self.upload_size
+            ):
                 for task_item in task_list:
                     upload_data = self.pipeline.build(task_item.data)
                     data_info.setdefault(task_item.data.table, []).append(upload_data)
+                    task_ids.add(task_item.task_id)
 
                 for table, datas in data_info.items():
                     try:
                         self.pipeline.upload(table, datas)
                     except Exception as e:
                         logger.info(e)
-            except Exception as e:
-                logger.info(e)
-
-            if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
-                time.sleep(self.wait_seconds)
-
-        # data_info, seeds = {}, []
-        # try:
-        #     for _ in range(self.upload_size):
-        #         item = self.upload.pop()
-        #         if not item:
-        #             break
-        #         # seeds.append(item.seed)
-        #         data = self.pipeline.build(item)
-        #         data_info.setdefault(item.table, []).append(data)
-        #     for table, datas in data_info.items():
-        #         try:
-        #             self.pipeline.upload(table, datas)
-        #             # TODO: 上传打点
-        #         except Exception as e:
-        #             logger.info(e)
-        # except Exception as e:
-        #     logger.info(e)
-        # if self.upload.length < self.upload_size:
-        #     time.sleep(self.wait_seconds)
+
+            self.task_queue.remove(task_ids)
+        except Exception as e:
+            logger.info(e)
+
+        if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
+            time.sleep(self.wait_seconds)
 
     def run(self):
         self.callback_register(self.upload_data, tag="Uploader")

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.24}/cobweb/schedulers/scheduler_with_redis.py

@@ -52,7 +52,8 @@ class RedisScheduler(Scheduler):
             time.sleep(self.scheduler_wait_seconds)
             return
 
-        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size:
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size\
+                or self.task_queue.length() > 5 * self.todo_queue_size:
             time.sleep(self.todo_queue_full_wait_seconds)
             return
 

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.24}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.23
+Version: 3.1.24
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.24}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="3.1.23",
+    version="3.1.24",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",