cobweb-launcher 3.1.23__py3-none-any.whl → 3.1.24__py3-none-any.whl

cobweb/launchers/uploader.py

@@ -39,45 +39,28 @@ class Uploader(threading.Thread):
 
     @check_pause
     def upload_data(self):
-        if task_list := self.task_queue.get_task_by_status(
-                status=Status.UPLOAD, limit=self.upload_size
-        ):
-            try:
-                data_info, task_ids = dict(), set()
+        try:
+            data_info, task_ids = dict(), set()
+            if task_list := self.task_queue.get_task_by_status(
+                    status=Status.UPLOAD, limit=self.upload_size
+            ):
                 for task_item in task_list:
                     upload_data = self.pipeline.build(task_item.data)
                     data_info.setdefault(task_item.data.table, []).append(upload_data)
+                    task_ids.add(task_item.task_id)
 
                 for table, datas in data_info.items():
                     try:
                         self.pipeline.upload(table, datas)
                     except Exception as e:
                         logger.info(e)
-            except Exception as e:
-                logger.info(e)
-
-            if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
-                time.sleep(self.wait_seconds)
-
-        # data_info, seeds = {}, []
-        # try:
-        #     for _ in range(self.upload_size):
-        #         item = self.upload.pop()
-        #         if not item:
-        #             break
-        #         # seeds.append(item.seed)
-        #         data = self.pipeline.build(item)
-        #         data_info.setdefault(item.table, []).append(data)
-        #     for table, datas in data_info.items():
-        #         try:
-        #             self.pipeline.upload(table, datas)
-        #             # TODO: 上传打点
-        #         except Exception as e:
-        #             logger.info(e)
-        # except Exception as e:
-        #     logger.info(e)
-        # if self.upload.length < self.upload_size:
-        #     time.sleep(self.wait_seconds)
+
+            self.task_queue.remove(task_ids)
+        except Exception as e:
+            logger.info(e)
+
+        if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
+            time.sleep(self.wait_seconds)
 
     def run(self):
         self.callback_register(self.upload_data, tag="Uploader")

cobweb/schedulers/scheduler_with_redis.py

@@ -52,7 +52,8 @@ class RedisScheduler(Scheduler):
             time.sleep(self.scheduler_wait_seconds)
             return
 
-        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size:
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size\
+                or self.task_queue.length() > 5 * self.todo_queue_size:
             time.sleep(self.todo_queue_full_wait_seconds)
             return
 

{cobweb_launcher-3.1.23.dist-info → cobweb_launcher-3.1.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.23
+Version: 3.1.24
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.23.dist-info → cobweb_launcher-3.1.24.dist-info}/RECORD

@@ -19,22 +19,22 @@ cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BX
 cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
 cobweb/launchers/distributor.py,sha256=I5QBs2hFiyGGkqLLkMw9uzf4_oRW2JvahNW9yc866cc,6748
 cobweb/launchers/launcher.py,sha256=Shb6o6MAM38d32ybW2gY6qpGmhuiV7jo9TDh0f7rud8,5694
-cobweb/launchers/uploader.py,sha256=dDBv6Vfy1ciaTAJA3TebJV-2oM3OMrqTfzpNX8VGv-0,2766
+cobweb/launchers/uploader.py,sha256=QwJOmG7jq2T5sRzrT386zJ0YYNz-hAv0i6GOpoEaRdU,2075
 cobweb/pipelines/__init__.py,sha256=rtkaaCZ4u1XcxpkDLHztETQjEcLZ_6DXTHjdfcJlyxQ,97
 cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,357
 cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
 cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
 cobweb/schedulers/scheduler.py,sha256=Of-BjbBh679R6glc12Kc8iugeERCSusP7jolpCc1UMI,1740
-cobweb/schedulers/scheduler_with_redis.py,sha256=SUiEjYhzbbzc5kt_zpK8bXaEjIpwqC-JBk8ApHcVa18,7149
+cobweb/schedulers/scheduler_with_redis.py,sha256=dafg8jllatBiTz8J-qjTo80Xw80jkdHFW-wKoyaH7G0,7221
 cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
 cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
 cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.23.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.23.dist-info/METADATA,sha256=QwwqDS7cSVmiivRXj_Kgu2BZW527APBQ-Qe6frnjIls,5998
-cobweb_launcher-3.1.23.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.23.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.23.dist-info/RECORD,,
+cobweb_launcher-3.1.24.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.24.dist-info/METADATA,sha256=THQpa-rStFvhQNEMWq392PlRrPH_hxDDf1_-c5lFIGA,5998
+cobweb_launcher-3.1.24.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.24.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.24.dist-info/RECORD,,