cobweb-launcher 3.1.23__py3-none-any.whl → 3.1.25__py3-none-any.whl

cobweb/base/task_queue.py

@@ -10,8 +10,8 @@ class Status(Enum):
     PENDING = 0     # 待处理
     PROCESSING = 1  # 处理中
     FINISHED = 2    # 已完成
-    INSERT = 3      # 失败
-    UPLOAD = 4     # 过期
+    INSERT = 3      # 新增
+    UPLOAD = 4     # 上传
 
 
 @dataclass
@@ -138,9 +138,10 @@ class TaskQueue:
             if data:
                 task_item.data = data
 
-            for tid in task_item.children_ids:
-                if self._tasks[tid].status == Status.INSERT:
-                    del self._tasks[tid]
+            if task_item.status != Status.FINISHED:
+                for tid in task_item.children_ids:
+                    if self._tasks[tid].status == Status.INSERT:
+                        del self._tasks[tid]
 
             task_item.children_ids = []
             self._tasks[task_id] = task_item
@@ -176,4 +177,4 @@ class TaskQueue:
     #                 expired_ids.append(seed_id)
     #         for seed_id in expired_ids:
     #             self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
-    #         print(f"清理了 {len(expired_ids)} 个过期种子")
+    #         print(f"清理了 {len(expired_ids)} 个过期种子")

cobweb/constant.py

@@ -51,7 +51,7 @@ class LogTemplate:
     launcher_pro_polling = """
 ----------------------- start - 轮训日志: {task} -----------------
         内存队列 
-            种子数:  {doing_len}
+            消费中:  {doing_len}
             待消费:  {todo_len}
             已消费:  {done_len}
         redis队列 

cobweb/launchers/uploader.py

@@ -39,45 +39,28 @@ class Uploader(threading.Thread):
 
     @check_pause
     def upload_data(self):
-        if task_list := self.task_queue.get_task_by_status(
-                status=Status.UPLOAD, limit=self.upload_size
-        ):
-            try:
-                data_info, task_ids = dict(), set()
+        try:
+            data_info, task_ids = dict(), set()
+            if task_list := self.task_queue.get_task_by_status(
+                    status=Status.UPLOAD, limit=self.upload_size
+            ):
                 for task_item in task_list:
                     upload_data = self.pipeline.build(task_item.data)
                     data_info.setdefault(task_item.data.table, []).append(upload_data)
+                    task_ids.add(task_item.task_id)
 
                 for table, datas in data_info.items():
                     try:
                         self.pipeline.upload(table, datas)
                     except Exception as e:
                         logger.info(e)
-            except Exception as e:
-                logger.info(e)
-
-            if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
-                time.sleep(self.wait_seconds)
-
-        # data_info, seeds = {}, []
-        # try:
-        #     for _ in range(self.upload_size):
-        #         item = self.upload.pop()
-        #         if not item:
-        #             break
-        #         # seeds.append(item.seed)
-        #         data = self.pipeline.build(item)
-        #         data_info.setdefault(item.table, []).append(data)
-        #     for table, datas in data_info.items():
-        #         try:
-        #             self.pipeline.upload(table, datas)
-        #             # TODO: 上传打点
-        #         except Exception as e:
-        #             logger.info(e)
-        # except Exception as e:
-        #     logger.info(e)
-        # if self.upload.length < self.upload_size:
-        #     time.sleep(self.wait_seconds)
+
+            self.task_queue.remove(task_ids)
+        except Exception as e:
+            logger.info(e)
+
+        if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
+            time.sleep(self.wait_seconds)
 
     def run(self):
         self.callback_register(self.upload_data, tag="Uploader")

cobweb/schedulers/scheduler_with_redis.py

@@ -52,7 +52,8 @@ class RedisScheduler(Scheduler):
             time.sleep(self.scheduler_wait_seconds)
             return
 
-        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size:
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size\
+                or self.task_queue.length() > 5 * self.todo_queue_size:
             time.sleep(self.todo_queue_full_wait_seconds)
             return
 

{cobweb_launcher-3.1.23.dist-info → cobweb_launcher-3.1.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.23
+Version: 3.1.25
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -177,12 +177,16 @@ app.start()
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
 
 
+
+

{cobweb_launcher-3.1.23.dist-info → cobweb_launcher-3.1.25.dist-info}/RECORD

@@ -1,5 +1,5 @@
 cobweb/__init__.py,sha256=YdBi3uytEFRXan155xU1kKMpiUKUupO2RGeJyXmH0zk,129
-cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
+cobweb/constant.py,sha256=s6W4Fz3DhH-4RutoWnR2bylL8eU44rc-CeOsovj87I0,2839
 cobweb/setting.py,sha256=rHPQfc4a1xMTbkt3_KXBfUomhYcbTXogsz7ew-QsqHw,1670
 cobweb/base/__init__.py,sha256=NanSxJr0WsqjqCNOQAlxlkt-vQEsERHYBzacFC057oI,222
 cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
@@ -8,7 +8,7 @@ cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
 cobweb/base/request.py,sha256=MBYYjWpbRQRulPG0zPbK0DO3LKmScqQ4tBzFXekYkao,2652
 cobweb/base/response.py,sha256=g8e5H0hEiRfqseh3nD7t6a1rhIJYRMV7nI47kqNOd-U,446
 cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
-cobweb/base/task_queue.py,sha256=3ScPKnjlPEuuCzWyG9D2iHiND3L9lLM7fo1LNOkw8CY,6337
+cobweb/base/task_queue.py,sha256=2MqGpHGNmK5B-kqv7z420RWyihzB9zgDHJUiLsmtzOI,6402
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/crawler.py,sha256=ZZVZJ17RWuvzUFGLjqdvyVZpmuq-ynslJwXQzdm_UdQ,709
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
@@ -19,22 +19,22 @@ cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BX
 cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
 cobweb/launchers/distributor.py,sha256=I5QBs2hFiyGGkqLLkMw9uzf4_oRW2JvahNW9yc866cc,6748
 cobweb/launchers/launcher.py,sha256=Shb6o6MAM38d32ybW2gY6qpGmhuiV7jo9TDh0f7rud8,5694
-cobweb/launchers/uploader.py,sha256=dDBv6Vfy1ciaTAJA3TebJV-2oM3OMrqTfzpNX8VGv-0,2766
+cobweb/launchers/uploader.py,sha256=QwJOmG7jq2T5sRzrT386zJ0YYNz-hAv0i6GOpoEaRdU,2075
 cobweb/pipelines/__init__.py,sha256=rtkaaCZ4u1XcxpkDLHztETQjEcLZ_6DXTHjdfcJlyxQ,97
 cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,357
 cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
 cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
 cobweb/schedulers/scheduler.py,sha256=Of-BjbBh679R6glc12Kc8iugeERCSusP7jolpCc1UMI,1740
-cobweb/schedulers/scheduler_with_redis.py,sha256=SUiEjYhzbbzc5kt_zpK8bXaEjIpwqC-JBk8ApHcVa18,7149
+cobweb/schedulers/scheduler_with_redis.py,sha256=dafg8jllatBiTz8J-qjTo80Xw80jkdHFW-wKoyaH7G0,7221
 cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
 cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
 cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.23.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.23.dist-info/METADATA,sha256=QwwqDS7cSVmiivRXj_Kgu2BZW527APBQ-Qe6frnjIls,5998
-cobweb_launcher-3.1.23.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.23.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.23.dist-info/RECORD,,
+cobweb_launcher-3.1.25.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.25.dist-info/METADATA,sha256=QkLxxH-8qIdxnqsEB6W-dZjX4PtnoYqnCemFTXzgyNg,6051
+cobweb_launcher-3.1.25.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.25.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.25.dist-info/RECORD,,