cobweb-launcher 3.1.23__tar.gz → 3.1.25__tar.gz

{cobweb-launcher-3.1.23/cobweb_launcher.egg-info → cobweb-launcher-3.1.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.23
+Version: 3.1.25
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -174,12 +174,16 @@ app.start()
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
 
 
+
+

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25}/README.md

@@ -159,10 +159,14 @@ app.start()
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25}/cobweb/base/task_queue.py

@@ -10,8 +10,8 @@ class Status(Enum):
     PENDING = 0     # 待处理
     PROCESSING = 1  # 处理中
     FINISHED = 2    # 已完成
-    INSERT = 3      # 失败
-    UPLOAD = 4     # 过期
+    INSERT = 3      # 新增
+    UPLOAD = 4     # 上传
 
 
 @dataclass
@@ -138,9 +138,10 @@ class TaskQueue:
             if data:
                 task_item.data = data
 
-            for tid in task_item.children_ids:
-                if self._tasks[tid].status == Status.INSERT:
-                    del self._tasks[tid]
+            if task_item.status != Status.FINISHED:
+                for tid in task_item.children_ids:
+                    if self._tasks[tid].status == Status.INSERT:
+                        del self._tasks[tid]
 
             task_item.children_ids = []
             self._tasks[task_id] = task_item
@@ -176,4 +177,4 @@ class TaskQueue:
     #                 expired_ids.append(seed_id)
     #         for seed_id in expired_ids:
     #             self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
-    #         print(f"清理了 {len(expired_ids)} 个过期种子")
+    #         print(f"清理了 {len(expired_ids)} 个过期种子")

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25}/cobweb/constant.py

@@ -51,7 +51,7 @@ class LogTemplate:
     launcher_pro_polling = """
 ----------------------- start - 轮训日志: {task} -----------------
         内存队列 
-            种子数:  {doing_len}
+            消费中:  {doing_len}
             待消费:  {todo_len}
             已消费:  {done_len}
         redis队列 

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25}/cobweb/launchers/uploader.py

@@ -39,45 +39,28 @@ class Uploader(threading.Thread):
 
     @check_pause
     def upload_data(self):
-        if task_list := self.task_queue.get_task_by_status(
-                status=Status.UPLOAD, limit=self.upload_size
-        ):
-            try:
-                data_info, task_ids = dict(), set()
+        try:
+            data_info, task_ids = dict(), set()
+            if task_list := self.task_queue.get_task_by_status(
+                    status=Status.UPLOAD, limit=self.upload_size
+            ):
                 for task_item in task_list:
                     upload_data = self.pipeline.build(task_item.data)
                     data_info.setdefault(task_item.data.table, []).append(upload_data)
+                    task_ids.add(task_item.task_id)
 
                 for table, datas in data_info.items():
                     try:
                         self.pipeline.upload(table, datas)
                     except Exception as e:
                         logger.info(e)
-            except Exception as e:
-                logger.info(e)
-
-            if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
-                time.sleep(self.wait_seconds)
-
-        # data_info, seeds = {}, []
-        # try:
-        #     for _ in range(self.upload_size):
-        #         item = self.upload.pop()
-        #         if not item:
-        #             break
-        #         # seeds.append(item.seed)
-        #         data = self.pipeline.build(item)
-        #         data_info.setdefault(item.table, []).append(data)
-        #     for table, datas in data_info.items():
-        #         try:
-        #             self.pipeline.upload(table, datas)
-        #             # TODO: 上传打点
-        #         except Exception as e:
-        #             logger.info(e)
-        # except Exception as e:
-        #     logger.info(e)
-        # if self.upload.length < self.upload_size:
-        #     time.sleep(self.wait_seconds)
+
+            self.task_queue.remove(task_ids)
+        except Exception as e:
+            logger.info(e)
+
+        if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
+            time.sleep(self.wait_seconds)
 
     def run(self):
         self.callback_register(self.upload_data, tag="Uploader")

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25}/cobweb/schedulers/scheduler_with_redis.py

@@ -52,7 +52,8 @@ class RedisScheduler(Scheduler):
             time.sleep(self.scheduler_wait_seconds)
             return
 
-        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size:
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size\
+                or self.task_queue.length() > 5 * self.todo_queue_size:
             time.sleep(self.todo_queue_full_wait_seconds)
             return
 

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.23
+Version: 3.1.25
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -174,12 +174,16 @@ app.start()
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
 
 
+
+

{cobweb-launcher-3.1.23 → cobweb-launcher-3.1.25}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="3.1.23",
+    version="3.1.25",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",