PyPI - cobweb-launcher - Versions diffs - 3.1.24__tar.gz → 3.1.25__tar.gz - Mend

cobweb-launcher 3.1.24tar.gz → 3.1.25tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (45) hide show

{cobweb-launcher-3.1.24/cobweb_launcher.egg-info → cobweb-launcher-3.1.25}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.24
+Version: 3.1.25
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -174,12 +174,16 @@ app.start()
 > &nbsp;&nbsp;&nbsp;&nbsp;upload_item["text"] = item.response.text
 > &nbsp;&nbsp;&nbsp;&nbsp;yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

{cobweb-launcher-3.1.24 → cobweb-launcher-3.1.25}/README.md RENAMED Viewed

@@ -159,10 +159,14 @@ app.start()
 > &nbsp;&nbsp;&nbsp;&nbsp;upload_item["text"] = item.response.text
 > &nbsp;&nbsp;&nbsp;&nbsp;yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

{cobweb-launcher-3.1.24 → cobweb-launcher-3.1.25}/cobweb/base/task_queue.py RENAMED Viewed

@@ -10,8 +10,8 @@ class Status(Enum):
     PENDING = 0     # 待处理
     PROCESSING = 1  # 处理中
     FINISHED = 2    # 已完成
-    INSERT = 3      # 失败
-    UPLOAD = 4     # 过期
+    INSERT = 3      # 新增
+    UPLOAD = 4     # 上传
 @dataclass
@@ -138,9 +138,10 @@ class TaskQueue:
             if data:
                 task_item.data = data
-            for tid in task_item.children_ids:
-                if self._tasks[tid].status == Status.INSERT:
-                    del self._tasks[tid]
+            if task_item.status != Status.FINISHED:
+                for tid in task_item.children_ids:
+                    if self._tasks[tid].status == Status.INSERT:
+                        del self._tasks[tid]
             task_item.children_ids = []
             self._tasks[task_id] = task_item
@@ -176,4 +177,4 @@ class TaskQueue:
     #                 expired_ids.append(seed_id)
     #         for seed_id in expired_ids:
     #             self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
-    #         print(f"清理了 {len(expired_ids)} 个过期种子")
+    #         print(f"清理了 {len(expired_ids)} 个过期种子")

{cobweb-launcher-3.1.24 → cobweb-launcher-3.1.25}/cobweb/constant.py RENAMED Viewed

@@ -51,7 +51,7 @@ class LogTemplate:
     launcher_pro_polling = """
 ----------------------- start - 轮训日志: {task} -----------------
         内存队列
-            种子数:  {doing_len}
+            消费中:  {doing_len}
             待消费:  {todo_len}
             已消费:  {done_len}
         redis队列

{cobweb-launcher-3.1.24 → cobweb-launcher-3.1.25/cobweb_launcher.egg-info}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.24
+Version: 3.1.25
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -174,12 +174,16 @@ app.start()
 > &nbsp;&nbsp;&nbsp;&nbsp;upload_item["text"] = item.response.text
 > &nbsp;&nbsp;&nbsp;&nbsp;yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 ## todo
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

{cobweb-launcher-3.1.24 → cobweb-launcher-3.1.25}/setup.py RENAMED Viewed

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 setup(
     name="cobweb-launcher",
-    version="3.1.24",
+    version="3.1.25",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",