cobweb-launcher 3.1.24__py3-none-any.whl → 3.1.25__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. cobweb/base/task_queue.py +7 -6
  2. cobweb/constant.py +1 -1
  3. {cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/METADATA +9 -5
  4. {cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/RECORD +7 -7
  5. {cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/LICENSE +0 -0
  6. {cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/WHEEL +0 -0
  7. {cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/top_level.txt +0 -0
cobweb/base/task_queue.py CHANGED
@@ -10,8 +10,8 @@ class Status(Enum):
10
10
  PENDING = 0 # 待处理
11
11
  PROCESSING = 1 # 处理中
12
12
  FINISHED = 2 # 已完成
13
- INSERT = 3 # 失败
14
- UPLOAD = 4 # 过期
13
+ INSERT = 3 # 新增
14
+ UPLOAD = 4 # 上传
15
15
 
16
16
 
17
17
  @dataclass
@@ -138,9 +138,10 @@ class TaskQueue:
138
138
  if data:
139
139
  task_item.data = data
140
140
 
141
- for tid in task_item.children_ids:
142
- if self._tasks[tid].status == Status.INSERT:
143
- del self._tasks[tid]
141
+ if task_item.status != Status.FINISHED:
142
+ for tid in task_item.children_ids:
143
+ if self._tasks[tid].status == Status.INSERT:
144
+ del self._tasks[tid]
144
145
 
145
146
  task_item.children_ids = []
146
147
  self._tasks[task_id] = task_item
@@ -176,4 +177,4 @@ class TaskQueue:
176
177
  # expired_ids.append(seed_id)
177
178
  # for seed_id in expired_ids:
178
179
  # self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
179
- # print(f"清理了 {len(expired_ids)} 个过期种子")
180
+ # print(f"清理了 {len(expired_ids)} 个过期种子")
cobweb/constant.py CHANGED
@@ -51,7 +51,7 @@ class LogTemplate:
51
51
  launcher_pro_polling = """
52
52
  ----------------------- start - 轮训日志: {task} -----------------
53
53
  内存队列
54
- 种子数: {doing_len}
54
+ 消费中: {doing_len}
55
55
  待消费: {todo_len}
56
56
  已消费: {done_len}
57
57
  redis队列
{cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.24
3
+ Version: 3.1.25
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
@@ -177,12 +177,16 @@ app.start()
177
177
  >     upload_item["text"] = item.response.text
178
178
  >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
179
179
  ## todo
180
- - 队列优化完善，使用queue的机制wait()同步各模块执行？
181
- - 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
182
- - 去重过滤（布隆过滤器等）
183
- - 单机防丢失
180
+ - [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
181
+ - [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
182
+ - [ ] 去重过滤（布隆过滤器等）
183
+ - [ ] 请求检验
184
+ - [ ] 异常回调
185
+ - [ ] 失败回调
184
186
 
185
187
  > 未更新流程图！！！
186
188
  ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
187
189
 
188
190
 
191
+
192
+
{cobweb_launcher-3.1.24.dist-info → cobweb_launcher-3.1.25.dist-info}/RECORD RENAMED
@@ -1,5 +1,5 @@
1
1
  cobweb/__init__.py,sha256=YdBi3uytEFRXan155xU1kKMpiUKUupO2RGeJyXmH0zk,129
2
- cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
2
+ cobweb/constant.py,sha256=s6W4Fz3DhH-4RutoWnR2bylL8eU44rc-CeOsovj87I0,2839
3
3
  cobweb/setting.py,sha256=rHPQfc4a1xMTbkt3_KXBfUomhYcbTXogsz7ew-QsqHw,1670
4
4
  cobweb/base/__init__.py,sha256=NanSxJr0WsqjqCNOQAlxlkt-vQEsERHYBzacFC057oI,222
5
5
  cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
@@ -8,7 +8,7 @@ cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
8
8
  cobweb/base/request.py,sha256=MBYYjWpbRQRulPG0zPbK0DO3LKmScqQ4tBzFXekYkao,2652
9
9
  cobweb/base/response.py,sha256=g8e5H0hEiRfqseh3nD7t6a1rhIJYRMV7nI47kqNOd-U,446
10
10
  cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
11
- cobweb/base/task_queue.py,sha256=3ScPKnjlPEuuCzWyG9D2iHiND3L9lLM7fo1LNOkw8CY,6337
11
+ cobweb/base/task_queue.py,sha256=2MqGpHGNmK5B-kqv7z420RWyihzB9zgDHJUiLsmtzOI,6402
12
12
  cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
13
13
  cobweb/crawlers/crawler.py,sha256=ZZVZJ17RWuvzUFGLjqdvyVZpmuq-ynslJwXQzdm_UdQ,709
14
14
  cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
@@ -33,8 +33,8 @@ cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,11
33
33
  cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
34
34
  cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
35
35
  cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
36
- cobweb_launcher-3.1.24.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
37
- cobweb_launcher-3.1.24.dist-info/METADATA,sha256=THQpa-rStFvhQNEMWq392PlRrPH_hxDDf1_-c5lFIGA,5998
38
- cobweb_launcher-3.1.24.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
39
- cobweb_launcher-3.1.24.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
40
- cobweb_launcher-3.1.24.dist-info/RECORD,,
36
+ cobweb_launcher-3.1.25.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
37
+ cobweb_launcher-3.1.25.dist-info/METADATA,sha256=QkLxxH-8qIdxnqsEB6W-dZjX4PtnoYqnCemFTXzgyNg,6051
38
+ cobweb_launcher-3.1.25.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
39
+ cobweb_launcher-3.1.25.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
40
+ cobweb_launcher-3.1.25.dist-info/RECORD,,