tasklite-engine 1.2.1__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tasklite-engine might be problematic. Click here for more details.
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/PKG-INFO +15 -7
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/README.md +14 -6
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/__init__.py +1 -1
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/backend/base.py +72 -25
- tasklite_engine-1.3.0/tasklite/backend/memory.py +343 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/backend/sqlite_backend.py +92 -22
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/channel.py +271 -202
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/completion.py +31 -2
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/config.py +56 -13
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/console.py +113 -3
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/dispatch.py +83 -43
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/governor.py +57 -46
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/policy.py +24 -21
- tasklite_engine-1.3.0/tasklite/engine/recovery.py +391 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/resource.py +37 -1
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/runtime.py +106 -81
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/scheduler.py +31 -7
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/session.py +5 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/store.py +27 -71
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/types.py +22 -1
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/exceptions.py +7 -2
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/hooks.py +3 -3
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/models/context.py +20 -5
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/models/job.py +92 -113
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/models/state.py +54 -24
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/pipeline.py +86 -11
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/taxonomy.py +211 -62
- tasklite_engine-1.3.0/tasklite/testing.py +96 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/utils/injective.py +67 -12
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/utils/ipc.py +180 -29
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/utils/jsonutil.py +28 -7
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/wrappers/discovery.py +47 -27
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/wrappers/http.py +264 -30
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite_engine.egg-info/PKG-INFO +15 -7
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite_engine.egg-info/SOURCES.txt +1 -0
- tasklite_engine-1.2.1/tasklite/backend/memory.py +0 -224
- tasklite_engine-1.2.1/tasklite/engine/recovery.py +0 -250
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/LICENSE +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/pyproject.toml +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/setup.cfg +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/backend/__init__.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/contrib/__init__.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/__init__.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/inflight.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/engine/pacing.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/models/__init__.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/py.typed +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/utils/__init__.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/utils/lockfile.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite/wrappers/__init__.py +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite_engine.egg-info/dependency_links.txt +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite_engine.egg-info/requires.txt +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tasklite_engine.egg-info/top_level.txt +0 -0
- {tasklite_engine-1.2.1 → tasklite_engine-1.3.0}/tests/test_packaging.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasklite-engine
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
@@ -37,7 +37,7 @@ Dynamic: license-file
|
|
|
37
37
|
[](LICENSE)
|
|
38
38
|
[]()
|
|
39
39
|
[-orange.svg)]()
|
|
40
|
-
[]()
|
|
41
41
|
|
|
42
42
|
---
|
|
43
43
|
|
|
@@ -152,11 +152,13 @@ flowchart TD
|
|
|
152
152
|
### 安装
|
|
153
153
|
|
|
154
154
|
要求 **Python ≥ 3.9**,零外部依赖:
|
|
155
|
-
|
|
155
|
+
|
|
156
156
|
```bash
|
|
157
157
|
pip install tasklite-engine
|
|
158
158
|
```
|
|
159
159
|
|
|
160
|
+
> **三方命名对照**:仓库名 `tasklite`,PyPI 包名 `tasklite-engine`(PyPI 的 `tasklite` 名已被无关项目占用),Python 导入名 `tasklite`——安装 `tasklite-engine`、`import tasklite`。
|
|
161
|
+
|
|
160
162
|
### 最小示例
|
|
161
163
|
|
|
162
164
|
```python
|
|
@@ -269,6 +271,7 @@ def fetch_handler(job, ctx):
|
|
|
269
271
|
|
|
270
272
|
```python
|
|
271
273
|
import json
|
|
274
|
+
from pathlib import Path
|
|
272
275
|
from tasklite import TaskLite, Job, RateLimitResource
|
|
273
276
|
|
|
274
277
|
def agent_worker(job: Job, ctx):
|
|
@@ -276,9 +279,10 @@ def agent_worker(job: Job, ctx):
|
|
|
276
279
|
prompt = job.payload["prompt"]
|
|
277
280
|
result = call_llm(prompt)
|
|
278
281
|
|
|
279
|
-
#
|
|
280
|
-
|
|
281
|
-
|
|
282
|
+
# declare_output 返回解析后的绝对路径字符串(str),用 Path 包装后写文件,
|
|
283
|
+
# 保证写入位置与框架校验/清理位置一致
|
|
284
|
+
out_path = Path(ctx.declare_output(f"./results/{job.job_id}.json", cleanup_on_fail=True))
|
|
285
|
+
out_path.write_text(json.dumps(result, ensure_ascii=False))
|
|
282
286
|
return True, {"tokens": result.get("usage", 0)}
|
|
283
287
|
|
|
284
288
|
pipeline = TaskLite(name="agent_batch", state_dir="./agent_state", max_workers=8)
|
|
@@ -355,12 +359,16 @@ tasklite/
|
|
|
355
359
|
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryMachine)
|
|
356
360
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
357
361
|
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
|
|
362
|
+
│ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
|
|
363
|
+
│ ├── governor.py # 死锁归因仲裁与依赖宽限 (DeadlockGovernor)
|
|
364
|
+
│ ├── inflight.py # 在途作业租约追踪 (InFlightTracker)
|
|
358
365
|
│ └── resource.py # 令牌桶限速与并发容量资源
|
|
359
366
|
├── backend/ # SQLite WAL 强一致事务持久化后端
|
|
360
367
|
├── models/ # Job / TaskContext / PipelineState 数据模型
|
|
361
368
|
├── taxonomy.py # 错误分类法与校验分类深模块 (ErrorTaxonomy)
|
|
369
|
+
├── testing.py # fake_ctx:handler 单测的官方 TaskContext 构造器
|
|
362
370
|
├── wrappers/ # discovery.py(增量扫描)/ http.py(网络守卫与快照)
|
|
363
|
-
└── utils/ # jsonutil
|
|
371
|
+
└── utils/ # injective(单射 %XX 转义)/ ipc(IPC 信号与降级落盘)/ jsonutil(禁NaN)/ lockfile
|
|
364
372
|
```
|
|
365
373
|
|
|
366
374
|
---
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
[](LICENSE)
|
|
9
9
|
[]()
|
|
10
10
|
[-orange.svg)]()
|
|
11
|
-
[]()
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
|
@@ -123,11 +123,13 @@ flowchart TD
|
|
|
123
123
|
### 安装
|
|
124
124
|
|
|
125
125
|
要求 **Python ≥ 3.9**,零外部依赖:
|
|
126
|
-
|
|
126
|
+
|
|
127
127
|
```bash
|
|
128
128
|
pip install tasklite-engine
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
+
> **三方命名对照**:仓库名 `tasklite`,PyPI 包名 `tasklite-engine`(PyPI 的 `tasklite` 名已被无关项目占用),Python 导入名 `tasklite`——安装 `tasklite-engine`、`import tasklite`。
|
|
132
|
+
|
|
131
133
|
### 最小示例
|
|
132
134
|
|
|
133
135
|
```python
|
|
@@ -240,6 +242,7 @@ def fetch_handler(job, ctx):
|
|
|
240
242
|
|
|
241
243
|
```python
|
|
242
244
|
import json
|
|
245
|
+
from pathlib import Path
|
|
243
246
|
from tasklite import TaskLite, Job, RateLimitResource
|
|
244
247
|
|
|
245
248
|
def agent_worker(job: Job, ctx):
|
|
@@ -247,9 +250,10 @@ def agent_worker(job: Job, ctx):
|
|
|
247
250
|
prompt = job.payload["prompt"]
|
|
248
251
|
result = call_llm(prompt)
|
|
249
252
|
|
|
250
|
-
#
|
|
251
|
-
|
|
252
|
-
|
|
253
|
+
# declare_output 返回解析后的绝对路径字符串(str),用 Path 包装后写文件,
|
|
254
|
+
# 保证写入位置与框架校验/清理位置一致
|
|
255
|
+
out_path = Path(ctx.declare_output(f"./results/{job.job_id}.json", cleanup_on_fail=True))
|
|
256
|
+
out_path.write_text(json.dumps(result, ensure_ascii=False))
|
|
253
257
|
return True, {"tokens": result.get("usage", 0)}
|
|
254
258
|
|
|
255
259
|
pipeline = TaskLite(name="agent_batch", state_dir="./agent_state", max_workers=8)
|
|
@@ -326,12 +330,16 @@ tasklite/
|
|
|
326
330
|
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryMachine)
|
|
327
331
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
328
332
|
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
|
|
333
|
+
│ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
|
|
334
|
+
│ ├── governor.py # 死锁归因仲裁与依赖宽限 (DeadlockGovernor)
|
|
335
|
+
│ ├── inflight.py # 在途作业租约追踪 (InFlightTracker)
|
|
329
336
|
│ └── resource.py # 令牌桶限速与并发容量资源
|
|
330
337
|
├── backend/ # SQLite WAL 强一致事务持久化后端
|
|
331
338
|
├── models/ # Job / TaskContext / PipelineState 数据模型
|
|
332
339
|
├── taxonomy.py # 错误分类法与校验分类深模块 (ErrorTaxonomy)
|
|
340
|
+
├── testing.py # fake_ctx:handler 单测的官方 TaskContext 构造器
|
|
333
341
|
├── wrappers/ # discovery.py(增量扫描)/ http.py(网络守卫与快照)
|
|
334
|
-
└── utils/ # jsonutil
|
|
342
|
+
└── utils/ # injective(单射 %XX 转义)/ ipc(IPC 信号与降级落盘)/ jsonutil(禁NaN)/ lockfile
|
|
335
343
|
```
|
|
336
344
|
|
|
337
345
|
---
|
|
@@ -2,30 +2,32 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
from abc import ABC, abstractmethod
|
|
5
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
6
|
-
|
|
7
|
-
from ..taxonomy import
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
5
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
6
|
+
|
|
7
|
+
from ..taxonomy import classify_error_type
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_queue_replacement(jobs: Any) -> None:
|
|
11
|
+
"""整表替换集统一形状校验(save_queue / replace_queue_atomic 共用)。
|
|
12
|
+
|
|
13
|
+
契约:替换集必须是 job dict 的 list/tuple(与 load_queue 行形态一致);
|
|
14
|
+
None(compute 漏写 return 的笔误形态)或非序列、元素非 dict 均属契约
|
|
15
|
+
违约,fail-loud 抛 TypeError。
|
|
16
|
+
不变式:双腿必须在任何写变(DELETE / 赋值)之前调用本校验——None 若
|
|
17
|
+
混过校验,SQLite 腿「DELETE 全表成功 + INSERT 全跳过」会静默清空整条
|
|
18
|
+
队列且事务正常提交,与 Memory 腿抛 TypeError 且队列原状形成行为分叉。
|
|
19
|
+
"""
|
|
20
|
+
if not isinstance(jobs, (list, tuple)):
|
|
21
|
+
raise TypeError(
|
|
22
|
+
f"queue replacement must be a list of job dicts, "
|
|
23
|
+
f"got {type(jobs).__name__}"
|
|
24
|
+
)
|
|
25
|
+
for i, job in enumerate(jobs):
|
|
26
|
+
if not isinstance(job, dict):
|
|
27
|
+
raise TypeError(
|
|
28
|
+
f"queue replacement item #{i} must be a job dict, "
|
|
29
|
+
f"got {type(job).__name__}"
|
|
30
|
+
)
|
|
29
31
|
|
|
30
32
|
|
|
31
33
|
|
|
@@ -49,7 +51,38 @@ class AbstractStateBackend(ABC):
|
|
|
49
51
|
def load_queue(self) -> List[Dict[str, Any]]: ...
|
|
50
52
|
|
|
51
53
|
@abstractmethod
|
|
52
|
-
def save_queue(self, jobs: List[Dict[str, Any]]) -> None:
|
|
54
|
+
def save_queue(self, jobs: List[Dict[str, Any]]) -> None:
|
|
55
|
+
"""整表重写队列(测试装配 / replace_queue_atomic 的事务内步骤)。
|
|
56
|
+
|
|
57
|
+
红线:这是「无读基准的整表覆盖」——跨进程并发的 enqueue/commit
|
|
58
|
+
若提交在本方法执行前的任意时刻,其已落盘行会被无条件抹除。进程内
|
|
59
|
+
任何「先读磁盘真相再决定写什么」的合并保存(崩溃恢复路径)严禁
|
|
60
|
+
直接调用本方法,必须走 ``replace_queue_atomic`` 把读-改-写收敛进
|
|
61
|
+
单个写事务。
|
|
62
|
+
|
|
63
|
+
替换集经 ``validate_queue_replacement`` 在写变前校验:None / 非序列 /
|
|
64
|
+
元素非 dict 抛 TypeError,队列保持调用前状态(双腿一致)。
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
@abstractmethod
|
|
68
|
+
def replace_queue_atomic(
|
|
69
|
+
self,
|
|
70
|
+
compute: Callable[[List[Dict[str, Any]]], List[Dict[str, Any]]],
|
|
71
|
+
) -> None:
|
|
72
|
+
"""读-改-写收敛的整表替换:磁盘真相读取与写回在单个写事务内完成。
|
|
73
|
+
|
|
74
|
+
``compute`` 在写锁内接收按序排列的磁盘队列快照,返回替换后的完整
|
|
75
|
+
队列。不变式:并发方(跨进程 enqueue / delta commit)要么先于本
|
|
76
|
+
事务提交(其行进入磁盘真相、参与合并,绝不丢失),要么排队等本
|
|
77
|
+
事务提交后再落盘——「读真相 → 计算 → 写回」之间不存在无锁窗口,
|
|
78
|
+
陈旧快照永远无法覆盖窗口期他进程已应答的新写入。
|
|
79
|
+
|
|
80
|
+
``compute`` 必须是纯计算(锁内执行,不得调用任何后端写方法,否则
|
|
81
|
+
跨连接写请求会以写锁互斥死等)。单事务原子:``compute`` 抛异常或
|
|
82
|
+
写失败时整体回滚,磁盘保持调用前状态,异常向外传播。``compute``
|
|
83
|
+
返回的替换集经 ``validate_queue_replacement`` 在写变前校验(None /
|
|
84
|
+
非序列 / 元素非 dict 抛 TypeError,双腿一致,磁盘保持原状)。
|
|
85
|
+
"""
|
|
53
86
|
|
|
54
87
|
@abstractmethod
|
|
55
88
|
def commit_job_success(
|
|
@@ -138,6 +171,17 @@ class AbstractStateBackend(ABC):
|
|
|
138
171
|
返回 True 成功;False 时 on-disk 队列不变,调用方走崩溃契约。
|
|
139
172
|
"""
|
|
140
173
|
|
|
174
|
+
@abstractmethod
|
|
175
|
+
def delete_queue_uids(self, uids: List[str]) -> int:
|
|
176
|
+
"""按 uid 定向批量删除队列行(加载期 repair 的差量落盘唯一出口)。
|
|
177
|
+
|
|
178
|
+
与 save_queue 的全表重写相对:只 DELETE 指定 uid 的行,其余行原样
|
|
179
|
+
保留。不变式:删除集在调用前确定,不在删除集内的行(含并发进程
|
|
180
|
+
刚入队的新行)无论与本事务先后提交都必然存活——陈旧加载快照永远
|
|
181
|
+
不会覆盖他进程的新写入。单事务原子;失败抛异常(不吞),磁盘保持
|
|
182
|
+
调用前状态(残留行由下次加载重新判定,天然幂等)。返回实际删除行数。
|
|
183
|
+
"""
|
|
184
|
+
|
|
141
185
|
@abstractmethod
|
|
142
186
|
def get_meta(self, key: str) -> Optional[str]:
|
|
143
187
|
"""读取一条框架级元数据(如 fencing 的 last_run_id)。无则返回 None。"""
|
|
@@ -189,6 +233,9 @@ class AbstractStateBackend(ABC):
|
|
|
189
233
|
用途:媒体/数据资产项目的存档迁移(硬链接 + wall 种子),
|
|
190
234
|
不必裸 SQL INSERT 框架内部表。返回实际写入行数。
|
|
191
235
|
幂等:已存在的 uid 被覆盖(meta 重置为空)。
|
|
236
|
+
不变式(wall/failed 全局互斥):已在 failed 的 uid 拒绝种子——
|
|
237
|
+
冲突整体拒绝(零写入、抛 ValueError),不静默清除 DLQ 记录;
|
|
238
|
+
持久层与管理 API(OpsConsole.seed_wall)两层同契约。
|
|
192
239
|
"""
|
|
193
240
|
|
|
194
241
|
@abstractmethod
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""纯内存状态后端适配器(InMemoryStateBackend)。
|
|
2
|
+
|
|
3
|
+
实现 AbstractStateBackend 完整契约,提供零文件系统 IO 的纯内存状态存储。
|
|
4
|
+
适用于瞬态管线、单元测试、沙盒执行与 CI 矩阵测试。
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import copy
|
|
9
|
+
import logging
|
|
10
|
+
import threading
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
13
|
+
|
|
14
|
+
from .base import (
|
|
15
|
+
AbstractStateBackend,
|
|
16
|
+
classify_error_type,
|
|
17
|
+
validate_queue_replacement,
|
|
18
|
+
)
|
|
19
|
+
from ..models.state import uid_from_job_dict
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger("tasklite")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class InMemoryStateBackend(AbstractStateBackend):
|
|
25
|
+
"""纯内存状态后端。提供快照隔离的 delta 事务语义,与 SQLite 后端行为完全对齐。"""
|
|
26
|
+
|
|
27
|
+
def __init__(self) -> None:
|
|
28
|
+
self._lock = threading.RLock()
|
|
29
|
+
self._wall: Dict[str, Dict[str, Any]] = {}
|
|
30
|
+
self._failed: Dict[str, Dict[str, Any]] = {}
|
|
31
|
+
self._cursors: Dict[str, str] = {}
|
|
32
|
+
self._queue: List[Dict[str, Any]] = []
|
|
33
|
+
self._meta: Dict[str, str] = {}
|
|
34
|
+
|
|
35
|
+
def load_wall(self) -> Dict[str, Dict[str, Any]]:
|
|
36
|
+
with self._lock:
|
|
37
|
+
return copy.deepcopy(self._wall)
|
|
38
|
+
|
|
39
|
+
def load_failed(self) -> Dict[str, Dict[str, Any]]:
|
|
40
|
+
with self._lock:
|
|
41
|
+
return copy.deepcopy(self._failed)
|
|
42
|
+
|
|
43
|
+
def load_cursors(self) -> Dict[str, str]:
|
|
44
|
+
with self._lock:
|
|
45
|
+
return dict(self._cursors)
|
|
46
|
+
|
|
47
|
+
def load_queue(self) -> List[Dict[str, Any]]:
|
|
48
|
+
with self._lock:
|
|
49
|
+
return copy.deepcopy(self._queue)
|
|
50
|
+
|
|
51
|
+
def _dedup_copy(self, jobs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
52
|
+
"""整表替换行的单一出口(save_queue 与 replace_queue_atomic 共用)。
|
|
53
|
+
|
|
54
|
+
替换集先经 ``validate_queue_replacement`` 校验(与 SQLite 腿同一出口、
|
|
55
|
+
同一时机——赋值之前 fail-loud,`_queue` 保持调用前状态)。
|
|
56
|
+
保存兜底去重:重复 uid 保留首条 + 告警,与 SQLite 腿
|
|
57
|
+
_rewrite_queue_rows 同语义;条目一律 deepcopy,杜绝外部可变别名
|
|
58
|
+
穿透快照隔离。
|
|
59
|
+
"""
|
|
60
|
+
validate_queue_replacement(jobs)
|
|
61
|
+
seen = set()
|
|
62
|
+
clean = []
|
|
63
|
+
for j in jobs:
|
|
64
|
+
u = uid_from_job_dict(j)
|
|
65
|
+
if u in seen:
|
|
66
|
+
logger.warning(f"save_queue: duplicate uid {u} dropped (kept first).")
|
|
67
|
+
continue
|
|
68
|
+
seen.add(u)
|
|
69
|
+
clean.append(copy.deepcopy(j))
|
|
70
|
+
return clean
|
|
71
|
+
|
|
72
|
+
def save_queue(self, jobs: List[Dict[str, Any]]) -> None:
|
|
73
|
+
with self._lock:
|
|
74
|
+
self._queue = self._dedup_copy(jobs)
|
|
75
|
+
|
|
76
|
+
def replace_queue_atomic(
|
|
77
|
+
self,
|
|
78
|
+
compute: Callable[[List[Dict[str, Any]]], List[Dict[str, Any]]],
|
|
79
|
+
) -> None:
|
|
80
|
+
with self._lock:
|
|
81
|
+
# 锁内读真相 → 纯计算 → 替换:compute 抛异常时 _queue 赋值
|
|
82
|
+
# 未发生,队列保持调用前状态(对齐 SQLite 腿事务回滚)。
|
|
83
|
+
self._queue = self._dedup_copy(compute(copy.deepcopy(self._queue)))
|
|
84
|
+
|
|
85
|
+
def _build_dlq_meta(self, meta: Optional[dict], prev: Any) -> Dict[str, Any]:
|
|
86
|
+
"""计算 DLQ 行终值(纯函数,不变更任何状态):_attempt 计数 + error_type + failed_at。
|
|
87
|
+
|
|
88
|
+
不变式:``_attempt`` 是写入事件计数而非逻辑失败次数;既有计数损坏
|
|
89
|
+
(非 dict 记录或非 int 计数)时静默重置为 1 并照常提交,绝不因脏计数
|
|
90
|
+
抛 TypeError——与 SQLite 后端「损坏行重置计数、写入成功」行为对齐。
|
|
91
|
+
``merged`` 已显式携带 ``_attempt`` 时保留调用方值(计数由最先 DLQ
|
|
92
|
+
该 uid 的路径权威给定)。
|
|
93
|
+
"""
|
|
94
|
+
merged = copy.deepcopy(meta or {})
|
|
95
|
+
if "error_type" not in merged:
|
|
96
|
+
merged["error_type"] = classify_error_type(merged)
|
|
97
|
+
if "failed_at" not in merged:
|
|
98
|
+
merged["failed_at"] = datetime.now(timezone.utc).isoformat()
|
|
99
|
+
if not isinstance(prev, dict):
|
|
100
|
+
merged["_attempt"] = 1
|
|
101
|
+
return merged
|
|
102
|
+
prev_attempt = prev.get("_attempt")
|
|
103
|
+
if isinstance(prev_attempt, int):
|
|
104
|
+
merged["_attempt"] = prev_attempt + 1
|
|
105
|
+
elif "_attempt" not in merged:
|
|
106
|
+
merged["_attempt"] = 1
|
|
107
|
+
return merged
|
|
108
|
+
|
|
109
|
+
def _write_dlq_entry(self, uid: str, meta: Optional[dict]) -> None:
|
|
110
|
+
"""DLQ 写入单一出口:终值经 _build_dlq_meta 计算后落变。"""
|
|
111
|
+
self._failed[uid] = self._build_dlq_meta(meta, self._failed.get(uid))
|
|
112
|
+
|
|
113
|
+
def commit_job_success(
|
|
114
|
+
self,
|
|
115
|
+
uid: str,
|
|
116
|
+
result_meta: dict,
|
|
117
|
+
*,
|
|
118
|
+
spawned_jobs: List[Dict[str, Any]] = (),
|
|
119
|
+
cursor_updates: Optional[Dict[str, str]] = None,
|
|
120
|
+
) -> bool:
|
|
121
|
+
with self._lock:
|
|
122
|
+
try:
|
|
123
|
+
# 校验先行:全部 deepcopy 与冲突判定在任何变更前完成。
|
|
124
|
+
# 不变式:返回 False / 抛异常 ⇒ wall/queue/failed/cursors 与
|
|
125
|
+
# 调用前完全一致(对齐 SQLite 事务回滚;store 的 3-strike
|
|
126
|
+
# 崩溃契约以「后端未变」为前提做重启重建)。
|
|
127
|
+
wall_meta = copy.deepcopy(result_meta or {})
|
|
128
|
+
spawned_copy = [copy.deepcopy(j) for j in spawned_jobs]
|
|
129
|
+
remaining = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
130
|
+
if spawned_copy:
|
|
131
|
+
remaining_uids = {uid_from_job_dict(j) for j in remaining}
|
|
132
|
+
seen: set = set()
|
|
133
|
+
for sj in spawned_copy:
|
|
134
|
+
suid = uid_from_job_dict(sj)
|
|
135
|
+
# spawned uid 撞上删除 popped 后的队列既有条目,或
|
|
136
|
+
# 批内自相重复,均判定内存/磁盘漂移 → False 走崩溃
|
|
137
|
+
# 契约,绝不产出重复队列条目(对齐 SQLite
|
|
138
|
+
# INSERT OR IGNORE + rowcount 漂移检测)。
|
|
139
|
+
if suid in remaining_uids or suid in seen:
|
|
140
|
+
logger.critical(
|
|
141
|
+
f"commit_job_success for {uid}: spawned job {suid} "
|
|
142
|
+
f"conflicts with queue (drift). Returning False."
|
|
143
|
+
)
|
|
144
|
+
return False
|
|
145
|
+
seen.add(suid)
|
|
146
|
+
cursor_sets: Dict[str, str] = {}
|
|
147
|
+
cursor_dels: set = set()
|
|
148
|
+
if cursor_updates:
|
|
149
|
+
for k, v in cursor_updates.items():
|
|
150
|
+
if v is None:
|
|
151
|
+
cursor_dels.add(k)
|
|
152
|
+
else:
|
|
153
|
+
cursor_sets[k] = str(v)
|
|
154
|
+
# 校验全部通过,统一落变(落变段不再调用任何可失败操作)
|
|
155
|
+
self._wall[uid] = wall_meta
|
|
156
|
+
self._queue = spawned_copy + remaining # 队首插入 spawned 并保序
|
|
157
|
+
# 成功 commit 清理 failed 同名残行:与「job 最终状态唯一」
|
|
158
|
+
# 语义一致,防 wall∩failed 并存污染 _attempt 计数。
|
|
159
|
+
self._failed.pop(uid, None)
|
|
160
|
+
self._cursors.update(cursor_sets)
|
|
161
|
+
for k in cursor_dels:
|
|
162
|
+
self._cursors.pop(k, None)
|
|
163
|
+
return True
|
|
164
|
+
except Exception as e:
|
|
165
|
+
logger.critical(f"Failed to commit job success for {uid}: {e}")
|
|
166
|
+
return False
|
|
167
|
+
|
|
168
|
+
def commit_job_failure(self, uid: str, result_meta: dict) -> bool:
|
|
169
|
+
with self._lock:
|
|
170
|
+
try:
|
|
171
|
+
# 校验先行:DLQ 终值计算与队列 uid 提取全部在变更前完成,
|
|
172
|
+
# 任一步失败 ⇒ failed/queue/wall 整体不变(对齐 SQLite 回滚)。
|
|
173
|
+
new_meta = self._build_dlq_meta(result_meta, self._failed.get(uid))
|
|
174
|
+
remaining = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
175
|
+
self._failed[uid] = new_meta
|
|
176
|
+
self._queue = remaining
|
|
177
|
+
# 同一事务语义内清理 wall 旧记录:rerun 任务重跑失败时旧成功
|
|
178
|
+
# 记录作废(最终状态唯一),防 wall∩failed 并存。
|
|
179
|
+
self._wall.pop(uid, None)
|
|
180
|
+
return True
|
|
181
|
+
except Exception as e:
|
|
182
|
+
logger.critical(f"Failed to commit job failure for {uid}: {e}")
|
|
183
|
+
return False
|
|
184
|
+
|
|
185
|
+
def commit_retry(
|
|
186
|
+
self,
|
|
187
|
+
popped_uid: str,
|
|
188
|
+
requeued_job: Dict[str, Any],
|
|
189
|
+
*,
|
|
190
|
+
front: bool = False,
|
|
191
|
+
) -> bool:
|
|
192
|
+
with self._lock:
|
|
193
|
+
try:
|
|
194
|
+
requeued_copy = copy.deepcopy(requeued_job)
|
|
195
|
+
ruid = uid_from_job_dict(requeued_copy)
|
|
196
|
+
remaining = [j for j in self._queue if uid_from_job_dict(j) != popped_uid]
|
|
197
|
+
# 对齐 SQLite 普通 INSERT 冲突回滚:requeued uid 撞上删除
|
|
198
|
+
# popped 后的既有条目 → False 走崩溃契约,绝不产出重复条目;
|
|
199
|
+
# uid == popped_uid 属同 job 重插,安全放行。
|
|
200
|
+
if ruid != popped_uid and any(uid_from_job_dict(j) == ruid for j in remaining):
|
|
201
|
+
logger.critical(
|
|
202
|
+
f"commit_retry for {popped_uid}: requeued uid {ruid} "
|
|
203
|
+
f"conflicts with existing queue entry. Returning False."
|
|
204
|
+
)
|
|
205
|
+
return False
|
|
206
|
+
if front:
|
|
207
|
+
self._queue = [requeued_copy] + remaining
|
|
208
|
+
else:
|
|
209
|
+
self._queue = remaining + [requeued_copy]
|
|
210
|
+
return True
|
|
211
|
+
except Exception as e:
|
|
212
|
+
logger.critical(f"Failed to commit retry for {popped_uid}: {e}")
|
|
213
|
+
return False
|
|
214
|
+
|
|
215
|
+
def commit_bulk_failure(self, uids_metas: List[Tuple[str, dict]]) -> bool:
|
|
216
|
+
with self._lock:
|
|
217
|
+
try:
|
|
218
|
+
# 校验先行:全部 DLQ 行计算在任何变更前完成——任一行失败则
|
|
219
|
+
# queue/wall/failed 整体不变(对齐 SQLite 事务回滚,杜绝
|
|
220
|
+
# 「队列已整体删除、DLQ 未落」的半成品失败态)。
|
|
221
|
+
new_entries: List[Tuple[str, Dict[str, Any]]] = []
|
|
222
|
+
overlay: Dict[str, Any] = {}
|
|
223
|
+
for uid, meta in uids_metas:
|
|
224
|
+
# 批内同 uid 多次出现时模拟 SQLite 同事务顺序写:
|
|
225
|
+
# 后一行读取前一行结果,_attempt 连续递增。
|
|
226
|
+
prev = overlay.get(uid, self._failed.get(uid))
|
|
227
|
+
entry = self._build_dlq_meta(meta, prev)
|
|
228
|
+
overlay[uid] = entry
|
|
229
|
+
new_entries.append((uid, entry))
|
|
230
|
+
fail_uids = {u for u, _ in uids_metas}
|
|
231
|
+
remaining = [j for j in self._queue if uid_from_job_dict(j) not in fail_uids]
|
|
232
|
+
for uid, entry in new_entries:
|
|
233
|
+
self._failed[uid] = entry
|
|
234
|
+
# rerun 任务被级联/死锁批量 DLQ 时 wall 旧成功记录作废
|
|
235
|
+
# (最终状态唯一),与 commit_job_failure 对称。
|
|
236
|
+
self._wall.pop(uid, None)
|
|
237
|
+
self._queue = remaining
|
|
238
|
+
return True
|
|
239
|
+
except Exception as e:
|
|
240
|
+
logger.critical(f"Failed to commit bulk failure: {e}")
|
|
241
|
+
return False
|
|
242
|
+
|
|
243
|
+
def append_failed(self, uid: str, payload: Optional[Dict[str, Any]] = None) -> None:
|
|
244
|
+
with self._lock:
|
|
245
|
+
self._write_dlq_entry(uid, payload or {})
|
|
246
|
+
|
|
247
|
+
def commit_skip(self, uid: str) -> bool:
|
|
248
|
+
with self._lock:
|
|
249
|
+
self._queue = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
250
|
+
return True
|
|
251
|
+
|
|
252
|
+
def delete_queue_uids(self, uids: List[str]) -> int:
|
|
253
|
+
"""按 uid 定向批量删除队列条目,与 SQLite 腿同语义:不触碰其余条目。"""
|
|
254
|
+
if not uids:
|
|
255
|
+
return 0
|
|
256
|
+
with self._lock:
|
|
257
|
+
del_set = set(uids)
|
|
258
|
+
kept = [j for j in self._queue if uid_from_job_dict(j) not in del_set]
|
|
259
|
+
removed = len(self._queue) - len(kept)
|
|
260
|
+
self._queue = kept
|
|
261
|
+
return removed
|
|
262
|
+
|
|
263
|
+
def get_meta(self, key: str) -> Optional[str]:
|
|
264
|
+
with self._lock:
|
|
265
|
+
return self._meta.get(key)
|
|
266
|
+
|
|
267
|
+
def set_meta(self, key: str, value: str) -> None:
|
|
268
|
+
with self._lock:
|
|
269
|
+
self._meta[key] = str(value)
|
|
270
|
+
|
|
271
|
+
def enqueue_jobs(self, jobs: List[Dict[str, Any]], *, front: bool = False) -> List[str]:
|
|
272
|
+
if not jobs:
|
|
273
|
+
return []
|
|
274
|
+
with self._lock:
|
|
275
|
+
existing = {uid_from_job_dict(j) for j in self._queue}
|
|
276
|
+
fresh: List[Dict[str, Any]] = []
|
|
277
|
+
batch_seen = set()
|
|
278
|
+
for j in jobs:
|
|
279
|
+
u = uid_from_job_dict(j)
|
|
280
|
+
if u in existing or u in batch_seen:
|
|
281
|
+
continue
|
|
282
|
+
batch_seen.add(u)
|
|
283
|
+
fresh.append(copy.deepcopy(j))
|
|
284
|
+
if not fresh:
|
|
285
|
+
return []
|
|
286
|
+
if front:
|
|
287
|
+
self._queue[0:0] = fresh
|
|
288
|
+
else:
|
|
289
|
+
self._queue.extend(fresh)
|
|
290
|
+
return [uid_from_job_dict(j) for j in fresh]
|
|
291
|
+
|
|
292
|
+
def delete_failed(self, uids: List[str]) -> int:
|
|
293
|
+
with self._lock:
|
|
294
|
+
del_set = set(uids)
|
|
295
|
+
count = 0
|
|
296
|
+
for u in del_set:
|
|
297
|
+
if u in self._failed:
|
|
298
|
+
del self._failed[u]
|
|
299
|
+
count += 1
|
|
300
|
+
return count
|
|
301
|
+
|
|
302
|
+
def delete_wall(self, uids: List[str]) -> int:
|
|
303
|
+
with self._lock:
|
|
304
|
+
del_set = set(uids)
|
|
305
|
+
count = 0
|
|
306
|
+
for u in del_set:
|
|
307
|
+
if u in self._wall:
|
|
308
|
+
del self._wall[u]
|
|
309
|
+
count += 1
|
|
310
|
+
return count
|
|
311
|
+
|
|
312
|
+
def seed_wall(self, uids: List[str]) -> int:
|
|
313
|
+
"""把 uid 批量写入 wall(meta 空 dict)。
|
|
314
|
+
|
|
315
|
+
不变式:wall/failed 全局互斥——已在 failed 的 uid 拒绝种子;
|
|
316
|
+
先查后写(同锁内),冲突整体拒绝、零写入,不静默清除 DLQ 记录。
|
|
317
|
+
"""
|
|
318
|
+
with self._lock:
|
|
319
|
+
conflict = sorted({u for u in uids if u in self._failed})
|
|
320
|
+
if conflict:
|
|
321
|
+
raise ValueError(
|
|
322
|
+
f"seed_wall refuses uid(s) already in failed: {conflict}; "
|
|
323
|
+
f"wall/failed must stay disjoint"
|
|
324
|
+
)
|
|
325
|
+
count = 0
|
|
326
|
+
for u in uids:
|
|
327
|
+
self._wall[u] = {}
|
|
328
|
+
count += 1
|
|
329
|
+
return count
|
|
330
|
+
|
|
331
|
+
def seed_cursor(self, key: str, value: str) -> None:
|
|
332
|
+
"""预填一个 cursor(UPSERT,幂等)。
|
|
333
|
+
|
|
334
|
+
与 SQLiteStateBackend.seed_cursor 同契约 fail-loud:key 非空 str、
|
|
335
|
+
value 必须 str——静默 str() 强转会让同一非法入参在本腿写 '123'、
|
|
336
|
+
sqlite 腿抛异常(跨后端行为分歧),且与内存镜像值型漂移。
|
|
337
|
+
"""
|
|
338
|
+
if not isinstance(key, str) or not key:
|
|
339
|
+
raise TypeError(f"cursor key must be a non-empty str, got {key!r}")
|
|
340
|
+
if not isinstance(value, str):
|
|
341
|
+
raise TypeError(f"cursor value must be str, got {type(value).__name__}")
|
|
342
|
+
with self._lock:
|
|
343
|
+
self._cursors[key] = value
|