tasklite-engine 1.3.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tasklite_engine-1.3.0/tasklite_engine.egg-info → tasklite_engine-1.4.0}/PKG-INFO +5 -6
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/README.md +3 -3
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/pyproject.toml +13 -2
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/__init__.py +1 -3
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/backend/base.py +18 -18
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/backend/memory.py +36 -37
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/backend/sqlite_backend.py +24 -25
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/__init__.py +1 -1
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/channel.py +55 -53
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/completion.py +15 -16
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/config.py +25 -25
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/console.py +8 -8
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/dispatch.py +10 -10
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/governor.py +21 -27
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/inflight.py +20 -56
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/policy.py +21 -36
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/recovery.py +2 -14
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/resource.py +41 -44
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/runtime.py +14 -39
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/scheduler.py +34 -29
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/session.py +9 -9
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/store.py +63 -63
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/types.py +5 -5
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/exceptions.py +0 -16
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/models/context.py +26 -27
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/models/job.py +18 -17
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/models/state.py +32 -32
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/pipeline.py +49 -48
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/taxonomy.py +36 -57
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/testing.py +15 -14
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/utils/injective.py +8 -8
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/utils/ipc.py +40 -41
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/utils/lockfile.py +4 -4
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/wrappers/discovery.py +19 -18
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/wrappers/http.py +9 -8
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0/tasklite_engine.egg-info}/PKG-INFO +5 -6
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tests/test_packaging.py +1 -1
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/LICENSE +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/setup.cfg +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/backend/__init__.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/contrib/__init__.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/engine/pacing.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/hooks.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/models/__init__.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/py.typed +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/utils/__init__.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/utils/jsonutil.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite/wrappers/__init__.py +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite_engine.egg-info/SOURCES.txt +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite_engine.egg-info/dependency_links.txt +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite_engine.egg-info/requires.txt +0 -0
- {tasklite_engine-1.3.0 → tasklite_engine-1.4.0}/tasklite_engine.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasklite-engine
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
@@ -9,14 +9,13 @@ Keywords: task-queue,task-orchestration,background-jobs,subprocess,process-isola
|
|
|
9
9
|
Classifier: Intended Audience :: Developers
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.14
|
|
18
17
|
Classifier: Typing :: Typed
|
|
19
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.10
|
|
20
19
|
Description-Content-Type: text/markdown
|
|
21
20
|
License-File: LICENSE
|
|
22
21
|
Provides-Extra: dev
|
|
@@ -33,7 +32,7 @@ Dynamic: license-file
|
|
|
33
32
|
> 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
|
|
34
33
|
> 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用类似 **systemd** 的守护与物理进程隔离架构,内置 SQLite WAL 事务持久化,子进程崩溃或断电不坏库,重启自动断点续跑。
|
|
35
34
|
|
|
36
|
-
[](https://www.python.org/)
|
|
37
36
|
[](LICENSE)
|
|
38
37
|
[]()
|
|
39
38
|
[-orange.svg)]()
|
|
@@ -151,7 +150,7 @@ flowchart TD
|
|
|
151
150
|
|
|
152
151
|
### 安装
|
|
153
152
|
|
|
154
|
-
要求 **Python ≥ 3.
|
|
153
|
+
要求 **Python ≥ 3.10**,零外部依赖:
|
|
155
154
|
|
|
156
155
|
```bash
|
|
157
156
|
pip install tasklite-engine
|
|
@@ -356,7 +355,7 @@ tasklite/
|
|
|
356
355
|
│ ├── console.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
|
|
357
356
|
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
358
357
|
│ ├── completion.py # 任务完成与提交 (CompletionMachine)
|
|
359
|
-
│ ├── recovery.py # 崩溃检测与恢复 (
|
|
358
|
+
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
|
|
360
359
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
361
360
|
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
|
|
362
361
|
│ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
> 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
|
|
5
5
|
> 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用类似 **systemd** 的守护与物理进程隔离架构,内置 SQLite WAL 事务持久化,子进程崩溃或断电不坏库,重启自动断点续跑。
|
|
6
6
|
|
|
7
|
-
[](https://www.python.org/)
|
|
8
8
|
[](LICENSE)
|
|
9
9
|
[]()
|
|
10
10
|
[-orange.svg)]()
|
|
@@ -122,7 +122,7 @@ flowchart TD
|
|
|
122
122
|
|
|
123
123
|
### 安装
|
|
124
124
|
|
|
125
|
-
要求 **Python ≥ 3.
|
|
125
|
+
要求 **Python ≥ 3.10**,零外部依赖:
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
128
|
pip install tasklite-engine
|
|
@@ -327,7 +327,7 @@ tasklite/
|
|
|
327
327
|
│ ├── console.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
|
|
328
328
|
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
329
329
|
│ ├── completion.py # 任务完成与提交 (CompletionMachine)
|
|
330
|
-
│ ├── recovery.py # 崩溃检测与恢复 (
|
|
330
|
+
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
|
|
331
331
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
332
332
|
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
|
|
333
333
|
│ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
|
|
@@ -7,7 +7,7 @@ name = "tasklite-engine"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
|
|
9
9
|
description = "A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
11
|
license = "MIT"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
|
|
@@ -17,7 +17,6 @@ classifiers = [
|
|
|
17
17
|
"Intended Audience :: Developers",
|
|
18
18
|
"Operating System :: OS Independent",
|
|
19
19
|
"Programming Language :: Python :: 3",
|
|
20
|
-
"Programming Language :: Python :: 3.9",
|
|
21
20
|
"Programming Language :: Python :: 3.10",
|
|
22
21
|
"Programming Language :: Python :: 3.11",
|
|
23
22
|
"Programming Language :: Python :: 3.12",
|
|
@@ -53,6 +52,18 @@ addopts = "--strict-markers"
|
|
|
53
52
|
timeout = 60
|
|
54
53
|
norecursedirs = ["mutants", ".git", ".hypothesis", "__pycache__"]
|
|
55
54
|
|
|
55
|
+
[tool.mypy]
|
|
56
|
+
python_version = "3.10"
|
|
57
|
+
files = ["tasklite"]
|
|
58
|
+
check_untyped_defs = true
|
|
59
|
+
no_implicit_optional = true
|
|
60
|
+
warn_redundant_casts = true
|
|
61
|
+
warn_unused_ignores = true
|
|
62
|
+
|
|
63
|
+
[[tool.mypy.overrides]]
|
|
64
|
+
module = "requests"
|
|
65
|
+
ignore_missing_imports = true
|
|
66
|
+
|
|
56
67
|
[tool.coverage.run]
|
|
57
68
|
source = ["tasklite"]
|
|
58
69
|
omit = ["*/tests/*"]
|
|
@@ -4,7 +4,7 @@ A lightweight, battle-hardened task orchestration engine with zero external
|
|
|
4
4
|
dependencies, process isolation, and ACID persistence.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "1.
|
|
7
|
+
__version__ = "1.4.0"
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
# Public API - Core
|
|
@@ -50,7 +50,6 @@ from .taxonomy import (
|
|
|
50
50
|
ErrorCategory,
|
|
51
51
|
ErrorClassification,
|
|
52
52
|
ErrorTaxonomy,
|
|
53
|
-
TransientRegistry,
|
|
54
53
|
ValidationErrorItem,
|
|
55
54
|
ValidationResult,
|
|
56
55
|
classify_error_type,
|
|
@@ -78,7 +77,6 @@ __all__ = [
|
|
|
78
77
|
"RateLimitHit",
|
|
79
78
|
"FATAL_EXCEPTIONS",
|
|
80
79
|
"TRANSIENT_EXCEPTIONS",
|
|
81
|
-
"TransientRegistry",
|
|
82
80
|
"classify_exception",
|
|
83
81
|
"is_transient_exception",
|
|
84
82
|
"WORKER_RESOURCE",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
from abc import ABC, abstractmethod
|
|
5
|
-
from typing import Any, Callable,
|
|
5
|
+
from typing import Any, Callable, Mapping, Sequence
|
|
6
6
|
|
|
7
7
|
from ..taxonomy import classify_error_type
|
|
8
8
|
|
|
@@ -39,19 +39,19 @@ class AbstractStateBackend(ABC):
|
|
|
39
39
|
"""
|
|
40
40
|
|
|
41
41
|
@abstractmethod
|
|
42
|
-
def load_wall(self) ->
|
|
42
|
+
def load_wall(self) -> dict[str, dict[str, Any]]: ...
|
|
43
43
|
|
|
44
44
|
@abstractmethod
|
|
45
|
-
def load_failed(self) ->
|
|
45
|
+
def load_failed(self) -> dict[str, dict[str, Any]]: ...
|
|
46
46
|
|
|
47
47
|
@abstractmethod
|
|
48
|
-
def load_cursors(self) ->
|
|
48
|
+
def load_cursors(self) -> dict[str, str]: ...
|
|
49
49
|
|
|
50
50
|
@abstractmethod
|
|
51
|
-
def load_queue(self) ->
|
|
51
|
+
def load_queue(self) -> list[dict[str, Any]]: ...
|
|
52
52
|
|
|
53
53
|
@abstractmethod
|
|
54
|
-
def save_queue(self, jobs:
|
|
54
|
+
def save_queue(self, jobs: list[dict[str, Any]]) -> None:
|
|
55
55
|
"""整表重写队列(测试装配 / replace_queue_atomic 的事务内步骤)。
|
|
56
56
|
|
|
57
57
|
红线:这是「无读基准的整表覆盖」——跨进程并发的 enqueue/commit
|
|
@@ -67,7 +67,7 @@ class AbstractStateBackend(ABC):
|
|
|
67
67
|
@abstractmethod
|
|
68
68
|
def replace_queue_atomic(
|
|
69
69
|
self,
|
|
70
|
-
compute: Callable[[
|
|
70
|
+
compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
|
|
71
71
|
) -> None:
|
|
72
72
|
"""读-改-写收敛的整表替换:磁盘真相读取与写回在单个写事务内完成。
|
|
73
73
|
|
|
@@ -90,8 +90,8 @@ class AbstractStateBackend(ABC):
|
|
|
90
90
|
uid: str,
|
|
91
91
|
result_meta: dict,
|
|
92
92
|
*,
|
|
93
|
-
spawned_jobs:
|
|
94
|
-
cursor_updates:
|
|
93
|
+
spawned_jobs: Sequence[dict[str, Any]] = (),
|
|
94
|
+
cursor_updates: Mapping[str, str | None] | None = None,
|
|
95
95
|
) -> bool:
|
|
96
96
|
"""Persist a successful job as a delta, atomically.
|
|
97
97
|
|
|
@@ -128,7 +128,7 @@ class AbstractStateBackend(ABC):
|
|
|
128
128
|
def commit_retry(
|
|
129
129
|
self,
|
|
130
130
|
popped_uid: str,
|
|
131
|
-
requeued_job:
|
|
131
|
+
requeued_job: dict[str, Any],
|
|
132
132
|
*,
|
|
133
133
|
front: bool = False,
|
|
134
134
|
) -> bool:
|
|
@@ -142,7 +142,7 @@ class AbstractStateBackend(ABC):
|
|
|
142
142
|
@abstractmethod
|
|
143
143
|
def commit_bulk_failure(
|
|
144
144
|
self,
|
|
145
|
-
uids_metas:
|
|
145
|
+
uids_metas: list[tuple[str, dict]],
|
|
146
146
|
) -> bool:
|
|
147
147
|
"""Best-effort bulk mark-failed (deadlock case), as a delta.
|
|
148
148
|
|
|
@@ -154,7 +154,7 @@ class AbstractStateBackend(ABC):
|
|
|
154
154
|
"""
|
|
155
155
|
|
|
156
156
|
@abstractmethod
|
|
157
|
-
def append_failed(self, uid: str, payload:
|
|
157
|
+
def append_failed(self, uid: str, payload: dict[str, Any] | None = None) -> None:
|
|
158
158
|
"""Append a record to the DLQ (failed log) without touching the queue.
|
|
159
159
|
|
|
160
160
|
Used for out-of-band DLQ appends (e.g. user-driven inspection).
|
|
@@ -172,7 +172,7 @@ class AbstractStateBackend(ABC):
|
|
|
172
172
|
"""
|
|
173
173
|
|
|
174
174
|
@abstractmethod
|
|
175
|
-
def delete_queue_uids(self, uids:
|
|
175
|
+
def delete_queue_uids(self, uids: list[str]) -> int:
|
|
176
176
|
"""按 uid 定向批量删除队列行(加载期 repair 的差量落盘唯一出口)。
|
|
177
177
|
|
|
178
178
|
与 save_queue 的全表重写相对:只 DELETE 指定 uid 的行,其余行原样
|
|
@@ -183,7 +183,7 @@ class AbstractStateBackend(ABC):
|
|
|
183
183
|
"""
|
|
184
184
|
|
|
185
185
|
@abstractmethod
|
|
186
|
-
def get_meta(self, key: str) ->
|
|
186
|
+
def get_meta(self, key: str) -> str | None:
|
|
187
187
|
"""读取一条框架级元数据(如 fencing 的 last_run_id)。无则返回 None。"""
|
|
188
188
|
|
|
189
189
|
@abstractmethod
|
|
@@ -191,7 +191,7 @@ class AbstractStateBackend(ABC):
|
|
|
191
191
|
"""写入一条框架级元数据(UPSERT 语义)。失败抛异常(不吞)。"""
|
|
192
192
|
|
|
193
193
|
@abstractmethod
|
|
194
|
-
def enqueue_jobs(self, jobs:
|
|
194
|
+
def enqueue_jobs(self, jobs: list[dict[str, Any]], *, front: bool = False) -> list[str]:
|
|
195
195
|
"""批量增量入队(enqueue 与 run 并发时不做全量覆盖)。
|
|
196
196
|
|
|
197
197
|
Effects: 按 front 在队列头/尾插入 jobs(保序),**单事务原子**。
|
|
@@ -205,7 +205,7 @@ class AbstractStateBackend(ABC):
|
|
|
205
205
|
"""
|
|
206
206
|
|
|
207
207
|
@abstractmethod
|
|
208
|
-
def delete_failed(self, uids:
|
|
208
|
+
def delete_failed(self, uids: list[str]) -> int:
|
|
209
209
|
"""从 DLQ 批量删除指定 uid(clear_dlq/clear_history 的后端实现)。
|
|
210
210
|
|
|
211
211
|
返回实际删除行数。不触碰 queue/wall。仅限 run() 之外调用
|
|
@@ -213,7 +213,7 @@ class AbstractStateBackend(ABC):
|
|
|
213
213
|
"""
|
|
214
214
|
|
|
215
215
|
@abstractmethod
|
|
216
|
-
def delete_wall(self, uids:
|
|
216
|
+
def delete_wall(self, uids: list[str]) -> int:
|
|
217
217
|
"""从 wall 批量删除指定 uid(clear_history 的后端实现 + commit 路径事务内清理)。返回实际删除行数。
|
|
218
218
|
|
|
219
219
|
两种调用上下文:
|
|
@@ -227,7 +227,7 @@ class AbstractStateBackend(ABC):
|
|
|
227
227
|
"""
|
|
228
228
|
|
|
229
229
|
@abstractmethod
|
|
230
|
-
def seed_wall(self, uids:
|
|
230
|
+
def seed_wall(self, uids: list[str]) -> int:
|
|
231
231
|
"""把 uid 批量写入 wall(meta 为空 dict)——存档迁移标记「已处理」。
|
|
232
232
|
|
|
233
233
|
用途:媒体/数据资产项目的存档迁移(硬链接 + wall 种子),
|
|
@@ -9,7 +9,7 @@ import copy
|
|
|
9
9
|
import logging
|
|
10
10
|
import threading
|
|
11
11
|
from datetime import datetime, timezone
|
|
12
|
-
from typing import Any, Callable,
|
|
12
|
+
from typing import Any, Callable, Mapping, Sequence
|
|
13
13
|
|
|
14
14
|
from .base import (
|
|
15
15
|
AbstractStateBackend,
|
|
@@ -26,29 +26,29 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
26
26
|
|
|
27
27
|
def __init__(self) -> None:
|
|
28
28
|
self._lock = threading.RLock()
|
|
29
|
-
self._wall:
|
|
30
|
-
self._failed:
|
|
31
|
-
self._cursors:
|
|
32
|
-
self._queue:
|
|
33
|
-
self._meta:
|
|
29
|
+
self._wall: dict[str, dict[str, Any]] = {}
|
|
30
|
+
self._failed: dict[str, dict[str, Any]] = {}
|
|
31
|
+
self._cursors: dict[str, str] = {}
|
|
32
|
+
self._queue: list[dict[str, Any]] = []
|
|
33
|
+
self._meta: dict[str, str] = {}
|
|
34
34
|
|
|
35
|
-
def load_wall(self) ->
|
|
35
|
+
def load_wall(self) -> dict[str, dict[str, Any]]:
|
|
36
36
|
with self._lock:
|
|
37
37
|
return copy.deepcopy(self._wall)
|
|
38
38
|
|
|
39
|
-
def load_failed(self) ->
|
|
39
|
+
def load_failed(self) -> dict[str, dict[str, Any]]:
|
|
40
40
|
with self._lock:
|
|
41
41
|
return copy.deepcopy(self._failed)
|
|
42
42
|
|
|
43
|
-
def load_cursors(self) ->
|
|
43
|
+
def load_cursors(self) -> dict[str, str]:
|
|
44
44
|
with self._lock:
|
|
45
45
|
return dict(self._cursors)
|
|
46
46
|
|
|
47
|
-
def load_queue(self) ->
|
|
47
|
+
def load_queue(self) -> list[dict[str, Any]]:
|
|
48
48
|
with self._lock:
|
|
49
49
|
return copy.deepcopy(self._queue)
|
|
50
50
|
|
|
51
|
-
def _dedup_copy(self, jobs:
|
|
51
|
+
def _dedup_copy(self, jobs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
52
52
|
"""整表替换行的单一出口(save_queue 与 replace_queue_atomic 共用)。
|
|
53
53
|
|
|
54
54
|
替换集先经 ``validate_queue_replacement`` 校验(与 SQLite 腿同一出口、
|
|
@@ -69,27 +69,26 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
69
69
|
clean.append(copy.deepcopy(j))
|
|
70
70
|
return clean
|
|
71
71
|
|
|
72
|
-
def save_queue(self, jobs:
|
|
72
|
+
def save_queue(self, jobs: list[dict[str, Any]]) -> None:
|
|
73
73
|
with self._lock:
|
|
74
74
|
self._queue = self._dedup_copy(jobs)
|
|
75
75
|
|
|
76
76
|
def replace_queue_atomic(
|
|
77
77
|
self,
|
|
78
|
-
compute: Callable[[
|
|
78
|
+
compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
|
|
79
79
|
) -> None:
|
|
80
80
|
with self._lock:
|
|
81
81
|
# 锁内读真相 → 纯计算 → 替换:compute 抛异常时 _queue 赋值
|
|
82
82
|
# 未发生,队列保持调用前状态(对齐 SQLite 腿事务回滚)。
|
|
83
83
|
self._queue = self._dedup_copy(compute(copy.deepcopy(self._queue)))
|
|
84
84
|
|
|
85
|
-
def _build_dlq_meta(self, meta:
|
|
85
|
+
def _build_dlq_meta(self, meta: dict | None, prev: Any) -> dict[str, Any]:
|
|
86
86
|
"""计算 DLQ 行终值(纯函数,不变更任何状态):_attempt 计数 + error_type + failed_at。
|
|
87
87
|
|
|
88
|
-
不变式:``_attempt``
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
该 uid 的路径权威给定)。
|
|
88
|
+
不变式:``_attempt`` 是写入事件计数而非逻辑失败次数;既有计数
|
|
89
|
+
无效(非 dict 记录或非 int 计数)时静默重置为 1 并照常提交,绝不
|
|
90
|
+
因脏计数抛 TypeError——与 SQLite 后端「损坏行重置计数、写入成功」
|
|
91
|
+
行为对齐。
|
|
93
92
|
"""
|
|
94
93
|
merged = copy.deepcopy(meta or {})
|
|
95
94
|
if "error_type" not in merged:
|
|
@@ -102,11 +101,11 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
102
101
|
prev_attempt = prev.get("_attempt")
|
|
103
102
|
if isinstance(prev_attempt, int):
|
|
104
103
|
merged["_attempt"] = prev_attempt + 1
|
|
105
|
-
|
|
106
|
-
merged["_attempt"] = 1
|
|
104
|
+
else:
|
|
105
|
+
merged["_attempt"] = 1 # 既有记录无有效计数:从 1 重新计数
|
|
107
106
|
return merged
|
|
108
107
|
|
|
109
|
-
def _write_dlq_entry(self, uid: str, meta:
|
|
108
|
+
def _write_dlq_entry(self, uid: str, meta: dict | None) -> None:
|
|
110
109
|
"""DLQ 写入单一出口:终值经 _build_dlq_meta 计算后落变。"""
|
|
111
110
|
self._failed[uid] = self._build_dlq_meta(meta, self._failed.get(uid))
|
|
112
111
|
|
|
@@ -115,8 +114,8 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
115
114
|
uid: str,
|
|
116
115
|
result_meta: dict,
|
|
117
116
|
*,
|
|
118
|
-
spawned_jobs:
|
|
119
|
-
cursor_updates:
|
|
117
|
+
spawned_jobs: Sequence[dict[str, Any]] = (),
|
|
118
|
+
cursor_updates: Mapping[str, str | None] | None = None,
|
|
120
119
|
) -> bool:
|
|
121
120
|
with self._lock:
|
|
122
121
|
try:
|
|
@@ -143,7 +142,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
143
142
|
)
|
|
144
143
|
return False
|
|
145
144
|
seen.add(suid)
|
|
146
|
-
cursor_sets:
|
|
145
|
+
cursor_sets: dict[str, str] = {}
|
|
147
146
|
cursor_dels: set = set()
|
|
148
147
|
if cursor_updates:
|
|
149
148
|
for k, v in cursor_updates.items():
|
|
@@ -185,7 +184,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
185
184
|
def commit_retry(
|
|
186
185
|
self,
|
|
187
186
|
popped_uid: str,
|
|
188
|
-
requeued_job:
|
|
187
|
+
requeued_job: dict[str, Any],
|
|
189
188
|
*,
|
|
190
189
|
front: bool = False,
|
|
191
190
|
) -> bool:
|
|
@@ -212,14 +211,14 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
212
211
|
logger.critical(f"Failed to commit retry for {popped_uid}: {e}")
|
|
213
212
|
return False
|
|
214
213
|
|
|
215
|
-
def commit_bulk_failure(self, uids_metas:
|
|
214
|
+
def commit_bulk_failure(self, uids_metas: list[tuple[str, dict]]) -> bool:
|
|
216
215
|
with self._lock:
|
|
217
216
|
try:
|
|
218
217
|
# 校验先行:全部 DLQ 行计算在任何变更前完成——任一行失败则
|
|
219
218
|
# queue/wall/failed 整体不变(对齐 SQLite 事务回滚,杜绝
|
|
220
219
|
# 「队列已整体删除、DLQ 未落」的半成品失败态)。
|
|
221
|
-
new_entries:
|
|
222
|
-
overlay:
|
|
220
|
+
new_entries: list[tuple[str, dict[str, Any]]] = []
|
|
221
|
+
overlay: dict[str, Any] = {}
|
|
223
222
|
for uid, meta in uids_metas:
|
|
224
223
|
# 批内同 uid 多次出现时模拟 SQLite 同事务顺序写:
|
|
225
224
|
# 后一行读取前一行结果,_attempt 连续递增。
|
|
@@ -240,7 +239,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
240
239
|
logger.critical(f"Failed to commit bulk failure: {e}")
|
|
241
240
|
return False
|
|
242
241
|
|
|
243
|
-
def append_failed(self, uid: str, payload:
|
|
242
|
+
def append_failed(self, uid: str, payload: dict[str, Any] | None = None) -> None:
|
|
244
243
|
with self._lock:
|
|
245
244
|
self._write_dlq_entry(uid, payload or {})
|
|
246
245
|
|
|
@@ -249,7 +248,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
249
248
|
self._queue = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
250
249
|
return True
|
|
251
250
|
|
|
252
|
-
def delete_queue_uids(self, uids:
|
|
251
|
+
def delete_queue_uids(self, uids: list[str]) -> int:
|
|
253
252
|
"""按 uid 定向批量删除队列条目,与 SQLite 腿同语义:不触碰其余条目。"""
|
|
254
253
|
if not uids:
|
|
255
254
|
return 0
|
|
@@ -260,7 +259,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
260
259
|
self._queue = kept
|
|
261
260
|
return removed
|
|
262
261
|
|
|
263
|
-
def get_meta(self, key: str) ->
|
|
262
|
+
def get_meta(self, key: str) -> str | None:
|
|
264
263
|
with self._lock:
|
|
265
264
|
return self._meta.get(key)
|
|
266
265
|
|
|
@@ -268,12 +267,12 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
268
267
|
with self._lock:
|
|
269
268
|
self._meta[key] = str(value)
|
|
270
269
|
|
|
271
|
-
def enqueue_jobs(self, jobs:
|
|
270
|
+
def enqueue_jobs(self, jobs: list[dict[str, Any]], *, front: bool = False) -> list[str]:
|
|
272
271
|
if not jobs:
|
|
273
272
|
return []
|
|
274
273
|
with self._lock:
|
|
275
274
|
existing = {uid_from_job_dict(j) for j in self._queue}
|
|
276
|
-
fresh:
|
|
275
|
+
fresh: list[dict[str, Any]] = []
|
|
277
276
|
batch_seen = set()
|
|
278
277
|
for j in jobs:
|
|
279
278
|
u = uid_from_job_dict(j)
|
|
@@ -289,7 +288,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
289
288
|
self._queue.extend(fresh)
|
|
290
289
|
return [uid_from_job_dict(j) for j in fresh]
|
|
291
290
|
|
|
292
|
-
def delete_failed(self, uids:
|
|
291
|
+
def delete_failed(self, uids: list[str]) -> int:
|
|
293
292
|
with self._lock:
|
|
294
293
|
del_set = set(uids)
|
|
295
294
|
count = 0
|
|
@@ -299,7 +298,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
299
298
|
count += 1
|
|
300
299
|
return count
|
|
301
300
|
|
|
302
|
-
def delete_wall(self, uids:
|
|
301
|
+
def delete_wall(self, uids: list[str]) -> int:
|
|
303
302
|
with self._lock:
|
|
304
303
|
del_set = set(uids)
|
|
305
304
|
count = 0
|
|
@@ -309,7 +308,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
309
308
|
count += 1
|
|
310
309
|
return count
|
|
311
310
|
|
|
312
|
-
def seed_wall(self, uids:
|
|
311
|
+
def seed_wall(self, uids: list[str]) -> int:
|
|
313
312
|
"""把 uid 批量写入 wall(meta 空 dict)。
|
|
314
313
|
|
|
315
314
|
不变式:wall/failed 全局互斥——已在 failed 的 uid 拒绝种子;
|