tasklite-engine 1.3.1__tar.gz → 1.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {tasklite_engine-1.3.1/tasklite_engine.egg-info → tasklite_engine-1.4.1}/PKG-INFO +6 -6
  2. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/README.md +3 -3
  3. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/pyproject.toml +14 -3
  4. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/__init__.py +8 -3
  5. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/base.py +26 -18
  6. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/memory.py +52 -39
  7. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/sqlite_backend.py +74 -33
  8. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/__init__.py +1 -1
  9. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/channel.py +55 -53
  10. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/completion.py +15 -16
  11. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/config.py +25 -25
  12. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/console.py +27 -11
  13. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/dispatch.py +10 -10
  14. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/governor.py +21 -27
  15. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/inflight.py +20 -56
  16. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/policy.py +21 -36
  17. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/recovery.py +2 -14
  18. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/resource.py +41 -44
  19. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/runtime.py +14 -39
  20. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/scheduler.py +34 -29
  21. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/session.py +9 -9
  22. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/store.py +97 -67
  23. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/types.py +5 -5
  24. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/exceptions.py +0 -16
  25. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/context.py +26 -27
  26. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/job.py +18 -17
  27. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/state.py +32 -32
  28. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/pipeline.py +56 -50
  29. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/taxonomy.py +36 -57
  30. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/testing.py +15 -14
  31. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/injective.py +8 -8
  32. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/ipc.py +40 -41
  33. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/lockfile.py +4 -4
  34. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/wrappers/discovery.py +19 -18
  35. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/wrappers/http.py +10 -9
  36. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1/tasklite_engine.egg-info}/PKG-INFO +6 -6
  37. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/requires.txt +1 -0
  38. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tests/test_packaging.py +1 -1
  39. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/LICENSE +0 -0
  40. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/setup.cfg +0 -0
  41. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/__init__.py +0 -0
  42. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/contrib/__init__.py +0 -0
  43. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/pacing.py +0 -0
  44. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/hooks.py +0 -0
  45. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/__init__.py +0 -0
  46. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/py.typed +0 -0
  47. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/__init__.py +0 -0
  48. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/jsonutil.py +0 -0
  49. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/wrappers/__init__.py +0 -0
  50. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/SOURCES.txt +0 -0
  51. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/dependency_links.txt +0 -0
  52. {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tasklite-engine
3
- Version: 1.3.1
3
+ Version: 1.4.1
4
4
  Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
5
5
  License-Expression: MIT
6
6
  Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
@@ -9,14 +9,13 @@ Keywords: task-queue,task-orchestration,background-jobs,subprocess,process-isola
9
9
  Classifier: Intended Audience :: Developers
10
10
  Classifier: Operating System :: OS Independent
11
11
  Classifier: Programming Language :: Python :: 3
12
- Classifier: Programming Language :: Python :: 3.9
13
12
  Classifier: Programming Language :: Python :: 3.10
14
13
  Classifier: Programming Language :: Python :: 3.11
15
14
  Classifier: Programming Language :: Python :: 3.12
16
15
  Classifier: Programming Language :: Python :: 3.13
17
16
  Classifier: Programming Language :: Python :: 3.14
18
17
  Classifier: Typing :: Typed
19
- Requires-Python: >=3.9
18
+ Requires-Python: >=3.10
20
19
  Description-Content-Type: text/markdown
21
20
  License-File: LICENSE
22
21
  Provides-Extra: dev
@@ -25,6 +24,7 @@ Requires-Dist: hypothesis>=6.0; extra == "dev"
25
24
  Requires-Dist: mutmut>=3.0; extra == "dev"
26
25
  Requires-Dist: pytest-timeout>=2.0; extra == "dev"
27
26
  Requires-Dist: pytest-cov>=4.0; extra == "dev"
27
+ Requires-Dist: requests>=2.28; extra == "dev"
28
28
  Dynamic: license-file
29
29
 
30
30
  # ⚡ TaskLite — 零依赖、进程隔离的轻量任务编排引擎
@@ -33,7 +33,7 @@ Dynamic: license-file
33
33
  > 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
34
34
  > 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用类似 **systemd** 的守护与物理进程隔离架构,内置 SQLite WAL 事务持久化,子进程崩溃或断电不坏库,重启自动断点续跑。
35
35
 
36
- [![Python Version](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
36
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/)
37
37
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
38
38
  [![Zero Dependencies](https://img.shields.io/badge/dependencies-0%20external-brightgreen.svg)]()
39
39
  [![ACID Persistence](https://img.shields.io/badge/persistence-SQLite%20WAL%20(ACID)-orange.svg)]()
@@ -151,7 +151,7 @@ flowchart TD
151
151
 
152
152
  ### 安装
153
153
 
154
- 要求 **Python ≥ 3.9**,零外部依赖:
154
+ 要求 **Python ≥ 3.10**,零外部依赖:
155
155
 
156
156
  ```bash
157
157
  pip install tasklite-engine
@@ -356,7 +356,7 @@ tasklite/
356
356
  │ ├── console.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
357
357
  │ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
358
358
  │ ├── completion.py # 任务完成与提交 (CompletionMachine)
359
- │ ├── recovery.py # 崩溃检测与恢复 (RecoveryMachine)
359
+ │ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
360
360
  │ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
361
361
  │ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
362
362
  │ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
@@ -4,7 +4,7 @@
4
4
  > 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
5
5
  > 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用类似 **systemd** 的守护与物理进程隔离架构,内置 SQLite WAL 事务持久化,子进程崩溃或断电不坏库,重启自动断点续跑。
6
6
 
7
- [![Python Version](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
7
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/)
8
8
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
9
9
  [![Zero Dependencies](https://img.shields.io/badge/dependencies-0%20external-brightgreen.svg)]()
10
10
  [![ACID Persistence](https://img.shields.io/badge/persistence-SQLite%20WAL%20(ACID)-orange.svg)]()
@@ -122,7 +122,7 @@ flowchart TD
122
122
 
123
123
  ### 安装
124
124
 
125
- 要求 **Python ≥ 3.9**,零外部依赖:
125
+ 要求 **Python ≥ 3.10**,零外部依赖:
126
126
 
127
127
  ```bash
128
128
  pip install tasklite-engine
@@ -327,7 +327,7 @@ tasklite/
327
327
  │ ├── console.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
328
328
  │ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
329
329
  │ ├── completion.py # 任务完成与提交 (CompletionMachine)
330
- │ ├── recovery.py # 崩溃检测与恢复 (RecoveryMachine)
330
+ │ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
331
331
  │ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
332
332
  │ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
333
333
  │ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
@@ -7,7 +7,7 @@ name = "tasklite-engine"
7
7
  dynamic = ["version"]
8
8
 
9
9
  description = "A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence"
10
- requires-python = ">=3.9"
10
+ requires-python = ">=3.10"
11
11
  license = "MIT"
12
12
  readme = "README.md"
13
13
 
@@ -17,7 +17,6 @@ classifiers = [
17
17
  "Intended Audience :: Developers",
18
18
  "Operating System :: OS Independent",
19
19
  "Programming Language :: Python :: 3",
20
- "Programming Language :: Python :: 3.9",
21
20
  "Programming Language :: Python :: 3.10",
22
21
  "Programming Language :: Python :: 3.11",
23
22
  "Programming Language :: Python :: 3.12",
@@ -31,7 +30,7 @@ Homepage = "https://github.com/CaptainLi321/tasklite"
31
30
  Repository = "https://github.com/CaptainLi321/tasklite"
32
31
 
33
32
  [project.optional-dependencies]
34
- dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0"]
33
+ dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0", "requests>=2.28"]
35
34
 
36
35
  [tool.setuptools.packages.find]
37
36
  include = ["tasklite", "tasklite.*"]
@@ -53,6 +52,18 @@ addopts = "--strict-markers"
53
52
  timeout = 60
54
53
  norecursedirs = ["mutants", ".git", ".hypothesis", "__pycache__"]
55
54
 
55
+ [tool.mypy]
56
+ python_version = "3.10"
57
+ files = ["tasklite"]
58
+ check_untyped_defs = true
59
+ no_implicit_optional = true
60
+ warn_redundant_casts = true
61
+ warn_unused_ignores = true
62
+
63
+ [[tool.mypy.overrides]]
64
+ module = "requests"
65
+ ignore_missing_imports = true
66
+
56
67
  [tool.coverage.run]
57
68
  source = ["tasklite"]
58
69
  omit = ["*/tests/*"]
@@ -4,7 +4,7 @@ A lightweight, battle-hardened task orchestration engine with zero external
4
4
  dependencies, process isolation, and ACID persistence.
5
5
  """
6
6
 
7
- __version__ = "1.3.1"
7
+ __version__ = "1.4.1"
8
8
 
9
9
 
10
10
  # Public API - Core
@@ -50,7 +50,6 @@ from .taxonomy import (
50
50
  ErrorCategory,
51
51
  ErrorClassification,
52
52
  ErrorTaxonomy,
53
- TransientRegistry,
54
53
  ValidationErrorItem,
55
54
  ValidationResult,
56
55
  classify_error_type,
@@ -66,6 +65,10 @@ from . import contrib
66
65
  # Public API - Resources
67
66
  from .engine.resource import CapacityResource, RateLimitResource, Resource
68
67
 
68
+ # Public API - Handler 注册表与 JSON 序列化
69
+ from .engine.types import HandlerEntry
70
+ from .utils import jsonutil
71
+
69
72
  __all__ = [
70
73
  # Core
71
74
  "TaskLite",
@@ -78,7 +81,6 @@ __all__ = [
78
81
  "RateLimitHit",
79
82
  "FATAL_EXCEPTIONS",
80
83
  "TRANSIENT_EXCEPTIONS",
81
- "TransientRegistry",
82
84
  "classify_exception",
83
85
  "is_transient_exception",
84
86
  "WORKER_RESOURCE",
@@ -121,6 +123,9 @@ __all__ = [
121
123
  "sanitize_job_component",
122
124
  "sanitize_content_id",
123
125
  "safe_uid_filename",
126
+ # Handler 注册表与 JSON 序列化
127
+ "HandlerEntry",
128
+ "jsonutil",
124
129
  # Contrib
125
130
  "contrib",
126
131
  # Resources
@@ -2,7 +2,7 @@
2
2
  from __future__ import annotations
3
3
 
4
4
  from abc import ABC, abstractmethod
5
- from typing import Any, Callable, Dict, List, Optional, Tuple
5
+ from typing import Any, Callable, Mapping, Sequence
6
6
 
7
7
  from ..taxonomy import classify_error_type
8
8
 
@@ -39,19 +39,23 @@ class AbstractStateBackend(ABC):
39
39
  """
40
40
 
41
41
  @abstractmethod
42
- def load_wall(self) -> Dict[str, Dict[str, Any]]: ...
42
+ def load_wall(self) -> dict[str, dict[str, Any]]: ...
43
43
 
44
44
  @abstractmethod
45
- def load_failed(self) -> Dict[str, Dict[str, Any]]: ...
45
+ def load_failed(self) -> dict[str, dict[str, Any]]: ...
46
46
 
47
47
  @abstractmethod
48
- def load_cursors(self) -> Dict[str, str]: ...
48
+ def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
49
+ """读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
49
50
 
50
51
  @abstractmethod
51
- def load_queue(self) -> List[Dict[str, Any]]: ...
52
+ def load_cursors(self) -> dict[str, str]: ...
52
53
 
53
54
  @abstractmethod
54
- def save_queue(self, jobs: List[Dict[str, Any]]) -> None:
55
+ def load_queue(self) -> list[dict[str, Any]]: ...
56
+
57
+ @abstractmethod
58
+ def save_queue(self, jobs: list[dict[str, Any]]) -> None:
55
59
  """整表重写队列(测试装配 / replace_queue_atomic 的事务内步骤)。
56
60
 
57
61
  红线:这是「无读基准的整表覆盖」——跨进程并发的 enqueue/commit
@@ -67,7 +71,7 @@ class AbstractStateBackend(ABC):
67
71
  @abstractmethod
68
72
  def replace_queue_atomic(
69
73
  self,
70
- compute: Callable[[List[Dict[str, Any]]], List[Dict[str, Any]]],
74
+ compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
71
75
  ) -> None:
72
76
  """读-改-写收敛的整表替换:磁盘真相读取与写回在单个写事务内完成。
73
77
 
@@ -90,8 +94,8 @@ class AbstractStateBackend(ABC):
90
94
  uid: str,
91
95
  result_meta: dict,
92
96
  *,
93
- spawned_jobs: List[Dict[str, Any]] = (),
94
- cursor_updates: Optional[Dict[str, str]] = None,
97
+ spawned_jobs: Sequence[dict[str, Any]] = (),
98
+ cursor_updates: Mapping[str, str | None] | None = None,
95
99
  ) -> bool:
96
100
  """Persist a successful job as a delta, atomically.
97
101
 
@@ -113,10 +117,14 @@ class AbstractStateBackend(ABC):
113
117
  self,
114
118
  uid: str,
115
119
  result_meta: dict,
120
+ job_payload: dict[str, Any] | None = None,
116
121
  ) -> bool:
117
122
  """Persist a failed job to the DLQ as a delta, atomically.
118
123
 
119
124
  Effects: failed_dlq INSERT/REPLACE uid -> result_meta; queue DELETE uid.
125
+ ``job_payload`` is the original business payload snapshot stored
126
+ alongside the failure meta (for retry tooling); None keeps any
127
+ previously stored snapshot intact.
120
128
  Returns True on success. Returns False if persistence failed — on
121
129
  failure the backend MUST leave the on-disk queue unchanged (the popped
122
130
  uid remains on disk); the caller is responsible for deciding whether to
@@ -128,7 +136,7 @@ class AbstractStateBackend(ABC):
128
136
  def commit_retry(
129
137
  self,
130
138
  popped_uid: str,
131
- requeued_job: Dict[str, Any],
139
+ requeued_job: dict[str, Any],
132
140
  *,
133
141
  front: bool = False,
134
142
  ) -> bool:
@@ -142,7 +150,7 @@ class AbstractStateBackend(ABC):
142
150
  @abstractmethod
143
151
  def commit_bulk_failure(
144
152
  self,
145
- uids_metas: List[Tuple[str, dict]],
153
+ uids_metas: list[tuple[str, dict]],
146
154
  ) -> bool:
147
155
  """Best-effort bulk mark-failed (deadlock case), as a delta.
148
156
 
@@ -154,7 +162,7 @@ class AbstractStateBackend(ABC):
154
162
  """
155
163
 
156
164
  @abstractmethod
157
- def append_failed(self, uid: str, payload: Optional[Dict[str, Any]] = None) -> None:
165
+ def append_failed(self, uid: str, payload: dict[str, Any] | None = None) -> None:
158
166
  """Append a record to the DLQ (failed log) without touching the queue.
159
167
 
160
168
  Used for out-of-band DLQ appends (e.g. user-driven inspection).
@@ -172,7 +180,7 @@ class AbstractStateBackend(ABC):
172
180
  """
173
181
 
174
182
  @abstractmethod
175
- def delete_queue_uids(self, uids: List[str]) -> int:
183
+ def delete_queue_uids(self, uids: list[str]) -> int:
176
184
  """按 uid 定向批量删除队列行(加载期 repair 的差量落盘唯一出口)。
177
185
 
178
186
  与 save_queue 的全表重写相对:只 DELETE 指定 uid 的行,其余行原样
@@ -183,7 +191,7 @@ class AbstractStateBackend(ABC):
183
191
  """
184
192
 
185
193
  @abstractmethod
186
- def get_meta(self, key: str) -> Optional[str]:
194
+ def get_meta(self, key: str) -> str | None:
187
195
  """读取一条框架级元数据(如 fencing 的 last_run_id)。无则返回 None。"""
188
196
 
189
197
  @abstractmethod
@@ -191,7 +199,7 @@ class AbstractStateBackend(ABC):
191
199
  """写入一条框架级元数据(UPSERT 语义)。失败抛异常(不吞)。"""
192
200
 
193
201
  @abstractmethod
194
- def enqueue_jobs(self, jobs: List[Dict[str, Any]], *, front: bool = False) -> List[str]:
202
+ def enqueue_jobs(self, jobs: list[dict[str, Any]], *, front: bool = False) -> list[str]:
195
203
  """批量增量入队(enqueue 与 run 并发时不做全量覆盖)。
196
204
 
197
205
  Effects: 按 front 在队列头/尾插入 jobs(保序),**单事务原子**。
@@ -205,7 +213,7 @@ class AbstractStateBackend(ABC):
205
213
  """
206
214
 
207
215
  @abstractmethod
208
- def delete_failed(self, uids: List[str]) -> int:
216
+ def delete_failed(self, uids: list[str]) -> int:
209
217
  """从 DLQ 批量删除指定 uid(clear_dlq/clear_history 的后端实现)。
210
218
 
211
219
  返回实际删除行数。不触碰 queue/wall。仅限 run() 之外调用
@@ -213,7 +221,7 @@ class AbstractStateBackend(ABC):
213
221
  """
214
222
 
215
223
  @abstractmethod
216
- def delete_wall(self, uids: List[str]) -> int:
224
+ def delete_wall(self, uids: list[str]) -> int:
217
225
  """从 wall 批量删除指定 uid(clear_history 的后端实现 + commit 路径事务内清理)。返回实际删除行数。
218
226
 
219
227
  两种调用上下文:
@@ -227,7 +235,7 @@ class AbstractStateBackend(ABC):
227
235
  """
228
236
 
229
237
  @abstractmethod
230
- def seed_wall(self, uids: List[str]) -> int:
238
+ def seed_wall(self, uids: list[str]) -> int:
231
239
  """把 uid 批量写入 wall(meta 为空 dict)——存档迁移标记「已处理」。
232
240
 
233
241
  用途:媒体/数据资产项目的存档迁移(硬链接 + wall 种子),
@@ -9,7 +9,7 @@ import copy
9
9
  import logging
10
10
  import threading
11
11
  from datetime import datetime, timezone
12
- from typing import Any, Callable, Dict, List, Optional, Tuple
12
+ from typing import Any, Callable, Mapping, Sequence
13
13
 
14
14
  from .base import (
15
15
  AbstractStateBackend,
@@ -26,29 +26,35 @@ class InMemoryStateBackend(AbstractStateBackend):
26
26
 
27
27
  def __init__(self) -> None:
28
28
  self._lock = threading.RLock()
29
- self._wall: Dict[str, Dict[str, Any]] = {}
30
- self._failed: Dict[str, Dict[str, Any]] = {}
31
- self._cursors: Dict[str, str] = {}
32
- self._queue: List[Dict[str, Any]] = []
33
- self._meta: Dict[str, str] = {}
34
-
35
- def load_wall(self) -> Dict[str, Dict[str, Any]]:
29
+ self._wall: dict[str, dict[str, Any]] = {}
30
+ self._failed: dict[str, dict[str, Any]] = {}
31
+ self._failed_payloads: dict[str, dict[str, Any]] = {}
32
+ self._cursors: dict[str, str] = {}
33
+ self._queue: list[dict[str, Any]] = []
34
+ self._meta: dict[str, str] = {}
35
+
36
+ def load_wall(self) -> dict[str, dict[str, Any]]:
36
37
  with self._lock:
37
38
  return copy.deepcopy(self._wall)
38
39
 
39
- def load_failed(self) -> Dict[str, Dict[str, Any]]:
40
+ def load_failed(self) -> dict[str, dict[str, Any]]:
40
41
  with self._lock:
41
42
  return copy.deepcopy(self._failed)
42
43
 
43
- def load_cursors(self) -> Dict[str, str]:
44
+ def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
45
+ """读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
46
+ with self._lock:
47
+ return copy.deepcopy(self._failed_payloads)
48
+
49
+ def load_cursors(self) -> dict[str, str]:
44
50
  with self._lock:
45
51
  return dict(self._cursors)
46
52
 
47
- def load_queue(self) -> List[Dict[str, Any]]:
53
+ def load_queue(self) -> list[dict[str, Any]]:
48
54
  with self._lock:
49
55
  return copy.deepcopy(self._queue)
50
56
 
51
- def _dedup_copy(self, jobs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
57
+ def _dedup_copy(self, jobs: list[dict[str, Any]]) -> list[dict[str, Any]]:
52
58
  """整表替换行的单一出口(save_queue 与 replace_queue_atomic 共用)。
53
59
 
54
60
  替换集先经 ``validate_queue_replacement`` 校验(与 SQLite 腿同一出口、
@@ -69,27 +75,26 @@ class InMemoryStateBackend(AbstractStateBackend):
69
75
  clean.append(copy.deepcopy(j))
70
76
  return clean
71
77
 
72
- def save_queue(self, jobs: List[Dict[str, Any]]) -> None:
78
+ def save_queue(self, jobs: list[dict[str, Any]]) -> None:
73
79
  with self._lock:
74
80
  self._queue = self._dedup_copy(jobs)
75
81
 
76
82
  def replace_queue_atomic(
77
83
  self,
78
- compute: Callable[[List[Dict[str, Any]]], List[Dict[str, Any]]],
84
+ compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
79
85
  ) -> None:
80
86
  with self._lock:
81
87
  # 锁内读真相 → 纯计算 → 替换:compute 抛异常时 _queue 赋值
82
88
  # 未发生,队列保持调用前状态(对齐 SQLite 腿事务回滚)。
83
89
  self._queue = self._dedup_copy(compute(copy.deepcopy(self._queue)))
84
90
 
85
- def _build_dlq_meta(self, meta: Optional[dict], prev: Any) -> Dict[str, Any]:
91
+ def _build_dlq_meta(self, meta: dict | None, prev: Any) -> dict[str, Any]:
86
92
  """计算 DLQ 行终值(纯函数,不变更任何状态):_attempt 计数 + error_type + failed_at。
87
93
 
88
- 不变式:``_attempt`` 是写入事件计数而非逻辑失败次数;既有计数损坏
89
- (非 dict 记录或非 int 计数)时静默重置为 1 并照常提交,绝不因脏计数
90
- 抛 TypeError——与 SQLite 后端「损坏行重置计数、写入成功」行为对齐。
91
- ``merged`` 已显式携带 ``_attempt`` 时保留调用方值(计数由最先 DLQ
92
- 该 uid 的路径权威给定)。
94
+ 不变式:``_attempt`` 是写入事件计数而非逻辑失败次数;既有计数
95
+ 无效(非 dict 记录或非 int 计数)时静默重置为 1 并照常提交,绝不
96
+ 因脏计数抛 TypeError——与 SQLite 后端「损坏行重置计数、写入成功」
97
+ 行为对齐。
93
98
  """
94
99
  merged = copy.deepcopy(meta or {})
95
100
  if "error_type" not in merged:
@@ -102,11 +107,11 @@ class InMemoryStateBackend(AbstractStateBackend):
102
107
  prev_attempt = prev.get("_attempt")
103
108
  if isinstance(prev_attempt, int):
104
109
  merged["_attempt"] = prev_attempt + 1
105
- elif "_attempt" not in merged:
106
- merged["_attempt"] = 1
110
+ else:
111
+ merged["_attempt"] = 1 # 既有记录无有效计数:从 1 重新计数
107
112
  return merged
108
113
 
109
- def _write_dlq_entry(self, uid: str, meta: Optional[dict]) -> None:
114
+ def _write_dlq_entry(self, uid: str, meta: dict | None) -> None:
110
115
  """DLQ 写入单一出口:终值经 _build_dlq_meta 计算后落变。"""
111
116
  self._failed[uid] = self._build_dlq_meta(meta, self._failed.get(uid))
112
117
 
@@ -115,8 +120,8 @@ class InMemoryStateBackend(AbstractStateBackend):
115
120
  uid: str,
116
121
  result_meta: dict,
117
122
  *,
118
- spawned_jobs: List[Dict[str, Any]] = (),
119
- cursor_updates: Optional[Dict[str, str]] = None,
123
+ spawned_jobs: Sequence[dict[str, Any]] = (),
124
+ cursor_updates: Mapping[str, str | None] | None = None,
120
125
  ) -> bool:
121
126
  with self._lock:
122
127
  try:
@@ -143,7 +148,7 @@ class InMemoryStateBackend(AbstractStateBackend):
143
148
  )
144
149
  return False
145
150
  seen.add(suid)
146
- cursor_sets: Dict[str, str] = {}
151
+ cursor_sets: dict[str, str] = {}
147
152
  cursor_dels: set = set()
148
153
  if cursor_updates:
149
154
  for k, v in cursor_updates.items():
@@ -157,6 +162,7 @@ class InMemoryStateBackend(AbstractStateBackend):
157
162
  # 成功 commit 清理 failed 同名残行:与「job 最终状态唯一」
158
163
  # 语义一致,防 wall∩failed 并存污染 _attempt 计数。
159
164
  self._failed.pop(uid, None)
165
+ self._failed_payloads.pop(uid, None)
160
166
  self._cursors.update(cursor_sets)
161
167
  for k in cursor_dels:
162
168
  self._cursors.pop(k, None)
@@ -165,7 +171,9 @@ class InMemoryStateBackend(AbstractStateBackend):
165
171
  logger.critical(f"Failed to commit job success for {uid}: {e}")
166
172
  return False
167
173
 
168
- def commit_job_failure(self, uid: str, result_meta: dict) -> bool:
174
+ def commit_job_failure(
175
+ self, uid: str, result_meta: dict, job_payload: dict[str, Any] | None = None
176
+ ) -> bool:
169
177
  with self._lock:
170
178
  try:
171
179
  # 校验先行:DLQ 终值计算与队列 uid 提取全部在变更前完成,
@@ -173,6 +181,10 @@ class InMemoryStateBackend(AbstractStateBackend):
173
181
  new_meta = self._build_dlq_meta(result_meta, self._failed.get(uid))
174
182
  remaining = [j for j in self._queue if uid_from_job_dict(j) != uid]
175
183
  self._failed[uid] = new_meta
184
+ # payload 快照 latest-wins;None 表示本路径无 payload,保留既有
185
+ # 快照(与 SQLite 后端 _write_dlq_row 的保留语义对齐)。
186
+ if job_payload is not None:
187
+ self._failed_payloads[uid] = copy.deepcopy(job_payload)
176
188
  self._queue = remaining
177
189
  # 同一事务语义内清理 wall 旧记录:rerun 任务重跑失败时旧成功
178
190
  # 记录作废(最终状态唯一),防 wall∩failed 并存。
@@ -185,7 +197,7 @@ class InMemoryStateBackend(AbstractStateBackend):
185
197
  def commit_retry(
186
198
  self,
187
199
  popped_uid: str,
188
- requeued_job: Dict[str, Any],
200
+ requeued_job: dict[str, Any],
189
201
  *,
190
202
  front: bool = False,
191
203
  ) -> bool:
@@ -212,14 +224,14 @@ class InMemoryStateBackend(AbstractStateBackend):
212
224
  logger.critical(f"Failed to commit retry for {popped_uid}: {e}")
213
225
  return False
214
226
 
215
- def commit_bulk_failure(self, uids_metas: List[Tuple[str, dict]]) -> bool:
227
+ def commit_bulk_failure(self, uids_metas: list[tuple[str, dict]]) -> bool:
216
228
  with self._lock:
217
229
  try:
218
230
  # 校验先行:全部 DLQ 行计算在任何变更前完成——任一行失败则
219
231
  # queue/wall/failed 整体不变(对齐 SQLite 事务回滚,杜绝
220
232
  # 「队列已整体删除、DLQ 未落」的半成品失败态)。
221
- new_entries: List[Tuple[str, Dict[str, Any]]] = []
222
- overlay: Dict[str, Any] = {}
233
+ new_entries: list[tuple[str, dict[str, Any]]] = []
234
+ overlay: dict[str, Any] = {}
223
235
  for uid, meta in uids_metas:
224
236
  # 批内同 uid 多次出现时模拟 SQLite 同事务顺序写:
225
237
  # 后一行读取前一行结果,_attempt 连续递增。
@@ -240,7 +252,7 @@ class InMemoryStateBackend(AbstractStateBackend):
240
252
  logger.critical(f"Failed to commit bulk failure: {e}")
241
253
  return False
242
254
 
243
- def append_failed(self, uid: str, payload: Optional[Dict[str, Any]] = None) -> None:
255
+ def append_failed(self, uid: str, payload: dict[str, Any] | None = None) -> None:
244
256
  with self._lock:
245
257
  self._write_dlq_entry(uid, payload or {})
246
258
 
@@ -249,7 +261,7 @@ class InMemoryStateBackend(AbstractStateBackend):
249
261
  self._queue = [j for j in self._queue if uid_from_job_dict(j) != uid]
250
262
  return True
251
263
 
252
- def delete_queue_uids(self, uids: List[str]) -> int:
264
+ def delete_queue_uids(self, uids: list[str]) -> int:
253
265
  """按 uid 定向批量删除队列条目,与 SQLite 腿同语义:不触碰其余条目。"""
254
266
  if not uids:
255
267
  return 0
@@ -260,7 +272,7 @@ class InMemoryStateBackend(AbstractStateBackend):
260
272
  self._queue = kept
261
273
  return removed
262
274
 
263
- def get_meta(self, key: str) -> Optional[str]:
275
+ def get_meta(self, key: str) -> str | None:
264
276
  with self._lock:
265
277
  return self._meta.get(key)
266
278
 
@@ -268,12 +280,12 @@ class InMemoryStateBackend(AbstractStateBackend):
268
280
  with self._lock:
269
281
  self._meta[key] = str(value)
270
282
 
271
- def enqueue_jobs(self, jobs: List[Dict[str, Any]], *, front: bool = False) -> List[str]:
283
+ def enqueue_jobs(self, jobs: list[dict[str, Any]], *, front: bool = False) -> list[str]:
272
284
  if not jobs:
273
285
  return []
274
286
  with self._lock:
275
287
  existing = {uid_from_job_dict(j) for j in self._queue}
276
- fresh: List[Dict[str, Any]] = []
288
+ fresh: list[dict[str, Any]] = []
277
289
  batch_seen = set()
278
290
  for j in jobs:
279
291
  u = uid_from_job_dict(j)
@@ -289,17 +301,18 @@ class InMemoryStateBackend(AbstractStateBackend):
289
301
  self._queue.extend(fresh)
290
302
  return [uid_from_job_dict(j) for j in fresh]
291
303
 
292
- def delete_failed(self, uids: List[str]) -> int:
304
+ def delete_failed(self, uids: list[str]) -> int:
293
305
  with self._lock:
294
306
  del_set = set(uids)
295
307
  count = 0
296
308
  for u in del_set:
297
309
  if u in self._failed:
298
310
  del self._failed[u]
311
+ self._failed_payloads.pop(u, None)
299
312
  count += 1
300
313
  return count
301
314
 
302
- def delete_wall(self, uids: List[str]) -> int:
315
+ def delete_wall(self, uids: list[str]) -> int:
303
316
  with self._lock:
304
317
  del_set = set(uids)
305
318
  count = 0
@@ -309,7 +322,7 @@ class InMemoryStateBackend(AbstractStateBackend):
309
322
  count += 1
310
323
  return count
311
324
 
312
- def seed_wall(self, uids: List[str]) -> int:
325
+ def seed_wall(self, uids: list[str]) -> int:
313
326
  """把 uid 批量写入 wall(meta 空 dict)。
314
327
 
315
328
  不变式:wall/failed 全局互斥——已在 failed 的 uid 拒绝种子;