tasklite-engine 1.3.1__tar.gz → 1.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tasklite_engine-1.3.1/tasklite_engine.egg-info → tasklite_engine-1.4.1}/PKG-INFO +6 -6
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/README.md +3 -3
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/pyproject.toml +14 -3
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/__init__.py +8 -3
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/base.py +26 -18
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/memory.py +52 -39
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/sqlite_backend.py +74 -33
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/__init__.py +1 -1
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/channel.py +55 -53
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/completion.py +15 -16
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/config.py +25 -25
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/console.py +27 -11
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/dispatch.py +10 -10
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/governor.py +21 -27
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/inflight.py +20 -56
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/policy.py +21 -36
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/recovery.py +2 -14
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/resource.py +41 -44
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/runtime.py +14 -39
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/scheduler.py +34 -29
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/session.py +9 -9
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/store.py +97 -67
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/types.py +5 -5
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/exceptions.py +0 -16
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/context.py +26 -27
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/job.py +18 -17
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/state.py +32 -32
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/pipeline.py +56 -50
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/taxonomy.py +36 -57
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/testing.py +15 -14
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/injective.py +8 -8
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/ipc.py +40 -41
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/lockfile.py +4 -4
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/wrappers/discovery.py +19 -18
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/wrappers/http.py +10 -9
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1/tasklite_engine.egg-info}/PKG-INFO +6 -6
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/requires.txt +1 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tests/test_packaging.py +1 -1
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/LICENSE +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/setup.cfg +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/backend/__init__.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/contrib/__init__.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/engine/pacing.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/hooks.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/models/__init__.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/py.typed +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/__init__.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/utils/jsonutil.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite/wrappers/__init__.py +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/SOURCES.txt +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/dependency_links.txt +0 -0
- {tasklite_engine-1.3.1 → tasklite_engine-1.4.1}/tasklite_engine.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasklite-engine
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.1
|
|
4
4
|
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
@@ -9,14 +9,13 @@ Keywords: task-queue,task-orchestration,background-jobs,subprocess,process-isola
|
|
|
9
9
|
Classifier: Intended Audience :: Developers
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.14
|
|
18
17
|
Classifier: Typing :: Typed
|
|
19
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.10
|
|
20
19
|
Description-Content-Type: text/markdown
|
|
21
20
|
License-File: LICENSE
|
|
22
21
|
Provides-Extra: dev
|
|
@@ -25,6 +24,7 @@ Requires-Dist: hypothesis>=6.0; extra == "dev"
|
|
|
25
24
|
Requires-Dist: mutmut>=3.0; extra == "dev"
|
|
26
25
|
Requires-Dist: pytest-timeout>=2.0; extra == "dev"
|
|
27
26
|
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
27
|
+
Requires-Dist: requests>=2.28; extra == "dev"
|
|
28
28
|
Dynamic: license-file
|
|
29
29
|
|
|
30
30
|
# ⚡ TaskLite — 零依赖、进程隔离的轻量任务编排引擎
|
|
@@ -33,7 +33,7 @@ Dynamic: license-file
|
|
|
33
33
|
> 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
|
|
34
34
|
> 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用类似 **systemd** 的守护与物理进程隔离架构,内置 SQLite WAL 事务持久化,子进程崩溃或断电不坏库,重启自动断点续跑。
|
|
35
35
|
|
|
36
|
-
[](https://www.python.org/)
|
|
37
37
|
[](LICENSE)
|
|
38
38
|
[]()
|
|
39
39
|
[-orange.svg)]()
|
|
@@ -151,7 +151,7 @@ flowchart TD
|
|
|
151
151
|
|
|
152
152
|
### 安装
|
|
153
153
|
|
|
154
|
-
要求 **Python ≥ 3.
|
|
154
|
+
要求 **Python ≥ 3.10**,零外部依赖:
|
|
155
155
|
|
|
156
156
|
```bash
|
|
157
157
|
pip install tasklite-engine
|
|
@@ -356,7 +356,7 @@ tasklite/
|
|
|
356
356
|
│ ├── console.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
|
|
357
357
|
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
358
358
|
│ ├── completion.py # 任务完成与提交 (CompletionMachine)
|
|
359
|
-
│ ├── recovery.py # 崩溃检测与恢复 (
|
|
359
|
+
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
|
|
360
360
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
361
361
|
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
|
|
362
362
|
│ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
> 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
|
|
5
5
|
> 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用类似 **systemd** 的守护与物理进程隔离架构,内置 SQLite WAL 事务持久化,子进程崩溃或断电不坏库,重启自动断点续跑。
|
|
6
6
|
|
|
7
|
-
[](https://www.python.org/)
|
|
8
8
|
[](LICENSE)
|
|
9
9
|
[]()
|
|
10
10
|
[-orange.svg)]()
|
|
@@ -122,7 +122,7 @@ flowchart TD
|
|
|
122
122
|
|
|
123
123
|
### 安装
|
|
124
124
|
|
|
125
|
-
要求 **Python ≥ 3.
|
|
125
|
+
要求 **Python ≥ 3.10**,零外部依赖:
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
128
|
pip install tasklite-engine
|
|
@@ -327,7 +327,7 @@ tasklite/
|
|
|
327
327
|
│ ├── console.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
|
|
328
328
|
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
329
329
|
│ ├── completion.py # 任务完成与提交 (CompletionMachine)
|
|
330
|
-
│ ├── recovery.py # 崩溃检测与恢复 (
|
|
330
|
+
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
|
|
331
331
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
332
332
|
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler)
|
|
333
333
|
│ ├── policy.py # 准入预检、重试退避与 rerun 决策 (ExecutionPolicy)
|
|
@@ -7,7 +7,7 @@ name = "tasklite-engine"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
|
|
9
9
|
description = "A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
11
|
license = "MIT"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
|
|
@@ -17,7 +17,6 @@ classifiers = [
|
|
|
17
17
|
"Intended Audience :: Developers",
|
|
18
18
|
"Operating System :: OS Independent",
|
|
19
19
|
"Programming Language :: Python :: 3",
|
|
20
|
-
"Programming Language :: Python :: 3.9",
|
|
21
20
|
"Programming Language :: Python :: 3.10",
|
|
22
21
|
"Programming Language :: Python :: 3.11",
|
|
23
22
|
"Programming Language :: Python :: 3.12",
|
|
@@ -31,7 +30,7 @@ Homepage = "https://github.com/CaptainLi321/tasklite"
|
|
|
31
30
|
Repository = "https://github.com/CaptainLi321/tasklite"
|
|
32
31
|
|
|
33
32
|
[project.optional-dependencies]
|
|
34
|
-
dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0"]
|
|
33
|
+
dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0", "requests>=2.28"]
|
|
35
34
|
|
|
36
35
|
[tool.setuptools.packages.find]
|
|
37
36
|
include = ["tasklite", "tasklite.*"]
|
|
@@ -53,6 +52,18 @@ addopts = "--strict-markers"
|
|
|
53
52
|
timeout = 60
|
|
54
53
|
norecursedirs = ["mutants", ".git", ".hypothesis", "__pycache__"]
|
|
55
54
|
|
|
55
|
+
[tool.mypy]
|
|
56
|
+
python_version = "3.10"
|
|
57
|
+
files = ["tasklite"]
|
|
58
|
+
check_untyped_defs = true
|
|
59
|
+
no_implicit_optional = true
|
|
60
|
+
warn_redundant_casts = true
|
|
61
|
+
warn_unused_ignores = true
|
|
62
|
+
|
|
63
|
+
[[tool.mypy.overrides]]
|
|
64
|
+
module = "requests"
|
|
65
|
+
ignore_missing_imports = true
|
|
66
|
+
|
|
56
67
|
[tool.coverage.run]
|
|
57
68
|
source = ["tasklite"]
|
|
58
69
|
omit = ["*/tests/*"]
|
|
@@ -4,7 +4,7 @@ A lightweight, battle-hardened task orchestration engine with zero external
|
|
|
4
4
|
dependencies, process isolation, and ACID persistence.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "1.
|
|
7
|
+
__version__ = "1.4.1"
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
# Public API - Core
|
|
@@ -50,7 +50,6 @@ from .taxonomy import (
|
|
|
50
50
|
ErrorCategory,
|
|
51
51
|
ErrorClassification,
|
|
52
52
|
ErrorTaxonomy,
|
|
53
|
-
TransientRegistry,
|
|
54
53
|
ValidationErrorItem,
|
|
55
54
|
ValidationResult,
|
|
56
55
|
classify_error_type,
|
|
@@ -66,6 +65,10 @@ from . import contrib
|
|
|
66
65
|
# Public API - Resources
|
|
67
66
|
from .engine.resource import CapacityResource, RateLimitResource, Resource
|
|
68
67
|
|
|
68
|
+
# Public API - Handler 注册表与 JSON 序列化
|
|
69
|
+
from .engine.types import HandlerEntry
|
|
70
|
+
from .utils import jsonutil
|
|
71
|
+
|
|
69
72
|
__all__ = [
|
|
70
73
|
# Core
|
|
71
74
|
"TaskLite",
|
|
@@ -78,7 +81,6 @@ __all__ = [
|
|
|
78
81
|
"RateLimitHit",
|
|
79
82
|
"FATAL_EXCEPTIONS",
|
|
80
83
|
"TRANSIENT_EXCEPTIONS",
|
|
81
|
-
"TransientRegistry",
|
|
82
84
|
"classify_exception",
|
|
83
85
|
"is_transient_exception",
|
|
84
86
|
"WORKER_RESOURCE",
|
|
@@ -121,6 +123,9 @@ __all__ = [
|
|
|
121
123
|
"sanitize_job_component",
|
|
122
124
|
"sanitize_content_id",
|
|
123
125
|
"safe_uid_filename",
|
|
126
|
+
# Handler 注册表与 JSON 序列化
|
|
127
|
+
"HandlerEntry",
|
|
128
|
+
"jsonutil",
|
|
124
129
|
# Contrib
|
|
125
130
|
"contrib",
|
|
126
131
|
# Resources
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
from abc import ABC, abstractmethod
|
|
5
|
-
from typing import Any, Callable,
|
|
5
|
+
from typing import Any, Callable, Mapping, Sequence
|
|
6
6
|
|
|
7
7
|
from ..taxonomy import classify_error_type
|
|
8
8
|
|
|
@@ -39,19 +39,23 @@ class AbstractStateBackend(ABC):
|
|
|
39
39
|
"""
|
|
40
40
|
|
|
41
41
|
@abstractmethod
|
|
42
|
-
def load_wall(self) ->
|
|
42
|
+
def load_wall(self) -> dict[str, dict[str, Any]]: ...
|
|
43
43
|
|
|
44
44
|
@abstractmethod
|
|
45
|
-
def load_failed(self) ->
|
|
45
|
+
def load_failed(self) -> dict[str, dict[str, Any]]: ...
|
|
46
46
|
|
|
47
47
|
@abstractmethod
|
|
48
|
-
def
|
|
48
|
+
def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
|
|
49
|
+
"""读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
|
|
49
50
|
|
|
50
51
|
@abstractmethod
|
|
51
|
-
def
|
|
52
|
+
def load_cursors(self) -> dict[str, str]: ...
|
|
52
53
|
|
|
53
54
|
@abstractmethod
|
|
54
|
-
def
|
|
55
|
+
def load_queue(self) -> list[dict[str, Any]]: ...
|
|
56
|
+
|
|
57
|
+
@abstractmethod
|
|
58
|
+
def save_queue(self, jobs: list[dict[str, Any]]) -> None:
|
|
55
59
|
"""整表重写队列(测试装配 / replace_queue_atomic 的事务内步骤)。
|
|
56
60
|
|
|
57
61
|
红线:这是「无读基准的整表覆盖」——跨进程并发的 enqueue/commit
|
|
@@ -67,7 +71,7 @@ class AbstractStateBackend(ABC):
|
|
|
67
71
|
@abstractmethod
|
|
68
72
|
def replace_queue_atomic(
|
|
69
73
|
self,
|
|
70
|
-
compute: Callable[[
|
|
74
|
+
compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
|
|
71
75
|
) -> None:
|
|
72
76
|
"""读-改-写收敛的整表替换:磁盘真相读取与写回在单个写事务内完成。
|
|
73
77
|
|
|
@@ -90,8 +94,8 @@ class AbstractStateBackend(ABC):
|
|
|
90
94
|
uid: str,
|
|
91
95
|
result_meta: dict,
|
|
92
96
|
*,
|
|
93
|
-
spawned_jobs:
|
|
94
|
-
cursor_updates:
|
|
97
|
+
spawned_jobs: Sequence[dict[str, Any]] = (),
|
|
98
|
+
cursor_updates: Mapping[str, str | None] | None = None,
|
|
95
99
|
) -> bool:
|
|
96
100
|
"""Persist a successful job as a delta, atomically.
|
|
97
101
|
|
|
@@ -113,10 +117,14 @@ class AbstractStateBackend(ABC):
|
|
|
113
117
|
self,
|
|
114
118
|
uid: str,
|
|
115
119
|
result_meta: dict,
|
|
120
|
+
job_payload: dict[str, Any] | None = None,
|
|
116
121
|
) -> bool:
|
|
117
122
|
"""Persist a failed job to the DLQ as a delta, atomically.
|
|
118
123
|
|
|
119
124
|
Effects: failed_dlq INSERT/REPLACE uid -> result_meta; queue DELETE uid.
|
|
125
|
+
``job_payload`` is the original business payload snapshot stored
|
|
126
|
+
alongside the failure meta (for retry tooling); None keeps any
|
|
127
|
+
previously stored snapshot intact.
|
|
120
128
|
Returns True on success. Returns False if persistence failed — on
|
|
121
129
|
failure the backend MUST leave the on-disk queue unchanged (the popped
|
|
122
130
|
uid remains on disk); the caller is responsible for deciding whether to
|
|
@@ -128,7 +136,7 @@ class AbstractStateBackend(ABC):
|
|
|
128
136
|
def commit_retry(
|
|
129
137
|
self,
|
|
130
138
|
popped_uid: str,
|
|
131
|
-
requeued_job:
|
|
139
|
+
requeued_job: dict[str, Any],
|
|
132
140
|
*,
|
|
133
141
|
front: bool = False,
|
|
134
142
|
) -> bool:
|
|
@@ -142,7 +150,7 @@ class AbstractStateBackend(ABC):
|
|
|
142
150
|
@abstractmethod
|
|
143
151
|
def commit_bulk_failure(
|
|
144
152
|
self,
|
|
145
|
-
uids_metas:
|
|
153
|
+
uids_metas: list[tuple[str, dict]],
|
|
146
154
|
) -> bool:
|
|
147
155
|
"""Best-effort bulk mark-failed (deadlock case), as a delta.
|
|
148
156
|
|
|
@@ -154,7 +162,7 @@ class AbstractStateBackend(ABC):
|
|
|
154
162
|
"""
|
|
155
163
|
|
|
156
164
|
@abstractmethod
|
|
157
|
-
def append_failed(self, uid: str, payload:
|
|
165
|
+
def append_failed(self, uid: str, payload: dict[str, Any] | None = None) -> None:
|
|
158
166
|
"""Append a record to the DLQ (failed log) without touching the queue.
|
|
159
167
|
|
|
160
168
|
Used for out-of-band DLQ appends (e.g. user-driven inspection).
|
|
@@ -172,7 +180,7 @@ class AbstractStateBackend(ABC):
|
|
|
172
180
|
"""
|
|
173
181
|
|
|
174
182
|
@abstractmethod
|
|
175
|
-
def delete_queue_uids(self, uids:
|
|
183
|
+
def delete_queue_uids(self, uids: list[str]) -> int:
|
|
176
184
|
"""按 uid 定向批量删除队列行(加载期 repair 的差量落盘唯一出口)。
|
|
177
185
|
|
|
178
186
|
与 save_queue 的全表重写相对:只 DELETE 指定 uid 的行,其余行原样
|
|
@@ -183,7 +191,7 @@ class AbstractStateBackend(ABC):
|
|
|
183
191
|
"""
|
|
184
192
|
|
|
185
193
|
@abstractmethod
|
|
186
|
-
def get_meta(self, key: str) ->
|
|
194
|
+
def get_meta(self, key: str) -> str | None:
|
|
187
195
|
"""读取一条框架级元数据(如 fencing 的 last_run_id)。无则返回 None。"""
|
|
188
196
|
|
|
189
197
|
@abstractmethod
|
|
@@ -191,7 +199,7 @@ class AbstractStateBackend(ABC):
|
|
|
191
199
|
"""写入一条框架级元数据(UPSERT 语义)。失败抛异常(不吞)。"""
|
|
192
200
|
|
|
193
201
|
@abstractmethod
|
|
194
|
-
def enqueue_jobs(self, jobs:
|
|
202
|
+
def enqueue_jobs(self, jobs: list[dict[str, Any]], *, front: bool = False) -> list[str]:
|
|
195
203
|
"""批量增量入队(enqueue 与 run 并发时不做全量覆盖)。
|
|
196
204
|
|
|
197
205
|
Effects: 按 front 在队列头/尾插入 jobs(保序),**单事务原子**。
|
|
@@ -205,7 +213,7 @@ class AbstractStateBackend(ABC):
|
|
|
205
213
|
"""
|
|
206
214
|
|
|
207
215
|
@abstractmethod
|
|
208
|
-
def delete_failed(self, uids:
|
|
216
|
+
def delete_failed(self, uids: list[str]) -> int:
|
|
209
217
|
"""从 DLQ 批量删除指定 uid(clear_dlq/clear_history 的后端实现)。
|
|
210
218
|
|
|
211
219
|
返回实际删除行数。不触碰 queue/wall。仅限 run() 之外调用
|
|
@@ -213,7 +221,7 @@ class AbstractStateBackend(ABC):
|
|
|
213
221
|
"""
|
|
214
222
|
|
|
215
223
|
@abstractmethod
|
|
216
|
-
def delete_wall(self, uids:
|
|
224
|
+
def delete_wall(self, uids: list[str]) -> int:
|
|
217
225
|
"""从 wall 批量删除指定 uid(clear_history 的后端实现 + commit 路径事务内清理)。返回实际删除行数。
|
|
218
226
|
|
|
219
227
|
两种调用上下文:
|
|
@@ -227,7 +235,7 @@ class AbstractStateBackend(ABC):
|
|
|
227
235
|
"""
|
|
228
236
|
|
|
229
237
|
@abstractmethod
|
|
230
|
-
def seed_wall(self, uids:
|
|
238
|
+
def seed_wall(self, uids: list[str]) -> int:
|
|
231
239
|
"""把 uid 批量写入 wall(meta 为空 dict)——存档迁移标记「已处理」。
|
|
232
240
|
|
|
233
241
|
用途:媒体/数据资产项目的存档迁移(硬链接 + wall 种子),
|
|
@@ -9,7 +9,7 @@ import copy
|
|
|
9
9
|
import logging
|
|
10
10
|
import threading
|
|
11
11
|
from datetime import datetime, timezone
|
|
12
|
-
from typing import Any, Callable,
|
|
12
|
+
from typing import Any, Callable, Mapping, Sequence
|
|
13
13
|
|
|
14
14
|
from .base import (
|
|
15
15
|
AbstractStateBackend,
|
|
@@ -26,29 +26,35 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
26
26
|
|
|
27
27
|
def __init__(self) -> None:
|
|
28
28
|
self._lock = threading.RLock()
|
|
29
|
-
self._wall:
|
|
30
|
-
self._failed:
|
|
31
|
-
self.
|
|
32
|
-
self.
|
|
33
|
-
self.
|
|
34
|
-
|
|
35
|
-
|
|
29
|
+
self._wall: dict[str, dict[str, Any]] = {}
|
|
30
|
+
self._failed: dict[str, dict[str, Any]] = {}
|
|
31
|
+
self._failed_payloads: dict[str, dict[str, Any]] = {}
|
|
32
|
+
self._cursors: dict[str, str] = {}
|
|
33
|
+
self._queue: list[dict[str, Any]] = []
|
|
34
|
+
self._meta: dict[str, str] = {}
|
|
35
|
+
|
|
36
|
+
def load_wall(self) -> dict[str, dict[str, Any]]:
|
|
36
37
|
with self._lock:
|
|
37
38
|
return copy.deepcopy(self._wall)
|
|
38
39
|
|
|
39
|
-
def load_failed(self) ->
|
|
40
|
+
def load_failed(self) -> dict[str, dict[str, Any]]:
|
|
40
41
|
with self._lock:
|
|
41
42
|
return copy.deepcopy(self._failed)
|
|
42
43
|
|
|
43
|
-
def
|
|
44
|
+
def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
|
|
45
|
+
"""读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
|
|
46
|
+
with self._lock:
|
|
47
|
+
return copy.deepcopy(self._failed_payloads)
|
|
48
|
+
|
|
49
|
+
def load_cursors(self) -> dict[str, str]:
|
|
44
50
|
with self._lock:
|
|
45
51
|
return dict(self._cursors)
|
|
46
52
|
|
|
47
|
-
def load_queue(self) ->
|
|
53
|
+
def load_queue(self) -> list[dict[str, Any]]:
|
|
48
54
|
with self._lock:
|
|
49
55
|
return copy.deepcopy(self._queue)
|
|
50
56
|
|
|
51
|
-
def _dedup_copy(self, jobs:
|
|
57
|
+
def _dedup_copy(self, jobs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
52
58
|
"""整表替换行的单一出口(save_queue 与 replace_queue_atomic 共用)。
|
|
53
59
|
|
|
54
60
|
替换集先经 ``validate_queue_replacement`` 校验(与 SQLite 腿同一出口、
|
|
@@ -69,27 +75,26 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
69
75
|
clean.append(copy.deepcopy(j))
|
|
70
76
|
return clean
|
|
71
77
|
|
|
72
|
-
def save_queue(self, jobs:
|
|
78
|
+
def save_queue(self, jobs: list[dict[str, Any]]) -> None:
|
|
73
79
|
with self._lock:
|
|
74
80
|
self._queue = self._dedup_copy(jobs)
|
|
75
81
|
|
|
76
82
|
def replace_queue_atomic(
|
|
77
83
|
self,
|
|
78
|
-
compute: Callable[[
|
|
84
|
+
compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
|
|
79
85
|
) -> None:
|
|
80
86
|
with self._lock:
|
|
81
87
|
# 锁内读真相 → 纯计算 → 替换:compute 抛异常时 _queue 赋值
|
|
82
88
|
# 未发生,队列保持调用前状态(对齐 SQLite 腿事务回滚)。
|
|
83
89
|
self._queue = self._dedup_copy(compute(copy.deepcopy(self._queue)))
|
|
84
90
|
|
|
85
|
-
def _build_dlq_meta(self, meta:
|
|
91
|
+
def _build_dlq_meta(self, meta: dict | None, prev: Any) -> dict[str, Any]:
|
|
86
92
|
"""计算 DLQ 行终值(纯函数,不变更任何状态):_attempt 计数 + error_type + failed_at。
|
|
87
93
|
|
|
88
|
-
不变式:``_attempt``
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
该 uid 的路径权威给定)。
|
|
94
|
+
不变式:``_attempt`` 是写入事件计数而非逻辑失败次数;既有计数
|
|
95
|
+
无效(非 dict 记录或非 int 计数)时静默重置为 1 并照常提交,绝不
|
|
96
|
+
因脏计数抛 TypeError——与 SQLite 后端「损坏行重置计数、写入成功」
|
|
97
|
+
行为对齐。
|
|
93
98
|
"""
|
|
94
99
|
merged = copy.deepcopy(meta or {})
|
|
95
100
|
if "error_type" not in merged:
|
|
@@ -102,11 +107,11 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
102
107
|
prev_attempt = prev.get("_attempt")
|
|
103
108
|
if isinstance(prev_attempt, int):
|
|
104
109
|
merged["_attempt"] = prev_attempt + 1
|
|
105
|
-
|
|
106
|
-
merged["_attempt"] = 1
|
|
110
|
+
else:
|
|
111
|
+
merged["_attempt"] = 1 # 既有记录无有效计数:从 1 重新计数
|
|
107
112
|
return merged
|
|
108
113
|
|
|
109
|
-
def _write_dlq_entry(self, uid: str, meta:
|
|
114
|
+
def _write_dlq_entry(self, uid: str, meta: dict | None) -> None:
|
|
110
115
|
"""DLQ 写入单一出口:终值经 _build_dlq_meta 计算后落变。"""
|
|
111
116
|
self._failed[uid] = self._build_dlq_meta(meta, self._failed.get(uid))
|
|
112
117
|
|
|
@@ -115,8 +120,8 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
115
120
|
uid: str,
|
|
116
121
|
result_meta: dict,
|
|
117
122
|
*,
|
|
118
|
-
spawned_jobs:
|
|
119
|
-
cursor_updates:
|
|
123
|
+
spawned_jobs: Sequence[dict[str, Any]] = (),
|
|
124
|
+
cursor_updates: Mapping[str, str | None] | None = None,
|
|
120
125
|
) -> bool:
|
|
121
126
|
with self._lock:
|
|
122
127
|
try:
|
|
@@ -143,7 +148,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
143
148
|
)
|
|
144
149
|
return False
|
|
145
150
|
seen.add(suid)
|
|
146
|
-
cursor_sets:
|
|
151
|
+
cursor_sets: dict[str, str] = {}
|
|
147
152
|
cursor_dels: set = set()
|
|
148
153
|
if cursor_updates:
|
|
149
154
|
for k, v in cursor_updates.items():
|
|
@@ -157,6 +162,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
157
162
|
# 成功 commit 清理 failed 同名残行:与「job 最终状态唯一」
|
|
158
163
|
# 语义一致,防 wall∩failed 并存污染 _attempt 计数。
|
|
159
164
|
self._failed.pop(uid, None)
|
|
165
|
+
self._failed_payloads.pop(uid, None)
|
|
160
166
|
self._cursors.update(cursor_sets)
|
|
161
167
|
for k in cursor_dels:
|
|
162
168
|
self._cursors.pop(k, None)
|
|
@@ -165,7 +171,9 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
165
171
|
logger.critical(f"Failed to commit job success for {uid}: {e}")
|
|
166
172
|
return False
|
|
167
173
|
|
|
168
|
-
def commit_job_failure(
|
|
174
|
+
def commit_job_failure(
|
|
175
|
+
self, uid: str, result_meta: dict, job_payload: dict[str, Any] | None = None
|
|
176
|
+
) -> bool:
|
|
169
177
|
with self._lock:
|
|
170
178
|
try:
|
|
171
179
|
# 校验先行:DLQ 终值计算与队列 uid 提取全部在变更前完成,
|
|
@@ -173,6 +181,10 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
173
181
|
new_meta = self._build_dlq_meta(result_meta, self._failed.get(uid))
|
|
174
182
|
remaining = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
175
183
|
self._failed[uid] = new_meta
|
|
184
|
+
# payload 快照 latest-wins;None 表示本路径无 payload,保留既有
|
|
185
|
+
# 快照(与 SQLite 后端 _write_dlq_row 的保留语义对齐)。
|
|
186
|
+
if job_payload is not None:
|
|
187
|
+
self._failed_payloads[uid] = copy.deepcopy(job_payload)
|
|
176
188
|
self._queue = remaining
|
|
177
189
|
# 同一事务语义内清理 wall 旧记录:rerun 任务重跑失败时旧成功
|
|
178
190
|
# 记录作废(最终状态唯一),防 wall∩failed 并存。
|
|
@@ -185,7 +197,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
185
197
|
def commit_retry(
|
|
186
198
|
self,
|
|
187
199
|
popped_uid: str,
|
|
188
|
-
requeued_job:
|
|
200
|
+
requeued_job: dict[str, Any],
|
|
189
201
|
*,
|
|
190
202
|
front: bool = False,
|
|
191
203
|
) -> bool:
|
|
@@ -212,14 +224,14 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
212
224
|
logger.critical(f"Failed to commit retry for {popped_uid}: {e}")
|
|
213
225
|
return False
|
|
214
226
|
|
|
215
|
-
def commit_bulk_failure(self, uids_metas:
|
|
227
|
+
def commit_bulk_failure(self, uids_metas: list[tuple[str, dict]]) -> bool:
|
|
216
228
|
with self._lock:
|
|
217
229
|
try:
|
|
218
230
|
# 校验先行:全部 DLQ 行计算在任何变更前完成——任一行失败则
|
|
219
231
|
# queue/wall/failed 整体不变(对齐 SQLite 事务回滚,杜绝
|
|
220
232
|
# 「队列已整体删除、DLQ 未落」的半成品失败态)。
|
|
221
|
-
new_entries:
|
|
222
|
-
overlay:
|
|
233
|
+
new_entries: list[tuple[str, dict[str, Any]]] = []
|
|
234
|
+
overlay: dict[str, Any] = {}
|
|
223
235
|
for uid, meta in uids_metas:
|
|
224
236
|
# 批内同 uid 多次出现时模拟 SQLite 同事务顺序写:
|
|
225
237
|
# 后一行读取前一行结果,_attempt 连续递增。
|
|
@@ -240,7 +252,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
240
252
|
logger.critical(f"Failed to commit bulk failure: {e}")
|
|
241
253
|
return False
|
|
242
254
|
|
|
243
|
-
def append_failed(self, uid: str, payload:
|
|
255
|
+
def append_failed(self, uid: str, payload: dict[str, Any] | None = None) -> None:
|
|
244
256
|
with self._lock:
|
|
245
257
|
self._write_dlq_entry(uid, payload or {})
|
|
246
258
|
|
|
@@ -249,7 +261,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
249
261
|
self._queue = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
250
262
|
return True
|
|
251
263
|
|
|
252
|
-
def delete_queue_uids(self, uids:
|
|
264
|
+
def delete_queue_uids(self, uids: list[str]) -> int:
|
|
253
265
|
"""按 uid 定向批量删除队列条目,与 SQLite 腿同语义:不触碰其余条目。"""
|
|
254
266
|
if not uids:
|
|
255
267
|
return 0
|
|
@@ -260,7 +272,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
260
272
|
self._queue = kept
|
|
261
273
|
return removed
|
|
262
274
|
|
|
263
|
-
def get_meta(self, key: str) ->
|
|
275
|
+
def get_meta(self, key: str) -> str | None:
|
|
264
276
|
with self._lock:
|
|
265
277
|
return self._meta.get(key)
|
|
266
278
|
|
|
@@ -268,12 +280,12 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
268
280
|
with self._lock:
|
|
269
281
|
self._meta[key] = str(value)
|
|
270
282
|
|
|
271
|
-
def enqueue_jobs(self, jobs:
|
|
283
|
+
def enqueue_jobs(self, jobs: list[dict[str, Any]], *, front: bool = False) -> list[str]:
|
|
272
284
|
if not jobs:
|
|
273
285
|
return []
|
|
274
286
|
with self._lock:
|
|
275
287
|
existing = {uid_from_job_dict(j) for j in self._queue}
|
|
276
|
-
fresh:
|
|
288
|
+
fresh: list[dict[str, Any]] = []
|
|
277
289
|
batch_seen = set()
|
|
278
290
|
for j in jobs:
|
|
279
291
|
u = uid_from_job_dict(j)
|
|
@@ -289,17 +301,18 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
289
301
|
self._queue.extend(fresh)
|
|
290
302
|
return [uid_from_job_dict(j) for j in fresh]
|
|
291
303
|
|
|
292
|
-
def delete_failed(self, uids:
|
|
304
|
+
def delete_failed(self, uids: list[str]) -> int:
|
|
293
305
|
with self._lock:
|
|
294
306
|
del_set = set(uids)
|
|
295
307
|
count = 0
|
|
296
308
|
for u in del_set:
|
|
297
309
|
if u in self._failed:
|
|
298
310
|
del self._failed[u]
|
|
311
|
+
self._failed_payloads.pop(u, None)
|
|
299
312
|
count += 1
|
|
300
313
|
return count
|
|
301
314
|
|
|
302
|
-
def delete_wall(self, uids:
|
|
315
|
+
def delete_wall(self, uids: list[str]) -> int:
|
|
303
316
|
with self._lock:
|
|
304
317
|
del_set = set(uids)
|
|
305
318
|
count = 0
|
|
@@ -309,7 +322,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
309
322
|
count += 1
|
|
310
323
|
return count
|
|
311
324
|
|
|
312
|
-
def seed_wall(self, uids:
|
|
325
|
+
def seed_wall(self, uids: list[str]) -> int:
|
|
313
326
|
"""把 uid 批量写入 wall(meta 空 dict)。
|
|
314
327
|
|
|
315
328
|
不变式:wall/failed 全局互斥——已在 failed 的 uid 拒绝种子;
|