tasklite-engine 1.4.0__tar.gz → 1.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tasklite_engine-1.4.0/tasklite_engine.egg-info → tasklite_engine-1.4.2}/PKG-INFO +2 -1
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/pyproject.toml +1 -1
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/__init__.py +8 -1
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/backend/base.py +8 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/backend/memory.py +15 -1
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/backend/sqlite_backend.py +50 -8
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/console.py +20 -4
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/store.py +34 -4
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/pipeline.py +7 -2
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/wrappers/http.py +1 -1
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2/tasklite_engine.egg-info}/PKG-INFO +2 -1
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite_engine.egg-info/requires.txt +1 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/LICENSE +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/README.md +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/setup.cfg +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/backend/__init__.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/contrib/__init__.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/__init__.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/channel.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/completion.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/config.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/dispatch.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/governor.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/inflight.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/pacing.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/policy.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/recovery.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/resource.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/runtime.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/scheduler.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/session.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/engine/types.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/exceptions.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/hooks.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/models/__init__.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/models/context.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/models/job.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/models/state.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/py.typed +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/taxonomy.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/testing.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/utils/__init__.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/utils/injective.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/utils/ipc.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/utils/jsonutil.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/utils/lockfile.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/wrappers/__init__.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite/wrappers/discovery.py +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite_engine.egg-info/SOURCES.txt +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite_engine.egg-info/dependency_links.txt +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite_engine.egg-info/top_level.txt +0 -0
- {tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tests/test_packaging.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasklite-engine
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.2
|
|
4
4
|
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
@@ -24,6 +24,7 @@ Requires-Dist: hypothesis>=6.0; extra == "dev"
|
|
|
24
24
|
Requires-Dist: mutmut>=3.0; extra == "dev"
|
|
25
25
|
Requires-Dist: pytest-timeout>=2.0; extra == "dev"
|
|
26
26
|
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
27
|
+
Requires-Dist: requests>=2.28; extra == "dev"
|
|
27
28
|
Dynamic: license-file
|
|
28
29
|
|
|
29
30
|
# ⚡ TaskLite — 零依赖、进程隔离的轻量任务编排引擎
|
|
@@ -30,7 +30,7 @@ Homepage = "https://github.com/CaptainLi321/tasklite"
|
|
|
30
30
|
Repository = "https://github.com/CaptainLi321/tasklite"
|
|
31
31
|
|
|
32
32
|
[project.optional-dependencies]
|
|
33
|
-
dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0"]
|
|
33
|
+
dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0", "requests>=2.28"]
|
|
34
34
|
|
|
35
35
|
[tool.setuptools.packages.find]
|
|
36
36
|
include = ["tasklite", "tasklite.*"]
|
|
@@ -4,7 +4,7 @@ A lightweight, battle-hardened task orchestration engine with zero external
|
|
|
4
4
|
dependencies, process isolation, and ACID persistence.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "1.4.
|
|
7
|
+
__version__ = "1.4.2"
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
# Public API - Core
|
|
@@ -65,6 +65,10 @@ from . import contrib
|
|
|
65
65
|
# Public API - Resources
|
|
66
66
|
from .engine.resource import CapacityResource, RateLimitResource, Resource
|
|
67
67
|
|
|
68
|
+
# Public API - Handler 注册表与 JSON 序列化
|
|
69
|
+
from .engine.types import HandlerEntry
|
|
70
|
+
from .utils import jsonutil
|
|
71
|
+
|
|
68
72
|
__all__ = [
|
|
69
73
|
# Core
|
|
70
74
|
"TaskLite",
|
|
@@ -119,6 +123,9 @@ __all__ = [
|
|
|
119
123
|
"sanitize_job_component",
|
|
120
124
|
"sanitize_content_id",
|
|
121
125
|
"safe_uid_filename",
|
|
126
|
+
# Handler 注册表与 JSON 序列化
|
|
127
|
+
"HandlerEntry",
|
|
128
|
+
"jsonutil",
|
|
122
129
|
# Contrib
|
|
123
130
|
"contrib",
|
|
124
131
|
# Resources
|
|
@@ -44,6 +44,10 @@ class AbstractStateBackend(ABC):
|
|
|
44
44
|
@abstractmethod
|
|
45
45
|
def load_failed(self) -> dict[str, dict[str, Any]]: ...
|
|
46
46
|
|
|
47
|
+
@abstractmethod
|
|
48
|
+
def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
|
|
49
|
+
"""读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
|
|
50
|
+
|
|
47
51
|
@abstractmethod
|
|
48
52
|
def load_cursors(self) -> dict[str, str]: ...
|
|
49
53
|
|
|
@@ -113,10 +117,14 @@ class AbstractStateBackend(ABC):
|
|
|
113
117
|
self,
|
|
114
118
|
uid: str,
|
|
115
119
|
result_meta: dict,
|
|
120
|
+
job_payload: dict[str, Any] | None = None,
|
|
116
121
|
) -> bool:
|
|
117
122
|
"""Persist a failed job to the DLQ as a delta, atomically.
|
|
118
123
|
|
|
119
124
|
Effects: failed_dlq INSERT/REPLACE uid -> result_meta; queue DELETE uid.
|
|
125
|
+
``job_payload`` is the original business payload snapshot stored
|
|
126
|
+
alongside the failure meta (for retry tooling); None keeps any
|
|
127
|
+
previously stored snapshot intact.
|
|
120
128
|
Returns True on success. Returns False if persistence failed — on
|
|
121
129
|
failure the backend MUST leave the on-disk queue unchanged (the popped
|
|
122
130
|
uid remains on disk); the caller is responsible for deciding whether to
|
|
@@ -28,6 +28,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
28
28
|
self._lock = threading.RLock()
|
|
29
29
|
self._wall: dict[str, dict[str, Any]] = {}
|
|
30
30
|
self._failed: dict[str, dict[str, Any]] = {}
|
|
31
|
+
self._failed_payloads: dict[str, dict[str, Any]] = {}
|
|
31
32
|
self._cursors: dict[str, str] = {}
|
|
32
33
|
self._queue: list[dict[str, Any]] = []
|
|
33
34
|
self._meta: dict[str, str] = {}
|
|
@@ -40,6 +41,11 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
40
41
|
with self._lock:
|
|
41
42
|
return copy.deepcopy(self._failed)
|
|
42
43
|
|
|
44
|
+
def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
|
|
45
|
+
"""读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
|
|
46
|
+
with self._lock:
|
|
47
|
+
return copy.deepcopy(self._failed_payloads)
|
|
48
|
+
|
|
43
49
|
def load_cursors(self) -> dict[str, str]:
|
|
44
50
|
with self._lock:
|
|
45
51
|
return dict(self._cursors)
|
|
@@ -156,6 +162,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
156
162
|
# 成功 commit 清理 failed 同名残行:与「job 最终状态唯一」
|
|
157
163
|
# 语义一致,防 wall∩failed 并存污染 _attempt 计数。
|
|
158
164
|
self._failed.pop(uid, None)
|
|
165
|
+
self._failed_payloads.pop(uid, None)
|
|
159
166
|
self._cursors.update(cursor_sets)
|
|
160
167
|
for k in cursor_dels:
|
|
161
168
|
self._cursors.pop(k, None)
|
|
@@ -164,7 +171,9 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
164
171
|
logger.critical(f"Failed to commit job success for {uid}: {e}")
|
|
165
172
|
return False
|
|
166
173
|
|
|
167
|
-
def commit_job_failure(
|
|
174
|
+
def commit_job_failure(
|
|
175
|
+
self, uid: str, result_meta: dict, job_payload: dict[str, Any] | None = None
|
|
176
|
+
) -> bool:
|
|
168
177
|
with self._lock:
|
|
169
178
|
try:
|
|
170
179
|
# 校验先行:DLQ 终值计算与队列 uid 提取全部在变更前完成,
|
|
@@ -172,6 +181,10 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
172
181
|
new_meta = self._build_dlq_meta(result_meta, self._failed.get(uid))
|
|
173
182
|
remaining = [j for j in self._queue if uid_from_job_dict(j) != uid]
|
|
174
183
|
self._failed[uid] = new_meta
|
|
184
|
+
# payload 快照 latest-wins;None 表示本路径无 payload,保留既有
|
|
185
|
+
# 快照(与 SQLite 后端 _write_dlq_row 的保留语义对齐)。
|
|
186
|
+
if job_payload is not None:
|
|
187
|
+
self._failed_payloads[uid] = copy.deepcopy(job_payload)
|
|
175
188
|
self._queue = remaining
|
|
176
189
|
# 同一事务语义内清理 wall 旧记录:rerun 任务重跑失败时旧成功
|
|
177
190
|
# 记录作废(最终状态唯一),防 wall∩failed 并存。
|
|
@@ -295,6 +308,7 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
295
308
|
for u in del_set:
|
|
296
309
|
if u in self._failed:
|
|
297
310
|
del self._failed[u]
|
|
311
|
+
self._failed_payloads.pop(u, None)
|
|
298
312
|
count += 1
|
|
299
313
|
return count
|
|
300
314
|
|
|
@@ -21,7 +21,7 @@ logger = logging.getLogger("tasklite")
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
_SCHEMA_VERSION =
|
|
24
|
+
_SCHEMA_VERSION = 2
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class SQLiteStateBackend(AbstractStateBackend):
|
|
@@ -106,9 +106,11 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
106
106
|
conn.execute('''
|
|
107
107
|
CREATE TABLE IF NOT EXISTS failed_dlq (
|
|
108
108
|
uid TEXT PRIMARY KEY,
|
|
109
|
-
payload TEXT
|
|
109
|
+
payload TEXT,
|
|
110
|
+
job_payload TEXT
|
|
110
111
|
)
|
|
111
112
|
''')
|
|
113
|
+
self._ensure_dlq_schema(conn)
|
|
112
114
|
self._ensure_queue_schema(conn)
|
|
113
115
|
conn.execute('''
|
|
114
116
|
CREATE TABLE IF NOT EXISTS cursors (
|
|
@@ -147,6 +149,16 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
147
149
|
logger.critical(f"Failed to write meta '{key}' in {self.path.name}: {e}")
|
|
148
150
|
raise
|
|
149
151
|
|
|
152
|
+
def _ensure_dlq_schema(self, conn) -> None:
|
|
153
|
+
"""确保 failed_dlq 具备 job_payload 快照列(v1 → v2 的加列迁移)。
|
|
154
|
+
|
|
155
|
+
加列是纯增量变更,旧读方(只 SELECT uid/payload)不受影响;
|
|
156
|
+
旧行该列为 NULL,属合法状态。
|
|
157
|
+
"""
|
|
158
|
+
cols = {c[1] for c in conn.execute("PRAGMA table_info(failed_dlq)").fetchall()}
|
|
159
|
+
if "job_payload" not in cols:
|
|
160
|
+
conn.execute("ALTER TABLE failed_dlq ADD COLUMN job_payload TEXT")
|
|
161
|
+
|
|
150
162
|
def _ensure_queue_schema(self, conn) -> None:
|
|
151
163
|
"""确保 queue 表使用当前 schema(uid PK + seq 保序列)。
|
|
152
164
|
|
|
@@ -229,6 +241,22 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
229
241
|
except json.JSONDecodeError as e:
|
|
230
242
|
raise RuntimeError(f"Corrupted failed DLQ payload in {self.path.name}: {e}") from e
|
|
231
243
|
|
|
244
|
+
def load_failed_payloads(self) -> dict[str, dict[str, Any]]:
|
|
245
|
+
"""读取 DLQ 各 uid 的原始业务 payload 快照(无快照的 uid 不出现)。"""
|
|
246
|
+
if not self.path.exists():
|
|
247
|
+
return {}
|
|
248
|
+
try:
|
|
249
|
+
with self._get_conn() as conn:
|
|
250
|
+
cursor = conn.execute(
|
|
251
|
+
'SELECT uid, job_payload FROM failed_dlq '
|
|
252
|
+
'WHERE job_payload IS NOT NULL'
|
|
253
|
+
)
|
|
254
|
+
return {row[0]: loads(row[1]) for row in cursor}
|
|
255
|
+
except (sqlite3.Error, OSError) as e:
|
|
256
|
+
raise RuntimeError(f"Failed to load DLQ payloads from {self.path.name}: {e}") from e
|
|
257
|
+
except json.JSONDecodeError as e:
|
|
258
|
+
raise RuntimeError(f"Corrupted DLQ job_payload in {self.path.name}: {e}") from e
|
|
259
|
+
|
|
232
260
|
def load_queue(self) -> list[dict[str, Any]]:
|
|
233
261
|
if not self.path.exists():
|
|
234
262
|
return []
|
|
@@ -414,7 +442,7 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
414
442
|
return False
|
|
415
443
|
return True
|
|
416
444
|
|
|
417
|
-
def _write_dlq_row(self, conn, uid: str, meta: dict) -> None:
|
|
445
|
+
def _write_dlq_row(self, conn, uid: str, meta: dict, job_payload: dict | None = None) -> None:
|
|
418
446
|
"""DLQ 行写入的单一出口。
|
|
419
447
|
|
|
420
448
|
读既有 ``_attempt`` 计数 → 递增或初始化 → INSERT OR REPLACE。
|
|
@@ -432,12 +460,16 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
432
460
|
推导)与 ``failed_at``(UTC ISO 时间戳)。所有 DLQ 写入路径(含死锁/级联/
|
|
433
461
|
重试耗尽)都带分类与时间,list_dlq() 查询可直接按类型过滤。
|
|
434
462
|
|
|
463
|
+
``job_payload`` 是原始业务 payload 的 JSON 快照(补跑用);传 None
|
|
464
|
+
时保留既有快照不覆盖——无 payload 的写入路径(bulk 级联/append)
|
|
465
|
+
不得抹掉此前快照。
|
|
466
|
+
|
|
435
467
|
调用方必须已通过 ``BEGIN IMMEDIATE`` 持写事务:SELECT 旧计数 →
|
|
436
468
|
递增 → REPLACE 的序列在 autocommit 下会丢并发计数(两个连接
|
|
437
469
|
各基于同一旧值 +1 落盘,只留一次)。
|
|
438
470
|
"""
|
|
439
471
|
row = conn.execute(
|
|
440
|
-
'SELECT payload FROM failed_dlq WHERE uid = ?', (uid,)
|
|
472
|
+
'SELECT payload, job_payload FROM failed_dlq WHERE uid = ?', (uid,)
|
|
441
473
|
).fetchone()
|
|
442
474
|
merged = dict(meta or {})
|
|
443
475
|
if "error_type" not in merged:
|
|
@@ -455,10 +487,17 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
455
487
|
merged["_attempt"] = 1 # 既有记录损坏:从 1 重新计数
|
|
456
488
|
else:
|
|
457
489
|
merged["_attempt"] = 1
|
|
458
|
-
|
|
459
|
-
|
|
490
|
+
effective_payload = job_payload if job_payload is not None else (
|
|
491
|
+
loads(row[1]) if row is not None and row[1] is not None else None
|
|
492
|
+
)
|
|
493
|
+
conn.execute(
|
|
494
|
+
'INSERT OR REPLACE INTO failed_dlq (uid, payload, job_payload) VALUES (?, ?, ?)',
|
|
495
|
+
(uid, dumps(merged), dumps(effective_payload) if effective_payload is not None else None),
|
|
496
|
+
)
|
|
460
497
|
|
|
461
|
-
def commit_job_failure(
|
|
498
|
+
def commit_job_failure(
|
|
499
|
+
self, uid: str, result_meta: dict, job_payload: dict | None = None
|
|
500
|
+
) -> bool:
|
|
462
501
|
"""原子 delta:写 DLQ + 删除 popped uid + 清理 wall 旧记录。失败时 on-disk 队列不变。
|
|
463
502
|
|
|
464
503
|
同 uid 多次失败时保留失败历史——DLQ 是 INSERT OR REPLACE
|
|
@@ -466,6 +505,9 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
466
505
|
``_attempt`` 计数并递增后合并写入,让同一 uid 的失败历史可观测。
|
|
467
506
|
写盘经 ``_write_dlq_row`` 单一出口(_attempt 计数路径收敛)。
|
|
468
507
|
|
|
508
|
+
``job_payload`` 为原始业务 payload 快照,随 DLQ 行一并落盘(补跑
|
|
509
|
+
无需外部反查);None 表示本次路径无 payload 可存(保留既有快照)。
|
|
510
|
+
|
|
469
511
|
同一事务内 ``DELETE FROM wall``——rerun 任务重跑
|
|
470
512
|
失败时 wall 里的旧成功记录必须作废(最终状态唯一);否则磁盘
|
|
471
513
|
wall∩failed 并存 → 下次 run DEBUG 断言崩。事务内单出口、原子、
|
|
@@ -475,7 +517,7 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
475
517
|
with self._get_conn() as conn:
|
|
476
518
|
# _attempt 读-递增-写必须持写事务串行化(详见 _get_conn)。
|
|
477
519
|
conn.execute('BEGIN IMMEDIATE')
|
|
478
|
-
self._write_dlq_row(conn, uid, result_meta)
|
|
520
|
+
self._write_dlq_row(conn, uid, result_meta, job_payload)
|
|
479
521
|
conn.execute('DELETE FROM queue WHERE uid = ?', (uid,))
|
|
480
522
|
conn.execute('DELETE FROM wall WHERE uid = ?', (uid,))
|
|
481
523
|
except Exception as e:
|
|
@@ -8,7 +8,7 @@ from __future__ import annotations
|
|
|
8
8
|
import logging
|
|
9
9
|
import time
|
|
10
10
|
from dataclasses import dataclass
|
|
11
|
-
from typing import Sequence
|
|
11
|
+
from typing import Callable, Sequence
|
|
12
12
|
|
|
13
13
|
from .store import DLQEntry, StateStore
|
|
14
14
|
from .resource import META_RESOURCE_SUSPENDS
|
|
@@ -108,6 +108,7 @@ class OpsConsole:
|
|
|
108
108
|
def list_dlq(self) -> list[DLQEntry]:
|
|
109
109
|
"""只读查询 DLQ,返回结构化条目(uid / error_type / error / attempts / failed_at / meta)。"""
|
|
110
110
|
failed = self._backend.load_failed()
|
|
111
|
+
payloads = self._backend.load_failed_payloads()
|
|
111
112
|
entries: list[DLQEntry] = []
|
|
112
113
|
for uid, meta in sorted(failed.items()):
|
|
113
114
|
if not isinstance(meta, dict):
|
|
@@ -119,6 +120,7 @@ class OpsConsole:
|
|
|
119
120
|
attempts=0,
|
|
120
121
|
failed_at=None,
|
|
121
122
|
meta={},
|
|
123
|
+
job_payload=payloads.get(uid),
|
|
122
124
|
)
|
|
123
125
|
)
|
|
124
126
|
continue
|
|
@@ -133,6 +135,7 @@ class OpsConsole:
|
|
|
133
135
|
attempts=attempts,
|
|
134
136
|
failed_at=meta.get("failed_at"),
|
|
135
137
|
meta=dict(meta),
|
|
138
|
+
job_payload=payloads.get(uid),
|
|
136
139
|
)
|
|
137
140
|
)
|
|
138
141
|
return entries
|
|
@@ -177,8 +180,15 @@ class OpsConsole:
|
|
|
177
180
|
targets: str | Sequence[str],
|
|
178
181
|
*,
|
|
179
182
|
where: Sequence[str] = ("wall", "failed"),
|
|
183
|
+
predicate: Callable[[str], bool] | None = None,
|
|
180
184
|
) -> int:
|
|
181
|
-
"""从 wall 和/或 DLQ 删除条目。
|
|
185
|
+
"""从 wall 和/或 DLQ 删除条目。
|
|
186
|
+
|
|
187
|
+
``predicate`` 提供官方的「Python 侧判定 + 定向批删」通道:对
|
|
188
|
+
targets 命中的 uid 逐条调用,返回 False 则保留。判定在本进程
|
|
189
|
+
完成、删除仍走后端批量删除接口,业务无需再裸连数据库绕过
|
|
190
|
+
WAL 校验与写事务纪律。
|
|
191
|
+
"""
|
|
182
192
|
if isinstance(targets, str):
|
|
183
193
|
patterns = [targets]
|
|
184
194
|
elif isinstance(targets, (list, tuple)):
|
|
@@ -186,13 +196,18 @@ class OpsConsole:
|
|
|
186
196
|
else:
|
|
187
197
|
raise TypeError(
|
|
188
198
|
f"targets must be a str or a list/tuple of str, "
|
|
189
|
-
f"got {type(targets).__name__}"
|
|
199
|
+
f"got {type(targets).__name__} ({targets!r})"
|
|
190
200
|
)
|
|
191
201
|
for p in patterns:
|
|
192
202
|
if not isinstance(p, str):
|
|
193
203
|
raise TypeError(
|
|
194
204
|
f"targets must contain only str, got {type(p).__name__} ({p!r})"
|
|
195
205
|
)
|
|
206
|
+
if predicate is not None and not callable(predicate):
|
|
207
|
+
raise TypeError(
|
|
208
|
+
f"predicate must be callable or None, "
|
|
209
|
+
f"got {type(predicate).__name__}"
|
|
210
|
+
)
|
|
196
211
|
if not isinstance(where, (list, tuple)):
|
|
197
212
|
raise TypeError(
|
|
198
213
|
f"where must be a sequence of 'wall'/'failed', got {type(where).__name__}"
|
|
@@ -206,10 +221,11 @@ class OpsConsole:
|
|
|
206
221
|
)
|
|
207
222
|
|
|
208
223
|
def _matches(uid: str) -> bool:
|
|
209
|
-
|
|
224
|
+
hit = any(
|
|
210
225
|
uid == p or (p.endswith("::") and uid.startswith(p))
|
|
211
226
|
for p in patterns
|
|
212
227
|
)
|
|
228
|
+
return hit and (predicate is None or predicate(uid))
|
|
213
229
|
|
|
214
230
|
state = self._store.state
|
|
215
231
|
total = 0
|
|
@@ -40,6 +40,7 @@ class DLQEntry(NamedTuple):
|
|
|
40
40
|
- ``attempts``: 累计写入 DLQ 的次数 (_attempt 计数)
|
|
41
41
|
- ``failed_at``: 最近一次失败时间 (UTC ISO 8601 字符串或 None)
|
|
42
42
|
- ``meta``: 完整 DLQ payload 字典视图
|
|
43
|
+
- ``job_payload``: 原始业务 payload 快照 (未随失败落盘快照时为 None)
|
|
43
44
|
"""
|
|
44
45
|
|
|
45
46
|
uid: str
|
|
@@ -48,6 +49,7 @@ class DLQEntry(NamedTuple):
|
|
|
48
49
|
attempts: int
|
|
49
50
|
failed_at: str | None
|
|
50
51
|
meta: dict[str, Any]
|
|
52
|
+
job_payload: dict[str, Any] | None = None
|
|
51
53
|
|
|
52
54
|
|
|
53
55
|
__all__ = [
|
|
@@ -391,7 +393,8 @@ class StateStore:
|
|
|
391
393
|
) -> FailureOutcome:
|
|
392
394
|
"""原子终态转移:永久失败(进入 DLQ)。"""
|
|
393
395
|
sanitized_meta = self._taxonomy.normalize_dlq_meta(error_meta)
|
|
394
|
-
|
|
396
|
+
job_payload = self._extract_job_payload(job_dict)
|
|
397
|
+
committed = self._backend.commit_job_failure(uid, sanitized_meta, job_payload)
|
|
395
398
|
|
|
396
399
|
if committed:
|
|
397
400
|
self._state.mark_failed(uid, sanitized_meta)
|
|
@@ -405,6 +408,28 @@ class StateStore:
|
|
|
405
408
|
self.commit_failed_crash(uid, sanitized_meta.get("error", "commit_job_failure"), job_dict)
|
|
406
409
|
return FailureOutcome(uid=uid, error_meta=sanitized_meta)
|
|
407
410
|
|
|
411
|
+
@staticmethod
|
|
412
|
+
def _extract_job_payload(job_dict: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
413
|
+
"""提取可 JSON 序列化的业务 payload 快照(DLQ 落盘用)。
|
|
414
|
+
|
|
415
|
+
不可序列化时降级为 None 并告警,绝不让快照失败把正常失败提交
|
|
416
|
+
打成 commit failure(那会误触 3-strike 崩溃契约)。
|
|
417
|
+
"""
|
|
418
|
+
if not isinstance(job_dict, dict):
|
|
419
|
+
return None
|
|
420
|
+
payload = job_dict.get("payload")
|
|
421
|
+
if not isinstance(payload, dict):
|
|
422
|
+
return None
|
|
423
|
+
try:
|
|
424
|
+
dumps(payload)
|
|
425
|
+
except (TypeError, ValueError):
|
|
426
|
+
logger.warning(
|
|
427
|
+
f"Job payload for {job_dict.get('job_id', '?')} is not "
|
|
428
|
+
f"JSON-serializable; DLQ payload snapshot skipped."
|
|
429
|
+
)
|
|
430
|
+
return None
|
|
431
|
+
return payload
|
|
432
|
+
|
|
408
433
|
def apply_retry(
|
|
409
434
|
self,
|
|
410
435
|
uid: str,
|
|
@@ -503,7 +528,8 @@ class StateStore:
|
|
|
503
528
|
failures = self._register_commit_failure(jd)
|
|
504
529
|
if failures >= self._threshold:
|
|
505
530
|
single_committed = self._backend.commit_job_failure(
|
|
506
|
-
uid, {"error": ERR_COMMIT_FAILURE_DLQ, "commit_failures": failures, "fatal": True}
|
|
531
|
+
uid, {"error": ERR_COMMIT_FAILURE_DLQ, "commit_failures": failures, "fatal": True},
|
|
532
|
+
job_payload=self._extract_job_payload(jd),
|
|
507
533
|
)
|
|
508
534
|
if single_committed:
|
|
509
535
|
queue = [j for j in queue if uid_from_job_dict(j) != uid]
|
|
@@ -582,7 +608,9 @@ class StateStore:
|
|
|
582
608
|
failures = self._register_commit_failure(jd)
|
|
583
609
|
if failures >= self._threshold:
|
|
584
610
|
dlq_meta = meta_by_uid[uid]
|
|
585
|
-
single_committed = self._backend.commit_job_failure(
|
|
611
|
+
single_committed = self._backend.commit_job_failure(
|
|
612
|
+
uid, dlq_meta, job_payload=self._extract_job_payload(jd)
|
|
613
|
+
)
|
|
586
614
|
if single_committed:
|
|
587
615
|
self.apply_failed(uid, dlq_meta, unregister=False)
|
|
588
616
|
self._record_stat("failed", 1)
|
|
@@ -641,7 +669,9 @@ class StateStore:
|
|
|
641
669
|
"fatal": True,
|
|
642
670
|
"commit_failures": failures,
|
|
643
671
|
}
|
|
644
|
-
dlq_committed = self._backend.commit_job_failure(
|
|
672
|
+
dlq_committed = self._backend.commit_job_failure(
|
|
673
|
+
uid, dlq_meta, job_payload=self._extract_job_payload(job_dict)
|
|
674
|
+
)
|
|
645
675
|
if dlq_committed:
|
|
646
676
|
self.apply_failed(uid, dlq_meta)
|
|
647
677
|
self._record_stat("failed", 1)
|
|
@@ -537,10 +537,15 @@ class TaskLite:
|
|
|
537
537
|
targets: str | Sequence[str],
|
|
538
538
|
*,
|
|
539
539
|
where: Sequence[str] = ("wall", "failed"),
|
|
540
|
+
predicate: Callable[[str], bool] | None = None,
|
|
540
541
|
) -> int:
|
|
541
|
-
"""从 wall 和/或 DLQ 删除条目。委托 OpsConsole。
|
|
542
|
+
"""从 wall 和/或 DLQ 删除条目。委托 OpsConsole。
|
|
543
|
+
|
|
544
|
+
``predicate`` 可选:对 targets 命中的 uid 逐条判定,返回 False
|
|
545
|
+
则保留(官方的 Python 侧判定 + 定向批删通道,替代裸 SQL 穿透)。
|
|
546
|
+
"""
|
|
542
547
|
self._ensure_not_running("clear_history")
|
|
543
|
-
return self._console.clear_history(targets, where=where)
|
|
548
|
+
return self._console.clear_history(targets, where=where, predicate=predicate)
|
|
544
549
|
|
|
545
550
|
def seed_wall(self, uids: Sequence[str]) -> int:
|
|
546
551
|
"""把 uid 批量写入 wall。委托 OpsConsole。"""
|
|
@@ -434,7 +434,7 @@ class http_guard:
|
|
|
434
434
|
|
|
435
435
|
Args:
|
|
436
436
|
ctx: 当前任务的 TaskContext(用于调用 `ctx.suspend_resource`)。
|
|
437
|
-
resource: 关联的限速资源名称(如 '
|
|
437
|
+
resource: 关联的限速资源名称(如 'api_main')。
|
|
438
438
|
policy: 状态码与异常分类规则器(默认使用标准 HttpPolicy)。
|
|
439
439
|
default_suspend_ttl: 遭遇 429 且未提供 Retry-After 时的默认挂起秒数。
|
|
440
440
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasklite-engine
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.2
|
|
4
4
|
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
@@ -24,6 +24,7 @@ Requires-Dist: hypothesis>=6.0; extra == "dev"
|
|
|
24
24
|
Requires-Dist: mutmut>=3.0; extra == "dev"
|
|
25
25
|
Requires-Dist: pytest-timeout>=2.0; extra == "dev"
|
|
26
26
|
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
27
|
+
Requires-Dist: requests>=2.28; extra == "dev"
|
|
27
28
|
Dynamic: license-file
|
|
28
29
|
|
|
29
30
|
# ⚡ TaskLite — 零依赖、进程隔离的轻量任务编排引擎
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tasklite_engine-1.4.0 → tasklite_engine-1.4.2}/tasklite_engine.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|