tasklite-engine 2.2.0__py3-none-any.whl → 3.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +30 -196
- tasklite/backend/__init__.py +2 -5
- tasklite/backend/base.py +156 -184
- tasklite/backend/memory.py +468 -231
- tasklite/backend/sqlite_backend.py +680 -408
- tasklite/engine/__init__.py +2 -2
- tasklite/engine/admission.py +85 -44
- tasklite/engine/channel.py +566 -458
- tasklite/engine/completion.py +535 -247
- tasklite/engine/config.py +7 -131
- tasklite/engine/dispatch.py +415 -321
- tasklite/engine/execution_access.py +39 -0
- tasklite/engine/files.py +52 -0
- tasklite/engine/governor.py +46 -204
- tasklite/engine/in_flight.py +15 -13
- tasklite/engine/ops.py +9 -347
- tasklite/engine/recovery.py +67 -267
- tasklite/engine/resource.py +256 -386
- tasklite/engine/runtime.py +527 -236
- tasklite/engine/scheduler.py +99 -90
- tasklite/engine/selection.py +53 -111
- tasklite/engine/session.py +49 -77
- tasklite/engine/store.py +595 -564
- tasklite/engine/types.py +66 -71
- tasklite/engine/wait.py +30 -16
- tasklite/exceptions.py +4 -57
- tasklite/hooks.py +3 -23
- tasklite/models/__init__.py +21 -6
- tasklite/models/artifact.py +386 -47
- tasklite/models/context.py +173 -203
- tasklite/models/job.py +115 -266
- tasklite/models/result.py +246 -0
- tasklite/models/state.py +97 -17
- tasklite/models/task.py +52 -105
- tasklite/pipeline.py +279 -534
- tasklite/resource.py +12 -0
- tasklite/selection.py +9 -97
- tasklite/snapshot.py +639 -0
- tasklite/testing.py +32 -34
- tasklite/triggers.py +5 -4
- tasklite/utils/artifact.py +85 -17
- tasklite/utils/clock.py +3 -3
- tasklite/utils/encoding.py +113 -18
- tasklite/utils/ipc.py +240 -242
- tasklite/utils/lockfile.py +33 -58
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +155 -158
- tasklite/wrappers/http.py +68 -481
- tasklite_engine-3.0.0.dev0.dist-info/METADATA +307 -0
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +58 -0
- tasklite/engine/errorclass.py +0 -971
- tasklite/models/attempt.py +0 -141
- tasklite_engine-2.2.0.dist-info/METADATA +0 -369
- tasklite_engine-2.2.0.dist-info/RECORD +0 -55
- {tasklite_engine-2.2.0.dist-info → tasklite_engine-3.0.0.dev0.dist-info}/WHEEL +0 -0
- {tasklite_engine-2.2.0.dist-info → tasklite_engine-3.0.0.dev0.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-2.2.0.dist-info → tasklite_engine-3.0.0.dev0.dist-info}/top_level.txt +0 -0
tasklite/backend/memory.py
CHANGED
|
@@ -9,18 +9,23 @@ from __future__ import annotations
|
|
|
9
9
|
|
|
10
10
|
import copy
|
|
11
11
|
import logging
|
|
12
|
+
import os
|
|
12
13
|
import threading
|
|
13
|
-
from
|
|
14
|
+
from pathlib import Path
|
|
14
15
|
from typing import Any, Callable, Mapping, Sequence
|
|
15
16
|
|
|
16
17
|
from .base import (
|
|
17
18
|
AbstractStateBackend,
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
QUEUE_EXECUTION_ID_KEY,
|
|
20
|
+
execution_id_matches_run,
|
|
21
|
+
public_queue_snapshot,
|
|
22
|
+
queue_snapshot_execution_id,
|
|
23
|
+
queue_snapshot_with_execution_id,
|
|
20
24
|
validate_queue_replacement,
|
|
21
25
|
)
|
|
22
|
-
from ..
|
|
26
|
+
from ..exceptions import TaskLiteError
|
|
23
27
|
from ..models.state import job_uid_from_snapshot
|
|
28
|
+
from ..utils.clock import utc_now_iso
|
|
24
29
|
|
|
25
30
|
logger = logging.getLogger("tasklite")
|
|
26
31
|
|
|
@@ -35,14 +40,17 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
35
40
|
|
|
36
41
|
def __init__(self) -> None:
|
|
37
42
|
self._lock = threading.RLock()
|
|
43
|
+
self._exec_lock = threading.Lock()
|
|
38
44
|
self._successes: dict[str, dict[str, Any]] = {}
|
|
39
45
|
self._failed: dict[str, dict[str, Any]] = {}
|
|
40
46
|
self._failure_payloads: dict[str, dict[str, Any]] = {}
|
|
41
47
|
self._cursors: dict[str, str] = {}
|
|
42
48
|
self._queue: list[dict[str, Any]] = []
|
|
43
49
|
self._engine_metadata: dict[str, str] = {}
|
|
44
|
-
self._attempts: dict[int,
|
|
50
|
+
self._attempts: dict[int, dict[str, Any]] = {}
|
|
45
51
|
self._attempt_seq = 1
|
|
52
|
+
self._job_sequences: dict[str, int] = {}
|
|
53
|
+
self._authorizations: dict[str, dict[str, Any]] = {}
|
|
46
54
|
|
|
47
55
|
# 全量加载 -------------------------------------------------------
|
|
48
56
|
|
|
@@ -65,7 +73,10 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
65
73
|
|
|
66
74
|
def load_queue(self) -> list[dict[str, Any]]:
|
|
67
75
|
with self._lock:
|
|
68
|
-
return
|
|
76
|
+
return [
|
|
77
|
+
public_queue_snapshot(j)
|
|
78
|
+
for j in self._queue if j.get("status", "pending") == "pending"
|
|
79
|
+
]
|
|
69
80
|
|
|
70
81
|
# 整表重写 -------------------------------------------------------
|
|
71
82
|
|
|
@@ -105,106 +116,6 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
105
116
|
|
|
106
117
|
# delta 提交 -----------------------------------------------------
|
|
107
118
|
|
|
108
|
-
def commit_job_success(
|
|
109
|
-
self,
|
|
110
|
-
job_uid: str,
|
|
111
|
-
result_metadata: dict,
|
|
112
|
-
*,
|
|
113
|
-
attempt_id: int | None = None,
|
|
114
|
-
execution_id: str | None = None,
|
|
115
|
-
finished_at: str | None = None,
|
|
116
|
-
spawned_job_snapshots: Sequence[dict[str, Any]] = (),
|
|
117
|
-
cursor_updates: Mapping[str, str | None] | None = None,
|
|
118
|
-
) -> bool:
|
|
119
|
-
with self._lock:
|
|
120
|
-
try:
|
|
121
|
-
# 校验先行:全部 deepcopy、attempts 行校验与冲突判定在任何变更前完成。
|
|
122
|
-
# 不变式:返回 False / 抛异常 ⇒ successes/attempts/queue/failed/cursors 与
|
|
123
|
-
# 调用前完全一致(对齐 SQLite 事务回滚;store 的连续失败
|
|
124
|
-
# 崩溃契约以「后端未变」为前提做重启重建)。
|
|
125
|
-
if not isinstance(attempt_id, int) or isinstance(attempt_id, bool):
|
|
126
|
-
logger.critical(
|
|
127
|
-
f"commit_job_success for {job_uid}: attempt_id must be an int, got {attempt_id!r}"
|
|
128
|
-
)
|
|
129
|
-
return False
|
|
130
|
-
if execution_id is not None and (not isinstance(execution_id, str) or not execution_id):
|
|
131
|
-
logger.critical(
|
|
132
|
-
f"commit_job_success for {job_uid}: execution_id must be a non-empty str or None, got {execution_id!r}"
|
|
133
|
-
)
|
|
134
|
-
return False
|
|
135
|
-
validate_attempt_finish("succeeded", finished_at, None)
|
|
136
|
-
current_attempt = self._attempts.get(attempt_id)
|
|
137
|
-
if current_attempt is None:
|
|
138
|
-
logger.critical(
|
|
139
|
-
f"commit_job_success for {job_uid}: attempt {attempt_id} does not exist"
|
|
140
|
-
)
|
|
141
|
-
return False
|
|
142
|
-
if current_attempt.job_uid != job_uid:
|
|
143
|
-
logger.critical(
|
|
144
|
-
f"commit_job_success for {job_uid}: attempt {attempt_id} job_uid mismatch "
|
|
145
|
-
f"(expected {job_uid!r}, got {current_attempt.job_uid!r})"
|
|
146
|
-
)
|
|
147
|
-
return False
|
|
148
|
-
if execution_id is not None and current_attempt.execution_id != execution_id:
|
|
149
|
-
logger.critical(
|
|
150
|
-
f"commit_job_success for {job_uid}: attempt {attempt_id} execution_id mismatch "
|
|
151
|
-
f"(expected {execution_id!r}, got {current_attempt.execution_id!r})"
|
|
152
|
-
)
|
|
153
|
-
return False
|
|
154
|
-
if current_attempt.outcome != "running":
|
|
155
|
-
logger.critical(
|
|
156
|
-
f"commit_job_success for {job_uid}: attempt {attempt_id} already finalized "
|
|
157
|
-
f"with outcome {current_attempt.outcome!r}"
|
|
158
|
-
)
|
|
159
|
-
return False
|
|
160
|
-
updated_attempt = replace(
|
|
161
|
-
current_attempt,
|
|
162
|
-
outcome="succeeded",
|
|
163
|
-
finished_at=finished_at,
|
|
164
|
-
error_message=None,
|
|
165
|
-
)
|
|
166
|
-
success_metadata = copy.deepcopy(result_metadata or {})
|
|
167
|
-
spawned_copy = [copy.deepcopy(j) for j in spawned_job_snapshots]
|
|
168
|
-
remaining = [j for j in self._queue if job_uid_from_snapshot(j) != job_uid]
|
|
169
|
-
if spawned_copy:
|
|
170
|
-
remaining_uids = {job_uid_from_snapshot(j) for j in remaining}
|
|
171
|
-
seen: set[str] = set()
|
|
172
|
-
for sj in spawned_copy:
|
|
173
|
-
suid = job_uid_from_snapshot(sj)
|
|
174
|
-
# spawned uid 撞上删除 popped 后的队列既有条目,或
|
|
175
|
-
# 批内自相重复,均判定内存/磁盘漂移 → False 走崩溃
|
|
176
|
-
# 契约,绝不产出重复队列条目(对齐 SQLite
|
|
177
|
-
# INSERT OR IGNORE + rowcount 漂移检测)。
|
|
178
|
-
if suid in remaining_uids or suid in seen:
|
|
179
|
-
logger.critical(
|
|
180
|
-
f"commit_job_success for {job_uid}: spawned job {suid} "
|
|
181
|
-
f"conflicts with queue (drift). Returning False."
|
|
182
|
-
)
|
|
183
|
-
return False
|
|
184
|
-
seen.add(suid)
|
|
185
|
-
cursor_sets: dict[str, str] = {}
|
|
186
|
-
cursor_dels: set[str] = set()
|
|
187
|
-
if cursor_updates:
|
|
188
|
-
for k, v in cursor_updates.items():
|
|
189
|
-
if v is None:
|
|
190
|
-
cursor_dels.add(k)
|
|
191
|
-
else:
|
|
192
|
-
cursor_sets[k] = str(v)
|
|
193
|
-
# 校验全部通过,统一落变(落变段不再调用任何可失败操作)
|
|
194
|
-
self._attempts[attempt_id] = updated_attempt
|
|
195
|
-
self._successes[job_uid] = success_metadata
|
|
196
|
-
self._queue = spawned_copy + remaining # 队首插入 spawned 并保序
|
|
197
|
-
# 成功 commit 清理 failed 同名残行:与「job 最终状态唯一」
|
|
198
|
-
# 语义一致,防 successes∩failed 并存。
|
|
199
|
-
self._failed.pop(job_uid, None)
|
|
200
|
-
self._failure_payloads.pop(job_uid, None)
|
|
201
|
-
self._cursors.update(cursor_sets)
|
|
202
|
-
for k in cursor_dels:
|
|
203
|
-
self._cursors.pop(k, None)
|
|
204
|
-
return True
|
|
205
|
-
except Exception as e:
|
|
206
|
-
logger.critical(f"Failed to commit job success for {job_uid}: {e}")
|
|
207
|
-
return False
|
|
208
119
|
|
|
209
120
|
def commit_job_failure(
|
|
210
121
|
self, job_uid: str, failure_metadata: dict, payload: dict[str, Any] | None = None
|
|
@@ -230,66 +141,25 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
230
141
|
return True
|
|
231
142
|
except Exception as e:
|
|
232
143
|
logger.critical(f"Failed to commit job failure for {job_uid}: {e}")
|
|
233
|
-
|
|
144
|
+
raise
|
|
234
145
|
|
|
235
|
-
def commit_retry(
|
|
236
|
-
self,
|
|
237
|
-
popped_job_uid: str,
|
|
238
|
-
requeued_job_snapshot: dict[str, Any],
|
|
239
|
-
*,
|
|
240
|
-
front: bool = False,
|
|
241
|
-
) -> bool:
|
|
242
|
-
with self._lock:
|
|
243
|
-
try:
|
|
244
|
-
requeued_copy = copy.deepcopy(requeued_job_snapshot)
|
|
245
|
-
ruid = job_uid_from_snapshot(requeued_copy)
|
|
246
|
-
remaining = [j for j in self._queue if job_uid_from_snapshot(j) != popped_job_uid]
|
|
247
|
-
# 对齐 SQLite 普通 INSERT 冲突回滚:requeued uid 撞上删除
|
|
248
|
-
# popped 后的既有条目 → False 走崩溃契约,绝不产出重复条目;
|
|
249
|
-
# uid == popped_job_uid 属同 job 重插(DELETE+INSERT),安全放行。
|
|
250
|
-
if ruid != popped_job_uid and any(job_uid_from_snapshot(j) == ruid for j in remaining):
|
|
251
|
-
logger.critical(
|
|
252
|
-
f"commit_retry for {popped_job_uid}: requeued uid {ruid} "
|
|
253
|
-
f"conflicts with existing queue entry. Returning False."
|
|
254
|
-
)
|
|
255
|
-
return False
|
|
256
|
-
if front:
|
|
257
|
-
self._queue = [requeued_copy] + remaining
|
|
258
|
-
else:
|
|
259
|
-
self._queue = remaining + [requeued_copy]
|
|
260
|
-
return True
|
|
261
|
-
except Exception as e:
|
|
262
|
-
logger.critical(f"Failed to commit retry for {popped_job_uid}: {e}")
|
|
263
|
-
return False
|
|
264
146
|
|
|
265
|
-
def commit_job_failures(self, job_uids_metadata: list[tuple[str, dict]]) ->
|
|
147
|
+
def commit_job_failures(self, job_uids_metadata: list[tuple[str, dict]]) -> None:
|
|
266
148
|
with self._lock:
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
self._successes.pop(job_uid, None)
|
|
279
|
-
self._queue = remaining
|
|
280
|
-
return True
|
|
281
|
-
except Exception as e:
|
|
282
|
-
logger.critical(f"Failed to commit bulk failure: {e}")
|
|
283
|
-
return False
|
|
284
|
-
|
|
285
|
-
def commit_skip(self, job_uid: str) -> bool:
|
|
149
|
+
# 所有可失败的复制与队列构造先于统一发布。
|
|
150
|
+
entries = [(job_uid, copy.deepcopy(meta or {})) for job_uid, meta in job_uids_metadata]
|
|
151
|
+
fail_uids = {u for u, _ in entries}
|
|
152
|
+
remaining = [j for j in self._queue if job_uid_from_snapshot(j) not in fail_uids]
|
|
153
|
+
failed = self._failed | dict(entries)
|
|
154
|
+
successes = {uid: meta for uid, meta in self._successes.items() if uid not in fail_uids}
|
|
155
|
+
self._failed = failed
|
|
156
|
+
self._successes = successes
|
|
157
|
+
self._queue = remaining
|
|
158
|
+
|
|
159
|
+
def commit_skip(self, job_uid: str) -> None:
|
|
286
160
|
with self._lock:
|
|
287
161
|
self._queue = [j for j in self._queue if job_uid_from_snapshot(j) != job_uid]
|
|
288
|
-
return True
|
|
289
162
|
|
|
290
|
-
def append_failed(self, job_uid: str, metadata: dict[str, Any] | None = None) -> None:
|
|
291
|
-
with self._lock:
|
|
292
|
-
self._failed[job_uid] = copy.deepcopy(metadata or {})
|
|
293
163
|
|
|
294
164
|
# 增量入队 -------------------------------------------------------
|
|
295
165
|
|
|
@@ -350,103 +220,470 @@ class InMemoryStateBackend(AbstractStateBackend):
|
|
|
350
220
|
count += 1
|
|
351
221
|
return count
|
|
352
222
|
|
|
353
|
-
def
|
|
354
|
-
|
|
223
|
+
def apply_manual_status(
|
|
224
|
+
self,
|
|
225
|
+
updates: Sequence[tuple[str, str, dict[str, Any], dict[str, Any] | None]],
|
|
226
|
+
) -> None:
|
|
227
|
+
"""批量原子应用人工状态变更 (job_uid, status, snapshot, metadata)。"""
|
|
228
|
+
if not updates:
|
|
229
|
+
return
|
|
230
|
+
with self._lock:
|
|
231
|
+
new_successes = copy.deepcopy(self._successes)
|
|
232
|
+
new_failed = copy.deepcopy(self._failed)
|
|
233
|
+
new_failure_payloads = copy.deepcopy(self._failure_payloads)
|
|
234
|
+
new_queue = copy.deepcopy(self._queue)
|
|
235
|
+
for job_uid, status, snapshot, metadata in updates:
|
|
236
|
+
snap_copy = copy.deepcopy(snapshot)
|
|
237
|
+
meta_copy = copy.deepcopy(metadata or {})
|
|
238
|
+
if status == "pending":
|
|
239
|
+
new_successes.pop(job_uid, None)
|
|
240
|
+
new_failed.pop(job_uid, None)
|
|
241
|
+
new_failure_payloads.pop(job_uid, None)
|
|
242
|
+
new_queue = [
|
|
243
|
+
j for j in new_queue
|
|
244
|
+
if job_uid_from_snapshot(j) != job_uid
|
|
245
|
+
]
|
|
246
|
+
snap_copy["status"] = "pending"
|
|
247
|
+
snap_copy["run_id"] = None
|
|
248
|
+
snap_copy["attempt_no"] = None
|
|
249
|
+
snap_copy["started_at"] = None
|
|
250
|
+
snap_copy.pop(QUEUE_EXECUTION_ID_KEY, None)
|
|
251
|
+
new_queue.append(snap_copy)
|
|
252
|
+
elif status == "succeeded":
|
|
253
|
+
new_queue = [
|
|
254
|
+
j for j in new_queue
|
|
255
|
+
if job_uid_from_snapshot(j) != job_uid
|
|
256
|
+
]
|
|
257
|
+
new_failed.pop(job_uid, None)
|
|
258
|
+
new_failure_payloads.pop(job_uid, None)
|
|
259
|
+
new_successes[job_uid] = meta_copy
|
|
260
|
+
elif status == "failed":
|
|
261
|
+
new_queue = [
|
|
262
|
+
j for j in new_queue
|
|
263
|
+
if job_uid_from_snapshot(j) != job_uid
|
|
264
|
+
]
|
|
265
|
+
new_successes.pop(job_uid, None)
|
|
266
|
+
new_failed[job_uid] = meta_copy
|
|
267
|
+
new_failure_payloads[job_uid] = snap_copy.get("payload") or {}
|
|
268
|
+
else:
|
|
269
|
+
raise ValueError(f"Unsupported manual status: {status!r}")
|
|
355
270
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
271
|
+
self._successes = new_successes
|
|
272
|
+
self._failed = new_failed
|
|
273
|
+
self._failure_payloads = new_failure_payloads
|
|
274
|
+
self._queue = new_queue
|
|
275
|
+
|
|
276
|
+
@staticmethod
|
|
277
|
+
def _snapshot_from_metadata(metadata: Any) -> dict[str, Any] | None:
|
|
278
|
+
if not isinstance(metadata, dict):
|
|
279
|
+
return None
|
|
280
|
+
snapshot = metadata.get("job_snapshot")
|
|
281
|
+
return snapshot if isinstance(snapshot, dict) else None
|
|
282
|
+
|
|
283
|
+
def load_known_job_snapshot(self, uid: str) -> dict[str, Any] | None:
|
|
284
|
+
"""按队列、成功、失败、Attempt 历史的优先级读取完整配置。"""
|
|
360
285
|
with self._lock:
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
f"seed_successes refuses uid(s) already in failed: {conflict}; "
|
|
365
|
-
f"successes/failed must stay disjoint"
|
|
366
|
-
)
|
|
367
|
-
count = 0
|
|
368
|
-
for u in job_uids:
|
|
369
|
-
self._successes[u] = {}
|
|
370
|
-
count += 1
|
|
371
|
-
return count
|
|
286
|
+
for snapshot in self._queue:
|
|
287
|
+
if job_uid_from_snapshot(snapshot) == uid:
|
|
288
|
+
return public_queue_snapshot(snapshot)
|
|
372
289
|
|
|
373
|
-
|
|
374
|
-
|
|
290
|
+
success_snapshot = self._snapshot_from_metadata(self._successes.get(uid))
|
|
291
|
+
if success_snapshot is not None:
|
|
292
|
+
return copy.deepcopy(success_snapshot)
|
|
375
293
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
294
|
+
failure_snapshot = self._snapshot_from_metadata(self._failed.get(uid))
|
|
295
|
+
if failure_snapshot is not None:
|
|
296
|
+
return copy.deepcopy(failure_snapshot)
|
|
297
|
+
|
|
298
|
+
for attempt_id in sorted(self._attempts, reverse=True):
|
|
299
|
+
attempt = self._attempts[attempt_id]
|
|
300
|
+
if attempt.get("job_uid") != uid:
|
|
301
|
+
continue
|
|
302
|
+
attempt_snapshot = attempt.get("job")
|
|
303
|
+
if isinstance(attempt_snapshot, dict):
|
|
304
|
+
return copy.deepcopy(attempt_snapshot)
|
|
305
|
+
return None
|
|
306
|
+
|
|
307
|
+
def load_known_job_snapshots(self) -> dict[str, dict[str, Any]]:
|
|
308
|
+
"""读取全部已知 Job 的完整配置,低优先级来源先写入。"""
|
|
309
|
+
with self._lock:
|
|
310
|
+
known: dict[str, dict[str, Any]] = {}
|
|
311
|
+
for attempt_id in sorted(self._attempts):
|
|
312
|
+
attempt = self._attempts[attempt_id]
|
|
313
|
+
uid = attempt.get("job_uid")
|
|
314
|
+
snapshot = attempt.get("job")
|
|
315
|
+
if isinstance(uid, str) and isinstance(snapshot, dict):
|
|
316
|
+
known[uid] = copy.deepcopy(snapshot)
|
|
317
|
+
|
|
318
|
+
for uid, metadata in self._failed.items():
|
|
319
|
+
snapshot = self._snapshot_from_metadata(metadata)
|
|
320
|
+
if snapshot is not None:
|
|
321
|
+
known[uid] = copy.deepcopy(snapshot)
|
|
322
|
+
|
|
323
|
+
for uid, metadata in self._successes.items():
|
|
324
|
+
snapshot = self._snapshot_from_metadata(metadata)
|
|
325
|
+
if snapshot is not None:
|
|
326
|
+
known[uid] = copy.deepcopy(snapshot)
|
|
327
|
+
|
|
328
|
+
for snapshot in self._queue:
|
|
329
|
+
known[job_uid_from_snapshot(snapshot)] = public_queue_snapshot(snapshot)
|
|
330
|
+
return known
|
|
331
|
+
|
|
332
|
+
def clear_history(self, uid: str, records: set[str]) -> int:
|
|
333
|
+
"""从 successes/failed/attempts 清除指定 job_uid 的记录,返回删除条数。"""
|
|
334
|
+
total = 0
|
|
335
|
+
with self._lock:
|
|
336
|
+
if "successes" in records and uid in self._successes:
|
|
337
|
+
del self._successes[uid]
|
|
338
|
+
total += 1
|
|
339
|
+
if "failures" in records:
|
|
340
|
+
if uid in self._failed:
|
|
341
|
+
del self._failed[uid]
|
|
342
|
+
total += 1
|
|
343
|
+
self._failure_payloads.pop(uid, None)
|
|
344
|
+
if "attempts" in records:
|
|
345
|
+
initial = len(self._attempts)
|
|
346
|
+
self._attempts = {
|
|
347
|
+
k: v for k, v in self._attempts.items()
|
|
348
|
+
if v.get("job_uid") != uid
|
|
349
|
+
}
|
|
350
|
+
total += initial - len(self._attempts)
|
|
351
|
+
return total
|
|
352
|
+
|
|
353
|
+
def get_cursor(self, key: str) -> str | None:
|
|
354
|
+
"""查询单个游标的最新已提交值,不存在返回 None。"""
|
|
380
355
|
if not isinstance(key, str) or not key:
|
|
381
356
|
raise TypeError(f"cursor key must be a non-empty str, got {key!r}")
|
|
382
|
-
if not isinstance(value, str):
|
|
383
|
-
raise TypeError(f"cursor value must be str, got {type(value).__name__}")
|
|
384
357
|
with self._lock:
|
|
385
|
-
self._cursors
|
|
358
|
+
return self._cursors.get(key)
|
|
359
|
+
|
|
360
|
+
def set_cursor(self, key: str, value: str | None) -> None:
|
|
361
|
+
"""设置或删除单个游标(None 为删除)。"""
|
|
362
|
+
if not isinstance(key, str) or not key:
|
|
363
|
+
raise TypeError(f"cursor key must be a non-empty str, got {key!r}")
|
|
364
|
+
if value is not None and not isinstance(value, str):
|
|
365
|
+
raise TypeError(f"cursor value must be str or None, got {type(value).__name__}")
|
|
366
|
+
with self._lock:
|
|
367
|
+
if value is None:
|
|
368
|
+
self._cursors.pop(key, None)
|
|
369
|
+
else:
|
|
370
|
+
self._cursors[key] = value
|
|
386
371
|
|
|
387
372
|
# 尝试轨迹(append-only 旁路观测面)--------------------------------
|
|
388
373
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
374
|
+
|
|
375
|
+
def load_latest_succeeded_finished_at(self, job_uid: str) -> str | None:
|
|
376
|
+
"""按 attempt_id 倒序读取指定 Job 最新成功 Attempt 的结算时间。"""
|
|
377
|
+
with self._lock:
|
|
378
|
+
for rec in reversed(list(self._attempts.values())):
|
|
379
|
+
if rec.get("job_uid") == job_uid and rec.get("outcome") == "success":
|
|
380
|
+
return rec["settled_at"]
|
|
381
|
+
return None
|
|
382
|
+
|
|
383
|
+
# 元数据 ---------------------------------------------------------
|
|
384
|
+
|
|
385
|
+
def get_engine_metadata(self, key: str) -> str | None:
|
|
386
|
+
with self._lock:
|
|
387
|
+
return self._engine_metadata.get(key)
|
|
388
|
+
|
|
389
|
+
def set_engine_metadata(self, key: str, value: str) -> None:
|
|
390
|
+
with self._lock:
|
|
391
|
+
self._engine_metadata[key] = str(value)
|
|
392
|
+
|
|
393
|
+
# 状态事务与编号 -------------------------------------------------
|
|
394
|
+
|
|
395
|
+
def mark_job_running(
|
|
396
|
+
self, job_uid: str, *, execution_id: str, run_id: str, started_at: str
|
|
397
|
+
) -> int:
|
|
398
|
+
if not isinstance(execution_id, str) or not execution_id:
|
|
399
|
+
raise ValueError("execution_id must be a non-empty str")
|
|
400
|
+
if not isinstance(run_id, str) or not run_id:
|
|
401
|
+
raise ValueError("run_id must be a non-empty str")
|
|
402
|
+
if not execution_id_matches_run(execution_id, run_id):
|
|
403
|
+
raise ValueError(f"execution_id {execution_id!r} does not belong to run {run_id!r}")
|
|
404
|
+
if not isinstance(started_at, str) or not started_at:
|
|
405
|
+
raise ValueError("started_at must be a non-empty str")
|
|
406
|
+
with self._lock:
|
|
407
|
+
job_idx = None
|
|
408
|
+
for idx, item in enumerate(self._queue):
|
|
409
|
+
if job_uid_from_snapshot(item) == job_uid:
|
|
410
|
+
job_idx = idx
|
|
411
|
+
break
|
|
412
|
+
if job_idx is None:
|
|
413
|
+
raise ValueError(f"Job {job_uid} not found in queue")
|
|
414
|
+
job_data = self._queue[job_idx]
|
|
415
|
+
if job_data.get("status") == "running":
|
|
416
|
+
raise ValueError(f"Job {job_uid} is already running")
|
|
417
|
+
|
|
418
|
+
next_no = self._job_sequences.get(job_uid, 0) + 1
|
|
419
|
+
self._job_sequences[job_uid] = next_no
|
|
420
|
+
|
|
421
|
+
updated_job = queue_snapshot_with_execution_id(job_data, execution_id)
|
|
422
|
+
updated_job["status"] = "running"
|
|
423
|
+
updated_job["run_id"] = run_id
|
|
424
|
+
updated_job["attempt_no"] = next_no
|
|
425
|
+
updated_job["started_at"] = started_at
|
|
426
|
+
self._queue[job_idx] = updated_job
|
|
427
|
+
return next_no
|
|
428
|
+
|
|
429
|
+
def get_attempt_sequence(self, job_uid: str) -> int:
|
|
392
430
|
with self._lock:
|
|
393
|
-
|
|
394
|
-
self._attempt_seq += 1
|
|
395
|
-
self._attempts[attempt_id] = record
|
|
396
|
-
return attempt_id
|
|
431
|
+
return self._job_sequences.get(job_uid, 0)
|
|
397
432
|
|
|
398
|
-
def
|
|
433
|
+
def clear_attempts(self, job_uid: str) -> int:
|
|
434
|
+
with self._lock:
|
|
435
|
+
initial = len(self._attempts)
|
|
436
|
+
self._attempts = {
|
|
437
|
+
k: v for k, v in self._attempts.items()
|
|
438
|
+
if v.get("job_uid") != job_uid
|
|
439
|
+
}
|
|
440
|
+
return initial - len(self._attempts)
|
|
441
|
+
|
|
442
|
+
def load_settled_attempts(self, job_uid: str) -> list[dict[str, Any]]:
|
|
443
|
+
with self._lock:
|
|
444
|
+
entries: list[dict[str, Any]] = []
|
|
445
|
+
for att in self._attempts.values():
|
|
446
|
+
if att.get("job_uid") == job_uid:
|
|
447
|
+
entries.append(copy.deepcopy(att))
|
|
448
|
+
entries.sort(key=lambda x: x.get("attempt_no", 0))
|
|
449
|
+
return entries
|
|
450
|
+
|
|
451
|
+
def load_queue_jobs(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
|
452
|
+
with self._lock:
|
|
453
|
+
results: list[dict[str, Any]] = []
|
|
454
|
+
for pos, j in enumerate(self._queue):
|
|
455
|
+
j_status = j.get("status", "pending")
|
|
456
|
+
if status is not None and j_status != status:
|
|
457
|
+
continue
|
|
458
|
+
results.append({
|
|
459
|
+
"job_uid": job_uid_from_snapshot(j),
|
|
460
|
+
"queue_position": pos,
|
|
461
|
+
"job": public_queue_snapshot(j),
|
|
462
|
+
"status": j_status,
|
|
463
|
+
"execution_id": queue_snapshot_execution_id(j),
|
|
464
|
+
"run_id": j.get("run_id"),
|
|
465
|
+
"attempt_no": j.get("attempt_no"),
|
|
466
|
+
"started_at": j.get("started_at"),
|
|
467
|
+
})
|
|
468
|
+
return results
|
|
469
|
+
|
|
470
|
+
def commit_job_settlement(
|
|
399
471
|
self,
|
|
400
|
-
attempt_id: int,
|
|
401
472
|
*,
|
|
473
|
+
job_uid: str,
|
|
402
474
|
outcome: str,
|
|
403
|
-
|
|
475
|
+
attempt_no: int,
|
|
476
|
+
execution_id: str,
|
|
477
|
+
run_id: str,
|
|
478
|
+
settled_at: str,
|
|
479
|
+
started_at: str | None = None,
|
|
480
|
+
job_snapshot: dict[str, Any] | None = None,
|
|
481
|
+
files: dict[str, Any] | None = None,
|
|
482
|
+
result: dict[str, Any] | None = None,
|
|
404
483
|
error_message: str | None = None,
|
|
405
|
-
|
|
406
|
-
|
|
484
|
+
success_metadata: dict[str, Any] | None = None,
|
|
485
|
+
spawned_job_snapshots: Sequence[dict[str, Any]] = (),
|
|
486
|
+
cursor_updates: Mapping[str, str | None] | None = None,
|
|
487
|
+
failure_metadata: dict[str, Any] | None = None,
|
|
488
|
+
payload: dict[str, Any] | None = None,
|
|
489
|
+
requeued_job_snapshot: dict[str, Any] | None = None,
|
|
490
|
+
record_attempt: bool = True,
|
|
491
|
+
) -> None:
|
|
492
|
+
if outcome not in ("success", "failure", "retry", "pending"):
|
|
493
|
+
raise ValueError(f"Invalid outcome: {outcome}")
|
|
494
|
+
if not isinstance(attempt_no, int) or isinstance(attempt_no, bool) or attempt_no < 1:
|
|
495
|
+
raise TypeError(f"attempt_no must be int >= 1, got {attempt_no!r}")
|
|
496
|
+
if not isinstance(execution_id, str) or not execution_id:
|
|
497
|
+
raise ValueError(f"execution_id must be non-empty str, got {execution_id!r}")
|
|
498
|
+
if not isinstance(run_id, str) or not run_id:
|
|
499
|
+
raise ValueError(f"run_id must be non-empty str, got {run_id!r}")
|
|
500
|
+
if not execution_id_matches_run(execution_id, run_id):
|
|
501
|
+
raise ValueError(f"execution_id {execution_id!r} does not belong to run {run_id!r}")
|
|
502
|
+
if not isinstance(settled_at, str) or not settled_at:
|
|
503
|
+
raise ValueError(f"settled_at must be non-empty str, got {settled_at!r}")
|
|
504
|
+
if started_at is not None and (not isinstance(started_at, str) or not started_at):
|
|
505
|
+
raise ValueError("started_at must be non-empty str or None")
|
|
506
|
+
if cursor_updates:
|
|
507
|
+
for k, v in cursor_updates.items():
|
|
508
|
+
if not isinstance(k, str) or not k:
|
|
509
|
+
raise ValueError(f"cursor key must be a non-empty str, got {k!r}")
|
|
510
|
+
if v is not None and not isinstance(v, str):
|
|
511
|
+
raise TypeError(f"cursor value must be str or None, got {type(v).__name__}")
|
|
512
|
+
if spawned_job_snapshots:
|
|
513
|
+
validate_queue_replacement(list(spawned_job_snapshots))
|
|
407
514
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
validate_attempt_finish(outcome, finished_at, error_message)
|
|
412
|
-
with self._lock:
|
|
413
|
-
current = self._attempts.get(attempt_id)
|
|
414
|
-
if current is None:
|
|
415
|
-
return False
|
|
416
|
-
if current.outcome != "running":
|
|
417
|
-
raise ValueError(
|
|
418
|
-
f"attempt {attempt_id} already finalized with outcome "
|
|
419
|
-
f"{current.outcome!r}; double finalization is a protocol violation"
|
|
420
|
-
)
|
|
421
|
-
self._attempts[attempt_id] = replace(
|
|
422
|
-
current, outcome=outcome, finished_at=finished_at, error_message=error_message
|
|
515
|
+
with self._lock:
|
|
516
|
+
running = next(
|
|
517
|
+
(item for item in self._queue if job_uid_from_snapshot(item) == job_uid), {}
|
|
423
518
|
)
|
|
424
|
-
|
|
519
|
+
if (
|
|
520
|
+
running.get("status"),
|
|
521
|
+
running.get("run_id"),
|
|
522
|
+
running.get("attempt_no"),
|
|
523
|
+
queue_snapshot_execution_id(running),
|
|
524
|
+
) != ("running", run_id, attempt_no, execution_id):
|
|
525
|
+
raise TaskLiteError(f"Settlement does not match the running execution for {job_uid}")
|
|
526
|
+
if any(
|
|
527
|
+
attempt.get("job_uid") == job_uid
|
|
528
|
+
and attempt.get("attempt_no") == attempt_no
|
|
529
|
+
for attempt in self._attempts.values()
|
|
530
|
+
):
|
|
531
|
+
raise TaskLiteError(f"Execution for {job_uid} attempt {attempt_no} is already settled")
|
|
532
|
+
|
|
533
|
+
new_attempts = dict(self._attempts)
|
|
534
|
+
new_attempt_id = self._attempt_seq
|
|
535
|
+
if record_attempt:
|
|
536
|
+
attempt_outcome = (
|
|
537
|
+
outcome if outcome in ("success", "failure", "retry") else "failure"
|
|
538
|
+
)
|
|
539
|
+
attempt_entry: dict[str, Any] = {
|
|
540
|
+
"attempt_id": new_attempt_id,
|
|
541
|
+
"job_uid": job_uid,
|
|
542
|
+
"attempt_no": attempt_no,
|
|
543
|
+
"run_id": run_id,
|
|
544
|
+
"outcome": attempt_outcome,
|
|
545
|
+
"started_at": started_at,
|
|
546
|
+
"settled_at": settled_at,
|
|
547
|
+
"job": copy.deepcopy(job_snapshot),
|
|
548
|
+
"files": copy.deepcopy(files),
|
|
549
|
+
"result": copy.deepcopy(result),
|
|
550
|
+
"error_message": error_message,
|
|
551
|
+
}
|
|
552
|
+
new_attempts[new_attempt_id] = attempt_entry
|
|
553
|
+
|
|
554
|
+
new_queue = list(self._queue)
|
|
555
|
+
new_successes = dict(self._successes)
|
|
556
|
+
new_failed = dict(self._failed)
|
|
557
|
+
new_failure_payloads = dict(self._failure_payloads)
|
|
558
|
+
new_cursors = dict(self._cursors)
|
|
559
|
+
|
|
560
|
+
if outcome == "success":
|
|
561
|
+
remaining = [j for j in new_queue if job_uid_from_snapshot(j) != job_uid]
|
|
562
|
+
if spawned_job_snapshots:
|
|
563
|
+
remaining_uids = {job_uid_from_snapshot(j) for j in remaining}
|
|
564
|
+
seen: set[str] = set()
|
|
565
|
+
for sj in spawned_job_snapshots:
|
|
566
|
+
suid = job_uid_from_snapshot(sj)
|
|
567
|
+
if suid in remaining_uids or suid in seen:
|
|
568
|
+
raise TaskLiteError(
|
|
569
|
+
f"Spawned job {suid} conflicts with queue for {job_uid}"
|
|
570
|
+
)
|
|
571
|
+
seen.add(suid)
|
|
572
|
+
for s in reversed(spawned_job_snapshots):
|
|
573
|
+
s_copy = copy.deepcopy(s)
|
|
574
|
+
s_copy.setdefault("status", "pending")
|
|
575
|
+
remaining.insert(0, s_copy)
|
|
576
|
+
new_queue = remaining
|
|
425
577
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
return [rec for rec in self._attempts.values() if rec.job_uid == job_uid]
|
|
578
|
+
new_successes[job_uid] = copy.deepcopy(success_metadata or {})
|
|
579
|
+
new_failed.pop(job_uid, None)
|
|
580
|
+
new_failure_payloads.pop(job_uid, None)
|
|
430
581
|
|
|
431
|
-
|
|
432
|
-
|
|
582
|
+
if cursor_updates:
|
|
583
|
+
for k, v in cursor_updates.items():
|
|
584
|
+
if v is None:
|
|
585
|
+
new_cursors.pop(k, None)
|
|
586
|
+
else:
|
|
587
|
+
new_cursors[k] = v
|
|
588
|
+
|
|
589
|
+
elif outcome == "failure":
|
|
590
|
+
new_queue = [j for j in new_queue if job_uid_from_snapshot(j) != job_uid]
|
|
591
|
+
new_failed[job_uid] = copy.deepcopy(failure_metadata or {})
|
|
592
|
+
if payload is not None:
|
|
593
|
+
new_failure_payloads[job_uid] = copy.deepcopy(payload)
|
|
594
|
+
new_successes.pop(job_uid, None)
|
|
595
|
+
|
|
596
|
+
elif outcome == "retry":
|
|
597
|
+
found = False
|
|
598
|
+
for idx, j in enumerate(new_queue):
|
|
599
|
+
if job_uid_from_snapshot(j) == job_uid:
|
|
600
|
+
target = copy.deepcopy(requeued_job_snapshot if requeued_job_snapshot is not None else j)
|
|
601
|
+
target["status"] = "pending"
|
|
602
|
+
target["run_id"] = None
|
|
603
|
+
target["attempt_no"] = target.get("attempt_no") or 1
|
|
604
|
+
target["started_at"] = None
|
|
605
|
+
target.pop(QUEUE_EXECUTION_ID_KEY, None)
|
|
606
|
+
new_queue[idx] = target
|
|
607
|
+
found = True
|
|
608
|
+
break
|
|
609
|
+
if not found and requeued_job_snapshot is not None:
|
|
610
|
+
target = copy.deepcopy(requeued_job_snapshot)
|
|
611
|
+
target["status"] = "pending"
|
|
612
|
+
new_queue.append(target)
|
|
613
|
+
|
|
614
|
+
elif outcome == "pending":
|
|
615
|
+
for idx, j in enumerate(new_queue):
|
|
616
|
+
if job_uid_from_snapshot(j) == job_uid:
|
|
617
|
+
target = copy.deepcopy(j)
|
|
618
|
+
target["status"] = "pending"
|
|
619
|
+
target["run_id"] = None
|
|
620
|
+
target["attempt_no"] = target.get("attempt_no") or 1
|
|
621
|
+
target["started_at"] = None
|
|
622
|
+
target.pop(QUEUE_EXECUTION_ID_KEY, None)
|
|
623
|
+
new_queue[idx] = target
|
|
624
|
+
break
|
|
625
|
+
|
|
626
|
+
# 统一原子落变
|
|
627
|
+
self._attempts = new_attempts
|
|
628
|
+
if record_attempt:
|
|
629
|
+
self._attempt_seq += 1
|
|
630
|
+
self._queue = new_queue
|
|
631
|
+
self._successes = new_successes
|
|
632
|
+
self._failed = new_failed
|
|
633
|
+
self._failure_payloads = new_failure_payloads
|
|
634
|
+
self._cursors = new_cursors
|
|
635
|
+
|
|
636
|
+
def record_authorization(
|
|
637
|
+
self,
|
|
638
|
+
path: str,
|
|
639
|
+
target_type: str,
|
|
640
|
+
*,
|
|
641
|
+
authorized_at: str | None = None,
|
|
642
|
+
) -> None:
|
|
643
|
+
norm_path = os.path.normpath(os.fspath(path))
|
|
644
|
+
if target_type not in ("file", "directory"):
|
|
645
|
+
raise ValueError(f"target_type must be 'file' or 'directory', got {target_type!r}")
|
|
646
|
+
ts = authorized_at if authorized_at is not None else utc_now_iso()
|
|
433
647
|
with self._lock:
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
return None
|
|
648
|
+
self._authorizations[norm_path] = {
|
|
649
|
+
"path": norm_path,
|
|
650
|
+
"target_type": target_type,
|
|
651
|
+
"authorized_at": ts,
|
|
652
|
+
}
|
|
440
653
|
|
|
441
|
-
|
|
654
|
+
def list_authorizations(self) -> list[dict[str, Any]]:
|
|
655
|
+
with self._lock:
|
|
656
|
+
records = list(self._authorizations.values())
|
|
657
|
+
return sorted(records, key=lambda r: (r["authorized_at"], r["path"]))
|
|
442
658
|
|
|
443
|
-
def
|
|
659
|
+
def revoke_authorization(self, path: str) -> bool:
|
|
660
|
+
norm_path = os.path.normpath(os.fspath(path))
|
|
444
661
|
with self._lock:
|
|
445
|
-
|
|
662
|
+
if norm_path in self._authorizations:
|
|
663
|
+
del self._authorizations[norm_path]
|
|
664
|
+
return True
|
|
665
|
+
return False
|
|
446
666
|
|
|
447
|
-
def
|
|
667
|
+
def is_path_authorized(self, path: str, target_type: str) -> bool:
|
|
668
|
+
norm_path = os.path.normpath(os.fspath(path))
|
|
669
|
+
target_p = Path(norm_path)
|
|
448
670
|
with self._lock:
|
|
449
|
-
|
|
671
|
+
exact = self._authorizations.get(norm_path)
|
|
672
|
+
if exact is not None and exact["target_type"] == target_type:
|
|
673
|
+
return True
|
|
674
|
+
for record in self._authorizations.values():
|
|
675
|
+
if record["target_type"] == "directory":
|
|
676
|
+
dir_path = Path(record["path"])
|
|
677
|
+
try:
|
|
678
|
+
if target_p.is_relative_to(dir_path):
|
|
679
|
+
return True
|
|
680
|
+
except (ValueError, TypeError):
|
|
681
|
+
pass
|
|
682
|
+
return False
|
|
683
|
+
|
|
684
|
+
def close(self) -> None:
|
|
685
|
+
"""关闭后端并释放资源(幂等)。"""
|
|
686
|
+
pass
|
|
450
687
|
|
|
451
688
|
|
|
452
689
|
__all__ = ["InMemoryStateBackend"]
|