tasklite-engine 2.2.0__py3-none-any.whl → 3.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. tasklite/__init__.py +30 -196
  2. tasklite/backend/__init__.py +2 -5
  3. tasklite/backend/base.py +156 -184
  4. tasklite/backend/memory.py +468 -231
  5. tasklite/backend/sqlite_backend.py +680 -408
  6. tasklite/engine/__init__.py +2 -2
  7. tasklite/engine/admission.py +85 -44
  8. tasklite/engine/channel.py +566 -458
  9. tasklite/engine/completion.py +535 -247
  10. tasklite/engine/config.py +7 -131
  11. tasklite/engine/dispatch.py +415 -321
  12. tasklite/engine/execution_access.py +39 -0
  13. tasklite/engine/files.py +52 -0
  14. tasklite/engine/governor.py +46 -204
  15. tasklite/engine/in_flight.py +15 -13
  16. tasklite/engine/ops.py +9 -347
  17. tasklite/engine/recovery.py +67 -267
  18. tasklite/engine/resource.py +256 -386
  19. tasklite/engine/runtime.py +527 -236
  20. tasklite/engine/scheduler.py +99 -90
  21. tasklite/engine/selection.py +53 -111
  22. tasklite/engine/session.py +49 -77
  23. tasklite/engine/store.py +595 -564
  24. tasklite/engine/types.py +66 -71
  25. tasklite/engine/wait.py +30 -16
  26. tasklite/exceptions.py +4 -57
  27. tasklite/hooks.py +3 -23
  28. tasklite/models/__init__.py +21 -6
  29. tasklite/models/artifact.py +386 -47
  30. tasklite/models/context.py +173 -203
  31. tasklite/models/job.py +115 -266
  32. tasklite/models/result.py +246 -0
  33. tasklite/models/state.py +97 -17
  34. tasklite/models/task.py +52 -105
  35. tasklite/pipeline.py +279 -534
  36. tasklite/resource.py +12 -0
  37. tasklite/selection.py +9 -97
  38. tasklite/snapshot.py +639 -0
  39. tasklite/testing.py +32 -34
  40. tasklite/triggers.py +5 -4
  41. tasklite/utils/artifact.py +85 -17
  42. tasklite/utils/clock.py +3 -3
  43. tasklite/utils/encoding.py +113 -18
  44. tasklite/utils/ipc.py +240 -242
  45. tasklite/utils/lockfile.py +33 -58
  46. tasklite/wrappers/__init__.py +1 -1
  47. tasklite/wrappers/discovery.py +155 -158
  48. tasklite/wrappers/http.py +68 -481
  49. tasklite_engine-3.0.0.dev0.dist-info/METADATA +307 -0
  50. tasklite_engine-3.0.0.dev0.dist-info/RECORD +58 -0
  51. tasklite/engine/errorclass.py +0 -971
  52. tasklite/models/attempt.py +0 -141
  53. tasklite_engine-2.2.0.dist-info/METADATA +0 -369
  54. tasklite_engine-2.2.0.dist-info/RECORD +0 -55
  55. {tasklite_engine-2.2.0.dist-info → tasklite_engine-3.0.0.dev0.dist-info}/WHEEL +0 -0
  56. {tasklite_engine-2.2.0.dist-info → tasklite_engine-3.0.0.dev0.dist-info}/licenses/LICENSE +0 -0
  57. {tasklite_engine-2.2.0.dist-info → tasklite_engine-3.0.0.dev0.dist-info}/top_level.txt +0 -0
@@ -9,18 +9,23 @@ from __future__ import annotations
9
9
 
10
10
  import copy
11
11
  import logging
12
+ import os
12
13
  import threading
13
- from dataclasses import replace
14
+ from pathlib import Path
14
15
  from typing import Any, Callable, Mapping, Sequence
15
16
 
16
17
  from .base import (
17
18
  AbstractStateBackend,
18
- validate_attempt_dispatch,
19
- validate_attempt_finish,
19
+ QUEUE_EXECUTION_ID_KEY,
20
+ execution_id_matches_run,
21
+ public_queue_snapshot,
22
+ queue_snapshot_execution_id,
23
+ queue_snapshot_with_execution_id,
20
24
  validate_queue_replacement,
21
25
  )
22
- from ..models.attempt import AttemptRecord
26
+ from ..exceptions import TaskLiteError
23
27
  from ..models.state import job_uid_from_snapshot
28
+ from ..utils.clock import utc_now_iso
24
29
 
25
30
  logger = logging.getLogger("tasklite")
26
31
 
@@ -35,14 +40,17 @@ class InMemoryStateBackend(AbstractStateBackend):
35
40
 
36
41
  def __init__(self) -> None:
37
42
  self._lock = threading.RLock()
43
+ self._exec_lock = threading.Lock()
38
44
  self._successes: dict[str, dict[str, Any]] = {}
39
45
  self._failed: dict[str, dict[str, Any]] = {}
40
46
  self._failure_payloads: dict[str, dict[str, Any]] = {}
41
47
  self._cursors: dict[str, str] = {}
42
48
  self._queue: list[dict[str, Any]] = []
43
49
  self._engine_metadata: dict[str, str] = {}
44
- self._attempts: dict[int, AttemptRecord] = {}
50
+ self._attempts: dict[int, dict[str, Any]] = {}
45
51
  self._attempt_seq = 1
52
+ self._job_sequences: dict[str, int] = {}
53
+ self._authorizations: dict[str, dict[str, Any]] = {}
46
54
 
47
55
  # 全量加载 -------------------------------------------------------
48
56
 
@@ -65,7 +73,10 @@ class InMemoryStateBackend(AbstractStateBackend):
65
73
 
66
74
  def load_queue(self) -> list[dict[str, Any]]:
67
75
  with self._lock:
68
- return copy.deepcopy(self._queue)
76
+ return [
77
+ public_queue_snapshot(j)
78
+ for j in self._queue if j.get("status", "pending") == "pending"
79
+ ]
69
80
 
70
81
  # 整表重写 -------------------------------------------------------
71
82
 
@@ -105,106 +116,6 @@ class InMemoryStateBackend(AbstractStateBackend):
105
116
 
106
117
  # delta 提交 -----------------------------------------------------
107
118
 
108
- def commit_job_success(
109
- self,
110
- job_uid: str,
111
- result_metadata: dict,
112
- *,
113
- attempt_id: int | None = None,
114
- execution_id: str | None = None,
115
- finished_at: str | None = None,
116
- spawned_job_snapshots: Sequence[dict[str, Any]] = (),
117
- cursor_updates: Mapping[str, str | None] | None = None,
118
- ) -> bool:
119
- with self._lock:
120
- try:
121
- # 校验先行:全部 deepcopy、attempts 行校验与冲突判定在任何变更前完成。
122
- # 不变式:返回 False / 抛异常 ⇒ successes/attempts/queue/failed/cursors 与
123
- # 调用前完全一致(对齐 SQLite 事务回滚;store 的连续失败
124
- # 崩溃契约以「后端未变」为前提做重启重建)。
125
- if not isinstance(attempt_id, int) or isinstance(attempt_id, bool):
126
- logger.critical(
127
- f"commit_job_success for {job_uid}: attempt_id must be an int, got {attempt_id!r}"
128
- )
129
- return False
130
- if execution_id is not None and (not isinstance(execution_id, str) or not execution_id):
131
- logger.critical(
132
- f"commit_job_success for {job_uid}: execution_id must be a non-empty str or None, got {execution_id!r}"
133
- )
134
- return False
135
- validate_attempt_finish("succeeded", finished_at, None)
136
- current_attempt = self._attempts.get(attempt_id)
137
- if current_attempt is None:
138
- logger.critical(
139
- f"commit_job_success for {job_uid}: attempt {attempt_id} does not exist"
140
- )
141
- return False
142
- if current_attempt.job_uid != job_uid:
143
- logger.critical(
144
- f"commit_job_success for {job_uid}: attempt {attempt_id} job_uid mismatch "
145
- f"(expected {job_uid!r}, got {current_attempt.job_uid!r})"
146
- )
147
- return False
148
- if execution_id is not None and current_attempt.execution_id != execution_id:
149
- logger.critical(
150
- f"commit_job_success for {job_uid}: attempt {attempt_id} execution_id mismatch "
151
- f"(expected {execution_id!r}, got {current_attempt.execution_id!r})"
152
- )
153
- return False
154
- if current_attempt.outcome != "running":
155
- logger.critical(
156
- f"commit_job_success for {job_uid}: attempt {attempt_id} already finalized "
157
- f"with outcome {current_attempt.outcome!r}"
158
- )
159
- return False
160
- updated_attempt = replace(
161
- current_attempt,
162
- outcome="succeeded",
163
- finished_at=finished_at,
164
- error_message=None,
165
- )
166
- success_metadata = copy.deepcopy(result_metadata or {})
167
- spawned_copy = [copy.deepcopy(j) for j in spawned_job_snapshots]
168
- remaining = [j for j in self._queue if job_uid_from_snapshot(j) != job_uid]
169
- if spawned_copy:
170
- remaining_uids = {job_uid_from_snapshot(j) for j in remaining}
171
- seen: set[str] = set()
172
- for sj in spawned_copy:
173
- suid = job_uid_from_snapshot(sj)
174
- # spawned uid 撞上删除 popped 后的队列既有条目,或
175
- # 批内自相重复,均判定内存/磁盘漂移 → False 走崩溃
176
- # 契约,绝不产出重复队列条目(对齐 SQLite
177
- # INSERT OR IGNORE + rowcount 漂移检测)。
178
- if suid in remaining_uids or suid in seen:
179
- logger.critical(
180
- f"commit_job_success for {job_uid}: spawned job {suid} "
181
- f"conflicts with queue (drift). Returning False."
182
- )
183
- return False
184
- seen.add(suid)
185
- cursor_sets: dict[str, str] = {}
186
- cursor_dels: set[str] = set()
187
- if cursor_updates:
188
- for k, v in cursor_updates.items():
189
- if v is None:
190
- cursor_dels.add(k)
191
- else:
192
- cursor_sets[k] = str(v)
193
- # 校验全部通过,统一落变(落变段不再调用任何可失败操作)
194
- self._attempts[attempt_id] = updated_attempt
195
- self._successes[job_uid] = success_metadata
196
- self._queue = spawned_copy + remaining # 队首插入 spawned 并保序
197
- # 成功 commit 清理 failed 同名残行:与「job 最终状态唯一」
198
- # 语义一致,防 successes∩failed 并存。
199
- self._failed.pop(job_uid, None)
200
- self._failure_payloads.pop(job_uid, None)
201
- self._cursors.update(cursor_sets)
202
- for k in cursor_dels:
203
- self._cursors.pop(k, None)
204
- return True
205
- except Exception as e:
206
- logger.critical(f"Failed to commit job success for {job_uid}: {e}")
207
- return False
208
119
 
209
120
  def commit_job_failure(
210
121
  self, job_uid: str, failure_metadata: dict, payload: dict[str, Any] | None = None
@@ -230,66 +141,25 @@ class InMemoryStateBackend(AbstractStateBackend):
230
141
  return True
231
142
  except Exception as e:
232
143
  logger.critical(f"Failed to commit job failure for {job_uid}: {e}")
233
- return False
144
+ raise
234
145
 
235
- def commit_retry(
236
- self,
237
- popped_job_uid: str,
238
- requeued_job_snapshot: dict[str, Any],
239
- *,
240
- front: bool = False,
241
- ) -> bool:
242
- with self._lock:
243
- try:
244
- requeued_copy = copy.deepcopy(requeued_job_snapshot)
245
- ruid = job_uid_from_snapshot(requeued_copy)
246
- remaining = [j for j in self._queue if job_uid_from_snapshot(j) != popped_job_uid]
247
- # 对齐 SQLite 普通 INSERT 冲突回滚:requeued uid 撞上删除
248
- # popped 后的既有条目 → False 走崩溃契约,绝不产出重复条目;
249
- # uid == popped_job_uid 属同 job 重插(DELETE+INSERT),安全放行。
250
- if ruid != popped_job_uid and any(job_uid_from_snapshot(j) == ruid for j in remaining):
251
- logger.critical(
252
- f"commit_retry for {popped_job_uid}: requeued uid {ruid} "
253
- f"conflicts with existing queue entry. Returning False."
254
- )
255
- return False
256
- if front:
257
- self._queue = [requeued_copy] + remaining
258
- else:
259
- self._queue = remaining + [requeued_copy]
260
- return True
261
- except Exception as e:
262
- logger.critical(f"Failed to commit retry for {popped_job_uid}: {e}")
263
- return False
264
146
 
265
- def commit_job_failures(self, job_uids_metadata: list[tuple[str, dict]]) -> bool:
147
+ def commit_job_failures(self, job_uids_metadata: list[tuple[str, dict]]) -> None:
266
148
  with self._lock:
267
- try:
268
- # 校验先行:全部失败行拷贝在任何变更前完成——任一行失败则
269
- # queue/successes/failed 整体不变(对齐 SQLite 事务回滚,杜绝
270
- # 「队列已整体删除、档案未落」的半成品失败态)。
271
- entries = [(job_uid, copy.deepcopy(meta or {})) for job_uid, meta in job_uids_metadata]
272
- fail_uids = {u for u, _ in job_uids_metadata}
273
- remaining = [j for j in self._queue if job_uid_from_snapshot(j) not in fail_uids]
274
- for job_uid, meta in entries:
275
- self._failed[job_uid] = meta
276
- # 级联/死锁批量失败时 successes 旧成功记录作废(最终状态
277
- # 唯一),与 commit_job_failure 对称。
278
- self._successes.pop(job_uid, None)
279
- self._queue = remaining
280
- return True
281
- except Exception as e:
282
- logger.critical(f"Failed to commit bulk failure: {e}")
283
- return False
284
-
285
- def commit_skip(self, job_uid: str) -> bool:
149
+ # 所有可失败的复制与队列构造先于统一发布。
150
+ entries = [(job_uid, copy.deepcopy(meta or {})) for job_uid, meta in job_uids_metadata]
151
+ fail_uids = {u for u, _ in entries}
152
+ remaining = [j for j in self._queue if job_uid_from_snapshot(j) not in fail_uids]
153
+ failed = self._failed | dict(entries)
154
+ successes = {uid: meta for uid, meta in self._successes.items() if uid not in fail_uids}
155
+ self._failed = failed
156
+ self._successes = successes
157
+ self._queue = remaining
158
+
159
+ def commit_skip(self, job_uid: str) -> None:
286
160
  with self._lock:
287
161
  self._queue = [j for j in self._queue if job_uid_from_snapshot(j) != job_uid]
288
- return True
289
162
 
290
- def append_failed(self, job_uid: str, metadata: dict[str, Any] | None = None) -> None:
291
- with self._lock:
292
- self._failed[job_uid] = copy.deepcopy(metadata or {})
293
163
 
294
164
  # 增量入队 -------------------------------------------------------
295
165
 
@@ -350,103 +220,470 @@ class InMemoryStateBackend(AbstractStateBackend):
350
220
  count += 1
351
221
  return count
352
222
 
353
- def seed_successes(self, job_uids: list[str]) -> int:
354
- """把 uid 批量写入 successes(meta 空 dict)。
223
+ def apply_manual_status(
224
+ self,
225
+ updates: Sequence[tuple[str, str, dict[str, Any], dict[str, Any] | None]],
226
+ ) -> None:
227
+ """批量原子应用人工状态变更 (job_uid, status, snapshot, metadata)。"""
228
+ if not updates:
229
+ return
230
+ with self._lock:
231
+ new_successes = copy.deepcopy(self._successes)
232
+ new_failed = copy.deepcopy(self._failed)
233
+ new_failure_payloads = copy.deepcopy(self._failure_payloads)
234
+ new_queue = copy.deepcopy(self._queue)
235
+ for job_uid, status, snapshot, metadata in updates:
236
+ snap_copy = copy.deepcopy(snapshot)
237
+ meta_copy = copy.deepcopy(metadata or {})
238
+ if status == "pending":
239
+ new_successes.pop(job_uid, None)
240
+ new_failed.pop(job_uid, None)
241
+ new_failure_payloads.pop(job_uid, None)
242
+ new_queue = [
243
+ j for j in new_queue
244
+ if job_uid_from_snapshot(j) != job_uid
245
+ ]
246
+ snap_copy["status"] = "pending"
247
+ snap_copy["run_id"] = None
248
+ snap_copy["attempt_no"] = None
249
+ snap_copy["started_at"] = None
250
+ snap_copy.pop(QUEUE_EXECUTION_ID_KEY, None)
251
+ new_queue.append(snap_copy)
252
+ elif status == "succeeded":
253
+ new_queue = [
254
+ j for j in new_queue
255
+ if job_uid_from_snapshot(j) != job_uid
256
+ ]
257
+ new_failed.pop(job_uid, None)
258
+ new_failure_payloads.pop(job_uid, None)
259
+ new_successes[job_uid] = meta_copy
260
+ elif status == "failed":
261
+ new_queue = [
262
+ j for j in new_queue
263
+ if job_uid_from_snapshot(j) != job_uid
264
+ ]
265
+ new_successes.pop(job_uid, None)
266
+ new_failed[job_uid] = meta_copy
267
+ new_failure_payloads[job_uid] = snap_copy.get("payload") or {}
268
+ else:
269
+ raise ValueError(f"Unsupported manual status: {status!r}")
355
270
 
356
- 不变式:successes/failed 全局互斥——已在 failed 的 uid 拒绝种子;
357
- 先查后写(同锁内),冲突整体拒绝、零写入,不静默清除失败档案
358
- 记录(失败历史须先经 delete_failed 显式清除)。
359
- """
271
+ self._successes = new_successes
272
+ self._failed = new_failed
273
+ self._failure_payloads = new_failure_payloads
274
+ self._queue = new_queue
275
+
276
+ @staticmethod
277
+ def _snapshot_from_metadata(metadata: Any) -> dict[str, Any] | None:
278
+ if not isinstance(metadata, dict):
279
+ return None
280
+ snapshot = metadata.get("job_snapshot")
281
+ return snapshot if isinstance(snapshot, dict) else None
282
+
283
+ def load_known_job_snapshot(self, uid: str) -> dict[str, Any] | None:
284
+ """按队列、成功、失败、Attempt 历史的优先级读取完整配置。"""
360
285
  with self._lock:
361
- conflict = sorted({u for u in job_uids if u in self._failed})
362
- if conflict:
363
- raise ValueError(
364
- f"seed_successes refuses uid(s) already in failed: {conflict}; "
365
- f"successes/failed must stay disjoint"
366
- )
367
- count = 0
368
- for u in job_uids:
369
- self._successes[u] = {}
370
- count += 1
371
- return count
286
+ for snapshot in self._queue:
287
+ if job_uid_from_snapshot(snapshot) == uid:
288
+ return public_queue_snapshot(snapshot)
372
289
 
373
- def seed_cursor(self, key: str, value: str) -> None:
374
- """预填一个 cursor(UPSERT,幂等)。
290
+ success_snapshot = self._snapshot_from_metadata(self._successes.get(uid))
291
+ if success_snapshot is not None:
292
+ return copy.deepcopy(success_snapshot)
375
293
 
376
- 与 SQLiteStateBackend.seed_cursor 同契约 fail-loud:key 非空 str、
377
- value 必须 str——静默 str() 强转会让同一非法入参在本腿写数字串、
378
- sqlite 腿抛异常(跨后端行为分歧),且与内存镜像值型漂移。
379
- """
294
+ failure_snapshot = self._snapshot_from_metadata(self._failed.get(uid))
295
+ if failure_snapshot is not None:
296
+ return copy.deepcopy(failure_snapshot)
297
+
298
+ for attempt_id in sorted(self._attempts, reverse=True):
299
+ attempt = self._attempts[attempt_id]
300
+ if attempt.get("job_uid") != uid:
301
+ continue
302
+ attempt_snapshot = attempt.get("job")
303
+ if isinstance(attempt_snapshot, dict):
304
+ return copy.deepcopy(attempt_snapshot)
305
+ return None
306
+
307
+ def load_known_job_snapshots(self) -> dict[str, dict[str, Any]]:
308
+ """读取全部已知 Job 的完整配置,低优先级来源先写入。"""
309
+ with self._lock:
310
+ known: dict[str, dict[str, Any]] = {}
311
+ for attempt_id in sorted(self._attempts):
312
+ attempt = self._attempts[attempt_id]
313
+ uid = attempt.get("job_uid")
314
+ snapshot = attempt.get("job")
315
+ if isinstance(uid, str) and isinstance(snapshot, dict):
316
+ known[uid] = copy.deepcopy(snapshot)
317
+
318
+ for uid, metadata in self._failed.items():
319
+ snapshot = self._snapshot_from_metadata(metadata)
320
+ if snapshot is not None:
321
+ known[uid] = copy.deepcopy(snapshot)
322
+
323
+ for uid, metadata in self._successes.items():
324
+ snapshot = self._snapshot_from_metadata(metadata)
325
+ if snapshot is not None:
326
+ known[uid] = copy.deepcopy(snapshot)
327
+
328
+ for snapshot in self._queue:
329
+ known[job_uid_from_snapshot(snapshot)] = public_queue_snapshot(snapshot)
330
+ return known
331
+
332
+ def clear_history(self, uid: str, records: set[str]) -> int:
333
+ """从 successes/failed/attempts 清除指定 job_uid 的记录,返回删除条数。"""
334
+ total = 0
335
+ with self._lock:
336
+ if "successes" in records and uid in self._successes:
337
+ del self._successes[uid]
338
+ total += 1
339
+ if "failures" in records:
340
+ if uid in self._failed:
341
+ del self._failed[uid]
342
+ total += 1
343
+ self._failure_payloads.pop(uid, None)
344
+ if "attempts" in records:
345
+ initial = len(self._attempts)
346
+ self._attempts = {
347
+ k: v for k, v in self._attempts.items()
348
+ if v.get("job_uid") != uid
349
+ }
350
+ total += initial - len(self._attempts)
351
+ return total
352
+
353
+ def get_cursor(self, key: str) -> str | None:
354
+ """查询单个游标的最新已提交值,不存在返回 None。"""
380
355
  if not isinstance(key, str) or not key:
381
356
  raise TypeError(f"cursor key must be a non-empty str, got {key!r}")
382
- if not isinstance(value, str):
383
- raise TypeError(f"cursor value must be str, got {type(value).__name__}")
384
357
  with self._lock:
385
- self._cursors[key] = value
358
+ return self._cursors.get(key)
359
+
360
+ def set_cursor(self, key: str, value: str | None) -> None:
361
+ """设置或删除单个游标(None 为删除)。"""
362
+ if not isinstance(key, str) or not key:
363
+ raise TypeError(f"cursor key must be a non-empty str, got {key!r}")
364
+ if value is not None and not isinstance(value, str):
365
+ raise TypeError(f"cursor value must be str or None, got {type(value).__name__}")
366
+ with self._lock:
367
+ if value is None:
368
+ self._cursors.pop(key, None)
369
+ else:
370
+ self._cursors[key] = value
386
371
 
387
372
  # 尝试轨迹(append-only 旁路观测面)--------------------------------
388
373
 
389
- def append_attempt(self, record: AttemptRecord) -> int:
390
- """派发即插行:分配自增轨迹 id 并落内存表(单语句语义,无读-改-写窗口)。"""
391
- validate_attempt_dispatch(record)
374
+
375
+ def load_latest_succeeded_finished_at(self, job_uid: str) -> str | None:
376
+ """按 attempt_id 倒序读取指定 Job 最新成功 Attempt 的结算时间。"""
377
+ with self._lock:
378
+ for rec in reversed(list(self._attempts.values())):
379
+ if rec.get("job_uid") == job_uid and rec.get("outcome") == "success":
380
+ return rec["settled_at"]
381
+ return None
382
+
383
+ # 元数据 ---------------------------------------------------------
384
+
385
+ def get_engine_metadata(self, key: str) -> str | None:
386
+ with self._lock:
387
+ return self._engine_metadata.get(key)
388
+
389
+ def set_engine_metadata(self, key: str, value: str) -> None:
390
+ with self._lock:
391
+ self._engine_metadata[key] = str(value)
392
+
393
+ # 状态事务与编号 -------------------------------------------------
394
+
395
+ def mark_job_running(
396
+ self, job_uid: str, *, execution_id: str, run_id: str, started_at: str
397
+ ) -> int:
398
+ if not isinstance(execution_id, str) or not execution_id:
399
+ raise ValueError("execution_id must be a non-empty str")
400
+ if not isinstance(run_id, str) or not run_id:
401
+ raise ValueError("run_id must be a non-empty str")
402
+ if not execution_id_matches_run(execution_id, run_id):
403
+ raise ValueError(f"execution_id {execution_id!r} does not belong to run {run_id!r}")
404
+ if not isinstance(started_at, str) or not started_at:
405
+ raise ValueError("started_at must be a non-empty str")
406
+ with self._lock:
407
+ job_idx = None
408
+ for idx, item in enumerate(self._queue):
409
+ if job_uid_from_snapshot(item) == job_uid:
410
+ job_idx = idx
411
+ break
412
+ if job_idx is None:
413
+ raise ValueError(f"Job {job_uid} not found in queue")
414
+ job_data = self._queue[job_idx]
415
+ if job_data.get("status") == "running":
416
+ raise ValueError(f"Job {job_uid} is already running")
417
+
418
+ next_no = self._job_sequences.get(job_uid, 0) + 1
419
+ self._job_sequences[job_uid] = next_no
420
+
421
+ updated_job = queue_snapshot_with_execution_id(job_data, execution_id)
422
+ updated_job["status"] = "running"
423
+ updated_job["run_id"] = run_id
424
+ updated_job["attempt_no"] = next_no
425
+ updated_job["started_at"] = started_at
426
+ self._queue[job_idx] = updated_job
427
+ return next_no
428
+
429
+ def get_attempt_sequence(self, job_uid: str) -> int:
392
430
  with self._lock:
393
- attempt_id = self._attempt_seq
394
- self._attempt_seq += 1
395
- self._attempts[attempt_id] = record
396
- return attempt_id
431
+ return self._job_sequences.get(job_uid, 0)
397
432
 
398
- def update_attempt(
433
+ def clear_attempts(self, job_uid: str) -> int:
434
+ with self._lock:
435
+ initial = len(self._attempts)
436
+ self._attempts = {
437
+ k: v for k, v in self._attempts.items()
438
+ if v.get("job_uid") != job_uid
439
+ }
440
+ return initial - len(self._attempts)
441
+
442
+ def load_settled_attempts(self, job_uid: str) -> list[dict[str, Any]]:
443
+ with self._lock:
444
+ entries: list[dict[str, Any]] = []
445
+ for att in self._attempts.values():
446
+ if att.get("job_uid") == job_uid:
447
+ entries.append(copy.deepcopy(att))
448
+ entries.sort(key=lambda x: x.get("attempt_no", 0))
449
+ return entries
450
+
451
+ def load_queue_jobs(self, *, status: str | None = None) -> list[dict[str, Any]]:
452
+ with self._lock:
453
+ results: list[dict[str, Any]] = []
454
+ for pos, j in enumerate(self._queue):
455
+ j_status = j.get("status", "pending")
456
+ if status is not None and j_status != status:
457
+ continue
458
+ results.append({
459
+ "job_uid": job_uid_from_snapshot(j),
460
+ "queue_position": pos,
461
+ "job": public_queue_snapshot(j),
462
+ "status": j_status,
463
+ "execution_id": queue_snapshot_execution_id(j),
464
+ "run_id": j.get("run_id"),
465
+ "attempt_no": j.get("attempt_no"),
466
+ "started_at": j.get("started_at"),
467
+ })
468
+ return results
469
+
470
+ def commit_job_settlement(
399
471
  self,
400
- attempt_id: int,
401
472
  *,
473
+ job_uid: str,
402
474
  outcome: str,
403
- finished_at: str,
475
+ attempt_no: int,
476
+ execution_id: str,
477
+ run_id: str,
478
+ settled_at: str,
479
+ started_at: str | None = None,
480
+ job_snapshot: dict[str, Any] | None = None,
481
+ files: dict[str, Any] | None = None,
482
+ result: dict[str, Any] | None = None,
404
483
  error_message: str | None = None,
405
- ) -> bool:
406
- """收尾轨迹行:仅允许把 running 行收敛为终态一次。
484
+ success_metadata: dict[str, Any] | None = None,
485
+ spawned_job_snapshots: Sequence[dict[str, Any]] = (),
486
+ cursor_updates: Mapping[str, str | None] | None = None,
487
+ failure_metadata: dict[str, Any] | None = None,
488
+ payload: dict[str, Any] | None = None,
489
+ requeued_job_snapshot: dict[str, Any] | None = None,
490
+ record_attempt: bool = True,
491
+ ) -> None:
492
+ if outcome not in ("success", "failure", "retry", "pending"):
493
+ raise ValueError(f"Invalid outcome: {outcome}")
494
+ if not isinstance(attempt_no, int) or isinstance(attempt_no, bool) or attempt_no < 1:
495
+ raise TypeError(f"attempt_no must be int >= 1, got {attempt_no!r}")
496
+ if not isinstance(execution_id, str) or not execution_id:
497
+ raise ValueError(f"execution_id must be non-empty str, got {execution_id!r}")
498
+ if not isinstance(run_id, str) or not run_id:
499
+ raise ValueError(f"run_id must be non-empty str, got {run_id!r}")
500
+ if not execution_id_matches_run(execution_id, run_id):
501
+ raise ValueError(f"execution_id {execution_id!r} does not belong to run {run_id!r}")
502
+ if not isinstance(settled_at, str) or not settled_at:
503
+ raise ValueError(f"settled_at must be non-empty str, got {settled_at!r}")
504
+ if started_at is not None and (not isinstance(started_at, str) or not started_at):
505
+ raise ValueError("started_at must be non-empty str or None")
506
+ if cursor_updates:
507
+ for k, v in cursor_updates.items():
508
+ if not isinstance(k, str) or not k:
509
+ raise ValueError(f"cursor key must be a non-empty str, got {k!r}")
510
+ if v is not None and not isinstance(v, str):
511
+ raise TypeError(f"cursor value must be str or None, got {type(v).__name__}")
512
+ if spawned_job_snapshots:
513
+ validate_queue_replacement(list(spawned_job_snapshots))
407
514
 
408
- 读-判-写在锁内串行完成;目标 id 不存在返回 False,对已收尾行
409
- 再次收尾抛 ValueError(双重终结是调用方协议缺陷,fail-loud)。
410
- """
411
- validate_attempt_finish(outcome, finished_at, error_message)
412
- with self._lock:
413
- current = self._attempts.get(attempt_id)
414
- if current is None:
415
- return False
416
- if current.outcome != "running":
417
- raise ValueError(
418
- f"attempt {attempt_id} already finalized with outcome "
419
- f"{current.outcome!r}; double finalization is a protocol violation"
420
- )
421
- self._attempts[attempt_id] = replace(
422
- current, outcome=outcome, finished_at=finished_at, error_message=error_message
515
+ with self._lock:
516
+ running = next(
517
+ (item for item in self._queue if job_uid_from_snapshot(item) == job_uid), {}
423
518
  )
424
- return True
519
+ if (
520
+ running.get("status"),
521
+ running.get("run_id"),
522
+ running.get("attempt_no"),
523
+ queue_snapshot_execution_id(running),
524
+ ) != ("running", run_id, attempt_no, execution_id):
525
+ raise TaskLiteError(f"Settlement does not match the running execution for {job_uid}")
526
+ if any(
527
+ attempt.get("job_uid") == job_uid
528
+ and attempt.get("attempt_no") == attempt_no
529
+ for attempt in self._attempts.values()
530
+ ):
531
+ raise TaskLiteError(f"Execution for {job_uid} attempt {attempt_no} is already settled")
532
+
533
+ new_attempts = dict(self._attempts)
534
+ new_attempt_id = self._attempt_seq
535
+ if record_attempt:
536
+ attempt_outcome = (
537
+ outcome if outcome in ("success", "failure", "retry") else "failure"
538
+ )
539
+ attempt_entry: dict[str, Any] = {
540
+ "attempt_id": new_attempt_id,
541
+ "job_uid": job_uid,
542
+ "attempt_no": attempt_no,
543
+ "run_id": run_id,
544
+ "outcome": attempt_outcome,
545
+ "started_at": started_at,
546
+ "settled_at": settled_at,
547
+ "job": copy.deepcopy(job_snapshot),
548
+ "files": copy.deepcopy(files),
549
+ "result": copy.deepcopy(result),
550
+ "error_message": error_message,
551
+ }
552
+ new_attempts[new_attempt_id] = attempt_entry
553
+
554
+ new_queue = list(self._queue)
555
+ new_successes = dict(self._successes)
556
+ new_failed = dict(self._failed)
557
+ new_failure_payloads = dict(self._failure_payloads)
558
+ new_cursors = dict(self._cursors)
559
+
560
+ if outcome == "success":
561
+ remaining = [j for j in new_queue if job_uid_from_snapshot(j) != job_uid]
562
+ if spawned_job_snapshots:
563
+ remaining_uids = {job_uid_from_snapshot(j) for j in remaining}
564
+ seen: set[str] = set()
565
+ for sj in spawned_job_snapshots:
566
+ suid = job_uid_from_snapshot(sj)
567
+ if suid in remaining_uids or suid in seen:
568
+ raise TaskLiteError(
569
+ f"Spawned job {suid} conflicts with queue for {job_uid}"
570
+ )
571
+ seen.add(suid)
572
+ for s in reversed(spawned_job_snapshots):
573
+ s_copy = copy.deepcopy(s)
574
+ s_copy.setdefault("status", "pending")
575
+ remaining.insert(0, s_copy)
576
+ new_queue = remaining
425
577
 
426
- def load_attempts(self, job_uid: str) -> list[AttemptRecord]:
427
- """按 job_uid 读取轨迹(dict 插入序 = 自增 id 序)。"""
428
- with self._lock:
429
- return [rec for rec in self._attempts.values() if rec.job_uid == job_uid]
578
+ new_successes[job_uid] = copy.deepcopy(success_metadata or {})
579
+ new_failed.pop(job_uid, None)
580
+ new_failure_payloads.pop(job_uid, None)
430
581
 
431
- def load_latest_succeeded_finished_at(self, job_uid: str) -> str | None:
432
- """按 attempt_id 倒序查找指定 job_uid 最新一条 succeeded 轨迹行的 finished_at。"""
582
+ if cursor_updates:
583
+ for k, v in cursor_updates.items():
584
+ if v is None:
585
+ new_cursors.pop(k, None)
586
+ else:
587
+ new_cursors[k] = v
588
+
589
+ elif outcome == "failure":
590
+ new_queue = [j for j in new_queue if job_uid_from_snapshot(j) != job_uid]
591
+ new_failed[job_uid] = copy.deepcopy(failure_metadata or {})
592
+ if payload is not None:
593
+ new_failure_payloads[job_uid] = copy.deepcopy(payload)
594
+ new_successes.pop(job_uid, None)
595
+
596
+ elif outcome == "retry":
597
+ found = False
598
+ for idx, j in enumerate(new_queue):
599
+ if job_uid_from_snapshot(j) == job_uid:
600
+ target = copy.deepcopy(requeued_job_snapshot if requeued_job_snapshot is not None else j)
601
+ target["status"] = "pending"
602
+ target["run_id"] = None
603
+ target["attempt_no"] = target.get("attempt_no") or 1
604
+ target["started_at"] = None
605
+ target.pop(QUEUE_EXECUTION_ID_KEY, None)
606
+ new_queue[idx] = target
607
+ found = True
608
+ break
609
+ if not found and requeued_job_snapshot is not None:
610
+ target = copy.deepcopy(requeued_job_snapshot)
611
+ target["status"] = "pending"
612
+ new_queue.append(target)
613
+
614
+ elif outcome == "pending":
615
+ for idx, j in enumerate(new_queue):
616
+ if job_uid_from_snapshot(j) == job_uid:
617
+ target = copy.deepcopy(j)
618
+ target["status"] = "pending"
619
+ target["run_id"] = None
620
+ target["attempt_no"] = target.get("attempt_no") or 1
621
+ target["started_at"] = None
622
+ target.pop(QUEUE_EXECUTION_ID_KEY, None)
623
+ new_queue[idx] = target
624
+ break
625
+
626
+ # 统一原子落变
627
+ self._attempts = new_attempts
628
+ if record_attempt:
629
+ self._attempt_seq += 1
630
+ self._queue = new_queue
631
+ self._successes = new_successes
632
+ self._failed = new_failed
633
+ self._failure_payloads = new_failure_payloads
634
+ self._cursors = new_cursors
635
+
636
+ def record_authorization(
637
+ self,
638
+ path: str,
639
+ target_type: str,
640
+ *,
641
+ authorized_at: str | None = None,
642
+ ) -> None:
643
+ norm_path = os.path.normpath(os.fspath(path))
644
+ if target_type not in ("file", "directory"):
645
+ raise ValueError(f"target_type must be 'file' or 'directory', got {target_type!r}")
646
+ ts = authorized_at if authorized_at is not None else utc_now_iso()
433
647
  with self._lock:
434
- for rec in reversed(self._attempts.values()):
435
- if rec.job_uid == job_uid and rec.outcome == "succeeded":
436
- if isinstance(rec.finished_at, str) and rec.finished_at:
437
- return rec.finished_at
438
- return None
439
- return None
648
+ self._authorizations[norm_path] = {
649
+ "path": norm_path,
650
+ "target_type": target_type,
651
+ "authorized_at": ts,
652
+ }
440
653
 
441
- # 元数据 ---------------------------------------------------------
654
+ def list_authorizations(self) -> list[dict[str, Any]]:
655
+ with self._lock:
656
+ records = list(self._authorizations.values())
657
+ return sorted(records, key=lambda r: (r["authorized_at"], r["path"]))
442
658
 
443
- def get_engine_metadata(self, key: str) -> str | None:
659
+ def revoke_authorization(self, path: str) -> bool:
660
+ norm_path = os.path.normpath(os.fspath(path))
444
661
  with self._lock:
445
- return self._engine_metadata.get(key)
662
+ if norm_path in self._authorizations:
663
+ del self._authorizations[norm_path]
664
+ return True
665
+ return False
446
666
 
447
- def set_engine_metadata(self, key: str, value: str) -> None:
667
+ def is_path_authorized(self, path: str, target_type: str) -> bool:
668
+ norm_path = os.path.normpath(os.fspath(path))
669
+ target_p = Path(norm_path)
448
670
  with self._lock:
449
- self._engine_metadata[key] = str(value)
671
+ exact = self._authorizations.get(norm_path)
672
+ if exact is not None and exact["target_type"] == target_type:
673
+ return True
674
+ for record in self._authorizations.values():
675
+ if record["target_type"] == "directory":
676
+ dir_path = Path(record["path"])
677
+ try:
678
+ if target_p.is_relative_to(dir_path):
679
+ return True
680
+ except (ValueError, TypeError):
681
+ pass
682
+ return False
683
+
684
+ def close(self) -> None:
685
+ """关闭后端并释放资源(幂等)。"""
686
+ pass
450
687
 
451
688
 
452
689
  __all__ = ["InMemoryStateBackend"]