tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
|
@@ -7,24 +7,38 @@
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import logging
|
|
10
|
-
import os
|
|
11
10
|
import sqlite3
|
|
11
|
+
from collections.abc import Iterator
|
|
12
12
|
from contextlib import contextmanager
|
|
13
13
|
from pathlib import Path
|
|
14
|
-
from typing import Any,
|
|
14
|
+
from typing import Any, Literal, Mapping, Sequence, cast
|
|
15
15
|
|
|
16
16
|
from .base import (
|
|
17
17
|
AbstractStateBackend,
|
|
18
|
-
|
|
18
|
+
FailureEntry,
|
|
19
|
+
GraphEditCommit,
|
|
20
|
+
is_valid_authorization_target_type,
|
|
21
|
+
normalize_authorization_path,
|
|
22
|
+
normalize_failure_entries,
|
|
19
23
|
public_queue_snapshot,
|
|
20
24
|
queue_snapshot_execution_id,
|
|
21
25
|
queue_snapshot_with_execution_id,
|
|
22
|
-
|
|
26
|
+
resolve_authorized_at,
|
|
27
|
+
validate_authorization_target_type,
|
|
28
|
+
validate_cursor_key,
|
|
29
|
+
validate_cursor_value,
|
|
30
|
+
validate_graph_edit_commit,
|
|
31
|
+
validate_mark_job_running_args,
|
|
32
|
+
validate_mark_job_running_state,
|
|
33
|
+
validate_recovery_attempt_state,
|
|
34
|
+
validate_running_settlement_state,
|
|
35
|
+
validate_settlement_args,
|
|
23
36
|
)
|
|
24
37
|
from ..exceptions import TaskLiteError
|
|
25
38
|
from ..models.state import job_uid_from_snapshot
|
|
26
|
-
from ..utils.
|
|
39
|
+
from ..utils.artifact import TargetType, evaluate_authorization
|
|
27
40
|
from ..utils.jsonutil import dumps, loads
|
|
41
|
+
from ..utils.lockfile import release_lock, try_acquire_lock
|
|
28
42
|
|
|
29
43
|
logger = logging.getLogger("tasklite")
|
|
30
44
|
|
|
@@ -45,6 +59,7 @@ _TABLE_COLUMNS: dict[str, tuple[str, ...]] = {
|
|
|
45
59
|
"job_sequences": ("job_uid", "last_attempt_no"),
|
|
46
60
|
"cursors": ("key", "value"),
|
|
47
61
|
"engine_metadata": ("key", "value"),
|
|
62
|
+
"authorizations": ("path", "target_type", "authorized_at"),
|
|
48
63
|
}
|
|
49
64
|
|
|
50
65
|
|
|
@@ -93,6 +108,51 @@ def _check_schema(
|
|
|
93
108
|
return version
|
|
94
109
|
|
|
95
110
|
|
|
111
|
+
def open_existing_sqlite_readonly(
|
|
112
|
+
db_path: str | Path,
|
|
113
|
+
*,
|
|
114
|
+
begin_transaction: bool = True,
|
|
115
|
+
timeout: float = 30.0,
|
|
116
|
+
) -> sqlite3.Connection:
|
|
117
|
+
"""校验并以强制只读 URI 打开已存在的 TaskLite Schema 6 状态库。
|
|
118
|
+
|
|
119
|
+
按顺序拒绝不存在路径、目录、0 字节空文件、非 SQLite 文件与未初始化/不兼容
|
|
120
|
+
Schema 库;全程零磁盘写入,不生成 ``-wal`` / ``-shm`` 旁路文件。
|
|
121
|
+
"""
|
|
122
|
+
path = Path(db_path)
|
|
123
|
+
if not path.exists():
|
|
124
|
+
raise TaskLiteError(f"Database file does not exist: {path}")
|
|
125
|
+
if path.is_dir():
|
|
126
|
+
raise TaskLiteError(f"Path is a directory, not a database file: {path}")
|
|
127
|
+
if path.stat().st_size == 0:
|
|
128
|
+
raise TaskLiteError(f"Uninitialized TaskLite database at {path}")
|
|
129
|
+
|
|
130
|
+
uri = path.resolve().as_uri() + "?mode=ro"
|
|
131
|
+
conn = sqlite3.connect(uri, uri=True, isolation_level=None, timeout=timeout)
|
|
132
|
+
try:
|
|
133
|
+
if begin_transaction:
|
|
134
|
+
conn.execute("BEGIN")
|
|
135
|
+
if _check_schema(conn, path) == 0:
|
|
136
|
+
raise TaskLiteError(f"Uninitialized TaskLite database at {path}")
|
|
137
|
+
except BaseException:
|
|
138
|
+
if begin_transaction:
|
|
139
|
+
try:
|
|
140
|
+
conn.execute("ROLLBACK")
|
|
141
|
+
except Exception:
|
|
142
|
+
pass
|
|
143
|
+
conn.close()
|
|
144
|
+
raise
|
|
145
|
+
return conn
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def verify_existing_sqlite_database(db_path: str | Path) -> Path:
|
|
149
|
+
"""只读探活已存在的 TaskLite Schema 6 状态库并立即释放连接(零磁盘写入)。"""
|
|
150
|
+
path = Path(db_path)
|
|
151
|
+
conn = open_existing_sqlite_readonly(path, begin_transaction=False)
|
|
152
|
+
conn.close()
|
|
153
|
+
return path
|
|
154
|
+
|
|
155
|
+
|
|
96
156
|
class SQLiteStateBackend(AbstractStateBackend):
|
|
97
157
|
"""原子 SQLite 状态后端:逐 job 单事务碎片化的 delta 提交。"""
|
|
98
158
|
|
|
@@ -102,28 +162,25 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
102
162
|
self._init_db()
|
|
103
163
|
|
|
104
164
|
@contextmanager
|
|
105
|
-
def
|
|
165
|
+
def exclusive_access(self) -> Iterator[None]:
|
|
166
|
+
"""非阻塞获取 ``<state_path>.lock`` 跨进程文件锁,离开作用域归还。"""
|
|
167
|
+
fd = try_acquire_lock(self.path, timeout=0)
|
|
168
|
+
if fd is None:
|
|
169
|
+
raise TaskLiteError(
|
|
170
|
+
f"Another run() or management operation is in progress for "
|
|
171
|
+
f"{self.path}; concurrent runs or management operations on the same state are forbidden."
|
|
172
|
+
)
|
|
173
|
+
try:
|
|
174
|
+
yield
|
|
175
|
+
finally:
|
|
176
|
+
release_lock(fd)
|
|
177
|
+
|
|
178
|
+
@contextmanager
|
|
179
|
+
def _get_conn(self) -> Iterator[sqlite3.Connection]:
|
|
106
180
|
"""连接上下文管理器:成功时 commit,异常时 rollback,始终 close。
|
|
107
181
|
|
|
108
|
-
``
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
``synchronous=FULL`` 是 per-connection 设置,每次新建连接都需设置;
|
|
112
|
-
``journal_mode=WAL`` 是 database-level,仅在 ``_init_db`` 中设置一次。
|
|
113
|
-
持久化保障:WAL+NORMAL 下 commit 不 fsync WAL,断电可回滚最后若干
|
|
114
|
-
事务——执行中 job 的回滚可被 at-least-once 重跑吸收,但 enqueue
|
|
115
|
-
应答后断电时 INSERT 事务消失且无 job 可重跑(任务静默蒸发)。
|
|
116
|
-
FULL 保证已应答事务落盘。
|
|
117
|
-
|
|
118
|
-
读-改-写事务纪律:``sqlite3`` 传统隔离模式只为 DML 隐式开事务,
|
|
119
|
-
SELECT 在 autocommit 下逐条取快照——「读 min/max(queue_position) → 分配 →
|
|
120
|
-
INSERT」「读 payload 快照 → REPLACE」「读 running 轨迹行 →
|
|
121
|
-
UPDATE」若不显式开事务,两个连接会基于同一份旧快照各算各的,
|
|
122
|
-
落盘出现重复 queue_position(idx_queue_position 非唯一,拦不住)或丢快照。此类
|
|
123
|
-
方法必须在任何 SELECT 之前显式 ``BEGIN IMMEDIATE``:先取写锁再
|
|
124
|
-
读,读与写在同一事务内;并发方要么先于本事务提交(本事务读到其
|
|
125
|
-
结果),要么排队等锁。锁等待由 connect(timeout=30.0) 的 busy
|
|
126
|
-
timeout 承担——并发方等待写锁而非立即报 database is locked。
|
|
182
|
+
``synchronous=FULL`` 保证已应答事务落盘;读-改-写方法必须在任意 SELECT
|
|
183
|
+
之前显式 ``BEGIN IMMEDIATE`` 串行化写事务。
|
|
127
184
|
"""
|
|
128
185
|
conn = sqlite3.connect(self.path, timeout=30.0)
|
|
129
186
|
try:
|
|
@@ -240,33 +297,27 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
240
297
|
''')
|
|
241
298
|
|
|
242
299
|
def _create_indexes(self, conn: sqlite3.Connection) -> None:
|
|
243
|
-
"""
|
|
244
|
-
在缺列上建索引时抛出费解的 no such column)。"""
|
|
300
|
+
"""建索引(在形状校验之后:异形表先给出明确的拒绝信息)。"""
|
|
245
301
|
conn.execute('CREATE INDEX IF NOT EXISTS idx_queue_position ON queue(queue_position)')
|
|
246
302
|
conn.execute('CREATE INDEX IF NOT EXISTS idx_attempts_job_uid ON attempts(job_uid)')
|
|
247
303
|
conn.execute('CREATE INDEX IF NOT EXISTS idx_queue_status ON queue(status)')
|
|
248
304
|
conn.execute('CREATE INDEX IF NOT EXISTS idx_attempts_job_uid_outcome ON attempts(job_uid, outcome)')
|
|
249
305
|
conn.execute('CREATE INDEX IF NOT EXISTS idx_attempts_job_uid_attempt_no ON attempts(job_uid, attempt_no)')
|
|
250
306
|
|
|
251
|
-
|
|
252
307
|
# 序号分配 ---------------------------------------------------------
|
|
253
308
|
|
|
254
309
|
def _queue_position_range(self, conn: sqlite3.Connection, count: int, front: bool) -> list[int]:
|
|
255
|
-
"""分配 count 个保序 queue_position
|
|
256
|
-
|
|
257
|
-
调用方必须已通过 ``BEGIN IMMEDIATE`` 持写事务:MIN/MAX 的读
|
|
258
|
-
与调用方随后的 INSERT 必须原子——否则两个连接读到同一 min/max
|
|
259
|
-
后各自分配,落盘重复 queue_position(idx_queue_position 非唯一索引,拦不住),
|
|
260
|
-
队头保序契约被破坏。
|
|
261
|
-
"""
|
|
310
|
+
"""分配 count 个保序 queue_position;调用方必须已持 ``BEGIN IMMEDIATE`` 写事务。"""
|
|
262
311
|
if count <= 0:
|
|
263
312
|
return []
|
|
264
|
-
min_queue_position, max_queue_position = conn.execute(
|
|
265
|
-
|
|
313
|
+
min_queue_position, max_queue_position = conn.execute(
|
|
314
|
+
'SELECT MIN(queue_position), MAX(queue_position) FROM queue'
|
|
315
|
+
).fetchone()
|
|
316
|
+
if min_queue_position is None:
|
|
266
317
|
min_queue_position, max_queue_position = 0, -1
|
|
267
318
|
if front:
|
|
268
319
|
base = min_queue_position - count
|
|
269
|
-
return list(range(base, min_queue_position))
|
|
320
|
+
return list(range(base, min_queue_position))
|
|
270
321
|
base = max_queue_position + 1
|
|
271
322
|
return list(range(base, base + count))
|
|
272
323
|
|
|
@@ -286,16 +337,6 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
286
337
|
cursor = conn.execute('SELECT job_uid, metadata FROM failed')
|
|
287
338
|
return {row[0]: loads(row[1]) for row in cursor}
|
|
288
339
|
|
|
289
|
-
def load_failure_payloads(self) -> dict[str, dict[str, Any]]:
|
|
290
|
-
"""读取失败档案各 job_uid 的原始业务 payload 快照(无快照的 job_uid 不出现)。"""
|
|
291
|
-
if not self.path.exists():
|
|
292
|
-
return {}
|
|
293
|
-
with self._get_conn() as conn:
|
|
294
|
-
cursor = conn.execute(
|
|
295
|
-
'SELECT job_uid, payload FROM failed WHERE payload IS NOT NULL'
|
|
296
|
-
)
|
|
297
|
-
return {row[0]: loads(row[1]) for row in cursor}
|
|
298
|
-
|
|
299
340
|
def load_cursors(self) -> dict[str, str]:
|
|
300
341
|
if not self.path.exists():
|
|
301
342
|
return {}
|
|
@@ -303,115 +344,16 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
303
344
|
cursor = conn.execute('SELECT key, value FROM cursors')
|
|
304
345
|
return {row[0]: row[1] for row in cursor}
|
|
305
346
|
|
|
306
|
-
def load_queue(self) -> list[dict[str, Any]]:
|
|
307
|
-
if not self.path.exists():
|
|
308
|
-
return []
|
|
309
|
-
with self._get_conn() as conn:
|
|
310
|
-
cursor = conn.execute(
|
|
311
|
-
"SELECT job_snapshot FROM queue WHERE status = 'pending' ORDER BY queue_position ASC"
|
|
312
|
-
)
|
|
313
|
-
return [public_queue_snapshot(loads(row[0])) for row in cursor]
|
|
314
|
-
|
|
315
|
-
# 整表重写 ---------------------------------------------------------
|
|
316
|
-
|
|
317
|
-
def _rewrite_queue_rows(self, conn: sqlite3.Connection, job_snapshots: list[dict[str, Any]]) -> None:
|
|
318
|
-
"""queue 表整表重写的单一出口(save_queue 与 replace_queue_atomic 共用)。
|
|
319
|
-
|
|
320
|
-
写变前先经 ``validate_queue_replacement`` 校验替换集(None/非法形状
|
|
321
|
-
fail-loud 抛 TypeError,不触发 DELETE)——否则 None 会让「DELETE
|
|
322
|
-
全表 + INSERT 全跳」静默清空队列且事务正常提交。
|
|
323
|
-
保存兜底去重:传入重复 job_uid 时保留首条 + 告警(而非 REPLACE 静默
|
|
324
|
-
覆盖为最后一条)——与加载期去重策略一致,杜绝队列 job_uid 索引与
|
|
325
|
-
queue list 的漂移。调用方必须已持写事务(显式或隐式)。
|
|
326
|
-
"""
|
|
327
|
-
validate_queue_replacement(job_snapshots)
|
|
328
|
-
conn.execute('DELETE FROM queue')
|
|
329
|
-
if job_snapshots:
|
|
330
|
-
seen: set[str] = set()
|
|
331
|
-
rows: list[tuple] = []
|
|
332
|
-
for j in job_snapshots:
|
|
333
|
-
u = job_uid_from_snapshot(j)
|
|
334
|
-
if u in seen:
|
|
335
|
-
logger.warning(
|
|
336
|
-
f"save_queue: duplicate job_uid {u} dropped (kept first)."
|
|
337
|
-
)
|
|
338
|
-
continue
|
|
339
|
-
seen.add(u)
|
|
340
|
-
rows.append((
|
|
341
|
-
u,
|
|
342
|
-
len(rows),
|
|
343
|
-
dumps(j),
|
|
344
|
-
j.get("status", "pending"),
|
|
345
|
-
j.get("run_id"),
|
|
346
|
-
j.get("attempt_no"),
|
|
347
|
-
j.get("started_at"),
|
|
348
|
-
))
|
|
349
|
-
conn.executemany(
|
|
350
|
-
'INSERT OR REPLACE INTO queue ('
|
|
351
|
-
'job_uid, queue_position, job_snapshot, status, run_id, attempt_no, started_at'
|
|
352
|
-
') VALUES (?, ?, ?, ?, ?, ?, ?)',
|
|
353
|
-
rows,
|
|
354
|
-
)
|
|
355
|
-
|
|
356
|
-
def save_queue(self, job_snapshots: list[dict[str, Any]]) -> None:
|
|
357
|
-
"""全量重写队列(测试装配 / replace_queue_atomic 事务内步骤,罕见 O(N))。
|
|
358
|
-
|
|
359
|
-
红线:无读基准的整表覆盖——崩溃恢复路径的「合并保存」严禁直接
|
|
360
|
-
调用本方法(读-改-写窗口会抹除窗口期他进程已应答的入队),
|
|
361
|
-
必须经 replace_queue_atomic 收敛进单个写事务。
|
|
362
|
-
"""
|
|
363
|
-
try:
|
|
364
|
-
with self._get_conn() as conn:
|
|
365
|
-
self._rewrite_queue_rows(conn, job_snapshots)
|
|
366
|
-
except Exception as e:
|
|
367
|
-
logger.critical(f"Failed to save queue to {self.path.name}: {e}")
|
|
368
|
-
raise
|
|
369
|
-
|
|
370
|
-
def replace_queue_atomic(
|
|
371
|
-
self,
|
|
372
|
-
compute: Callable[[list[dict[str, Any]]], list[dict[str, Any]]],
|
|
373
|
-
) -> None:
|
|
374
|
-
try:
|
|
375
|
-
with self._get_conn() as conn:
|
|
376
|
-
# 读-改-写事务纪律:先取写锁再读磁盘真相,compute 与写回
|
|
377
|
-
# 同事务——并发 enqueue 要么先于本事务提交(进入磁盘真相、
|
|
378
|
-
# 参与合并),要么等本事务提交后再落盘,绝无中间态覆盖。
|
|
379
|
-
conn.execute('BEGIN IMMEDIATE')
|
|
380
|
-
disk_queue = [
|
|
381
|
-
loads(row[0])
|
|
382
|
-
for row in conn.execute(
|
|
383
|
-
'SELECT job_snapshot FROM queue ORDER BY queue_position ASC'
|
|
384
|
-
)
|
|
385
|
-
]
|
|
386
|
-
self._rewrite_queue_rows(conn, compute(disk_queue))
|
|
387
|
-
except Exception as e:
|
|
388
|
-
logger.critical(f"Failed to replace queue atomically in {self.path.name}: {e}")
|
|
389
|
-
raise
|
|
390
|
-
|
|
391
347
|
# delta 提交 ---------------------------------------------------------
|
|
392
348
|
|
|
393
|
-
|
|
394
349
|
def _write_failed_row(
|
|
395
350
|
self,
|
|
396
351
|
conn: sqlite3.Connection,
|
|
397
352
|
job_uid: str,
|
|
398
|
-
meta: dict,
|
|
353
|
+
meta: dict[str, Any],
|
|
399
354
|
payload: dict[str, Any] | None = None,
|
|
400
355
|
) -> None:
|
|
401
|
-
"""
|
|
402
|
-
|
|
403
|
-
failed 按 job_uid 唯一终态:INSERT OR REPLACE(最终状态唯一);失败
|
|
404
|
-
次数与执行时间线由 attempts 旁路轨迹承接,本表不注入计数字段、
|
|
405
|
-
不解释 meta 内容。
|
|
406
|
-
|
|
407
|
-
``payload`` 是原始业务输入的 JSON 快照(补跑用);传 None
|
|
408
|
-
时保留既有快照不覆盖——无 payload 的写入路径(bulk 级联/append)
|
|
409
|
-
不得抹掉此前快照。
|
|
410
|
-
|
|
411
|
-
调用方必须已通过 ``BEGIN IMMEDIATE`` 持写事务:保留快照的
|
|
412
|
-
SELECT → REPLACE 序列在 autocommit 下会丢并发快照(两个连接各
|
|
413
|
-
基于同一旧行决定保留/覆盖,交错落盘后快照来源不确定)。
|
|
414
|
-
"""
|
|
356
|
+
"""失败档案行写入单出口(``payload=None`` 保留既有快照,须在 ``BEGIN IMMEDIATE`` 内调用)。"""
|
|
415
357
|
row = conn.execute(
|
|
416
358
|
'SELECT payload FROM failed WHERE job_uid = ?', (job_uid,)
|
|
417
359
|
).fetchone()
|
|
@@ -423,31 +365,29 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
423
365
|
)
|
|
424
366
|
conn.execute(
|
|
425
367
|
'INSERT OR REPLACE INTO failed (job_uid, metadata, payload) VALUES (?, ?, ?)',
|
|
426
|
-
(
|
|
427
|
-
|
|
368
|
+
(
|
|
369
|
+
job_uid,
|
|
370
|
+
dumps(merged),
|
|
371
|
+
dumps(effective_payload) if effective_payload is not None else None,
|
|
372
|
+
),
|
|
428
373
|
)
|
|
429
374
|
|
|
430
|
-
def
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
失败时 on-disk 队列不变。同一事务内 ``DELETE FROM successes``——重跑
|
|
436
|
-
任务失败时 successes 里的旧成功记录必须作废(最终状态唯一);否则磁盘
|
|
437
|
-
successes∩failed 并存,破坏全局互斥。事务内单出口、原子、幂等(once
|
|
438
|
-
任务失败时 successes 本无该行,DELETE no-op)。
|
|
439
|
-
"""
|
|
375
|
+
def commit_job_failures(self, entries: Sequence[FailureEntry]) -> None:
|
|
376
|
+
"""原子批量写失败档案 + 按 job_uid 删除队列行 + 清理 successes 对侧记录。"""
|
|
377
|
+
normalized = normalize_failure_entries(entries)
|
|
378
|
+
if not normalized:
|
|
379
|
+
return
|
|
440
380
|
try:
|
|
441
381
|
with self._get_conn() as conn:
|
|
442
|
-
# 快照保留的读-改-写必须持写事务串行化(详见 _get_conn)。
|
|
443
382
|
conn.execute('BEGIN IMMEDIATE')
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
383
|
+
for job_uid, meta, payload in normalized:
|
|
384
|
+
self._write_failed_row(conn, job_uid, meta, payload)
|
|
385
|
+
uid_params = [(job_uid,) for job_uid, _, _ in normalized]
|
|
386
|
+
conn.executemany('DELETE FROM queue WHERE job_uid = ?', uid_params)
|
|
387
|
+
conn.executemany('DELETE FROM successes WHERE job_uid = ?', uid_params)
|
|
447
388
|
except Exception as e:
|
|
448
|
-
logger.critical(f"Failed to commit
|
|
389
|
+
logger.critical(f"Failed to commit failure archive entries in {self.path.name}: {e}")
|
|
449
390
|
raise
|
|
450
|
-
return True
|
|
451
391
|
|
|
452
392
|
def commit_skip(self, job_uid: str) -> None:
|
|
453
393
|
"""移除 Trigger False 或重复候选;不写 successes/failed。"""
|
|
@@ -458,49 +398,14 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
458
398
|
logger.critical(f"Failed to commit skip for {job_uid} in {self.path.name}: {e}")
|
|
459
399
|
raise
|
|
460
400
|
|
|
461
|
-
|
|
462
|
-
def commit_job_failures(self, job_uids_metadata: list[tuple[str, dict]]) -> None:
|
|
463
|
-
"""原子 delta:批量写失败档案 + 按 job_uid 精准删除队列行 + 批量清 successes 旧记录。
|
|
464
|
-
|
|
465
|
-
与 commit_job_failure 对称,同一事务内批量 ``DELETE FROM successes``
|
|
466
|
-
——重跑任务被级联/死锁批量失败时,successes 旧成功记录作废(最终状态
|
|
467
|
-
唯一);剩余(未删除)行保持原 queue_position 顺序。失败时 on-disk 队列不变。
|
|
468
|
-
"""
|
|
469
|
-
try:
|
|
470
|
-
with self._get_conn() as conn:
|
|
471
|
-
# 快照保留的读-改-写必须持写事务串行化(详见 _get_conn)。
|
|
472
|
-
conn.execute('BEGIN IMMEDIATE')
|
|
473
|
-
for job_uid, meta in job_uids_metadata:
|
|
474
|
-
self._write_failed_row(conn, job_uid, meta)
|
|
475
|
-
conn.executemany(
|
|
476
|
-
'DELETE FROM queue WHERE job_uid = ?',
|
|
477
|
-
[(job_uid,) for job_uid, _ in job_uids_metadata],
|
|
478
|
-
)
|
|
479
|
-
conn.executemany(
|
|
480
|
-
'DELETE FROM successes WHERE job_uid = ?',
|
|
481
|
-
[(job_uid,) for job_uid, _ in job_uids_metadata],
|
|
482
|
-
)
|
|
483
|
-
except Exception as e:
|
|
484
|
-
logger.critical(f"Failed to commit bulk failure in {self.path.name}: {e}")
|
|
485
|
-
raise
|
|
486
|
-
|
|
487
|
-
|
|
488
401
|
# 增量入队 -----------------------------------------------------------
|
|
489
402
|
|
|
490
403
|
def enqueue_jobs(self, job_snapshots: list[dict[str, Any]], *, front: bool = False) -> list[str]:
|
|
491
|
-
"""批量增量入队:单事务原子插入,跳过重复 job_uid。
|
|
492
|
-
|
|
493
|
-
与 save_queue 的区别:不做 DELETE 全表重写——与运行中 run() 的
|
|
494
|
-
delta commit 并发时不覆盖其已提交变更(写入侧不覆盖,读-改-写
|
|
495
|
-
窗口仍由 replace_queue_atomic 收敛)。失败抛异常(不吞)。
|
|
496
|
-
"""
|
|
404
|
+
"""批量增量入队:单事务原子插入,跳过重复 job_uid。"""
|
|
497
405
|
if not job_snapshots:
|
|
498
406
|
return []
|
|
499
407
|
try:
|
|
500
408
|
with self._get_conn() as conn:
|
|
501
|
-
# 去重 SELECT 与 queue_position 分配读必须与后续 INSERT 同事务:
|
|
502
|
-
# 先取写锁再读,并发 front 入队才不会基于过期 min/max(queue_position)
|
|
503
|
-
# 分配出重复 queue_position(事务纪律详见 _get_conn docstring)。
|
|
504
409
|
conn.execute('BEGIN IMMEDIATE')
|
|
505
410
|
existing = {
|
|
506
411
|
row[0] for row in conn.execute('SELECT job_uid FROM queue')
|
|
@@ -509,8 +414,6 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
509
414
|
batch_seen: set[str] = set()
|
|
510
415
|
for j in job_snapshots:
|
|
511
416
|
u = job_uid_from_snapshot(j)
|
|
512
|
-
# 批次内重复(同一 enqueue 调用传相同 job_uid 两次)与
|
|
513
|
-
# 队列中已有的 job_uid 都跳过(首个存储,后续跳过)。
|
|
514
417
|
if u in existing or u in batch_seen:
|
|
515
418
|
continue
|
|
516
419
|
batch_seen.add(u)
|
|
@@ -539,194 +442,213 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
539
442
|
logger.critical(f"Failed to enqueue job_snapshots to {self.path.name}: {e}")
|
|
540
443
|
raise
|
|
541
444
|
|
|
542
|
-
#
|
|
445
|
+
# 管理写与查询 -------------------------------------------------------
|
|
543
446
|
|
|
544
|
-
def
|
|
545
|
-
"""
|
|
546
|
-
if not
|
|
547
|
-
return
|
|
447
|
+
def load_consistent_snapshot(self) -> dict[str, Any]:
|
|
448
|
+
"""在单一只读事务内原子捕获全部核心表一致快照(含 cursors 与 job_sequences)。"""
|
|
449
|
+
if not self.path.exists():
|
|
450
|
+
return {
|
|
451
|
+
"queue_jobs": [],
|
|
452
|
+
"successes": {},
|
|
453
|
+
"failed": {},
|
|
454
|
+
"failure_payloads": {},
|
|
455
|
+
"cursors": {},
|
|
456
|
+
"attempts": [],
|
|
457
|
+
"job_sequences": {},
|
|
458
|
+
}
|
|
459
|
+
conn = sqlite3.connect(self.path, isolation_level=None, timeout=30.0)
|
|
548
460
|
try:
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
461
|
+
conn.execute("BEGIN")
|
|
462
|
+
queue_rows: list[dict[str, Any]] = []
|
|
463
|
+
for row in conn.execute(
|
|
464
|
+
"SELECT job_uid, queue_position, job_snapshot, status, run_id, attempt_no, started_at "
|
|
465
|
+
"FROM queue ORDER BY queue_position ASC"
|
|
466
|
+
):
|
|
467
|
+
stored_job = loads(row[2])
|
|
468
|
+
if not isinstance(stored_job, dict):
|
|
469
|
+
raise RuntimeError(f"Corrupted queue job_snapshot for {row[0]!r}")
|
|
470
|
+
queue_rows.append({
|
|
471
|
+
"job_uid": row[0],
|
|
472
|
+
"queue_position": row[1],
|
|
473
|
+
"job": public_queue_snapshot(stored_job),
|
|
474
|
+
"status": row[3],
|
|
475
|
+
"execution_id": queue_snapshot_execution_id(stored_job),
|
|
476
|
+
"run_id": row[4],
|
|
477
|
+
"attempt_no": row[5],
|
|
478
|
+
"started_at": row[6],
|
|
479
|
+
})
|
|
557
480
|
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
)
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
481
|
+
successes = {
|
|
482
|
+
row[0]: loads(row[1])
|
|
483
|
+
for row in conn.execute("SELECT job_uid, metadata FROM successes")
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
failed: dict[str, dict[str, Any]] = {}
|
|
487
|
+
failure_payloads: dict[str, dict[str, Any]] = {}
|
|
488
|
+
for row in conn.execute("SELECT job_uid, metadata, payload FROM failed"):
|
|
489
|
+
failed[row[0]] = loads(row[1])
|
|
490
|
+
if row[2] is not None:
|
|
491
|
+
failure_payloads[row[0]] = loads(row[2])
|
|
492
|
+
|
|
493
|
+
cursors = {
|
|
494
|
+
row[0]: row[1]
|
|
495
|
+
for row in conn.execute("SELECT key, value FROM cursors")
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
attempts: list[dict[str, Any]] = []
|
|
499
|
+
for row in conn.execute(
|
|
500
|
+
"SELECT attempt_id, job_uid, attempt_no, run_id, outcome, started_at, settled_at, "
|
|
501
|
+
"job_snapshot, files, result, error_message "
|
|
502
|
+
"FROM attempts ORDER BY attempt_id ASC"
|
|
503
|
+
):
|
|
504
|
+
attempts.append({
|
|
505
|
+
"attempt_id": row[0],
|
|
506
|
+
"job_uid": row[1],
|
|
507
|
+
"attempt_no": row[2],
|
|
508
|
+
"run_id": row[3],
|
|
509
|
+
"outcome": row[4],
|
|
510
|
+
"started_at": row[5],
|
|
511
|
+
"settled_at": row[6],
|
|
512
|
+
"job": loads(row[7]) if row[7] else None,
|
|
513
|
+
"files": loads(row[8]) if row[8] else None,
|
|
514
|
+
"result": loads(row[9]) if row[9] else None,
|
|
515
|
+
"error_message": row[10],
|
|
516
|
+
})
|
|
571
517
|
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
518
|
+
job_sequences = {
|
|
519
|
+
row[0]: row[1]
|
|
520
|
+
for row in conn.execute("SELECT job_uid, last_attempt_no FROM job_sequences")
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
return {
|
|
524
|
+
"queue_jobs": queue_rows,
|
|
525
|
+
"successes": successes,
|
|
526
|
+
"failed": failed,
|
|
527
|
+
"failure_payloads": failure_payloads,
|
|
528
|
+
"cursors": cursors,
|
|
529
|
+
"attempts": attempts,
|
|
530
|
+
"job_sequences": job_sequences,
|
|
531
|
+
}
|
|
532
|
+
finally:
|
|
533
|
+
try:
|
|
534
|
+
conn.execute("ROLLBACK")
|
|
535
|
+
except Exception:
|
|
536
|
+
pass
|
|
537
|
+
conn.close()
|
|
585
538
|
|
|
586
|
-
def
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
) -> None:
|
|
590
|
-
"""批量原子应用人工状态变更 (job_uid, status, snapshot, metadata)。"""
|
|
591
|
-
if not updates:
|
|
539
|
+
def commit_graph_edit(self, commit: GraphEditCommit) -> None:
|
|
540
|
+
"""在单 BEGIN IMMEDIATE 事务内复核遗留 running 补写并按触碰 uid 精准更新三表(O(Δ))。"""
|
|
541
|
+
if not validate_graph_edit_commit(commit):
|
|
592
542
|
return
|
|
543
|
+
|
|
593
544
|
with self._get_conn() as conn:
|
|
594
545
|
conn.execute("BEGIN IMMEDIATE")
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
546
|
+
|
|
547
|
+
for rec in commit.recovery_attempts:
|
|
548
|
+
row = conn.execute(
|
|
549
|
+
"SELECT status, run_id, attempt_no, job_snapshot FROM queue WHERE job_uid = ?",
|
|
550
|
+
(rec.job_uid,),
|
|
551
|
+
).fetchone()
|
|
552
|
+
stored_job = loads(row[3]) if (row is not None and row[3]) else None
|
|
553
|
+
job_dict = stored_job if isinstance(stored_job, dict) else None
|
|
554
|
+
seq_row = conn.execute(
|
|
555
|
+
"SELECT last_attempt_no FROM job_sequences WHERE job_uid = ?",
|
|
556
|
+
(rec.job_uid,),
|
|
557
|
+
).fetchone()
|
|
558
|
+
settled = bool(
|
|
559
|
+
conn.execute(
|
|
560
|
+
"SELECT 1 FROM attempts WHERE job_uid = ? AND attempt_no = ?",
|
|
561
|
+
(rec.job_uid, rec.attempt_no),
|
|
601
562
|
).fetchone()
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
else
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
563
|
+
)
|
|
564
|
+
validate_recovery_attempt_state(
|
|
565
|
+
rec,
|
|
566
|
+
present_in_queue=row is not None,
|
|
567
|
+
queue_status=row[0] if (row is not None and job_dict is not None) else None,
|
|
568
|
+
queue_run_id=row[1] if (row is not None and job_dict is not None) else None,
|
|
569
|
+
queue_attempt_no=row[2] if (row is not None and job_dict is not None) else None,
|
|
570
|
+
queue_execution_id=(
|
|
571
|
+
queue_snapshot_execution_id(job_dict) if job_dict is not None else None
|
|
572
|
+
),
|
|
573
|
+
attempt_sequence=seq_row[0] if seq_row is not None else None,
|
|
574
|
+
already_settled=settled,
|
|
575
|
+
)
|
|
576
|
+
conn.execute(
|
|
577
|
+
"INSERT INTO attempts ("
|
|
578
|
+
"job_uid, attempt_no, run_id, outcome, started_at, settled_at, "
|
|
579
|
+
"job_snapshot, files, result, error_message"
|
|
580
|
+
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
581
|
+
(
|
|
582
|
+
rec.job_uid,
|
|
583
|
+
rec.attempt_no,
|
|
584
|
+
rec.run_id,
|
|
585
|
+
"failure",
|
|
586
|
+
rec.started_at,
|
|
587
|
+
rec.settled_at,
|
|
588
|
+
dumps(rec.job_snapshot) if rec.job_snapshot is not None else None,
|
|
589
|
+
dumps(rec.files) if rec.files is not None else None,
|
|
590
|
+
dumps(rec.result) if rec.result is not None else None,
|
|
591
|
+
rec.error_message,
|
|
592
|
+
),
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
if commit.queue_writes:
|
|
596
|
+
conn.executemany(
|
|
597
|
+
"DELETE FROM queue WHERE job_uid = ?",
|
|
598
|
+
[(uid,) for uid in commit.queue_writes],
|
|
599
|
+
)
|
|
600
|
+
to_insert = [
|
|
601
|
+
(uid, public_queue_snapshot(snap))
|
|
602
|
+
for uid, snap in commit.queue_writes.items()
|
|
603
|
+
if snap is not None
|
|
604
|
+
]
|
|
605
|
+
if to_insert:
|
|
606
|
+
positions = self._queue_position_range(conn, len(to_insert), front=False)
|
|
607
|
+
conn.executemany(
|
|
608
|
+
"INSERT INTO queue ("
|
|
609
|
+
"job_uid, queue_position, job_snapshot, status, run_id, attempt_no, started_at"
|
|
610
|
+
") VALUES (?, ?, ?, 'pending', NULL, NULL, NULL)",
|
|
611
|
+
[
|
|
612
|
+
(uid, pos, dumps(snap))
|
|
613
|
+
for (uid, snap), pos in zip(to_insert, positions)
|
|
614
|
+
],
|
|
615
|
+
)
|
|
616
|
+
|
|
617
|
+
for uid, meta in commit.success_writes.items():
|
|
618
|
+
if meta is None:
|
|
619
|
+
conn.execute("DELETE FROM successes WHERE job_uid = ?", (uid,))
|
|
620
|
+
else:
|
|
616
621
|
conn.execute(
|
|
617
622
|
"INSERT OR REPLACE INTO successes (job_uid, metadata) VALUES (?, ?)",
|
|
618
|
-
(
|
|
623
|
+
(uid, dumps(meta)),
|
|
619
624
|
)
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
625
|
+
|
|
626
|
+
for uid, entry in commit.failed_writes.items():
|
|
627
|
+
if entry is None:
|
|
628
|
+
conn.execute("DELETE FROM failed WHERE job_uid = ?", (uid,))
|
|
629
|
+
else:
|
|
630
|
+
fail_meta, payload = entry
|
|
624
631
|
conn.execute(
|
|
625
632
|
"INSERT OR REPLACE INTO failed (job_uid, metadata, payload) VALUES (?, ?, ?)",
|
|
626
|
-
(
|
|
633
|
+
(
|
|
634
|
+
uid,
|
|
635
|
+
dumps(fail_meta),
|
|
636
|
+
dumps(payload) if payload is not None else None,
|
|
637
|
+
),
|
|
627
638
|
)
|
|
628
|
-
else:
|
|
629
|
-
raise ValueError(f"Unsupported manual status: {status!r}")
|
|
630
|
-
|
|
631
|
-
@staticmethod
|
|
632
|
-
def _snapshot_from_metadata(raw: str | None) -> dict[str, Any] | None:
|
|
633
|
-
if not raw:
|
|
634
|
-
return None
|
|
635
|
-
metadata = loads(raw)
|
|
636
|
-
if not isinstance(metadata, dict):
|
|
637
|
-
return None
|
|
638
|
-
snapshot = metadata.get("job_snapshot")
|
|
639
|
-
return snapshot if isinstance(snapshot, dict) else None
|
|
640
|
-
|
|
641
|
-
def load_known_job_snapshot(self, uid: str) -> dict[str, Any] | None:
|
|
642
|
-
"""按队列、成功、失败、Attempt 历史的优先级读取完整配置。"""
|
|
643
|
-
with self._get_conn() as conn:
|
|
644
|
-
row = conn.execute(
|
|
645
|
-
"SELECT job_snapshot FROM queue WHERE job_uid = ?", (uid,)
|
|
646
|
-
).fetchone()
|
|
647
|
-
if row and row[0]:
|
|
648
|
-
snapshot = loads(row[0])
|
|
649
|
-
if isinstance(snapshot, dict):
|
|
650
|
-
return public_queue_snapshot(snapshot)
|
|
651
639
|
|
|
652
|
-
|
|
653
|
-
"SELECT
|
|
654
|
-
|
|
655
|
-
if row:
|
|
656
|
-
snapshot = self._snapshot_from_metadata(row[0])
|
|
657
|
-
if snapshot is not None:
|
|
658
|
-
return snapshot
|
|
659
|
-
|
|
660
|
-
row = conn.execute(
|
|
661
|
-
"SELECT metadata FROM failed WHERE job_uid = ?", (uid,)
|
|
662
|
-
).fetchone()
|
|
663
|
-
if row:
|
|
664
|
-
snapshot = self._snapshot_from_metadata(row[0])
|
|
665
|
-
if snapshot is not None:
|
|
666
|
-
return snapshot
|
|
667
|
-
|
|
668
|
-
row = conn.execute(
|
|
669
|
-
"SELECT job_snapshot FROM attempts "
|
|
670
|
-
"WHERE job_uid = ? AND job_snapshot IS NOT NULL "
|
|
671
|
-
"ORDER BY attempt_id DESC LIMIT 1",
|
|
672
|
-
(uid,),
|
|
640
|
+
overlap_row = conn.execute(
|
|
641
|
+
"SELECT s.job_uid FROM successes AS s "
|
|
642
|
+
"INNER JOIN failed AS f ON s.job_uid = f.job_uid LIMIT 1"
|
|
673
643
|
).fetchone()
|
|
674
|
-
if
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
return None
|
|
679
|
-
|
|
680
|
-
def load_known_job_snapshots(self) -> dict[str, dict[str, Any]]:
|
|
681
|
-
"""读取全部已知 Job 的完整配置,低优先级来源先写入。"""
|
|
682
|
-
known: dict[str, dict[str, Any]] = {}
|
|
683
|
-
with self._get_conn() as conn:
|
|
684
|
-
for uid, raw in conn.execute(
|
|
685
|
-
"SELECT job_uid, job_snapshot FROM attempts "
|
|
686
|
-
"WHERE job_snapshot IS NOT NULL ORDER BY attempt_id ASC"
|
|
687
|
-
):
|
|
688
|
-
snapshot = loads(raw) if raw else None
|
|
689
|
-
if isinstance(snapshot, dict):
|
|
690
|
-
known[uid] = snapshot
|
|
691
|
-
|
|
692
|
-
for uid, raw in conn.execute("SELECT job_uid, metadata FROM failed"):
|
|
693
|
-
snapshot = self._snapshot_from_metadata(raw)
|
|
694
|
-
if snapshot is not None:
|
|
695
|
-
known[uid] = snapshot
|
|
696
|
-
|
|
697
|
-
for uid, raw in conn.execute("SELECT job_uid, metadata FROM successes"):
|
|
698
|
-
snapshot = self._snapshot_from_metadata(raw)
|
|
699
|
-
if snapshot is not None:
|
|
700
|
-
known[uid] = snapshot
|
|
701
|
-
|
|
702
|
-
for uid, raw in conn.execute("SELECT job_uid, job_snapshot FROM queue"):
|
|
703
|
-
snapshot = loads(raw) if raw else None
|
|
704
|
-
if isinstance(snapshot, dict):
|
|
705
|
-
known[uid] = public_queue_snapshot(snapshot)
|
|
706
|
-
return known
|
|
707
|
-
|
|
708
|
-
def clear_history(self, uid: str, records: set[str]) -> int:
|
|
709
|
-
"""从 successes/failed/attempts 清除指定 job_uid 的记录,返回删除条数。"""
|
|
710
|
-
if not self.path.exists():
|
|
711
|
-
return 0
|
|
712
|
-
total = 0
|
|
713
|
-
with self._get_conn() as conn:
|
|
714
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
715
|
-
if "successes" in records:
|
|
716
|
-
cur = conn.execute("DELETE FROM successes WHERE job_uid = ?", (uid,))
|
|
717
|
-
total += cur.rowcount if cur.rowcount is not None else 0
|
|
718
|
-
if "failures" in records:
|
|
719
|
-
cur = conn.execute("DELETE FROM failed WHERE job_uid = ?", (uid,))
|
|
720
|
-
total += cur.rowcount if cur.rowcount is not None else 0
|
|
721
|
-
if "attempts" in records:
|
|
722
|
-
cur = conn.execute("DELETE FROM attempts WHERE job_uid = ?", (uid,))
|
|
723
|
-
total += cur.rowcount if cur.rowcount is not None else 0
|
|
724
|
-
return total
|
|
644
|
+
if overlap_row is not None:
|
|
645
|
+
raise TaskLiteError(
|
|
646
|
+
f"Invariant conflict: job {overlap_row[0]!r} cannot exist in both successes and failed"
|
|
647
|
+
)
|
|
725
648
|
|
|
726
649
|
def get_cursor(self, key: str) -> str | None:
|
|
727
650
|
"""查询单个游标的最新已提交值,不存在返回 None。"""
|
|
728
|
-
|
|
729
|
-
raise TypeError(f"cursor key must be a non-empty str, got {key!r}")
|
|
651
|
+
validate_cursor_key(key)
|
|
730
652
|
if not self.path.exists():
|
|
731
653
|
return None
|
|
732
654
|
with self._get_conn() as conn:
|
|
@@ -735,10 +657,8 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
735
657
|
|
|
736
658
|
def set_cursor(self, key: str, value: str | None) -> None:
|
|
737
659
|
"""设置或删除单个游标(None 为删除)。"""
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
if value is not None and not isinstance(value, str):
|
|
741
|
-
raise TypeError(f"cursor value must be str or None, got {type(value).__name__}")
|
|
660
|
+
validate_cursor_key(key)
|
|
661
|
+
validate_cursor_value(value)
|
|
742
662
|
try:
|
|
743
663
|
with self._get_conn() as conn:
|
|
744
664
|
conn.execute("BEGIN IMMEDIATE")
|
|
@@ -771,13 +691,6 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
771
691
|
|
|
772
692
|
# 元数据 ---------------------------------------------------------------
|
|
773
693
|
|
|
774
|
-
def get_engine_metadata(self, key: str) -> str | None:
|
|
775
|
-
if not self.path.exists():
|
|
776
|
-
return None
|
|
777
|
-
with self._get_conn() as conn:
|
|
778
|
-
row = conn.execute('SELECT value FROM engine_metadata WHERE key = ?', (key,)).fetchone()
|
|
779
|
-
return row[0] if row else None
|
|
780
|
-
|
|
781
694
|
def set_engine_metadata(self, key: str, value: str) -> None:
|
|
782
695
|
try:
|
|
783
696
|
with self._get_conn() as conn:
|
|
@@ -794,24 +707,21 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
794
707
|
def mark_job_running(
|
|
795
708
|
self, job_uid: str, *, execution_id: str, run_id: str, started_at: str
|
|
796
709
|
) -> int:
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
raise ValueError("run_id must be a non-empty str")
|
|
801
|
-
if not execution_id_matches_run(execution_id, run_id):
|
|
802
|
-
raise ValueError(f"execution_id {execution_id!r} does not belong to run {run_id!r}")
|
|
803
|
-
if not isinstance(started_at, str) or not started_at:
|
|
804
|
-
raise ValueError("started_at must be a non-empty str")
|
|
710
|
+
validate_mark_job_running_args(
|
|
711
|
+
execution_id=execution_id, run_id=run_id, started_at=started_at
|
|
712
|
+
)
|
|
805
713
|
with self._get_conn() as conn:
|
|
806
714
|
conn.execute('BEGIN IMMEDIATE')
|
|
807
715
|
row = conn.execute(
|
|
808
716
|
'SELECT status, job_snapshot FROM queue WHERE job_uid = ?',
|
|
809
717
|
(job_uid,),
|
|
810
718
|
).fetchone()
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
719
|
+
validate_mark_job_running_state(
|
|
720
|
+
job_uid,
|
|
721
|
+
present_in_queue=row is not None,
|
|
722
|
+
queue_status=row[0] if row is not None else None,
|
|
723
|
+
)
|
|
724
|
+
assert row is not None
|
|
815
725
|
job_snapshot = loads(row[1])
|
|
816
726
|
if not isinstance(job_snapshot, dict):
|
|
817
727
|
raise ValueError(f"Job {job_uid} has an invalid queue snapshot")
|
|
@@ -850,14 +760,6 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
850
760
|
).fetchone()
|
|
851
761
|
return row[0] if row is not None else 0
|
|
852
762
|
|
|
853
|
-
def clear_attempts(self, job_uid: str) -> int:
|
|
854
|
-
if not self.path.exists():
|
|
855
|
-
return 0
|
|
856
|
-
with self._get_conn() as conn:
|
|
857
|
-
conn.execute('BEGIN IMMEDIATE')
|
|
858
|
-
cur = conn.execute('DELETE FROM attempts WHERE job_uid = ?', (job_uid,))
|
|
859
|
-
return cur.rowcount
|
|
860
|
-
|
|
861
763
|
def load_settled_attempts(self, job_uid: str) -> list[dict[str, Any]]:
|
|
862
764
|
if not self.path.exists():
|
|
863
765
|
return []
|
|
@@ -939,29 +841,16 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
939
841
|
requeued_job_snapshot: dict[str, Any] | None = None,
|
|
940
842
|
record_attempt: bool = True,
|
|
941
843
|
) -> None:
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
raise ValueError(f"execution_id {execution_id!r} does not belong to run {run_id!r}")
|
|
953
|
-
if not isinstance(settled_at, str) or not settled_at:
|
|
954
|
-
raise ValueError(f"settled_at must be non-empty str, got {settled_at!r}")
|
|
955
|
-
if started_at is not None and (not isinstance(started_at, str) or not started_at):
|
|
956
|
-
raise ValueError("started_at must be non-empty str or None")
|
|
957
|
-
if cursor_updates:
|
|
958
|
-
for k, v in cursor_updates.items():
|
|
959
|
-
if not isinstance(k, str) or not k:
|
|
960
|
-
raise ValueError(f"cursor key must be a non-empty str, got {k!r}")
|
|
961
|
-
if v is not None and not isinstance(v, str):
|
|
962
|
-
raise TypeError(f"cursor value must be str or None, got {type(v).__name__}")
|
|
963
|
-
if spawned_job_snapshots:
|
|
964
|
-
validate_queue_replacement(list(spawned_job_snapshots))
|
|
844
|
+
validate_settlement_args(
|
|
845
|
+
outcome=outcome,
|
|
846
|
+
attempt_no=attempt_no,
|
|
847
|
+
execution_id=execution_id,
|
|
848
|
+
run_id=run_id,
|
|
849
|
+
settled_at=settled_at,
|
|
850
|
+
started_at=started_at,
|
|
851
|
+
cursor_updates=cursor_updates,
|
|
852
|
+
spawned_job_snapshots=spawned_job_snapshots,
|
|
853
|
+
)
|
|
965
854
|
|
|
966
855
|
with self._get_conn() as conn:
|
|
967
856
|
conn.execute('BEGIN IMMEDIATE')
|
|
@@ -971,24 +860,29 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
971
860
|
"FROM queue WHERE job_uid = ?",
|
|
972
861
|
(job_uid,),
|
|
973
862
|
).fetchone()
|
|
974
|
-
if running is None
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
)
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
)
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
863
|
+
stored_job = loads(running[3]) if (running is not None and running[3]) else None
|
|
864
|
+
job_dict = stored_job if isinstance(stored_job, dict) else None
|
|
865
|
+
settled = bool(
|
|
866
|
+
conn.execute(
|
|
867
|
+
"SELECT 1 FROM attempts WHERE job_uid = ? AND attempt_no = ?",
|
|
868
|
+
(job_uid, attempt_no),
|
|
869
|
+
).fetchone()
|
|
870
|
+
)
|
|
871
|
+
validate_running_settlement_state(
|
|
872
|
+
job_uid,
|
|
873
|
+
present_in_queue=running is not None and job_dict is not None,
|
|
874
|
+
queue_status=running[0] if (running is not None and job_dict is not None) else None,
|
|
875
|
+
queue_run_id=running[1] if (running is not None and job_dict is not None) else None,
|
|
876
|
+
queue_attempt_no=running[2] if (running is not None and job_dict is not None) else None,
|
|
877
|
+
queue_execution_id=(
|
|
878
|
+
queue_snapshot_execution_id(job_dict) if job_dict is not None else None
|
|
879
|
+
),
|
|
880
|
+
run_id=run_id,
|
|
881
|
+
attempt_no=attempt_no,
|
|
882
|
+
execution_id=execution_id,
|
|
883
|
+
already_settled=settled,
|
|
884
|
+
)
|
|
885
|
+
assert job_dict is not None
|
|
992
886
|
|
|
993
887
|
if record_attempt:
|
|
994
888
|
attempt_outcome = (
|
|
@@ -1068,30 +962,58 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
1068
962
|
conn.execute('DELETE FROM queue WHERE job_uid = ?', (job_uid,))
|
|
1069
963
|
|
|
1070
964
|
elif outcome == "retry":
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
else:
|
|
1082
|
-
conn.execute(
|
|
1083
|
-
'UPDATE queue SET job_snapshot = ?, status = ?, run_id = NULL, '
|
|
1084
|
-
'attempt_no = NULL, started_at = NULL WHERE job_uid = ?',
|
|
1085
|
-
(dumps(public_queue_snapshot(stored_job)), "pending", job_uid),
|
|
1086
|
-
)
|
|
965
|
+
target_snap = (
|
|
966
|
+
public_queue_snapshot(requeued_job_snapshot)
|
|
967
|
+
if requeued_job_snapshot is not None
|
|
968
|
+
else public_queue_snapshot(job_dict)
|
|
969
|
+
)
|
|
970
|
+
conn.execute(
|
|
971
|
+
'UPDATE queue SET job_snapshot = ?, status = ?, run_id = NULL, '
|
|
972
|
+
'attempt_no = NULL, started_at = NULL WHERE job_uid = ?',
|
|
973
|
+
(dumps(target_snap), "pending", job_uid),
|
|
974
|
+
)
|
|
1087
975
|
|
|
1088
976
|
elif outcome == "pending":
|
|
1089
977
|
conn.execute(
|
|
1090
978
|
'UPDATE queue SET job_snapshot = ?, status = ?, run_id = NULL, '
|
|
1091
979
|
'attempt_no = NULL, started_at = NULL WHERE job_uid = ?',
|
|
1092
|
-
(dumps(public_queue_snapshot(
|
|
980
|
+
(dumps(public_queue_snapshot(job_dict)), "pending", job_uid),
|
|
1093
981
|
)
|
|
1094
982
|
|
|
983
|
+
@staticmethod
|
|
984
|
+
def _load_auth_pairs(conn: sqlite3.Connection) -> list[tuple[str, TargetType]]:
|
|
985
|
+
rows = conn.execute('SELECT path, target_type FROM authorizations').fetchall()
|
|
986
|
+
return [(row[0], cast(TargetType, row[1])) for row in rows]
|
|
987
|
+
|
|
988
|
+
def grant_authorization(
|
|
989
|
+
self,
|
|
990
|
+
path: str,
|
|
991
|
+
target_type: Literal["file", "directory"],
|
|
992
|
+
*,
|
|
993
|
+
authorized_at: str | None = None,
|
|
994
|
+
) -> dict[str, Any]:
|
|
995
|
+
norm_path = normalize_authorization_path(path)
|
|
996
|
+
validated_type = validate_authorization_target_type(target_type, error_type=TaskLiteError)
|
|
997
|
+
ts = resolve_authorized_at(authorized_at)
|
|
998
|
+
with self._get_conn() as conn:
|
|
999
|
+
conn.execute('BEGIN IMMEDIATE')
|
|
1000
|
+
match = evaluate_authorization(
|
|
1001
|
+
self._load_auth_pairs(conn),
|
|
1002
|
+
norm_path,
|
|
1003
|
+
validated_type,
|
|
1004
|
+
)
|
|
1005
|
+
if match.grant_conflict is not None:
|
|
1006
|
+
raise TaskLiteError(match.grant_conflict)
|
|
1007
|
+
conn.execute(
|
|
1008
|
+
'INSERT INTO authorizations (path, target_type, authorized_at) VALUES (?, ?, ?)',
|
|
1009
|
+
(norm_path, validated_type, ts),
|
|
1010
|
+
)
|
|
1011
|
+
return {
|
|
1012
|
+
"path": norm_path,
|
|
1013
|
+
"target_type": validated_type,
|
|
1014
|
+
"authorized_at": ts,
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1095
1017
|
def record_authorization(
|
|
1096
1018
|
self,
|
|
1097
1019
|
path: str,
|
|
@@ -1099,14 +1021,21 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
1099
1021
|
*,
|
|
1100
1022
|
authorized_at: str | None = None,
|
|
1101
1023
|
) -> None:
|
|
1102
|
-
norm_path =
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
ts = authorized_at if authorized_at is not None else utc_now_iso()
|
|
1024
|
+
norm_path = normalize_authorization_path(path)
|
|
1025
|
+
validated_type = validate_authorization_target_type(target_type, error_type=ValueError)
|
|
1026
|
+
ts = resolve_authorized_at(authorized_at)
|
|
1106
1027
|
with self._get_conn() as conn:
|
|
1028
|
+
conn.execute('BEGIN IMMEDIATE')
|
|
1029
|
+
match = evaluate_authorization(
|
|
1030
|
+
self._load_auth_pairs(conn),
|
|
1031
|
+
norm_path,
|
|
1032
|
+
validated_type,
|
|
1033
|
+
)
|
|
1034
|
+
if match.exact_path is not None and match.exact_path != norm_path:
|
|
1035
|
+
conn.execute('DELETE FROM authorizations WHERE path = ?', (match.exact_path,))
|
|
1107
1036
|
conn.execute(
|
|
1108
1037
|
'INSERT OR REPLACE INTO authorizations (path, target_type, authorized_at) VALUES (?, ?, ?)',
|
|
1109
|
-
(norm_path,
|
|
1038
|
+
(norm_path, validated_type, ts),
|
|
1110
1039
|
)
|
|
1111
1040
|
|
|
1112
1041
|
def list_authorizations(self) -> list[dict[str, Any]]:
|
|
@@ -1120,35 +1049,34 @@ class SQLiteStateBackend(AbstractStateBackend):
|
|
|
1120
1049
|
]
|
|
1121
1050
|
|
|
1122
1051
|
def revoke_authorization(self, path: str) -> bool:
|
|
1123
|
-
norm_path =
|
|
1052
|
+
norm_path = normalize_authorization_path(path)
|
|
1124
1053
|
with self._get_conn() as conn:
|
|
1125
|
-
|
|
1054
|
+
conn.execute('BEGIN IMMEDIATE')
|
|
1055
|
+
match = evaluate_authorization(self._load_auth_pairs(conn), norm_path)
|
|
1056
|
+
if match.exact_path is None:
|
|
1057
|
+
return False
|
|
1058
|
+
cur = conn.execute('DELETE FROM authorizations WHERE path = ?', (match.exact_path,))
|
|
1126
1059
|
return bool(cur.rowcount and cur.rowcount > 0)
|
|
1127
1060
|
|
|
1128
1061
|
def is_path_authorized(self, path: str, target_type: str) -> bool:
|
|
1129
|
-
norm_path =
|
|
1130
|
-
|
|
1062
|
+
norm_path = normalize_authorization_path(path)
|
|
1063
|
+
if not is_valid_authorization_target_type(target_type):
|
|
1064
|
+
return False
|
|
1131
1065
|
with self._get_conn() as conn:
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
return True
|
|
1138
|
-
cur = conn.execute("SELECT path FROM authorizations WHERE target_type = 'directory'")
|
|
1139
|
-
for row in cur.fetchall():
|
|
1140
|
-
dir_path = Path(row[0])
|
|
1141
|
-
try:
|
|
1142
|
-
if target_p.is_relative_to(dir_path):
|
|
1143
|
-
return True
|
|
1144
|
-
except (ValueError, TypeError):
|
|
1145
|
-
pass
|
|
1146
|
-
return False
|
|
1066
|
+
return evaluate_authorization(
|
|
1067
|
+
self._load_auth_pairs(conn),
|
|
1068
|
+
norm_path,
|
|
1069
|
+
cast(TargetType, target_type),
|
|
1070
|
+
).covered
|
|
1147
1071
|
|
|
1148
1072
|
def close(self) -> None:
|
|
1149
1073
|
"""关闭后端并释放资源(幂等)。"""
|
|
1150
1074
|
pass
|
|
1151
1075
|
|
|
1152
1076
|
|
|
1153
|
-
__all__ = [
|
|
1154
|
-
|
|
1077
|
+
__all__ = [
|
|
1078
|
+
"SQLiteStateBackend",
|
|
1079
|
+
"TARGET_SCHEMA_VERSION",
|
|
1080
|
+
"open_existing_sqlite_readonly",
|
|
1081
|
+
"verify_existing_sqlite_database",
|
|
1082
|
+
]
|