tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
tasklite/snapshot.py
CHANGED
|
@@ -1,25 +1,38 @@
|
|
|
1
|
-
"""StateSnapshot
|
|
1
|
+
"""StateSnapshot:只读一致状态快照、拷贝模式与可读报告(ADR-0011 / ADR-0012)。
|
|
2
2
|
|
|
3
3
|
独立只读入口,支持从数据库路径或 TaskLite 实例打开;SQLite 与内存实例均
|
|
4
4
|
返回同一种只读快照。一个上下文提供一致快照,不创建/迁移/修复数据库,
|
|
5
5
|
关闭快照不关闭 TaskLite。
|
|
6
|
+
|
|
7
|
+
内部架构:单一深投影核心(StateSnapshot 统一承载优先级裁决、字段归一化与
|
|
8
|
+
过滤排序)+ 2 个薄数据适配器(SQLite 流式只读适配器与统一内存表视图适配器,
|
|
9
|
+
后者统一服务于 InMemoryStateBackend、SQLite copy 模式与 EditTransaction)。
|
|
6
10
|
"""
|
|
7
11
|
from __future__ import annotations
|
|
8
12
|
|
|
9
13
|
import copy
|
|
14
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
10
15
|
from pathlib import Path
|
|
11
|
-
import
|
|
12
|
-
from typing import TYPE_CHECKING, Any, Iterator, Mapping, Protocol
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Literal, Protocol
|
|
13
17
|
|
|
14
|
-
from .
|
|
18
|
+
from ._report import render_snapshot_report
|
|
19
|
+
from .backend.sqlite_backend import open_existing_sqlite_readonly
|
|
15
20
|
from .exceptions import TaskLiteError
|
|
16
|
-
from .models.
|
|
21
|
+
from .models.records import (
|
|
22
|
+
extract_archive_result,
|
|
23
|
+
extract_effective_job_config,
|
|
24
|
+
extract_job_snapshot_from_archive,
|
|
25
|
+
normalize_outer_files,
|
|
26
|
+
normalize_result_record,
|
|
27
|
+
)
|
|
17
28
|
from .utils.encoding import parse_job_uid
|
|
18
29
|
from .utils.jsonutil import loads
|
|
19
30
|
|
|
20
31
|
if TYPE_CHECKING:
|
|
21
32
|
from .pipeline import TaskLite
|
|
22
33
|
|
|
34
|
+
_VALID_SNAPSHOT_MODES = ("lazy", "copy")
|
|
35
|
+
|
|
23
36
|
|
|
24
37
|
def _format_iso_z(ts: str | None) -> str | None:
|
|
25
38
|
"""时间戳统一规范为 UTC ISO 8601 字符串(以 Z 结尾,None 表示未知)。"""
|
|
@@ -30,320 +43,273 @@ def _format_iso_z(ts: str | None) -> str | None:
|
|
|
30
43
|
return ts
|
|
31
44
|
|
|
32
45
|
|
|
33
|
-
def
|
|
34
|
-
"""提取有效配置快照(包含身份、payload、depends_on、files 及资源、预算、超时、Trigger)。"""
|
|
35
|
-
raw_files = snap.get("files") or {}
|
|
36
|
-
files_dict: dict[str, Any] = {}
|
|
37
|
-
for name, spec in raw_files.items():
|
|
38
|
-
if hasattr(spec, "to_dict"):
|
|
39
|
-
files_dict[name] = spec.to_dict()
|
|
40
|
-
elif isinstance(spec, dict):
|
|
41
|
-
files_dict[name] = copy.deepcopy(spec)
|
|
42
|
-
else:
|
|
43
|
-
files_dict[name] = spec
|
|
44
|
-
return {
|
|
45
|
-
"task_type": snap.get("task_type"),
|
|
46
|
-
"job_id": snap.get("job_id"),
|
|
47
|
-
"payload": copy.deepcopy(snap.get("payload") or {}),
|
|
48
|
-
"depends_on": list(snap.get("depends_on") or []),
|
|
49
|
-
"files": files_dict,
|
|
50
|
-
"resources": dict(snap.get("resources") or {}),
|
|
51
|
-
"max_retries_per_run": snap.get("max_retries_per_run"),
|
|
52
|
-
"timeout_seconds": snap.get("timeout_seconds"),
|
|
53
|
-
"trigger": snap.get("trigger"),
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def _normalize_outer_files(raw_files: Mapping[str, Any] | None) -> dict[str, Any]:
|
|
58
|
-
"""外层 files 规范化:按名称记录已解析预声明与动态声明并集(必须含绝对 path)。"""
|
|
59
|
-
if not raw_files:
|
|
60
|
-
return {}
|
|
61
|
-
normalized: dict[str, Any] = {}
|
|
62
|
-
for name, spec in raw_files.items():
|
|
63
|
-
if hasattr(spec, "to_dict"):
|
|
64
|
-
d = spec.to_dict()
|
|
65
|
-
elif isinstance(spec, dict):
|
|
66
|
-
d = copy.deepcopy(spec)
|
|
67
|
-
else:
|
|
68
|
-
continue
|
|
69
|
-
if d.get("path") is not None:
|
|
70
|
-
normalized[name] = {
|
|
71
|
-
"path": d["path"],
|
|
72
|
-
"role": d.get("role", "output"),
|
|
73
|
-
"delete_existing": bool(d.get("delete_existing", False)),
|
|
74
|
-
"delete_on_failure": bool(d.get("delete_on_failure", False)),
|
|
75
|
-
"delete_on_success": bool(d.get("delete_on_success", False)),
|
|
76
|
-
}
|
|
77
|
-
return normalized
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
def _normalize_result(raw_result: Any) -> dict[str, Any] | None:
|
|
81
|
-
"""结算结果规范化为 unified 字典形状(message / exception / metadata / annotations)。"""
|
|
82
|
-
if raw_result is None:
|
|
83
|
-
return None
|
|
84
|
-
if not isinstance(raw_result, dict):
|
|
85
|
-
return {
|
|
86
|
-
"message": str(raw_result),
|
|
87
|
-
"exception": None,
|
|
88
|
-
"metadata": {},
|
|
89
|
-
"annotations": {},
|
|
90
|
-
}
|
|
91
|
-
msg = raw_result.get("message")
|
|
92
|
-
if msg is not None:
|
|
93
|
-
msg = str(msg)
|
|
94
|
-
raw_exc = raw_result.get("exception")
|
|
95
|
-
exc_dict: dict[str, Any] | None = None
|
|
96
|
-
if isinstance(raw_exc, dict):
|
|
97
|
-
exc_dict = {
|
|
98
|
-
"type": str(raw_exc.get("type", "Exception")),
|
|
99
|
-
"message": str(raw_exc.get("message", "")),
|
|
100
|
-
"traceback": (
|
|
101
|
-
str(raw_exc["traceback"])
|
|
102
|
-
if raw_exc.get("traceback") is not None
|
|
103
|
-
else None
|
|
104
|
-
),
|
|
105
|
-
}
|
|
106
|
-
meta = raw_result.get("metadata")
|
|
107
|
-
meta_dict = copy.deepcopy(meta) if isinstance(meta, dict) else {}
|
|
108
|
-
ann = raw_result.get("annotations")
|
|
109
|
-
ann_dict = copy.deepcopy(ann) if isinstance(ann, dict) else {}
|
|
110
|
-
return {
|
|
111
|
-
"message": msg,
|
|
112
|
-
"exception": exc_dict,
|
|
113
|
-
"metadata": meta_dict,
|
|
114
|
-
"annotations": ann_dict,
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def _build_queued_job_dict(
|
|
46
|
+
def _project_queued_job(
|
|
119
47
|
uid: str,
|
|
120
48
|
job_snap: Mapping[str, Any],
|
|
121
49
|
status: str,
|
|
122
50
|
run_id: str | None,
|
|
123
51
|
attempt_no: int | None,
|
|
52
|
+
started_at: str | None = None,
|
|
124
53
|
) -> dict[str, Any]:
|
|
125
|
-
"""
|
|
54
|
+
"""队列记录投影:pending 不携带执行身份与开始时间,running 投影已落盘字段。"""
|
|
55
|
+
is_running = status == "running"
|
|
126
56
|
return {
|
|
127
57
|
"job_uid": uid,
|
|
128
58
|
"status": status,
|
|
129
|
-
"job":
|
|
130
|
-
"files":
|
|
59
|
+
"job": extract_effective_job_config(job_snap),
|
|
60
|
+
"files": normalize_outer_files(job_snap.get("files")),
|
|
131
61
|
"result": None,
|
|
132
|
-
"run_id": run_id if
|
|
133
|
-
"attempt_no": attempt_no if
|
|
134
|
-
"started_at": None,
|
|
62
|
+
"run_id": run_id if is_running else None,
|
|
63
|
+
"attempt_no": attempt_no if is_running else None,
|
|
64
|
+
"started_at": _format_iso_z(started_at) if is_running else None,
|
|
135
65
|
"settled_at": None,
|
|
136
66
|
}
|
|
137
67
|
|
|
138
68
|
|
|
139
|
-
def
|
|
69
|
+
def _project_succeeded_job(
|
|
140
70
|
uid: str,
|
|
141
|
-
succ_meta:
|
|
71
|
+
succ_meta: Mapping[str, Any],
|
|
142
72
|
) -> dict[str, Any]:
|
|
143
73
|
"""当前成功只投影自身档案,不从独立的 Attempt 历史补推事实。"""
|
|
144
|
-
job_snap =
|
|
145
|
-
outer_files =
|
|
74
|
+
job_snap = extract_job_snapshot_from_archive(uid, succ_meta, fill_missing_identity=False)
|
|
75
|
+
outer_files = normalize_outer_files(
|
|
146
76
|
succ_meta.get("files") or job_snap.get("files")
|
|
147
77
|
)
|
|
148
|
-
result = _normalize_result(
|
|
149
|
-
succ_meta.get("result")
|
|
150
|
-
or {
|
|
151
|
-
"message": None,
|
|
152
|
-
"exception": None,
|
|
153
|
-
"metadata": {},
|
|
154
|
-
"annotations": succ_meta.get("annotations") or {},
|
|
155
|
-
}
|
|
156
|
-
)
|
|
157
|
-
run_id = succ_meta.get("run_id")
|
|
158
|
-
attempt_no = succ_meta.get("attempt_no")
|
|
159
|
-
started_at = _format_iso_z(succ_meta.get("started_at"))
|
|
160
|
-
settled_at = _format_iso_z(succ_meta.get("settled_at"))
|
|
161
|
-
|
|
162
78
|
return {
|
|
163
79
|
"job_uid": uid,
|
|
164
80
|
"status": "succeeded",
|
|
165
|
-
"job":
|
|
81
|
+
"job": extract_effective_job_config(job_snap),
|
|
166
82
|
"files": outer_files,
|
|
167
|
-
"result":
|
|
168
|
-
"run_id": run_id,
|
|
169
|
-
"attempt_no": attempt_no,
|
|
170
|
-
"started_at": started_at,
|
|
171
|
-
"settled_at": settled_at,
|
|
83
|
+
"result": extract_archive_result(succ_meta, is_failure=False),
|
|
84
|
+
"run_id": succ_meta.get("run_id"),
|
|
85
|
+
"attempt_no": succ_meta.get("attempt_no"),
|
|
86
|
+
"started_at": _format_iso_z(succ_meta.get("started_at")),
|
|
87
|
+
"settled_at": _format_iso_z(succ_meta.get("settled_at")),
|
|
172
88
|
}
|
|
173
89
|
|
|
174
90
|
|
|
175
|
-
def
|
|
91
|
+
def _project_failed_job(
|
|
176
92
|
uid: str,
|
|
177
|
-
fail_meta:
|
|
178
|
-
payload:
|
|
93
|
+
fail_meta: Mapping[str, Any],
|
|
94
|
+
payload: Mapping[str, Any] | None,
|
|
179
95
|
) -> dict[str, Any]:
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
outer_files = _normalize_outer_files(
|
|
184
|
-
fail_meta.get("files") or job_snap.get("files")
|
|
96
|
+
"""当前失败档案投影:仅在 job_snapshot 缺 payload 时补回独立 payload 列。"""
|
|
97
|
+
job_snap = extract_job_snapshot_from_archive(
|
|
98
|
+
uid, fail_meta, payload=payload, fill_missing_identity=False
|
|
185
99
|
)
|
|
186
|
-
|
|
187
|
-
fail_meta.get("
|
|
188
|
-
or {
|
|
189
|
-
"message": fail_meta.get("error_message") or fail_meta.get("message"),
|
|
190
|
-
"annotations": fail_meta.get("annotations") or {},
|
|
191
|
-
}
|
|
100
|
+
outer_files = normalize_outer_files(
|
|
101
|
+
fail_meta.get("files") or job_snap.get("files")
|
|
192
102
|
)
|
|
193
|
-
run_id = fail_meta.get("run_id")
|
|
194
|
-
attempt_no = fail_meta.get("attempt_no")
|
|
195
|
-
started_at = _format_iso_z(fail_meta.get("started_at"))
|
|
196
|
-
settled_at = _format_iso_z(fail_meta.get("settled_at"))
|
|
197
|
-
|
|
198
103
|
return {
|
|
199
104
|
"job_uid": uid,
|
|
200
105
|
"status": "failed",
|
|
201
|
-
"job":
|
|
106
|
+
"job": extract_effective_job_config(job_snap),
|
|
202
107
|
"files": outer_files,
|
|
203
|
-
"result":
|
|
108
|
+
"result": extract_archive_result(fail_meta, is_failure=True),
|
|
109
|
+
"run_id": fail_meta.get("run_id"),
|
|
110
|
+
"attempt_no": fail_meta.get("attempt_no"),
|
|
111
|
+
"started_at": _format_iso_z(fail_meta.get("started_at")),
|
|
112
|
+
"settled_at": _format_iso_z(fail_meta.get("settled_at")),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _project_attempt(
|
|
117
|
+
uid: str,
|
|
118
|
+
attempt_no: int,
|
|
119
|
+
run_id: str,
|
|
120
|
+
outcome: str,
|
|
121
|
+
started_at: str | None,
|
|
122
|
+
settled_at: str | None,
|
|
123
|
+
job_snap: Mapping[str, Any],
|
|
124
|
+
files_data: Mapping[str, Any] | None,
|
|
125
|
+
result_data: Any,
|
|
126
|
+
) -> dict[str, Any]:
|
|
127
|
+
"""物理 Attempt 历史行投影为统一只读字典。"""
|
|
128
|
+
return {
|
|
129
|
+
"job_uid": uid,
|
|
130
|
+
"outcome": outcome,
|
|
131
|
+
"job": extract_effective_job_config(job_snap),
|
|
132
|
+
"files": normalize_outer_files(files_data),
|
|
133
|
+
"result": normalize_result_record(result_data),
|
|
204
134
|
"run_id": run_id,
|
|
205
135
|
"attempt_no": attempt_no,
|
|
206
|
-
"started_at": started_at,
|
|
207
|
-
"settled_at": settled_at,
|
|
136
|
+
"started_at": _format_iso_z(started_at),
|
|
137
|
+
"settled_at": _format_iso_z(settled_at),
|
|
208
138
|
}
|
|
209
139
|
|
|
210
140
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
141
|
+
def _matches_task_type(uid: str, task_type: str | None) -> bool:
|
|
142
|
+
if task_type is None:
|
|
143
|
+
return True
|
|
144
|
+
try:
|
|
145
|
+
tt, _ = parse_job_uid(uid)
|
|
146
|
+
except Exception:
|
|
147
|
+
return False
|
|
148
|
+
return tt == task_type
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _decode_json_dict(raw: Any) -> dict[str, Any]:
|
|
152
|
+
if isinstance(raw, str):
|
|
153
|
+
val = loads(raw) if raw else {}
|
|
154
|
+
return val if isinstance(val, dict) else {}
|
|
155
|
+
return raw if isinstance(raw, dict) else {}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _decode_json_optional_dict(raw: Any) -> dict[str, Any] | None:
|
|
159
|
+
if raw is None:
|
|
160
|
+
return None
|
|
161
|
+
if isinstance(raw, str):
|
|
162
|
+
if not raw:
|
|
163
|
+
return None
|
|
164
|
+
val = loads(raw)
|
|
165
|
+
return val if isinstance(val, dict) else None
|
|
166
|
+
return raw if isinstance(raw, dict) else None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _decode_attempt_result(raw_result: Any, error_message: str | None, *, fallback_empty_dict: bool) -> Any:
|
|
170
|
+
if raw_result is not None:
|
|
171
|
+
if isinstance(raw_result, str):
|
|
172
|
+
return loads(raw_result) if raw_result else ({"message": error_message} if fallback_empty_dict else None)
|
|
173
|
+
return raw_result
|
|
174
|
+
if error_message is not None:
|
|
175
|
+
return {"message": error_message}
|
|
176
|
+
return {"message": None} if fallback_empty_dict else None
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# ── 2 个薄数据适配器(流式 SQLite + 统一内存表视图)────────────────────────
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class _SnapshotAdapter(Protocol):
|
|
183
|
+
def get_queued(
|
|
184
|
+
self, uid: str
|
|
185
|
+
) -> tuple[Mapping[str, Any], str, str | None, int | None, str | None] | None: ...
|
|
186
|
+
def get_success(self, uid: str) -> Mapping[str, Any] | None: ...
|
|
187
|
+
def get_failed(self, uid: str) -> tuple[Mapping[str, Any], Mapping[str, Any] | None] | None: ...
|
|
188
|
+
def iter_queued(
|
|
189
|
+
self, *, status: str | None = None, task_type: str | None = None
|
|
190
|
+
) -> Iterator[tuple[str, Mapping[str, Any], str, str | None, int | None, str | None]]: ...
|
|
191
|
+
def iter_non_queued_successes(
|
|
192
|
+
self, *, task_type: str | None = None
|
|
193
|
+
) -> Iterator[tuple[str, Mapping[str, Any]]]: ...
|
|
194
|
+
def iter_non_queued_failed(
|
|
195
|
+
self, *, task_type: str | None = None
|
|
196
|
+
) -> Iterator[tuple[str, Mapping[str, Any], Mapping[str, Any] | None]]: ...
|
|
219
197
|
def iter_attempts(
|
|
220
|
-
self,
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
198
|
+
self, *, job_uid: str | None = None, task_type: str | None = None
|
|
199
|
+
) -> Iterator[
|
|
200
|
+
tuple[
|
|
201
|
+
str,
|
|
202
|
+
int,
|
|
203
|
+
str,
|
|
204
|
+
str,
|
|
205
|
+
str | None,
|
|
206
|
+
str | None,
|
|
207
|
+
Mapping[str, Any],
|
|
208
|
+
Mapping[str, Any] | None,
|
|
209
|
+
Any,
|
|
210
|
+
]
|
|
211
|
+
]: ...
|
|
225
212
|
def close(self) -> None: ...
|
|
226
213
|
|
|
227
214
|
|
|
228
|
-
class
|
|
229
|
-
"""
|
|
215
|
+
class _SqliteStreamingAdapter:
|
|
216
|
+
"""SQLite 流式只读适配器(mode='lazy'):仅持单一只读事务游标按需读取。"""
|
|
230
217
|
|
|
231
218
|
def __init__(self, db_path: Path) -> None:
|
|
232
|
-
|
|
233
|
-
raise TaskLiteError(f"Database file does not exist: {db_path}")
|
|
234
|
-
if db_path.is_dir():
|
|
235
|
-
raise TaskLiteError(f"Path is a directory, not a database file: {db_path}")
|
|
236
|
-
uri = db_path.resolve().as_uri() + "?mode=ro"
|
|
237
|
-
conn = sqlite3.connect(uri, uri=True, isolation_level=None, timeout=30.0)
|
|
238
|
-
try:
|
|
239
|
-
conn.execute("BEGIN")
|
|
240
|
-
if _check_schema(conn, db_path) == 0:
|
|
241
|
-
raise TaskLiteError(f"Uninitialized TaskLite database at {db_path}")
|
|
242
|
-
except BaseException:
|
|
243
|
-
conn.close()
|
|
244
|
-
raise
|
|
245
|
-
self._conn = conn
|
|
219
|
+
self._conn = open_existing_sqlite_readonly(db_path, begin_transaction=True)
|
|
246
220
|
self._closed = False
|
|
247
221
|
|
|
248
|
-
def
|
|
222
|
+
def get_queued(
|
|
223
|
+
self, uid: str
|
|
224
|
+
) -> tuple[Mapping[str, Any], str, str | None, int | None, str | None] | None:
|
|
249
225
|
row = self._conn.execute(
|
|
250
|
-
"SELECT
|
|
226
|
+
"SELECT job_snapshot, status, run_id, attempt_no, started_at "
|
|
251
227
|
"FROM queue WHERE job_uid = ?",
|
|
252
228
|
(uid,),
|
|
253
229
|
).fetchone()
|
|
254
|
-
if row:
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
230
|
+
if not row:
|
|
231
|
+
return None
|
|
232
|
+
return (
|
|
233
|
+
_decode_json_dict(row[0]),
|
|
234
|
+
row[1] if row[1] else "pending",
|
|
235
|
+
row[2],
|
|
236
|
+
row[3],
|
|
237
|
+
row[4],
|
|
238
|
+
)
|
|
258
239
|
|
|
259
|
-
|
|
240
|
+
def get_success(self, uid: str) -> Mapping[str, Any] | None:
|
|
241
|
+
row = self._conn.execute(
|
|
260
242
|
"SELECT metadata FROM successes WHERE job_uid = ?", (uid,)
|
|
261
243
|
).fetchone()
|
|
262
|
-
if
|
|
263
|
-
|
|
264
|
-
|
|
244
|
+
if not row:
|
|
245
|
+
return None
|
|
246
|
+
return _decode_json_dict(row[0])
|
|
265
247
|
|
|
266
|
-
|
|
248
|
+
def get_failed(self, uid: str) -> tuple[Mapping[str, Any], Mapping[str, Any] | None] | None:
|
|
249
|
+
row = self._conn.execute(
|
|
267
250
|
"SELECT metadata, payload FROM failed WHERE job_uid = ?", (uid,)
|
|
268
251
|
).fetchone()
|
|
269
|
-
if
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
return _build_failed_job_dict(uid, fail_meta, payload)
|
|
273
|
-
|
|
274
|
-
return None
|
|
252
|
+
if not row:
|
|
253
|
+
return None
|
|
254
|
+
return _decode_json_dict(row[0]), _decode_json_optional_dict(row[1])
|
|
275
255
|
|
|
276
|
-
def
|
|
277
|
-
self,
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
) -> Iterator[dict[str, Any]]:
|
|
282
|
-
q_rows = self._conn.execute(
|
|
283
|
-
"SELECT job_uid, queue_position, job_snapshot, status, run_id, attempt_no "
|
|
256
|
+
def iter_queued(
|
|
257
|
+
self, *, status: str | None = None, task_type: str | None = None
|
|
258
|
+
) -> Iterator[tuple[str, Mapping[str, Any], str, str | None, int | None, str | None]]:
|
|
259
|
+
cursor = self._conn.execute(
|
|
260
|
+
"SELECT job_uid, job_snapshot, status, run_id, attempt_no, started_at "
|
|
284
261
|
"FROM queue ORDER BY queue_position ASC"
|
|
285
262
|
)
|
|
286
|
-
|
|
287
|
-
for r in q_rows:
|
|
263
|
+
for r in cursor:
|
|
288
264
|
uid = r[0]
|
|
289
|
-
st = r[
|
|
265
|
+
st = r[2] if r[2] else "pending"
|
|
290
266
|
if status is not None and st != status:
|
|
291
267
|
continue
|
|
292
|
-
if
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
if status is None or status == "failed":
|
|
322
|
-
f_rows = self._conn.execute(
|
|
323
|
-
"SELECT f.job_uid, f.metadata, f.payload FROM failed AS f "
|
|
324
|
-
"WHERE NOT EXISTS ("
|
|
325
|
-
"SELECT 1 FROM queue AS q WHERE q.job_uid = f.job_uid"
|
|
326
|
-
")"
|
|
327
|
-
)
|
|
328
|
-
for r in f_rows:
|
|
329
|
-
uid = r[0]
|
|
330
|
-
if task_type is not None:
|
|
331
|
-
try:
|
|
332
|
-
tt, _ = parse_job_uid(uid)
|
|
333
|
-
except Exception:
|
|
334
|
-
continue
|
|
335
|
-
if tt != task_type:
|
|
336
|
-
continue
|
|
337
|
-
fail_meta = loads(r[1]) if r[1] else {}
|
|
338
|
-
payload = loads(r[2]) if r[2] else None
|
|
339
|
-
yield _build_failed_job_dict(uid, fail_meta, payload)
|
|
268
|
+
if not _matches_task_type(uid, task_type):
|
|
269
|
+
continue
|
|
270
|
+
yield uid, _decode_json_dict(r[1]), st, r[3], r[4], r[5]
|
|
271
|
+
|
|
272
|
+
def iter_non_queued_successes(
|
|
273
|
+
self, *, task_type: str | None = None
|
|
274
|
+
) -> Iterator[tuple[str, Mapping[str, Any]]]:
|
|
275
|
+
cursor = self._conn.execute(
|
|
276
|
+
"SELECT s.job_uid, s.metadata FROM successes AS s "
|
|
277
|
+
"WHERE NOT EXISTS (SELECT 1 FROM queue AS q WHERE q.job_uid = s.job_uid)"
|
|
278
|
+
)
|
|
279
|
+
for r in cursor:
|
|
280
|
+
uid = r[0]
|
|
281
|
+
if not _matches_task_type(uid, task_type):
|
|
282
|
+
continue
|
|
283
|
+
yield uid, _decode_json_dict(r[1])
|
|
284
|
+
|
|
285
|
+
def iter_non_queued_failed(
|
|
286
|
+
self, *, task_type: str | None = None
|
|
287
|
+
) -> Iterator[tuple[str, Mapping[str, Any], Mapping[str, Any] | None]]:
|
|
288
|
+
cursor = self._conn.execute(
|
|
289
|
+
"SELECT f.job_uid, f.metadata, f.payload FROM failed AS f "
|
|
290
|
+
"WHERE NOT EXISTS (SELECT 1 FROM queue AS q WHERE q.job_uid = f.job_uid)"
|
|
291
|
+
)
|
|
292
|
+
for r in cursor:
|
|
293
|
+
uid = r[0]
|
|
294
|
+
if not _matches_task_type(uid, task_type):
|
|
295
|
+
continue
|
|
296
|
+
yield uid, _decode_json_dict(r[1]), _decode_json_optional_dict(r[2])
|
|
340
297
|
|
|
341
298
|
def iter_attempts(
|
|
342
|
-
self,
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
299
|
+
self, *, job_uid: str | None = None, task_type: str | None = None
|
|
300
|
+
) -> Iterator[
|
|
301
|
+
tuple[
|
|
302
|
+
str,
|
|
303
|
+
int,
|
|
304
|
+
str,
|
|
305
|
+
str,
|
|
306
|
+
str | None,
|
|
307
|
+
str | None,
|
|
308
|
+
Mapping[str, Any],
|
|
309
|
+
Mapping[str, Any] | None,
|
|
310
|
+
Any,
|
|
311
|
+
]
|
|
312
|
+
]:
|
|
347
313
|
sql = (
|
|
348
314
|
"SELECT attempt_no, run_id, outcome, started_at, settled_at, "
|
|
349
315
|
"job_snapshot, files, result, error_message, job_uid "
|
|
@@ -357,27 +323,19 @@ class _SqliteSnapshotReader:
|
|
|
357
323
|
cursor = self._conn.execute(sql, params)
|
|
358
324
|
for row in cursor:
|
|
359
325
|
uid = row[9]
|
|
360
|
-
if
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
"job": _normalize_job_config(job_snap),
|
|
374
|
-
"files": _normalize_outer_files(files_data),
|
|
375
|
-
"result": _normalize_result(res_data),
|
|
376
|
-
"run_id": row[1],
|
|
377
|
-
"attempt_no": row[0],
|
|
378
|
-
"started_at": _format_iso_z(row[3]),
|
|
379
|
-
"settled_at": _format_iso_z(row[4]),
|
|
380
|
-
}
|
|
326
|
+
if not _matches_task_type(uid, task_type):
|
|
327
|
+
continue
|
|
328
|
+
yield (
|
|
329
|
+
uid,
|
|
330
|
+
row[0],
|
|
331
|
+
row[1],
|
|
332
|
+
row[2],
|
|
333
|
+
row[3],
|
|
334
|
+
row[4],
|
|
335
|
+
_decode_json_dict(row[5]),
|
|
336
|
+
_decode_json_dict(row[6]),
|
|
337
|
+
_decode_attempt_result(row[7], row[8], fallback_empty_dict=True),
|
|
338
|
+
)
|
|
381
339
|
|
|
382
340
|
def close(self) -> None:
|
|
383
341
|
if not self._closed:
|
|
@@ -392,142 +350,289 @@ class _SqliteSnapshotReader:
|
|
|
392
350
|
self._closed = True
|
|
393
351
|
|
|
394
352
|
|
|
395
|
-
class
|
|
396
|
-
"""
|
|
353
|
+
class _MemoryTableAdapter:
|
|
354
|
+
"""统一内存表视图适配器:同时服务 InMemoryStateBackend、SQLite copy 与 EditTransaction。
|
|
397
355
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
self._successes = copy.deepcopy(backend._successes)
|
|
402
|
-
self._failed = copy.deepcopy(backend._failed)
|
|
403
|
-
self._failure_payloads = copy.deepcopy(backend._failure_payloads)
|
|
404
|
-
self._attempts = copy.deepcopy(backend._attempts)
|
|
405
|
-
self._closed = False
|
|
356
|
+
大字段 JSON 列在 SQLite copy 模式下保持原始 str,仅在命中且通过过滤条件后
|
|
357
|
+
惰性解码,避免无谓全量反序列化并保证零数据库文件句柄占用。
|
|
358
|
+
"""
|
|
406
359
|
|
|
407
|
-
def
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
360
|
+
def __init__(
|
|
361
|
+
self,
|
|
362
|
+
*,
|
|
363
|
+
queue_rows: list[tuple[str, Any, str, str | None, int | None, str | None]],
|
|
364
|
+
successes: dict[str, Any],
|
|
365
|
+
failed: dict[str, tuple[Any, Any]],
|
|
366
|
+
attempts: list[tuple[str, int, str, str, str | None, str | None, Any, Any, Any, str | None]],
|
|
367
|
+
attempt_result_fallback_empty_dict: bool = False,
|
|
368
|
+
) -> None:
|
|
369
|
+
self._queue_rows = queue_rows
|
|
370
|
+
self._queue_by_uid = {row[0]: row for row in queue_rows}
|
|
371
|
+
self._successes = successes
|
|
372
|
+
self._failed = failed
|
|
373
|
+
self._attempts = attempts
|
|
374
|
+
self._attempt_result_fallback_empty_dict = attempt_result_fallback_empty_dict
|
|
375
|
+
self._closed = False
|
|
414
376
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
377
|
+
@classmethod
|
|
378
|
+
def from_sqlite_copy(cls, db_path: Path) -> _MemoryTableAdapter:
|
|
379
|
+
"""在单一只读事务内读取原始行并立即释放连接(mode='copy')。"""
|
|
380
|
+
conn = open_existing_sqlite_readonly(db_path, begin_transaction=True)
|
|
381
|
+
try:
|
|
382
|
+
q_raw = conn.execute(
|
|
383
|
+
"SELECT job_uid, job_snapshot, status, run_id, attempt_no, started_at "
|
|
384
|
+
"FROM queue ORDER BY queue_position ASC"
|
|
385
|
+
).fetchall()
|
|
386
|
+
s_raw = conn.execute("SELECT job_uid, metadata FROM successes").fetchall()
|
|
387
|
+
f_raw = conn.execute("SELECT job_uid, metadata, payload FROM failed").fetchall()
|
|
388
|
+
a_raw = conn.execute(
|
|
389
|
+
"SELECT job_uid, attempt_no, run_id, outcome, started_at, settled_at, "
|
|
390
|
+
"job_snapshot, files, result, error_message "
|
|
391
|
+
"FROM attempts ORDER BY attempt_id ASC"
|
|
392
|
+
).fetchall()
|
|
393
|
+
finally:
|
|
394
|
+
try:
|
|
395
|
+
conn.execute("ROLLBACK")
|
|
396
|
+
except Exception:
|
|
397
|
+
pass
|
|
398
|
+
try:
|
|
399
|
+
conn.close()
|
|
400
|
+
except Exception:
|
|
401
|
+
pass
|
|
418
402
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
403
|
+
queue_rows = [
|
|
404
|
+
(r[0], r[1], r[2] if r[2] else "pending", r[3], r[4], r[5])
|
|
405
|
+
for r in q_raw
|
|
406
|
+
]
|
|
407
|
+
successes = {r[0]: r[1] for r in s_raw}
|
|
408
|
+
failed = {r[0]: (r[1], r[2]) for r in f_raw}
|
|
409
|
+
attempts = [
|
|
410
|
+
(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9])
|
|
411
|
+
for r in a_raw
|
|
412
|
+
]
|
|
413
|
+
return cls(
|
|
414
|
+
queue_rows=queue_rows,
|
|
415
|
+
successes=successes,
|
|
416
|
+
failed=failed,
|
|
417
|
+
attempts=attempts,
|
|
418
|
+
attempt_result_fallback_empty_dict=True,
|
|
419
|
+
)
|
|
423
420
|
|
|
424
|
-
|
|
421
|
+
@classmethod
|
|
422
|
+
def from_consistent_snapshot(cls, snapshot: Mapping[str, Any]) -> _MemoryTableAdapter:
|
|
423
|
+
"""从内存后端 load_consistent_snapshot() 捕获的一致字典视图构建适配器。"""
|
|
424
|
+
queue_rows = [
|
|
425
|
+
(
|
|
426
|
+
row["job_uid"],
|
|
427
|
+
row["job"],
|
|
428
|
+
row["status"],
|
|
429
|
+
row["run_id"],
|
|
430
|
+
row["attempt_no"],
|
|
431
|
+
row.get("started_at"),
|
|
432
|
+
)
|
|
433
|
+
for row in snapshot["queue_jobs"]
|
|
434
|
+
]
|
|
435
|
+
successes = dict(snapshot["successes"])
|
|
436
|
+
failure_payloads = snapshot["failure_payloads"]
|
|
437
|
+
failed = {
|
|
438
|
+
uid: (meta, failure_payloads.get(uid))
|
|
439
|
+
for uid, meta in snapshot["failed"].items()
|
|
440
|
+
}
|
|
441
|
+
attempts = [
|
|
442
|
+
(
|
|
443
|
+
item["job_uid"],
|
|
444
|
+
item["attempt_no"],
|
|
445
|
+
item["run_id"],
|
|
446
|
+
item["outcome"],
|
|
447
|
+
item.get("started_at"),
|
|
448
|
+
item.get("settled_at"),
|
|
449
|
+
item.get("job") or {},
|
|
450
|
+
item.get("files"),
|
|
451
|
+
item.get("result"),
|
|
452
|
+
item.get("error_message"),
|
|
453
|
+
)
|
|
454
|
+
for item in snapshot["attempts"]
|
|
455
|
+
if isinstance(item.get("job_uid"), str) and item.get("job_uid")
|
|
456
|
+
]
|
|
457
|
+
return cls(
|
|
458
|
+
queue_rows=queue_rows,
|
|
459
|
+
successes=successes,
|
|
460
|
+
failed=failed,
|
|
461
|
+
attempts=attempts,
|
|
462
|
+
)
|
|
425
463
|
|
|
426
|
-
|
|
427
|
-
|
|
464
|
+
@classmethod
|
|
465
|
+
def from_edit_tables(
|
|
466
|
+
cls,
|
|
428
467
|
*,
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
468
|
+
queue_rows: Sequence[Mapping[str, Any]],
|
|
469
|
+
successes: Mapping[str, Mapping[str, Any]],
|
|
470
|
+
failed: Mapping[str, tuple[Mapping[str, Any], Mapping[str, Any] | None]],
|
|
471
|
+
attempts: Sequence[Mapping[str, Any]],
|
|
472
|
+
) -> _MemoryTableAdapter:
|
|
473
|
+
"""从 EditTransaction 三表工作图内存副本构建适配器。"""
|
|
474
|
+
q_list = [
|
|
475
|
+
(
|
|
476
|
+
row["job_uid"],
|
|
477
|
+
row["job"],
|
|
478
|
+
row["status"],
|
|
479
|
+
row.get("run_id"),
|
|
480
|
+
row.get("attempt_no"),
|
|
481
|
+
row.get("started_at"),
|
|
482
|
+
)
|
|
483
|
+
for row in queue_rows
|
|
484
|
+
]
|
|
485
|
+
a_list = [
|
|
486
|
+
(
|
|
487
|
+
item["job_uid"],
|
|
488
|
+
item["attempt_no"],
|
|
489
|
+
item["run_id"],
|
|
490
|
+
item["outcome"],
|
|
491
|
+
item.get("started_at"),
|
|
492
|
+
item.get("settled_at"),
|
|
493
|
+
item.get("job") or {},
|
|
494
|
+
item.get("files"),
|
|
495
|
+
item.get("result"),
|
|
496
|
+
item.get("error_message"),
|
|
497
|
+
)
|
|
498
|
+
for item in attempts
|
|
499
|
+
if isinstance(item.get("job_uid"), str) and item.get("job_uid")
|
|
500
|
+
]
|
|
501
|
+
return cls(
|
|
502
|
+
queue_rows=q_list,
|
|
503
|
+
successes=dict(successes),
|
|
504
|
+
failed=dict(failed),
|
|
505
|
+
attempts=a_list,
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
def get_queued(
|
|
509
|
+
self, uid: str
|
|
510
|
+
) -> tuple[Mapping[str, Any], str, str | None, int | None, str | None] | None:
|
|
511
|
+
row = self._queue_by_uid.get(uid)
|
|
512
|
+
if row is None:
|
|
513
|
+
return None
|
|
514
|
+
return _decode_json_dict(row[1]), row[2], row[3], row[4], row[5]
|
|
515
|
+
|
|
516
|
+
def get_success(self, uid: str) -> Mapping[str, Any] | None:
|
|
517
|
+
if uid not in self._successes:
|
|
518
|
+
return None
|
|
519
|
+
return _decode_json_dict(self._successes[uid])
|
|
520
|
+
|
|
521
|
+
def get_failed(self, uid: str) -> tuple[Mapping[str, Any], Mapping[str, Any] | None] | None:
|
|
522
|
+
if uid not in self._failed:
|
|
523
|
+
return None
|
|
524
|
+
raw_meta, raw_payload = self._failed[uid]
|
|
525
|
+
return _decode_json_dict(raw_meta), _decode_json_optional_dict(raw_payload)
|
|
526
|
+
|
|
527
|
+
def iter_queued(
|
|
528
|
+
self, *, status: str | None = None, task_type: str | None = None
|
|
529
|
+
) -> Iterator[tuple[str, Mapping[str, Any], str, str | None, int | None, str | None]]:
|
|
530
|
+
for row in self._queue_rows:
|
|
531
|
+
uid = row[0]
|
|
532
|
+
st = row[2]
|
|
437
533
|
if status is not None and st != status:
|
|
438
534
|
continue
|
|
439
|
-
if
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
except Exception:
|
|
443
|
-
continue
|
|
444
|
-
if tt != task_type:
|
|
445
|
-
continue
|
|
446
|
-
yield _build_queued_job_dict(
|
|
447
|
-
uid, j, st, j.get("run_id"), j.get("attempt_no"),
|
|
448
|
-
)
|
|
535
|
+
if not _matches_task_type(uid, task_type):
|
|
536
|
+
continue
|
|
537
|
+
yield uid, _decode_json_dict(row[1]), st, row[3], row[4], row[5]
|
|
449
538
|
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
if tt != task_type:
|
|
460
|
-
continue
|
|
461
|
-
yield _build_succeeded_job_dict(uid, succ_meta)
|
|
539
|
+
def iter_non_queued_successes(
|
|
540
|
+
self, *, task_type: str | None = None
|
|
541
|
+
) -> Iterator[tuple[str, Mapping[str, Any]]]:
|
|
542
|
+
for uid, raw_meta in self._successes.items():
|
|
543
|
+
if uid in self._queue_by_uid:
|
|
544
|
+
continue
|
|
545
|
+
if not _matches_task_type(uid, task_type):
|
|
546
|
+
continue
|
|
547
|
+
yield uid, _decode_json_dict(raw_meta)
|
|
462
548
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
if tt != task_type:
|
|
473
|
-
continue
|
|
474
|
-
payload = self._failure_payloads.get(uid)
|
|
475
|
-
yield _build_failed_job_dict(uid, fail_meta, payload)
|
|
549
|
+
def iter_non_queued_failed(
|
|
550
|
+
self, *, task_type: str | None = None
|
|
551
|
+
) -> Iterator[tuple[str, Mapping[str, Any], Mapping[str, Any] | None]]:
|
|
552
|
+
for uid, (raw_meta, raw_payload) in self._failed.items():
|
|
553
|
+
if uid in self._queue_by_uid:
|
|
554
|
+
continue
|
|
555
|
+
if not _matches_task_type(uid, task_type):
|
|
556
|
+
continue
|
|
557
|
+
yield uid, _decode_json_dict(raw_meta), _decode_json_optional_dict(raw_payload)
|
|
476
558
|
|
|
477
559
|
def iter_attempts(
|
|
478
|
-
self,
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
560
|
+
self, *, job_uid: str | None = None, task_type: str | None = None
|
|
561
|
+
) -> Iterator[
|
|
562
|
+
tuple[
|
|
563
|
+
str,
|
|
564
|
+
int,
|
|
565
|
+
str,
|
|
566
|
+
str,
|
|
567
|
+
str | None,
|
|
568
|
+
str | None,
|
|
569
|
+
Mapping[str, Any],
|
|
570
|
+
Mapping[str, Any] | None,
|
|
571
|
+
Any,
|
|
572
|
+
]
|
|
573
|
+
]:
|
|
484
574
|
if job_uid is not None:
|
|
485
|
-
|
|
486
|
-
|
|
575
|
+
rows = [r for r in self._attempts if r[0] == job_uid]
|
|
576
|
+
rows.sort(key=lambda r: r[1] if r[1] is not None else 0)
|
|
577
|
+
else:
|
|
578
|
+
rows = self._attempts
|
|
487
579
|
|
|
488
|
-
for
|
|
489
|
-
uid =
|
|
490
|
-
if not uid:
|
|
580
|
+
for row in rows:
|
|
581
|
+
uid = row[0]
|
|
582
|
+
if not _matches_task_type(uid, task_type):
|
|
491
583
|
continue
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
"attempt_no": item["attempt_no"],
|
|
508
|
-
"started_at": _format_iso_z(item["started_at"]),
|
|
509
|
-
"settled_at": _format_iso_z(item["settled_at"]),
|
|
510
|
-
}
|
|
584
|
+
yield (
|
|
585
|
+
uid,
|
|
586
|
+
row[1],
|
|
587
|
+
row[2],
|
|
588
|
+
row[3],
|
|
589
|
+
row[4],
|
|
590
|
+
row[5],
|
|
591
|
+
_decode_json_dict(row[6]),
|
|
592
|
+
_decode_json_optional_dict(row[7]),
|
|
593
|
+
_decode_attempt_result(
|
|
594
|
+
row[8],
|
|
595
|
+
row[9],
|
|
596
|
+
fallback_empty_dict=self._attempt_result_fallback_empty_dict,
|
|
597
|
+
),
|
|
598
|
+
)
|
|
511
599
|
|
|
512
600
|
def close(self) -> None:
|
|
513
601
|
if not self._closed:
|
|
514
|
-
self.
|
|
602
|
+
self._queue_rows.clear()
|
|
603
|
+
self._queue_by_uid.clear()
|
|
515
604
|
self._successes.clear()
|
|
516
605
|
self._failed.clear()
|
|
517
|
-
self._failure_payloads.clear()
|
|
518
606
|
self._attempts.clear()
|
|
519
607
|
self._closed = True
|
|
520
608
|
|
|
521
609
|
|
|
610
|
+
# ── 单一深投影核心 StateSnapshot ───────────────────────────────────────────
|
|
611
|
+
|
|
612
|
+
|
|
522
613
|
class StateSnapshot:
|
|
523
|
-
"""一致只读状态快照(JSON
|
|
614
|
+
"""一致只读状态快照(JSON 兼容字典查询与可读报告)。"""
|
|
524
615
|
|
|
525
|
-
def __init__(
|
|
616
|
+
def __init__(
|
|
617
|
+
self,
|
|
618
|
+
source: str | Path | "TaskLite",
|
|
619
|
+
*,
|
|
620
|
+
mode: Literal["lazy", "copy"] = "lazy",
|
|
621
|
+
) -> None:
|
|
622
|
+
if not isinstance(mode, str) or mode not in _VALID_SNAPSHOT_MODES:
|
|
623
|
+
raise TaskLiteError(
|
|
624
|
+
f"mode must be 'lazy' or 'copy', got {mode!r}"
|
|
625
|
+
)
|
|
526
626
|
self._source = source
|
|
527
627
|
self._closed = False
|
|
528
|
-
|
|
628
|
+
adapter: _SnapshotAdapter
|
|
529
629
|
if isinstance(source, (str, Path)):
|
|
530
|
-
|
|
630
|
+
db_path = Path(source)
|
|
631
|
+
adapter = (
|
|
632
|
+
_MemoryTableAdapter.from_sqlite_copy(db_path)
|
|
633
|
+
if mode == "copy"
|
|
634
|
+
else _SqliteStreamingAdapter(db_path)
|
|
635
|
+
)
|
|
531
636
|
else:
|
|
532
637
|
from .pipeline import TaskLite
|
|
533
638
|
|
|
@@ -536,21 +641,72 @@ class StateSnapshot:
|
|
|
536
641
|
"source must be a str, Path, or TaskLite instance, "
|
|
537
642
|
f"got {type(source).__name__}"
|
|
538
643
|
)
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
644
|
+
kind, payload = source._open_snapshot_source()
|
|
645
|
+
if kind == "sqlite":
|
|
646
|
+
assert isinstance(payload, Path)
|
|
647
|
+
adapter = (
|
|
648
|
+
_MemoryTableAdapter.from_sqlite_copy(payload)
|
|
649
|
+
if mode == "copy"
|
|
650
|
+
else _SqliteStreamingAdapter(payload)
|
|
542
651
|
)
|
|
543
|
-
backend = source._snapshot_backend()
|
|
544
|
-
if source.state_path is not None:
|
|
545
|
-
reader = _SqliteSnapshotReader(source.state_path)
|
|
546
652
|
else:
|
|
547
|
-
|
|
548
|
-
|
|
653
|
+
assert isinstance(payload, Mapping)
|
|
654
|
+
adapter = _MemoryTableAdapter.from_consistent_snapshot(payload)
|
|
655
|
+
self._adapter: _SnapshotAdapter = adapter
|
|
549
656
|
|
|
550
657
|
@classmethod
|
|
551
|
-
def open(
|
|
552
|
-
|
|
553
|
-
|
|
658
|
+
def open(
|
|
659
|
+
cls,
|
|
660
|
+
source: str | Path | "TaskLite",
|
|
661
|
+
*,
|
|
662
|
+
mode: Literal["lazy", "copy"] = "lazy",
|
|
663
|
+
) -> StateSnapshot:
|
|
664
|
+
"""打开只读快照(支持数据库路径或 TaskLite 实例;mode='lazy'|'copy')。"""
|
|
665
|
+
return cls(source, mode=mode)
|
|
666
|
+
|
|
667
|
+
@classmethod
|
|
668
|
+
def _from_tables(
|
|
669
|
+
cls,
|
|
670
|
+
*,
|
|
671
|
+
queue_rows: Sequence[Mapping[str, Any]],
|
|
672
|
+
successes: Mapping[str, Mapping[str, Any]],
|
|
673
|
+
failed: Mapping[str, tuple[Mapping[str, Any], Mapping[str, Any] | None]],
|
|
674
|
+
attempts: Sequence[Mapping[str, Any]],
|
|
675
|
+
source: Any = None,
|
|
676
|
+
) -> StateSnapshot:
|
|
677
|
+
"""包内受控表投影入口:将三表工作图与尝试轨迹副本封装为只读 StateSnapshot。"""
|
|
678
|
+
instance = cls.__new__(cls)
|
|
679
|
+
instance._source = source
|
|
680
|
+
instance._closed = False
|
|
681
|
+
instance._adapter = _MemoryTableAdapter.from_edit_tables(
|
|
682
|
+
queue_rows=queue_rows,
|
|
683
|
+
successes=successes,
|
|
684
|
+
failed=failed,
|
|
685
|
+
attempts=attempts,
|
|
686
|
+
)
|
|
687
|
+
return instance
|
|
688
|
+
|
|
689
|
+
@staticmethod
|
|
690
|
+
def report(
|
|
691
|
+
first: StateSnapshot,
|
|
692
|
+
second: StateSnapshot | None = None,
|
|
693
|
+
) -> str:
|
|
694
|
+
"""生成面向人类阅读的单快照状态报告或双快照增量对比报告。"""
|
|
695
|
+
if not isinstance(first, StateSnapshot):
|
|
696
|
+
raise TaskLiteError(
|
|
697
|
+
f"first must be a StateSnapshot instance, got {type(first).__name__}"
|
|
698
|
+
)
|
|
699
|
+
if first.closed:
|
|
700
|
+
raise TaskLiteError("Cannot generate report from a closed StateSnapshot")
|
|
701
|
+
if second is not None:
|
|
702
|
+
if not isinstance(second, StateSnapshot):
|
|
703
|
+
raise TaskLiteError(
|
|
704
|
+
"second must be a StateSnapshot instance or None, "
|
|
705
|
+
f"got {type(second).__name__}"
|
|
706
|
+
)
|
|
707
|
+
if second.closed:
|
|
708
|
+
raise TaskLiteError("Cannot generate report from a closed StateSnapshot")
|
|
709
|
+
return render_snapshot_report(first, second)
|
|
554
710
|
|
|
555
711
|
def get_job(self, uid: str) -> dict[str, Any] | None:
|
|
556
712
|
"""查询指定 Job 的当前状态与配置/结果投影(不存在返回 None)。"""
|
|
@@ -564,7 +720,22 @@ class StateSnapshot:
|
|
|
564
720
|
parse_job_uid(uid)
|
|
565
721
|
except Exception:
|
|
566
722
|
return None
|
|
567
|
-
|
|
723
|
+
|
|
724
|
+
queued = self._adapter.get_queued(uid)
|
|
725
|
+
if queued is not None:
|
|
726
|
+
job_snap, status, run_id, attempt_no, started_at = queued
|
|
727
|
+
return _project_queued_job(uid, job_snap, status, run_id, attempt_no, started_at)
|
|
728
|
+
|
|
729
|
+
succ_meta = self._adapter.get_success(uid)
|
|
730
|
+
if succ_meta is not None:
|
|
731
|
+
return _project_succeeded_job(uid, succ_meta)
|
|
732
|
+
|
|
733
|
+
failed_entry = self._adapter.get_failed(uid)
|
|
734
|
+
if failed_entry is not None:
|
|
735
|
+
fail_meta, payload = failed_entry
|
|
736
|
+
return _project_failed_job(uid, fail_meta, payload)
|
|
737
|
+
|
|
738
|
+
return None
|
|
568
739
|
|
|
569
740
|
def iter_jobs(
|
|
570
741
|
self,
|
|
@@ -582,10 +753,27 @@ class StateSnapshot:
|
|
|
582
753
|
if task_type is not None and not isinstance(task_type, str):
|
|
583
754
|
raise TaskLiteError(f"task_type must be a str or None, got {type(task_type).__name__}")
|
|
584
755
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
756
|
+
if status is None or status in ("pending", "running"):
|
|
757
|
+
for uid, job_snap, st, run_id, attempt_no, started_at in self._adapter.iter_queued(
|
|
758
|
+
status=status, task_type=task_type
|
|
759
|
+
):
|
|
760
|
+
if self._closed:
|
|
761
|
+
raise TaskLiteError("StateSnapshot is closed")
|
|
762
|
+
yield _project_queued_job(uid, job_snap, st, run_id, attempt_no, started_at)
|
|
763
|
+
|
|
764
|
+
if status is None or status == "succeeded":
|
|
765
|
+
for uid, succ_meta in self._adapter.iter_non_queued_successes(task_type=task_type):
|
|
766
|
+
if self._closed:
|
|
767
|
+
raise TaskLiteError("StateSnapshot is closed")
|
|
768
|
+
yield _project_succeeded_job(uid, succ_meta)
|
|
769
|
+
|
|
770
|
+
if status is None or status == "failed":
|
|
771
|
+
for uid, fail_meta, payload in self._adapter.iter_non_queued_failed(
|
|
772
|
+
task_type=task_type
|
|
773
|
+
):
|
|
774
|
+
if self._closed:
|
|
775
|
+
raise TaskLiteError("StateSnapshot is closed")
|
|
776
|
+
yield _project_failed_job(uid, fail_meta, payload)
|
|
589
777
|
|
|
590
778
|
def iter_attempts(
|
|
591
779
|
self,
|
|
@@ -601,15 +789,35 @@ class StateSnapshot:
|
|
|
601
789
|
if task_type is not None and not isinstance(task_type, str):
|
|
602
790
|
raise TaskLiteError(f"task_type must be a str or None, got {type(task_type).__name__}")
|
|
603
791
|
|
|
604
|
-
for
|
|
792
|
+
for (
|
|
793
|
+
uid,
|
|
794
|
+
attempt_no,
|
|
795
|
+
run_id,
|
|
796
|
+
outcome,
|
|
797
|
+
started_at,
|
|
798
|
+
settled_at,
|
|
799
|
+
job_snap,
|
|
800
|
+
files_data,
|
|
801
|
+
result_data,
|
|
802
|
+
) in self._adapter.iter_attempts(job_uid=job_uid, task_type=task_type):
|
|
605
803
|
if self._closed:
|
|
606
804
|
raise TaskLiteError("StateSnapshot is closed")
|
|
607
|
-
yield
|
|
805
|
+
yield _project_attempt(
|
|
806
|
+
uid,
|
|
807
|
+
attempt_no,
|
|
808
|
+
run_id,
|
|
809
|
+
outcome,
|
|
810
|
+
started_at,
|
|
811
|
+
settled_at,
|
|
812
|
+
job_snap,
|
|
813
|
+
files_data,
|
|
814
|
+
result_data,
|
|
815
|
+
)
|
|
608
816
|
|
|
609
817
|
def close(self) -> None:
|
|
610
818
|
"""关闭只读快照(幂等;不关闭底层的 TaskLite 实例)。"""
|
|
611
819
|
if not self._closed:
|
|
612
|
-
self.
|
|
820
|
+
self._adapter.close()
|
|
613
821
|
self._closed = True
|
|
614
822
|
|
|
615
823
|
@property
|
|
@@ -617,11 +825,6 @@ class StateSnapshot:
|
|
|
617
825
|
"""快照是否已关闭。"""
|
|
618
826
|
return self._closed
|
|
619
827
|
|
|
620
|
-
@property
|
|
621
|
-
def is_closed(self) -> bool:
|
|
622
|
-
"""快照是否已关闭。"""
|
|
623
|
-
return self._closed
|
|
624
|
-
|
|
625
828
|
@property
|
|
626
829
|
def source(self) -> Any:
|
|
627
830
|
"""打开快照的源对象或路径。"""
|