tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
tasklite/engine/store.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
"""v2 StateStore: 统一内存状态机与后端持久化事务的深模块。
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
依赖图级联分析。单一出口纪律:
|
|
3
|
+
内敛原子双写一致性、六集合全局互斥与依赖图级联分析。单一出口纪律:
|
|
5
4
|
|
|
6
5
|
- Job 终结事务唯一经由本模块的 apply_* 出口族收敛(成功 / 失败 / 重试 /
|
|
7
|
-
跳过 / 批量失败 / 级联)——backend commit + 内存镜像 +
|
|
6
|
+
跳过 / 批量失败 / 级联)——backend commit + 内存镜像 + 级联
|
|
8
7
|
在同一出口内对齐,内存永不领先磁盘;
|
|
9
8
|
- 失败终态内存登记唯一经由 ``mark_failed_memory``(successes/failed 互斥的
|
|
10
9
|
单点维护);
|
|
@@ -16,49 +15,37 @@ from __future__ import annotations
|
|
|
16
15
|
import copy
|
|
17
16
|
import logging
|
|
18
17
|
import os
|
|
19
|
-
from
|
|
18
|
+
from contextlib import AbstractContextManager
|
|
20
19
|
from dataclasses import dataclass, field
|
|
21
20
|
from pathlib import Path
|
|
22
|
-
from typing import Any, Callable,
|
|
21
|
+
from typing import Any, Callable, Literal, Mapping, Sequence
|
|
23
22
|
|
|
24
|
-
from ..backend.base import AbstractStateBackend
|
|
23
|
+
from ..backend.base import AbstractStateBackend, GraphEditCommit
|
|
25
24
|
from ..exceptions import TaskLiteError
|
|
26
|
-
from ..models.artifact import
|
|
27
|
-
from ..models.job import Job
|
|
28
|
-
from ..models.
|
|
25
|
+
from ..models.artifact import FileSpec, resolve_file_spec
|
|
26
|
+
from ..models.job import Job
|
|
27
|
+
from ..models.records import (
|
|
28
|
+
build_execution_failure_record,
|
|
29
|
+
build_execution_success_record,
|
|
30
|
+
extract_effective_job_config,
|
|
31
|
+
extract_serializable_payload,
|
|
32
|
+
normalize_job_intake_snapshot,
|
|
33
|
+
)
|
|
29
34
|
from ..models.state import PipelineState, job_uid_from_snapshot
|
|
30
|
-
from ..
|
|
35
|
+
from ..models.task import TaskRegistry
|
|
36
|
+
from ..utils.artifact import InspectedPath, PathPolicy
|
|
31
37
|
from ..utils.clock import parse_utc_iso_to_us, utc_now_iso
|
|
32
|
-
from ..utils.encoding import parse_job_uid
|
|
33
|
-
from ..utils.jsonutil import dumps, loads
|
|
34
38
|
from .admission import (
|
|
35
39
|
ImmediateRequeuePolicy,
|
|
36
40
|
RequeuePolicy,
|
|
37
41
|
TriggerRegistry,
|
|
38
|
-
extract_effective_job_config,
|
|
39
42
|
extract_trusted_metadata_timestamp_us,
|
|
40
|
-
is_trusted_success_metadata,
|
|
41
43
|
)
|
|
44
|
+
from .session import RunSession
|
|
42
45
|
|
|
43
46
|
logger = logging.getLogger("tasklite")
|
|
44
47
|
|
|
45
48
|
|
|
46
|
-
@dataclass(frozen=True)
|
|
47
|
-
class FailureEntry:
|
|
48
|
-
"""失败档案只读查询的结构化不可变条目(运维查询面的载荷形状)。
|
|
49
|
-
|
|
50
|
-
- ``job_uid``: 作业唯一标识 (task_type::job_id)
|
|
51
|
-
- ``error_message``: 失败说明
|
|
52
|
-
- ``metadata``: 完整失败档案 meta 字典视图
|
|
53
|
-
- ``payload``: 原始业务 payload 快照(失败落盘无可存快照时为 None)
|
|
54
|
-
"""
|
|
55
|
-
|
|
56
|
-
job_uid: str
|
|
57
|
-
error_message: str
|
|
58
|
-
metadata: dict[str, Any]
|
|
59
|
-
payload: dict[str, Any] | None = None
|
|
60
|
-
|
|
61
|
-
|
|
62
49
|
@dataclass(frozen=True)
|
|
63
50
|
class SuccessOutcome:
|
|
64
51
|
"""apply_success 原子转移结果。"""
|
|
@@ -105,78 +92,112 @@ class StateStore:
|
|
|
105
92
|
backend: AbstractStateBackend,
|
|
106
93
|
state: PipelineState | None = None,
|
|
107
94
|
*,
|
|
108
|
-
|
|
95
|
+
tasks: TaskRegistry | None = None,
|
|
109
96
|
trigger_registry: TriggerRegistry | None = None,
|
|
110
97
|
requeue_policy: RequeuePolicy | None = None,
|
|
111
98
|
base_dir: str | Path | None = None,
|
|
99
|
+
state_path: str | Path | None = None,
|
|
112
100
|
protected_roots: Sequence[str | Path] = (),
|
|
101
|
+
path_policy: PathPolicy | None = None,
|
|
102
|
+
session: RunSession | None = None,
|
|
113
103
|
) -> None:
|
|
114
104
|
self._backend = backend
|
|
115
105
|
self._state = state if state is not None else PipelineState({}, {}, {}, [])
|
|
116
|
-
self.
|
|
106
|
+
self._tasks = tasks
|
|
117
107
|
self._trigger_registry = (
|
|
118
|
-
trigger_registry
|
|
119
|
-
if trigger_registry is not None
|
|
120
|
-
else TriggerRegistry(base_dir=base_dir)
|
|
108
|
+
trigger_registry if trigger_registry is not None else TriggerRegistry()
|
|
121
109
|
)
|
|
122
110
|
# 重试节奏唯一出口:核心不做任何节奏计算,默认立即重入队。
|
|
123
111
|
self._requeue_policy = (
|
|
124
112
|
requeue_policy if requeue_policy is not None else ImmediateRequeuePolicy()
|
|
125
113
|
)
|
|
126
|
-
self.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
Path(
|
|
114
|
+
self._session = session if session is not None else RunSession()
|
|
115
|
+
|
|
116
|
+
fallback_base: str | Path = (
|
|
117
|
+
base_dir if base_dir is not None else Path(os.path.abspath(os.getcwd()))
|
|
130
118
|
)
|
|
119
|
+
if path_policy is not None:
|
|
120
|
+
if protected_roots or state_path is not None or path_policy.fixed_base_dir is None:
|
|
121
|
+
merged_roots = (*path_policy.protected_paths, *protected_roots)
|
|
122
|
+
effective_base: str | Path = (
|
|
123
|
+
path_policy.fixed_base_dir
|
|
124
|
+
if path_policy.fixed_base_dir is not None
|
|
125
|
+
else fallback_base
|
|
126
|
+
)
|
|
127
|
+
self._path_policy = PathPolicy(
|
|
128
|
+
base_dir=effective_base,
|
|
129
|
+
state_path=state_path,
|
|
130
|
+
protected_roots=merged_roots,
|
|
131
|
+
adapter=path_policy.adapter,
|
|
132
|
+
)
|
|
133
|
+
else:
|
|
134
|
+
self._path_policy = path_policy
|
|
135
|
+
else:
|
|
136
|
+
self._path_policy = PathPolicy(
|
|
137
|
+
base_dir=fallback_base,
|
|
138
|
+
state_path=state_path,
|
|
139
|
+
protected_roots=protected_roots,
|
|
140
|
+
)
|
|
131
141
|
self.authorizer: Callable[[str, str], bool | None] | None = None
|
|
132
142
|
|
|
133
143
|
@property
|
|
134
|
-
def
|
|
135
|
-
"""
|
|
136
|
-
return self.
|
|
137
|
-
|
|
138
|
-
# ── 内部统计 ────────────────────────────────────────────────
|
|
139
|
-
|
|
140
|
-
def _record_stat(self, key: str, amount: int = 1) -> None:
|
|
141
|
-
"""更新统计指标(单一真相源)。"""
|
|
142
|
-
if self._stats is not None:
|
|
143
|
-
self._stats[key] = self._stats.get(key, 0) + amount
|
|
144
|
+
def protected_roots(self) -> tuple[Path, ...]:
|
|
145
|
+
"""受保护路径集合(派生自统一 PathPolicy,单一事实源)。"""
|
|
146
|
+
return self._path_policy.protected_paths
|
|
144
147
|
|
|
145
148
|
@property
|
|
146
|
-
def
|
|
147
|
-
"""
|
|
148
|
-
return self.
|
|
149
|
-
|
|
149
|
+
def path_policy(self) -> PathPolicy:
|
|
150
|
+
"""统一路径规范化与受保护路径策略实例。"""
|
|
151
|
+
return self._path_policy
|
|
150
152
|
|
|
151
|
-
def
|
|
153
|
+
def resolve_file_spec(
|
|
152
154
|
self,
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
155
|
+
spec: FileSpec,
|
|
156
|
+
*,
|
|
157
|
+
depends_on: Sequence[str],
|
|
158
|
+
enforce_safety: bool | None = None,
|
|
159
|
+
) -> FileSpec:
|
|
160
|
+
"""经统一 PathPolicy 单次原子解析 FileSpec(上游引用、安全边界、存在性判定与路径绑定)。"""
|
|
161
|
+
def _lookup(producer_uid: str, output_name: str) -> Mapping[str, Any] | None:
|
|
162
|
+
artifacts = self._state.successes.get(producer_uid, {}).get("artifacts", {})
|
|
163
|
+
return artifacts.get(output_name) if isinstance(artifacts, Mapping) else None
|
|
164
|
+
|
|
165
|
+
return resolve_file_spec(
|
|
166
|
+
spec,
|
|
167
|
+
path_policy=self._path_policy,
|
|
168
|
+
depends_on=depends_on,
|
|
169
|
+
artifact_lookup=_lookup,
|
|
170
|
+
enforce_safety=enforce_safety,
|
|
171
|
+
)
|
|
159
172
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
173
|
+
@property
|
|
174
|
+
def base_dir(self) -> Path:
|
|
175
|
+
"""当前实例固定的相对路径解析基准目录(直接派生自 PathPolicy)。"""
|
|
176
|
+
assert self._path_policy.fixed_base_dir is not None
|
|
177
|
+
return Path(self._path_policy.fixed_base_dir)
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def session(self) -> RunSession:
|
|
181
|
+
"""当前关联的运行期会话(单次 Run 排除规则与生命周期状态真相源)。"""
|
|
182
|
+
return self._session
|
|
163
183
|
|
|
164
184
|
def _selection_excluded_uids(self) -> frozenset[str]:
|
|
165
185
|
"""当前队列中被排除拦截(Exclude 或临时排除)的合法作业 UID 集合。"""
|
|
166
|
-
|
|
186
|
+
exclude = self._session.exclude
|
|
187
|
+
temporary_uids = self._session.temporary_excluded_uids
|
|
188
|
+
if exclude is None and not temporary_uids:
|
|
167
189
|
return frozenset()
|
|
168
|
-
excluded: set[str] = set(
|
|
169
|
-
if
|
|
190
|
+
excluded: set[str] = set(temporary_uids)
|
|
191
|
+
if exclude is not None:
|
|
170
192
|
for job_snapshot in self._state.queue:
|
|
171
193
|
try:
|
|
172
194
|
job = Job.from_dict(job_snapshot)
|
|
173
195
|
except (KeyError, TypeError, ValueError, TaskLiteError):
|
|
174
196
|
continue
|
|
175
|
-
if
|
|
197
|
+
if exclude.matches_task_type(job.task_type):
|
|
176
198
|
excluded.add(job.uid)
|
|
177
199
|
return frozenset(excluded)
|
|
178
200
|
|
|
179
|
-
|
|
180
201
|
@staticmethod
|
|
181
202
|
def _attempt_error(metadata: Mapping[str, Any]) -> str | None:
|
|
182
203
|
"""从失败 meta 提取轨迹行错误串(空串收敛为 None)。"""
|
|
@@ -200,249 +221,85 @@ class StateStore:
|
|
|
200
221
|
self._backend.set_cursor(key, value)
|
|
201
222
|
self._state.set_cursor(key, value)
|
|
202
223
|
|
|
203
|
-
def
|
|
204
|
-
"""
|
|
205
|
-
return self._backend.
|
|
206
|
-
|
|
207
|
-
def
|
|
208
|
-
"""
|
|
209
|
-
return self._backend.
|
|
210
|
-
|
|
211
|
-
def
|
|
212
|
-
"""
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
def set_job_status(
|
|
230
|
-
self,
|
|
231
|
-
job_or_uid: Job | str,
|
|
232
|
-
status: str,
|
|
233
|
-
*,
|
|
234
|
-
recursive: bool,
|
|
235
|
-
note: str | None = None,
|
|
236
|
-
) -> None:
|
|
237
|
-
"""人工设置作业状态(pending/succeeded/failed),原子处理目标及递归下游。"""
|
|
238
|
-
if status not in ("pending", "succeeded", "failed"):
|
|
239
|
-
raise TaskLiteError(
|
|
240
|
-
f"status must be one of 'pending', 'succeeded', 'failed', got {status!r}"
|
|
241
|
-
)
|
|
242
|
-
if not isinstance(recursive, bool):
|
|
243
|
-
raise TaskLiteError(f"recursive must be a bool, got {type(recursive).__name__}")
|
|
244
|
-
if note is not None and not isinstance(note, str):
|
|
245
|
-
raise TaskLiteError(f"note must be a str or None, got {type(note).__name__}")
|
|
246
|
-
|
|
247
|
-
if isinstance(job_or_uid, Job):
|
|
248
|
-
target_job = job_or_uid
|
|
249
|
-
target_uid = job_or_uid.uid
|
|
250
|
-
elif isinstance(job_or_uid, str):
|
|
251
|
-
if not job_or_uid:
|
|
252
|
-
raise TaskLiteError("job_or_uid cannot be empty")
|
|
253
|
-
parse_job_uid(job_or_uid)
|
|
254
|
-
target_job = None
|
|
255
|
-
target_uid = job_or_uid
|
|
256
|
-
else:
|
|
257
|
-
raise TaskLiteError(
|
|
258
|
-
f"job_or_uid must be a Job or str, got {type(job_or_uid).__name__}"
|
|
259
|
-
)
|
|
260
|
-
|
|
261
|
-
if target_job is not None:
|
|
262
|
-
intake_snap = self.normalize_and_validate_job(target_job)
|
|
263
|
-
existing_snap = self.get_job_snapshot(target_uid)
|
|
264
|
-
if existing_snap is not None:
|
|
265
|
-
if extract_effective_job_config(existing_snap) != extract_effective_job_config(intake_snap):
|
|
266
|
-
raise TaskLiteError(
|
|
267
|
-
f"Job configuration conflict for UID {target_uid!r}: "
|
|
268
|
-
f"passed Job does not match existing effective configuration"
|
|
269
|
-
)
|
|
270
|
-
target_snap = existing_snap
|
|
271
|
-
else:
|
|
272
|
-
target_snap = intake_snap
|
|
273
|
-
else:
|
|
274
|
-
found = self.get_job_snapshot(target_uid)
|
|
275
|
-
if found is None:
|
|
276
|
-
raise TaskLiteError(f"No existing snapshot found for job UID {target_uid!r}")
|
|
277
|
-
target_snap = found
|
|
278
|
-
|
|
279
|
-
all_known = self.get_all_known_jobs()
|
|
280
|
-
all_known[target_uid] = target_snap
|
|
281
|
-
|
|
282
|
-
affected: list[tuple[str, dict[str, Any]]] = [(target_uid, target_snap)]
|
|
283
|
-
if recursive:
|
|
284
|
-
downstream_uids = self._find_downstream_uids(target_uid, all_known)
|
|
285
|
-
for child_uid in downstream_uids:
|
|
286
|
-
child_snap = all_known.get(child_uid)
|
|
287
|
-
if child_snap is not None:
|
|
288
|
-
affected.append((child_uid, child_snap))
|
|
289
|
-
|
|
290
|
-
updates: list[tuple[str, str, dict[str, Any], dict[str, Any] | None]] = []
|
|
291
|
-
for u, snap in affected:
|
|
292
|
-
if status == "pending":
|
|
293
|
-
updates.append((u, "pending", snap, None))
|
|
294
|
-
elif status == "succeeded":
|
|
295
|
-
annotations = {"completion_source": "manual"}
|
|
296
|
-
if note is not None:
|
|
297
|
-
annotations["note"] = note
|
|
298
|
-
succ_meta: dict[str, Any] = {
|
|
299
|
-
"job_snapshot": snap,
|
|
300
|
-
"annotations": annotations,
|
|
301
|
-
"settled_at": utc_now_iso(),
|
|
302
|
-
"run_id": None,
|
|
303
|
-
"attempt_no": None,
|
|
304
|
-
"started_at": None,
|
|
305
|
-
"artifacts": {},
|
|
306
|
-
"inputs": [],
|
|
307
|
-
}
|
|
308
|
-
updates.append((u, "succeeded", snap, succ_meta))
|
|
309
|
-
elif status == "failed":
|
|
310
|
-
fail_meta: dict[str, Any] = {
|
|
311
|
-
"job_snapshot": snap,
|
|
312
|
-
"annotations": {"completion_source": "manual"},
|
|
313
|
-
"error_message": note or "Manual status set to failed",
|
|
314
|
-
"settled_at": utc_now_iso(),
|
|
315
|
-
"run_id": None,
|
|
316
|
-
"attempt_no": None,
|
|
317
|
-
}
|
|
318
|
-
if note is not None:
|
|
319
|
-
fail_meta["note"] = note
|
|
320
|
-
updates.append((u, "failed", snap, fail_meta))
|
|
321
|
-
|
|
322
|
-
self._backend.apply_manual_status(updates)
|
|
323
|
-
for u, st, snap, up_meta in updates:
|
|
324
|
-
if st == "pending":
|
|
325
|
-
self._state.discard_success(u)
|
|
326
|
-
self._state.discard_failed(u)
|
|
327
|
-
self._state.remove_queued_uid(u)
|
|
328
|
-
self._state.spawn_jobs([snap], front=False)
|
|
329
|
-
elif st == "succeeded":
|
|
330
|
-
self._state.remove_queued_uid(u)
|
|
331
|
-
self._state.mark_success(u, up_meta or {})
|
|
332
|
-
elif st == "failed":
|
|
333
|
-
self._state.remove_queued_uid(u)
|
|
334
|
-
self._state.mark_failed(u, up_meta or {})
|
|
335
|
-
|
|
336
|
-
def clear_history(self, uid: str, records: Collection[str]) -> int:
|
|
337
|
-
"""从 successes/failed/attempts 清除指定 job_uid 的记录,返回删除条数。"""
|
|
338
|
-
if not isinstance(uid, str) or not uid:
|
|
339
|
-
raise TaskLiteError(f"uid must be a non-empty str, got {uid!r}")
|
|
340
|
-
try:
|
|
341
|
-
parse_job_uid(uid)
|
|
342
|
-
except Exception as exc:
|
|
343
|
-
raise TaskLiteError(
|
|
344
|
-
f"uid must be a valid canonical 'task_type::job_id' str, got {uid!r}"
|
|
345
|
-
) from exc
|
|
346
|
-
if not isinstance(records, (list, tuple, set, frozenset)):
|
|
347
|
-
raise TaskLiteError(
|
|
348
|
-
"records must be a collection of 'successes', 'failures', "
|
|
349
|
-
f"'attempts', got {type(records).__name__}"
|
|
350
|
-
)
|
|
351
|
-
record_set = set(records)
|
|
352
|
-
allowed = {"successes", "failures", "attempts"}
|
|
353
|
-
unknown = record_set - allowed
|
|
354
|
-
if not record_set:
|
|
355
|
-
raise TaskLiteError("records must not be empty")
|
|
356
|
-
if unknown:
|
|
357
|
-
raise TaskLiteError(
|
|
358
|
-
f"records contains unknown target(s): {sorted(unknown)!r}; "
|
|
359
|
-
f"allowed: {sorted(allowed)!r}"
|
|
360
|
-
)
|
|
361
|
-
total = self._backend.clear_history(uid, record_set)
|
|
362
|
-
if "successes" in record_set:
|
|
363
|
-
self._state.discard_success(uid)
|
|
364
|
-
if "failures" in record_set:
|
|
365
|
-
self._state.discard_failed(uid)
|
|
366
|
-
return total
|
|
224
|
+
def exclusive_access(self) -> AbstractContextManager[None]:
|
|
225
|
+
"""获取底层状态库的独占执行权上下文(委托后端独占执行权契约)。"""
|
|
226
|
+
return self._backend.exclusive_access()
|
|
227
|
+
|
|
228
|
+
def load_consistent_snapshot(self) -> dict[str, Any]:
|
|
229
|
+
"""在单一读事务或单锁内原子捕获全部核心表一致快照。"""
|
|
230
|
+
return self._backend.load_consistent_snapshot()
|
|
231
|
+
|
|
232
|
+
def apply_graph_edit(self, commit: GraphEditCommit) -> None:
|
|
233
|
+
"""原子提交图修整事务并在落盘成功后以单次一致快照同步内存状态镜像。"""
|
|
234
|
+
if (
|
|
235
|
+
not commit.queue_writes
|
|
236
|
+
and not commit.success_writes
|
|
237
|
+
and not commit.failed_writes
|
|
238
|
+
and not commit.recovery_attempts
|
|
239
|
+
):
|
|
240
|
+
return
|
|
241
|
+
self._backend.commit_graph_edit(commit)
|
|
242
|
+
snap = self._backend.load_consistent_snapshot()
|
|
243
|
+
self._state = PipelineState(
|
|
244
|
+
snap["successes"],
|
|
245
|
+
snap["failed"],
|
|
246
|
+
snap["cursors"],
|
|
247
|
+
[row["job"] for row in snap["queue_jobs"] if row["status"] == "pending"],
|
|
248
|
+
)
|
|
367
249
|
|
|
368
|
-
# ── 摄入管道(enqueue / spawn
|
|
250
|
+
# ── 摄入管道(enqueue / spawn / edit add-replace 单一入口)────────
|
|
369
251
|
|
|
370
252
|
def _require_registered_task(self, task_type: str) -> None:
|
|
371
253
|
"""新声明必须对应本实例已注册的 Task。"""
|
|
372
|
-
|
|
373
|
-
if tasks is None or task_type not in tasks:
|
|
254
|
+
if self._tasks is None or task_type not in self._tasks:
|
|
374
255
|
raise TaskLiteError(
|
|
375
256
|
f"unknown task_type {task_type!r} (register the task before "
|
|
376
257
|
f"submitting its jobs)"
|
|
377
258
|
)
|
|
378
259
|
|
|
379
|
-
def
|
|
380
|
-
"""
|
|
381
|
-
if isinstance(job, Job):
|
|
382
|
-
job_snapshot = copy.deepcopy(job.to_dict())
|
|
383
|
-
elif isinstance(job, dict):
|
|
384
|
-
if "task_type" not in job or "job_id" not in job:
|
|
385
|
-
raise TaskLiteError("Job dict must contain 'task_type' and 'job_id'")
|
|
386
|
-
job_snapshot = copy.deepcopy(job)
|
|
387
|
-
else:
|
|
388
|
-
raise TaskLiteError(
|
|
389
|
-
f"Expected Job or dict, got {type(job).__name__}"
|
|
390
|
-
)
|
|
391
|
-
|
|
392
|
-
self.finalize_intake_snapshot(job_snapshot, str(job_snapshot["task_type"]))
|
|
393
|
-
return job_snapshot
|
|
394
|
-
|
|
395
|
-
def _normalize_new_job_snapshot(self, job: Job) -> dict[str, Any]:
|
|
396
|
-
"""解析并校验新声明的有效配置,要求 Task 已注册。"""
|
|
260
|
+
def normalize_new_job_snapshot(self, job: Job, *, caller: str = "enqueue") -> dict[str, Any]:
|
|
261
|
+
"""摄入唯一入口:校验 Task 已注册,捕获独立快照并落实规格继承、FileSpec 解析与触发器校验。"""
|
|
397
262
|
if not isinstance(job, Job):
|
|
398
263
|
raise TaskLiteError(
|
|
399
|
-
f"
|
|
264
|
+
f"{caller}() expects a Job, got {type(job).__name__}"
|
|
400
265
|
)
|
|
401
266
|
self._require_registered_task(job.task_type)
|
|
402
267
|
job_snapshot = copy.deepcopy(job.to_dict())
|
|
403
268
|
self.finalize_intake_snapshot(job_snapshot, job.task_type)
|
|
404
269
|
return job_snapshot
|
|
405
270
|
|
|
406
|
-
def normalize_spawned_job(self, job: Job) -> dict[str, Any]:
|
|
407
|
-
"""动态子作业与 enqueue 共用摄入规则,非法声明由父结算整体拒绝。"""
|
|
408
|
-
self._require_registered_task(job.task_type)
|
|
409
|
-
job_snapshot = job.to_dict()
|
|
410
|
-
self.finalize_intake_snapshot(job_snapshot, job.task_type)
|
|
411
|
-
return job_snapshot
|
|
412
|
-
|
|
413
271
|
def finalize_intake_snapshot(self, job_snapshot: dict[str, Any], task_type: str) -> None:
|
|
414
|
-
"""
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
)
|
|
418
|
-
self._trigger_registry.normalize_job_snapshot(job_snapshot, task_type)
|
|
419
|
-
inject_worker_resource(job_snapshot)
|
|
272
|
+
"""落实 Task→Job 规格继承、FileSpec 解析、worker 槽位注入与已注册 Trigger 校验。"""
|
|
273
|
+
task = self._tasks.get(task_type) if self._tasks is not None else None
|
|
274
|
+
normalize_job_intake_snapshot(job_snapshot, task=task, base_dir=self.base_dir)
|
|
275
|
+
self._trigger_registry.require_registered(job_snapshot.get("trigger"))
|
|
420
276
|
|
|
421
277
|
def enqueue_job(self, job: Job) -> bool:
|
|
422
|
-
"""
|
|
423
|
-
|
|
424
|
-
raise TaskLiteError(
|
|
425
|
-
f"enqueue() expects a Job, got {type(job).__name__}"
|
|
426
|
-
)
|
|
427
|
-
job_snapshot = self._normalize_new_job_snapshot(job)
|
|
278
|
+
"""单作业入队摄入管道:单次一致快照校验全队列(含 pending/running)同 UID 幂等/冲突与依赖存在性。"""
|
|
279
|
+
job_snapshot = self.normalize_new_job_snapshot(job, caller="enqueue")
|
|
428
280
|
target_uid = job.uid
|
|
429
281
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
282
|
+
queue_rows = self._backend.load_queue_jobs()
|
|
283
|
+
|
|
284
|
+
for row in queue_rows:
|
|
285
|
+
if row.get("job_uid") != target_uid:
|
|
434
286
|
continue
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
287
|
+
queued = row.get("job")
|
|
288
|
+
if (
|
|
289
|
+
isinstance(queued, Mapping)
|
|
290
|
+
and extract_effective_job_config(queued) == extract_effective_job_config(job_snapshot)
|
|
291
|
+
):
|
|
292
|
+
return False
|
|
293
|
+
raise TaskLiteError(
|
|
294
|
+
f"Job {target_uid!r} is already queued with different effective configuration"
|
|
295
|
+
)
|
|
441
296
|
|
|
442
|
-
#
|
|
297
|
+
# 依赖引用存在性按最小读集查验:允许引用全队列(含遗留 running)及终态档案,
|
|
298
|
+
# 避免扫描无关的 attempts/cursors 表。
|
|
443
299
|
if job.depends_on:
|
|
444
|
-
|
|
445
|
-
|
|
300
|
+
existing_uids: set[str] = {
|
|
301
|
+
row["job_uid"] for row in queue_rows if isinstance(row.get("job_uid"), str)
|
|
302
|
+
}
|
|
446
303
|
existing_uids.update(self._backend.load_successes().keys())
|
|
447
304
|
existing_uids.update(self._backend.load_failed().keys())
|
|
448
305
|
for dep in job.depends_on:
|
|
@@ -450,7 +307,10 @@ class StateStore:
|
|
|
450
307
|
raise TaskLiteError(
|
|
451
308
|
f"Job {job.uid!r} references non-existent dependency {dep!r} in depends_on"
|
|
452
309
|
)
|
|
310
|
+
|
|
453
311
|
inserted = self._backend.enqueue_jobs([job_snapshot])
|
|
312
|
+
if inserted and target_uid not in self._state.queue_uids:
|
|
313
|
+
self._state.spawn_jobs([copy.deepcopy(job_snapshot)], front=False)
|
|
454
314
|
return bool(inserted)
|
|
455
315
|
|
|
456
316
|
def mark_job_running(
|
|
@@ -479,13 +339,7 @@ class StateStore:
|
|
|
479
339
|
def trusted_last_succeeded_at_us(
|
|
480
340
|
self, job_uid: str, *, success_metadata: Mapping[str, Any] | None = None
|
|
481
341
|
) -> int | None:
|
|
482
|
-
"""可信成功基线(整数微秒):无可信档案则返回 None(调用方保守执行)。
|
|
483
|
-
|
|
484
|
-
不变式:基线只取自同事务提交的 attempts succeeded 行;先以 successes
|
|
485
|
-
档案框架元数据(seed 的空字典、缺键或坏值一律不可信)为闸门,且校验
|
|
486
|
-
successes 框架时间戳与 attempts 实际完成时间戳严格一致,防止
|
|
487
|
-
clear_history 后残留的旧 attempts 成功行被拼接成伪基线。
|
|
488
|
-
"""
|
|
342
|
+
"""可信成功基线(整数微秒):无可信档案则返回 None(调用方保守执行)。"""
|
|
489
343
|
if success_metadata is None:
|
|
490
344
|
success_metadata = self._state.successes.get(job_uid)
|
|
491
345
|
meta_ts = extract_trusted_metadata_timestamp_us(success_metadata)
|
|
@@ -507,12 +361,7 @@ class StateStore:
|
|
|
507
361
|
def mark_failed_memory(
|
|
508
362
|
self, job_uid: str, metadata: dict[str, Any], *, unregister: bool = True
|
|
509
363
|
) -> None:
|
|
510
|
-
"""失败终态内存登记的唯一尾段(successes/failed 互斥单点维护)。
|
|
511
|
-
|
|
512
|
-
独立复制框架档案,再落 failed 集合——state.mark_failed
|
|
513
|
-
同步摘除 successes 同名行(成功/失败互删语义);``unregister=False``
|
|
514
|
-
供批量崩溃契约路径先登记、由调用方统一处置在途集合。
|
|
515
|
-
"""
|
|
364
|
+
"""失败终态内存登记的唯一尾段(successes/failed 互斥单点维护)。"""
|
|
516
365
|
normalized = copy.deepcopy(metadata)
|
|
517
366
|
self._state.mark_failed(job_uid, normalized)
|
|
518
367
|
if unregister:
|
|
@@ -520,8 +369,7 @@ class StateStore:
|
|
|
520
369
|
|
|
521
370
|
@property
|
|
522
371
|
def state(self) -> PipelineState:
|
|
523
|
-
"""底层 PipelineState
|
|
524
|
-
恢复编排层/机器群直读)。"""
|
|
372
|
+
"""底层 PipelineState 引用(内存状态唯一访问路径)。"""
|
|
525
373
|
return self._state
|
|
526
374
|
|
|
527
375
|
@property
|
|
@@ -534,41 +382,43 @@ class StateStore:
|
|
|
534
382
|
self._state = state if state is not None else PipelineState({}, {}, {}, [])
|
|
535
383
|
|
|
536
384
|
def set_backend(self, backend: AbstractStateBackend) -> None:
|
|
537
|
-
"""换库唯一锚点:backend 所有权归 StateStore 单持。
|
|
538
|
-
|
|
539
|
-
不变式:本方法是全引擎唯一的后端重绑定点——其余组件(runtime /
|
|
540
|
-
门面管理面)一律经 ``store.backend`` 只读派生,换库一处
|
|
541
|
-
动作、全部持有者必然同步。
|
|
542
|
-
"""
|
|
385
|
+
"""换库唯一锚点:backend 所有权归 StateStore 单持。"""
|
|
543
386
|
self._backend = backend
|
|
544
387
|
|
|
545
388
|
# ── 路径授权管理与判定 ──────────────────────────────────────────
|
|
546
389
|
|
|
547
|
-
def
|
|
548
|
-
"""
|
|
549
|
-
|
|
550
|
-
|
|
390
|
+
def _inspect_lookup(self, path: str | Path) -> InspectedPath:
|
|
391
|
+
"""经统一 PathPolicy 以 lookup 模式解析路径(保留叶目录项不跟随)。"""
|
|
392
|
+
try:
|
|
393
|
+
return self._path_policy.inspect(path, mode="lookup")
|
|
394
|
+
except ValueError as exc:
|
|
395
|
+
raise TaskLiteError(str(exc)) from exc
|
|
551
396
|
|
|
552
397
|
def is_path_protected(self, path: str | Path) -> bool:
|
|
553
|
-
"""
|
|
554
|
-
|
|
555
|
-
raw = Path(os.fspath(path))
|
|
556
|
-
target = (self.base_dir / raw).resolve() if not raw.is_absolute() else raw.resolve()
|
|
557
|
-
except Exception:
|
|
558
|
-
return True
|
|
559
|
-
for root in self.protected_roots:
|
|
560
|
-
try:
|
|
561
|
-
if root.is_relative_to(target) or target.is_relative_to(root):
|
|
562
|
-
return True
|
|
563
|
-
except (ValueError, TypeError):
|
|
564
|
-
pass
|
|
565
|
-
return False
|
|
398
|
+
"""检查路径是否与受保护路径集合(含 state_path 5 项)双向重叠。"""
|
|
399
|
+
return self._path_policy.is_protected(path)
|
|
566
400
|
|
|
567
401
|
def is_path_authorized(self, path: str | Path) -> bool:
|
|
568
402
|
"""检查指定规范绝对路径是否命中已有授权。"""
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
403
|
+
inspected = self._inspect_lookup(path)
|
|
404
|
+
return self._backend.is_path_authorized(inspected.canonical_path, inspected.target_type)
|
|
405
|
+
|
|
406
|
+
def grant_authorization(
|
|
407
|
+
self,
|
|
408
|
+
path: str | Path,
|
|
409
|
+
*,
|
|
410
|
+
target_type: Literal["file", "directory"],
|
|
411
|
+
) -> dict[str, Any]:
|
|
412
|
+
"""Run 外显式预授权单个文件或目录(重复或父子类型冲突抛 TaskLiteError)。"""
|
|
413
|
+
if type(target_type) is not str or target_type not in ("file", "directory"):
|
|
414
|
+
raise TaskLiteError(
|
|
415
|
+
f"target_type must be 'file' or 'directory', got {target_type!r}"
|
|
416
|
+
)
|
|
417
|
+
try:
|
|
418
|
+
inspected = self._path_policy.inspect(path, mode="strict")
|
|
419
|
+
except ValueError as exc:
|
|
420
|
+
raise TaskLiteError(str(exc)) from exc
|
|
421
|
+
return self._backend.grant_authorization(inspected.canonical_path, target_type)
|
|
572
422
|
|
|
573
423
|
def record_authorization(
|
|
574
424
|
self,
|
|
@@ -577,9 +427,11 @@ class StateStore:
|
|
|
577
427
|
*,
|
|
578
428
|
authorized_at: str | None = None,
|
|
579
429
|
) -> None:
|
|
580
|
-
"""
|
|
581
|
-
|
|
582
|
-
self._backend.record_authorization(
|
|
430
|
+
"""持久化记录一条文件系统路径授权(运行期支持 file -> directory 升级)。"""
|
|
431
|
+
inspected = self._inspect_lookup(path)
|
|
432
|
+
self._backend.record_authorization(
|
|
433
|
+
inspected.canonical_path, target_type, authorized_at=authorized_at
|
|
434
|
+
)
|
|
583
435
|
|
|
584
436
|
def list_authorizations(self) -> list[dict[str, Any]]:
|
|
585
437
|
"""查询全部已持久化的授权记录。"""
|
|
@@ -587,19 +439,20 @@ class StateStore:
|
|
|
587
439
|
|
|
588
440
|
def revoke_authorization(self, path: str | Path) -> bool:
|
|
589
441
|
"""按规范绝对路径精确撤销授权。"""
|
|
590
|
-
|
|
591
|
-
return self._backend.revoke_authorization(
|
|
442
|
+
inspected = self._inspect_lookup(path)
|
|
443
|
+
return self._backend.revoke_authorization(inspected.canonical_path)
|
|
592
444
|
|
|
593
445
|
def check_or_request_authorization(self, path: str | Path) -> bool | None:
|
|
594
446
|
"""按实际目标范围查询、询问并持久化授权;None 表示等待决策。"""
|
|
595
447
|
if self.is_path_protected(path):
|
|
596
448
|
return False
|
|
597
|
-
norm_path = str(self._authorization_target(path))
|
|
598
449
|
while True:
|
|
599
|
-
|
|
450
|
+
inspected = self._inspect_lookup(path)
|
|
451
|
+
norm_path = inspected.canonical_path
|
|
452
|
+
effective_type = inspected.target_type
|
|
600
453
|
if self._backend.is_path_authorized(norm_path, effective_type):
|
|
601
454
|
return True
|
|
602
|
-
if os.environ.get("
|
|
455
|
+
if os.environ.get("TASKLITE_DANGEROUS_AUTO_APPROVE") == "1":
|
|
603
456
|
decision: bool | None = True
|
|
604
457
|
elif self.authorizer is not None:
|
|
605
458
|
decision = self.authorizer(norm_path, effective_type)
|
|
@@ -609,7 +462,8 @@ class StateStore:
|
|
|
609
462
|
return None if decision is None else False
|
|
610
463
|
self._backend.record_authorization(norm_path, effective_type)
|
|
611
464
|
# 询问期间文件可能变成目录,保存的文件授权不能扩大为目录树。
|
|
612
|
-
|
|
465
|
+
after = self._inspect_lookup(path)
|
|
466
|
+
if effective_type == "directory" or after.target_type != "directory":
|
|
613
467
|
return True
|
|
614
468
|
|
|
615
469
|
# ── 事务性原子终态转移(apply_* 出口族)──────────────────────────
|
|
@@ -622,7 +476,7 @@ class StateStore:
|
|
|
622
476
|
spawned_job_snapshots: Sequence[dict[str, Any]] = (),
|
|
623
477
|
cursor_updates: Mapping[str, str | None] | None = None,
|
|
624
478
|
declared_inputs: Sequence[str] = (),
|
|
625
|
-
declared_artifacts:
|
|
479
|
+
declared_artifacts: Mapping[str, FileSpec | str] | None = None,
|
|
626
480
|
run_id: str | None = None,
|
|
627
481
|
job_snapshot: dict[str, Any] | None = None,
|
|
628
482
|
attempt_no: int = 1,
|
|
@@ -633,37 +487,19 @@ class StateStore:
|
|
|
633
487
|
) -> SuccessOutcome:
|
|
634
488
|
"""原子终态转移:成功(同事务终结对应 attempts 行 + successes 落盘 + 清失败档案残行 + 子任务/cursor)。"""
|
|
635
489
|
effective_finished_at = finished_at if finished_at is not None else utc_now_iso()
|
|
636
|
-
success_metadata = self._execution_record(result, files)
|
|
637
|
-
success_metadata.update(
|
|
638
|
-
run_id=run_id or "run", attempt_no=attempt_no,
|
|
639
|
-
started_at=started_at, settled_at=effective_finished_at,
|
|
640
|
-
)
|
|
641
|
-
declarations = tuple(declared_artifacts)
|
|
642
|
-
success_metadata["artifacts"] = {
|
|
643
|
-
declaration.name: {"path": declaration.path}
|
|
644
|
-
for declaration in declarations if declaration.name is not None
|
|
645
|
-
}
|
|
646
|
-
success_metadata["last_succeeded_at"] = effective_finished_at
|
|
647
|
-
if run_id is not None:
|
|
648
|
-
success_metadata["last_success_run_id"] = run_id
|
|
649
|
-
if job_snapshot is not None:
|
|
650
|
-
success_metadata["job_snapshot"] = copy.deepcopy(job_snapshot)
|
|
651
|
-
if declared_inputs:
|
|
652
|
-
input_paths = list(dict.fromkeys(
|
|
653
|
-
path for path in declared_inputs if isinstance(path, str) and path
|
|
654
|
-
))
|
|
655
|
-
if input_paths:
|
|
656
|
-
success_metadata["inputs"] = input_paths
|
|
657
|
-
|
|
658
|
-
# 增量记录 success_count(防御非 int 脏数据;重跑成功从对侧终态续数)
|
|
659
490
|
prev_meta = self._state.successes.get(job_uid) or self._state.failed.get(job_uid)
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
491
|
+
success_metadata = build_execution_success_record(
|
|
492
|
+
result=result,
|
|
493
|
+
files=files,
|
|
494
|
+
run_id=run_id,
|
|
495
|
+
attempt_no=attempt_no,
|
|
496
|
+
started_at=started_at,
|
|
497
|
+
settled_at=effective_finished_at,
|
|
498
|
+
declared_inputs=declared_inputs,
|
|
499
|
+
declared_artifacts=declared_artifacts,
|
|
500
|
+
job_snapshot=job_snapshot,
|
|
501
|
+
prev_meta=prev_meta,
|
|
502
|
+
)
|
|
667
503
|
|
|
668
504
|
spawned_snapshots = list(spawned_job_snapshots)
|
|
669
505
|
self._backend.commit_job_settlement(
|
|
@@ -689,7 +525,6 @@ class StateStore:
|
|
|
689
525
|
self._state.update_cursors(cursor_updates)
|
|
690
526
|
self._state.mark_success(job_uid, success_metadata)
|
|
691
527
|
self._state.unregister_in_flight(job_uid)
|
|
692
|
-
self._record_stat("succeeded", 1)
|
|
693
528
|
spawned_uids = [job_uid_from_snapshot(j) for j in spawned_snapshots]
|
|
694
529
|
return SuccessOutcome(job_uid=job_uid, success_metadata=success_metadata, spawned_uids=spawned_uids)
|
|
695
530
|
|
|
@@ -699,7 +534,6 @@ class StateStore:
|
|
|
699
534
|
failure_metadata: dict[str, Any],
|
|
700
535
|
*,
|
|
701
536
|
job_snapshot: dict[str, Any] | None = None,
|
|
702
|
-
count_as: str = "failed",
|
|
703
537
|
cascade: bool = True,
|
|
704
538
|
attempt_no: int | None = None,
|
|
705
539
|
run_id: str | None = None,
|
|
@@ -710,20 +544,19 @@ class StateStore:
|
|
|
710
544
|
execution_id: str | None = None,
|
|
711
545
|
) -> FailureOutcome:
|
|
712
546
|
"""原子终态转移:永久失败(进入失败档案 + successes 互删 + 级联下游)。"""
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
payload = self._extract_job_payload(job_snapshot)
|
|
547
|
+
effective_settled_at = settled_at if settled_at is not None else utc_now_iso()
|
|
548
|
+
normalized_meta = build_execution_failure_record(
|
|
549
|
+
failure_metadata,
|
|
550
|
+
job_snapshot=job_snapshot,
|
|
551
|
+
settled_at=effective_settled_at,
|
|
552
|
+
attempt_no=attempt_no,
|
|
553
|
+
run_id=run_id,
|
|
554
|
+
started_at=started_at,
|
|
555
|
+
files=files,
|
|
556
|
+
result=result,
|
|
557
|
+
execution_id=execution_id,
|
|
558
|
+
)
|
|
559
|
+
payload = extract_serializable_payload(job_snapshot)
|
|
727
560
|
if attempt_no is not None:
|
|
728
561
|
self._backend.commit_job_settlement(
|
|
729
562
|
job_uid=job_uid,
|
|
@@ -741,71 +574,14 @@ class StateStore:
|
|
|
741
574
|
payload=payload,
|
|
742
575
|
)
|
|
743
576
|
else:
|
|
744
|
-
self._backend.
|
|
577
|
+
self._backend.commit_job_failures([(job_uid, normalized_meta, payload)])
|
|
745
578
|
|
|
746
579
|
self.mark_failed_memory(job_uid, normalized_meta)
|
|
747
|
-
self._record_stat(count_as, 1)
|
|
748
580
|
cascaded_uids: list[str] = []
|
|
749
581
|
if cascade:
|
|
750
582
|
cascaded_uids = self.cascade_fail(job_uid)
|
|
751
583
|
return FailureOutcome(job_uid=job_uid, failure_metadata=normalized_meta, cascaded_uids=cascaded_uids)
|
|
752
584
|
|
|
753
|
-
@staticmethod
|
|
754
|
-
def _failure_record(
|
|
755
|
-
metadata: dict[str, Any], job_snapshot: dict[str, Any] | None,
|
|
756
|
-
) -> dict[str, Any]:
|
|
757
|
-
"""当前失败档案自足;无对应执行时不得借用历史 Attempt 的事实。"""
|
|
758
|
-
record = copy.deepcopy(metadata)
|
|
759
|
-
record.update(
|
|
760
|
-
settled_at=utc_now_iso(), run_id=None, attempt_no=None, started_at=None,
|
|
761
|
-
files=copy.deepcopy((job_snapshot or {}).get("files") or {}),
|
|
762
|
-
result={
|
|
763
|
-
"message": metadata.get("error_message") or metadata.get("message"),
|
|
764
|
-
"exception": copy.deepcopy(metadata.get("exception")),
|
|
765
|
-
"metadata": {},
|
|
766
|
-
"annotations": copy.deepcopy(metadata.get("annotations") or {}),
|
|
767
|
-
},
|
|
768
|
-
)
|
|
769
|
-
if job_snapshot is not None:
|
|
770
|
-
record["job_snapshot"] = copy.deepcopy(job_snapshot)
|
|
771
|
-
return record
|
|
772
|
-
|
|
773
|
-
@staticmethod
|
|
774
|
-
def _execution_record(
|
|
775
|
-
result: dict[str, Any], files: dict[str, Any] | None,
|
|
776
|
-
) -> dict[str, Any]:
|
|
777
|
-
"""业务数据只存在于 result.metadata;档案根部仅容纳框架字段。"""
|
|
778
|
-
result = copy.deepcopy(result)
|
|
779
|
-
result["annotations"] = {
|
|
780
|
-
**result.get("annotations", {}), "completion_source": "execution",
|
|
781
|
-
}
|
|
782
|
-
return {
|
|
783
|
-
"result": result,
|
|
784
|
-
"annotations": {"completion_source": "execution"},
|
|
785
|
-
"files": copy.deepcopy(files or {}),
|
|
786
|
-
}
|
|
787
|
-
|
|
788
|
-
@staticmethod
|
|
789
|
-
def _extract_job_payload(job_snapshot: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
790
|
-
"""提取可 JSON 序列化的业务 payload 快照(失败档案落盘用)。
|
|
791
|
-
|
|
792
|
-
不可序列化时降级为 None 并告警,不阻断失败提交。
|
|
793
|
-
"""
|
|
794
|
-
if not isinstance(job_snapshot, dict):
|
|
795
|
-
return None
|
|
796
|
-
payload = job_snapshot.get("payload")
|
|
797
|
-
if not isinstance(payload, dict):
|
|
798
|
-
return None
|
|
799
|
-
try:
|
|
800
|
-
dumps(payload)
|
|
801
|
-
except (TypeError, ValueError):
|
|
802
|
-
logger.warning(
|
|
803
|
-
f"Job payload for {job_snapshot.get('job_id', '?')} is not "
|
|
804
|
-
f"JSON-serializable; failure archive payload snapshot skipped."
|
|
805
|
-
)
|
|
806
|
-
return None
|
|
807
|
-
return payload
|
|
808
|
-
|
|
809
585
|
def apply_retry(
|
|
810
586
|
self,
|
|
811
587
|
job_uid: str,
|
|
@@ -839,7 +615,6 @@ class StateStore:
|
|
|
839
615
|
|
|
840
616
|
self._state.unregister_in_flight(job_uid)
|
|
841
617
|
self._state.requeue_jobs([retry_job_snapshot], front=plan.front)
|
|
842
|
-
self._record_stat("retried", 1)
|
|
843
618
|
return RetryOutcome(job_uid=job_uid, retry_job_snapshot=retry_job_snapshot)
|
|
844
619
|
|
|
845
620
|
def apply_pending(
|
|
@@ -886,18 +661,13 @@ class StateStore:
|
|
|
886
661
|
self._backend.commit_skip(job_uid)
|
|
887
662
|
self._state.remove_queued_uid(job_uid)
|
|
888
663
|
self._state.unregister_in_flight(job_uid)
|
|
889
|
-
self._record_stat("skipped", 1)
|
|
890
664
|
return SkipOutcome(job_uid=job_uid, was_known=True)
|
|
891
665
|
|
|
892
666
|
def apply_bulk_failure(
|
|
893
667
|
self,
|
|
894
668
|
uids_metas: Sequence[tuple[str, dict[str, Any]]],
|
|
895
669
|
) -> BulkFailureOutcome:
|
|
896
|
-
"""原子批量失败(死锁归因或批量熔断);剩余队列由失败集单点派生。
|
|
897
|
-
|
|
898
|
-
执行选择激活时按队列行 task_type 过滤被排除/跳过 uid:它们绝不进入
|
|
899
|
-
失败档案(过滤后为空则直接返回空结果,由运行期选择仲裁收敛)。
|
|
900
|
-
"""
|
|
670
|
+
"""原子批量失败(死锁归因或批量熔断);剩余队列由失败集单点派生。"""
|
|
901
671
|
excluded_uids = self._selection_excluded_uids()
|
|
902
672
|
if excluded_uids:
|
|
903
673
|
uids_metas = [
|
|
@@ -909,7 +679,14 @@ class StateStore:
|
|
|
909
679
|
)
|
|
910
680
|
queued = {job_uid_from_snapshot(j): j for j in self._state.queue}
|
|
911
681
|
normalized_metas = [
|
|
912
|
-
(
|
|
682
|
+
(
|
|
683
|
+
job_uid,
|
|
684
|
+
build_execution_failure_record(
|
|
685
|
+
metadata,
|
|
686
|
+
job_snapshot=queued.get(job_uid),
|
|
687
|
+
settled_at=utc_now_iso(),
|
|
688
|
+
),
|
|
689
|
+
)
|
|
913
690
|
for job_uid, metadata in uids_metas
|
|
914
691
|
]
|
|
915
692
|
self._backend.commit_job_failures(normalized_metas)
|
|
@@ -920,7 +697,6 @@ class StateStore:
|
|
|
920
697
|
if job_uid_from_snapshot(jd) not in failed_set
|
|
921
698
|
]
|
|
922
699
|
self._state.replace_queue(list(remaining_queue))
|
|
923
|
-
self._record_stat("failed", len(normalized_metas))
|
|
924
700
|
for job_uid, metadata in normalized_metas:
|
|
925
701
|
self.mark_failed_memory(job_uid, metadata)
|
|
926
702
|
return BulkFailureOutcome(
|
|
@@ -929,26 +705,25 @@ class StateStore:
|
|
|
929
705
|
)
|
|
930
706
|
|
|
931
707
|
def cascade_fail(self, failed_uid: str) -> list[str]:
|
|
932
|
-
"""
|
|
933
|
-
|
|
934
|
-
执行选择激活时,被排除作业是传播终止点:不进入级联失败集合,也不
|
|
935
|
-
向其下游继续传播(下游保持待依赖留队,由后续 run 消费)。
|
|
936
|
-
"""
|
|
708
|
+
"""父作业失败后级联标记全部下游为依赖失败。"""
|
|
937
709
|
excluded_uids = self._selection_excluded_uids()
|
|
938
710
|
cascade_uids = self._state.cascade_fail(
|
|
939
711
|
failed_uid, excluded_uids=excluded_uids
|
|
940
712
|
)
|
|
941
713
|
if excluded_uids:
|
|
942
|
-
# 兜底过滤:终态写入出口绝不接受被排除 uid(与 DFS 终止点双保险)。
|
|
943
714
|
cascade_uids = [job_uid for job_uid in cascade_uids if job_uid not in excluded_uids]
|
|
944
715
|
if not cascade_uids:
|
|
945
716
|
return []
|
|
946
717
|
queued = {job_uid_from_snapshot(j): j for j in self._state.queue}
|
|
947
718
|
cascade_metas = [
|
|
948
|
-
(
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
719
|
+
(
|
|
720
|
+
job_uid,
|
|
721
|
+
build_execution_failure_record(
|
|
722
|
+
{"error_message": 'Dependency failed or has no record', "failed_dependency": failed_uid},
|
|
723
|
+
job_snapshot=queued.get(job_uid),
|
|
724
|
+
settled_at=utc_now_iso(),
|
|
725
|
+
),
|
|
726
|
+
)
|
|
952
727
|
for job_uid in cascade_uids
|
|
953
728
|
]
|
|
954
729
|
self._backend.commit_job_failures(cascade_metas)
|
|
@@ -958,7 +733,6 @@ class StateStore:
|
|
|
958
733
|
if job_uid_from_snapshot(jd) not in cascade_set
|
|
959
734
|
]
|
|
960
735
|
self._state.replace_queue(remaining)
|
|
961
|
-
self._record_stat("cascade_failed", len(cascade_uids))
|
|
962
736
|
for job_uid, metadata in cascade_metas:
|
|
963
737
|
self.mark_failed_memory(job_uid, metadata)
|
|
964
738
|
return cascade_uids
|
|
@@ -966,7 +740,6 @@ class StateStore:
|
|
|
966
740
|
|
|
967
741
|
__all__ = [
|
|
968
742
|
"BulkFailureOutcome",
|
|
969
|
-
"FailureEntry",
|
|
970
743
|
"FailureOutcome",
|
|
971
744
|
"RetryOutcome",
|
|
972
745
|
"SkipOutcome",
|