tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
tasklite/engine/completion.py
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
retry/success/failure 三态事务提交。重试预算只统计本 Run 中显式
|
|
5
5
|
``TaskResult.retry`` 的消耗;候选来源与累计数量分开登记。
|
|
6
6
|
|
|
7
|
-
``
|
|
8
|
-
|
|
7
|
+
收尾清理期授权等待由 ``AuthorizationCoordinator`` 统一承接,本模块不自建
|
|
8
|
+
暂挂队列或控制流异常,亦不穿透访问底层进程句柄。
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
from __future__ import annotations
|
|
@@ -18,23 +18,19 @@ from dataclasses import dataclass
|
|
|
18
18
|
from typing import Any, Sequence, TYPE_CHECKING
|
|
19
19
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
|
-
from .admission import TriggerRegistry
|
|
22
21
|
from .channel import ExecutionChannel
|
|
23
|
-
from .session import RunSession
|
|
24
|
-
from .in_flight import InFlightTracker
|
|
25
22
|
from .resource import ResourceManager
|
|
23
|
+
from .session import RunSession
|
|
26
24
|
from .store import StateStore
|
|
27
25
|
|
|
28
26
|
from ..exceptions import TaskLiteError, _ArtifactIOError
|
|
27
|
+
from ..models.artifact import FileSpec
|
|
29
28
|
from ..models.job import Job
|
|
30
|
-
from ..models.
|
|
31
|
-
from
|
|
32
|
-
from ..utils.clock import utc_now_iso
|
|
33
|
-
from .admission import extract_effective_job_config
|
|
29
|
+
from ..models.records import build_result_record, extract_effective_job_config
|
|
30
|
+
from .authorization import AuthorizationCoordinator, CleanupAuthorizationDecision
|
|
34
31
|
from .channel import ExecutionResult, JobHandle
|
|
35
|
-
|
|
36
32
|
from .in_flight import InFlightJob, InFlightTracker
|
|
37
|
-
from .types import
|
|
33
|
+
from .types import AuthorizationStatus
|
|
38
34
|
|
|
39
35
|
logger = logging.getLogger("tasklite")
|
|
40
36
|
_CLEANUP_ERRORS = (OSError, ValueError, NotImplementedError)
|
|
@@ -46,10 +42,6 @@ class _PreparedCompletion:
|
|
|
46
42
|
spawned_job_snapshots: list[dict[str, Any]]
|
|
47
43
|
|
|
48
44
|
|
|
49
|
-
class _CleanupAuthorizationPending(Exception):
|
|
50
|
-
"""清理尚待授权,保留有效结果及在途归属。"""
|
|
51
|
-
|
|
52
|
-
|
|
53
45
|
class CompletionMachine:
|
|
54
46
|
"""结果提交 / 输出清理 / 资源释放 / 在飞结算的完成侧机器。"""
|
|
55
47
|
|
|
@@ -57,19 +49,27 @@ class CompletionMachine:
|
|
|
57
49
|
self,
|
|
58
50
|
*,
|
|
59
51
|
store: "StateStore",
|
|
60
|
-
trigger_registry: "TriggerRegistry | None" = None,
|
|
61
52
|
channel: "ExecutionChannel",
|
|
62
53
|
resources: "ResourceManager",
|
|
63
54
|
in_flight: "InFlightTracker",
|
|
64
55
|
session: "RunSession",
|
|
56
|
+
authorization: AuthorizationCoordinator | None = None,
|
|
65
57
|
) -> None:
|
|
66
58
|
self._store = store
|
|
67
|
-
self._trigger_registry = trigger_registry
|
|
68
59
|
self._channel = channel
|
|
69
60
|
self._resources = resources
|
|
70
61
|
self._in_flight = in_flight
|
|
71
62
|
self._session = session
|
|
72
|
-
self.
|
|
63
|
+
self._authorization = (
|
|
64
|
+
authorization
|
|
65
|
+
if authorization is not None
|
|
66
|
+
else AuthorizationCoordinator(
|
|
67
|
+
store=store,
|
|
68
|
+
channel=channel,
|
|
69
|
+
in_flight=in_flight,
|
|
70
|
+
session=session,
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
73
|
|
|
74
74
|
@staticmethod
|
|
75
75
|
def _cleanup_exception(exc: BaseException) -> dict[str, str]:
|
|
@@ -114,107 +114,52 @@ class CompletionMachine:
|
|
|
114
114
|
|
|
115
115
|
# ── 唯一收尾出口 ──────────────────────────────────────────────────
|
|
116
116
|
|
|
117
|
-
@property
|
|
118
|
-
def waiting_authorization(self) -> bool:
|
|
119
|
-
return bool(self._pending_cleanup)
|
|
120
|
-
|
|
121
|
-
def has_pending_result(self, handle: JobHandle) -> bool:
|
|
122
|
-
return handle.job_uid in self._pending_cleanup
|
|
123
|
-
|
|
124
|
-
def _authorize_cleanup(
|
|
125
|
-
self, entry: InFlightJob | None, result: ExecutionResult,
|
|
126
|
-
active_files: dict[str, FileSpec],
|
|
127
|
-
) -> None:
|
|
128
|
-
policy = ArtifactPathPolicy(
|
|
129
|
-
output_roots=getattr(self._channel, "output_roots", None),
|
|
130
|
-
protected_roots=self._channel.protected_roots,
|
|
131
|
-
)
|
|
132
|
-
for spec in active_files.values():
|
|
133
|
-
requested = spec.delete_on_success if result.success else spec.delete_on_failure
|
|
134
|
-
if not requested or spec.path is None:
|
|
135
|
-
continue
|
|
136
|
-
try:
|
|
137
|
-
if not policy.entry_exists(ArtifactDeclaration(spec.path)):
|
|
138
|
-
continue
|
|
139
|
-
except _CLEANUP_ERRORS:
|
|
140
|
-
continue
|
|
141
|
-
stopped = self._session.stop_mode is not StopMode.NONE
|
|
142
|
-
waiting = entry is not None and entry.job_uid in self._pending_cleanup
|
|
143
|
-
if stopped and waiting:
|
|
144
|
-
status = AuthorizationStatus.STOPPED
|
|
145
|
-
message = f"authorization wait stopped for {spec.path}"
|
|
146
|
-
else:
|
|
147
|
-
decision = self._store.check_or_request_authorization(spec.path)
|
|
148
|
-
stopped = self._session.stop_mode is not StopMode.NONE
|
|
149
|
-
if stopped and (waiting or decision is None):
|
|
150
|
-
status = AuthorizationStatus.STOPPED
|
|
151
|
-
message = f"authorization wait stopped for {spec.path}"
|
|
152
|
-
else:
|
|
153
|
-
if decision is True:
|
|
154
|
-
continue
|
|
155
|
-
if decision is None:
|
|
156
|
-
raise _CleanupAuthorizationPending
|
|
157
|
-
status = AuthorizationStatus.DENIED
|
|
158
|
-
message = f"authorization denied for {spec.path}"
|
|
159
|
-
result.authorization_outcome = AuthorizationOutcome(status, message, spec.path)
|
|
160
|
-
if entry is not None and entry.handle is not None:
|
|
161
|
-
self._channel.terminate_process(entry.handle)
|
|
162
|
-
raise _CleanupAuthorizationPending
|
|
163
|
-
# 后续询问回调可能改变此前目标,整组通过后再交付实际清理。
|
|
164
|
-
for spec in active_files.values():
|
|
165
|
-
requested = spec.delete_on_success if result.success else spec.delete_on_failure
|
|
166
|
-
if requested and spec.path and not self._store.is_path_authorized(spec.path):
|
|
167
|
-
try:
|
|
168
|
-
present = policy.entry_exists(ArtifactDeclaration(spec.path))
|
|
169
|
-
except _CLEANUP_ERRORS:
|
|
170
|
-
continue
|
|
171
|
-
if present:
|
|
172
|
-
raise _CleanupAuthorizationPending
|
|
173
|
-
|
|
174
117
|
def complete_job(
|
|
175
|
-
self,
|
|
176
|
-
|
|
118
|
+
self,
|
|
119
|
+
entry: InFlightJob,
|
|
120
|
+
result: ExecutionResult,
|
|
121
|
+
*,
|
|
122
|
+
prepared: _PreparedCompletion | None = None,
|
|
177
123
|
) -> None:
|
|
178
124
|
"""处理一个在飞作业的完成结果(唯一收尾出口)。
|
|
179
125
|
|
|
180
126
|
事务性提交与内存 apply 见 ``apply_result``(两条提交路径共用);
|
|
181
|
-
|
|
182
|
-
|
|
127
|
+
本方法在提交成功后注销在途条目并清除授权状态,资源释放在 ``apply_result``
|
|
128
|
+
内部恰好执行一次(提交前或异常兜底)。
|
|
183
129
|
|
|
184
|
-
|
|
185
|
-
「四集合真空」窗口(uid 不在 successes/failed/queue/in-flight 任一
|
|
186
|
-
集合),使 handler 自 spawn 同 uid 子任务时被 ``is_known`` 漏判。
|
|
187
|
-
注销时机下推到 ``apply_result`` 内部分支(retry 在重入队前、
|
|
130
|
+
身份非真空:在飞注销下推到 ``apply_result`` 内部分支(retry 在重入队前、
|
|
188
131
|
success 在 mark_success 后、failure 在 mark_failed 后),保证
|
|
189
132
|
uid 在 ``is_known`` 可达的代码段内始终属于至少一个集合。
|
|
190
|
-
|
|
191
|
-
声明只在业务结算完成后删除;提交前异常保留声明以便诊断。
|
|
192
133
|
"""
|
|
193
134
|
uid = entry.job_uid
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
135
|
+
exec_id = (
|
|
136
|
+
entry.execution_id
|
|
137
|
+
if entry.execution_id is not None
|
|
138
|
+
else (entry.handle.execution_id if entry.handle is not None else None)
|
|
139
|
+
)
|
|
140
|
+
settled = self.apply_result(
|
|
141
|
+
uid,
|
|
142
|
+
entry.job,
|
|
143
|
+
entry.job_snapshot,
|
|
144
|
+
result,
|
|
145
|
+
started_monotonic=entry.started_monotonic,
|
|
146
|
+
expect_in_flight=(entry.handle is not None),
|
|
147
|
+
execution_id=exec_id,
|
|
148
|
+
attempt_no=entry.attempt_no,
|
|
149
|
+
run_id=entry.run_id or self._session.run_id,
|
|
150
|
+
started_at=entry.started_at,
|
|
151
|
+
entry=entry,
|
|
152
|
+
prepared=prepared,
|
|
153
|
+
)
|
|
154
|
+
if settled:
|
|
155
|
+
self._in_flight.settle(uid, state=self._store.state)
|
|
156
|
+
self._authorization.clear_execution(exec_id, job_uid=uid)
|
|
215
157
|
|
|
216
158
|
def _prepare_completion(
|
|
217
|
-
self,
|
|
159
|
+
self,
|
|
160
|
+
uid: str,
|
|
161
|
+
job: Job,
|
|
162
|
+
result: ExecutionResult,
|
|
218
163
|
entry: InFlightJob | None,
|
|
219
164
|
) -> _PreparedCompletion:
|
|
220
165
|
store = self._store
|
|
@@ -223,7 +168,7 @@ class CompletionMachine:
|
|
|
223
168
|
active_files = dict(entry.active_files)
|
|
224
169
|
elif job.files:
|
|
225
170
|
active_files = {
|
|
226
|
-
k: v.
|
|
171
|
+
k: v._resolve_against(store.base_dir) if not v.is_output_ref else v
|
|
227
172
|
for k, v in job.files.items()
|
|
228
173
|
}
|
|
229
174
|
|
|
@@ -232,10 +177,7 @@ class CompletionMachine:
|
|
|
232
177
|
if result.artifact_error is not None:
|
|
233
178
|
raise _ArtifactIOError(result.artifact_error)
|
|
234
179
|
if result.success:
|
|
235
|
-
policy =
|
|
236
|
-
output_roots=getattr(self._channel, "output_roots", None),
|
|
237
|
-
protected_roots=self._channel.protected_roots,
|
|
238
|
-
)
|
|
180
|
+
policy = self._store.path_policy
|
|
239
181
|
for fname, spec in active_files.items():
|
|
240
182
|
if spec.role != "output":
|
|
241
183
|
continue
|
|
@@ -244,16 +186,16 @@ class CompletionMachine:
|
|
|
244
186
|
result.message = f"Missing output path for {fname!r}"
|
|
245
187
|
break
|
|
246
188
|
try:
|
|
247
|
-
present = policy.
|
|
248
|
-
|
|
249
|
-
)
|
|
189
|
+
present = policy.inspect(
|
|
190
|
+
spec.path, mode="strict", verify_unchanged=True
|
|
191
|
+
).exists
|
|
250
192
|
except _CLEANUP_ERRORS as exc:
|
|
251
193
|
result.success = False
|
|
252
194
|
result.message = f"Output verification failed: {exc}"
|
|
253
195
|
self._attach_cleanup_failure(result, exc)
|
|
254
196
|
break
|
|
255
197
|
if not present:
|
|
256
|
-
#
|
|
198
|
+
# strict inspect 已执行根、受保护路径与符号链接边界防护。若声明了成功删除,
|
|
257
199
|
# 契约允许工作进程在返回前原子 rename 移走该临时输出,缺失视为已移除。
|
|
258
200
|
if not spec.delete_on_success:
|
|
259
201
|
result.success = False
|
|
@@ -296,8 +238,8 @@ class CompletionMachine:
|
|
|
296
238
|
started_at: str | None = None,
|
|
297
239
|
entry: InFlightJob | None = None,
|
|
298
240
|
prepared: _PreparedCompletion | None = None,
|
|
299
|
-
) ->
|
|
300
|
-
"""
|
|
241
|
+
) -> bool:
|
|
242
|
+
"""事务性提交一个执行结果到后端并同步内存;若清理授权仍待决返回 False。
|
|
301
243
|
|
|
302
244
|
先提交后端(唯一真相源),commit 成功后才 apply 到内存 state。
|
|
303
245
|
三态分发:
|
|
@@ -312,98 +254,88 @@ class CompletionMachine:
|
|
|
312
254
|
f"at apply_result entry"
|
|
313
255
|
)
|
|
314
256
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
if
|
|
318
|
-
self.
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
257
|
+
released = False
|
|
258
|
+
try:
|
|
259
|
+
if prepared is None:
|
|
260
|
+
prepared = self._prepare_completion(uid, job, result, entry)
|
|
261
|
+
decision = self._authorization.evaluate_cleanup(
|
|
262
|
+
entry, result, prepared.active_files
|
|
263
|
+
)
|
|
264
|
+
if decision is CleanupAuthorizationDecision.PENDING:
|
|
265
|
+
if entry is not None:
|
|
266
|
+
self._authorization.defer_cleanup(entry, result, prepared)
|
|
267
|
+
return False
|
|
268
|
+
active_files = prepared.active_files
|
|
269
|
+
|
|
270
|
+
authorization_outcome = result.authorization_outcome
|
|
271
|
+
if (
|
|
272
|
+
authorization_outcome is not None
|
|
273
|
+
and authorization_outcome.status
|
|
274
|
+
in (AuthorizationStatus.DENIED, AuthorizationStatus.STOPPED)
|
|
275
|
+
):
|
|
276
|
+
# 不变式:资源释放先于 running 回退事务,Release 回调仍须观察 running。
|
|
277
|
+
if entry is not None:
|
|
278
|
+
released = True
|
|
279
|
+
entry.release_resources(self._resources)
|
|
280
|
+
self._store.apply_pending(
|
|
281
|
+
job_uid=uid,
|
|
282
|
+
job_snapshot=job_snapshot,
|
|
283
|
+
attempt_no=attempt_no,
|
|
284
|
+
execution_id=execution_id,
|
|
285
|
+
run_id=run_id or self._session.run_id,
|
|
286
|
+
started_at=started_at,
|
|
287
|
+
error_message=authorization_outcome.message,
|
|
288
|
+
files={k: v.to_dict() for k, v in active_files.items()},
|
|
289
|
+
result=build_result_record(
|
|
290
|
+
message=authorization_outcome.message,
|
|
291
|
+
annotations={
|
|
292
|
+
"authorization_status": authorization_outcome.status.value,
|
|
293
|
+
"path": authorization_outcome.path,
|
|
294
|
+
},
|
|
295
|
+
),
|
|
296
|
+
)
|
|
297
|
+
if authorization_outcome.status is AuthorizationStatus.DENIED:
|
|
298
|
+
self._session.temporarily_exclude_chain(uid, store.state)
|
|
299
|
+
result.will_retry = False
|
|
300
|
+
return True
|
|
301
|
+
|
|
302
|
+
spawned_job_snapshots = prepared.spawned_job_snapshots
|
|
303
|
+
|
|
304
|
+
# 声明式产物主进程清理
|
|
305
|
+
if active_files:
|
|
306
|
+
policy = self._store.path_policy
|
|
307
|
+
authorize_cb = self._store.backend.is_path_authorized
|
|
308
|
+
if result.success:
|
|
309
|
+
for _fname, spec in active_files.items():
|
|
310
|
+
if not (spec.delete_on_success and spec.path):
|
|
311
|
+
continue
|
|
312
|
+
try:
|
|
313
|
+
policy.delete(spec.path, authorize=authorize_cb)
|
|
314
|
+
except _CLEANUP_ERRORS as exc:
|
|
315
|
+
result.success = False
|
|
316
|
+
result.message = f"Success cleanup failed: {exc}"
|
|
317
|
+
self._attach_cleanup_failure(result, exc)
|
|
318
|
+
break
|
|
319
|
+
if not result.success:
|
|
320
|
+
spawned_job_snapshots = []
|
|
321
|
+
result.cursor_updates = {}
|
|
322
|
+
else:
|
|
323
|
+
for _fname, spec in active_files.items():
|
|
324
|
+
if not (spec.delete_on_failure and spec.path):
|
|
325
|
+
continue
|
|
326
|
+
try:
|
|
327
|
+
policy.delete(spec.path, authorize=authorize_cb)
|
|
328
|
+
except _CLEANUP_ERRORS as exc:
|
|
329
|
+
self._record_failure_cleanup_error(result, exc)
|
|
330
|
+
|
|
331
|
+
# 释放资源(必须在事务提交之前完成)
|
|
328
332
|
if entry is not None:
|
|
333
|
+
released = True
|
|
329
334
|
entry.release_resources(self._resources)
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
execution_id=execution_id,
|
|
335
|
-
run_id=run_id or self._session.run_id,
|
|
336
|
-
started_at=started_at,
|
|
337
|
-
error_message=authorization_outcome.message,
|
|
338
|
-
files={k: v.to_dict() for k, v in active_files.items()},
|
|
339
|
-
result={
|
|
340
|
-
"message": authorization_outcome.message,
|
|
341
|
-
"exception": None,
|
|
342
|
-
"metadata": {},
|
|
343
|
-
"annotations": {
|
|
344
|
-
"authorization_status": authorization_outcome.status.value,
|
|
345
|
-
"path": authorization_outcome.path,
|
|
346
|
-
},
|
|
347
|
-
},
|
|
348
|
-
)
|
|
349
|
-
if authorization_outcome.status is AuthorizationStatus.DENIED:
|
|
350
|
-
self._session.temporarily_exclude_chain(uid)
|
|
351
|
-
result.will_retry = False
|
|
352
|
-
return
|
|
353
|
-
|
|
354
|
-
spawned_job_snapshots = prepared.spawned_job_snapshots
|
|
355
|
-
|
|
356
|
-
# 声明式产物主进程清理
|
|
357
|
-
if active_files:
|
|
358
|
-
policy = ArtifactPathPolicy(
|
|
359
|
-
output_roots=getattr(self._channel, "output_roots", None),
|
|
360
|
-
protected_roots=self._channel.protected_roots,
|
|
361
|
-
authorization_check=lambda path, kind: self._store.backend.is_path_authorized(path, kind),
|
|
362
|
-
)
|
|
363
|
-
if result.success:
|
|
364
|
-
for fname, spec in active_files.items():
|
|
365
|
-
if not (spec.delete_on_success and spec.path):
|
|
366
|
-
continue
|
|
367
|
-
declaration = ArtifactDeclaration(spec.path, delete_on_success=True)
|
|
368
|
-
try:
|
|
369
|
-
present = policy.entry_exists(declaration)
|
|
370
|
-
except _CLEANUP_ERRORS as exc:
|
|
371
|
-
result.success = False
|
|
372
|
-
result.message = f"Success cleanup failed: {exc}"
|
|
373
|
-
self._attach_cleanup_failure(result, exc)
|
|
374
|
-
break
|
|
375
|
-
if not present:
|
|
376
|
-
continue
|
|
377
|
-
try:
|
|
378
|
-
policy.cleanup(declaration, success=True)
|
|
379
|
-
except _CLEANUP_ERRORS as exc:
|
|
380
|
-
result.success = False
|
|
381
|
-
result.message = f"Success cleanup failed: {exc}"
|
|
382
|
-
self._attach_cleanup_failure(result, exc)
|
|
383
|
-
break
|
|
384
|
-
if not result.success:
|
|
385
|
-
spawned_job_snapshots = []
|
|
386
|
-
result.cursor_updates = {}
|
|
387
|
-
else:
|
|
388
|
-
for fname, spec in active_files.items():
|
|
389
|
-
if not (spec.delete_on_failure and spec.path):
|
|
390
|
-
continue
|
|
391
|
-
declaration = ArtifactDeclaration(spec.path, delete_on_failure=True)
|
|
392
|
-
try:
|
|
393
|
-
present = policy.entry_exists(declaration)
|
|
394
|
-
except _CLEANUP_ERRORS as exc:
|
|
395
|
-
self._record_failure_cleanup_error(result, exc)
|
|
396
|
-
continue
|
|
397
|
-
if not present:
|
|
398
|
-
continue
|
|
399
|
-
try:
|
|
400
|
-
policy.cleanup(declaration, success=False)
|
|
401
|
-
except _CLEANUP_ERRORS as exc:
|
|
402
|
-
self._record_failure_cleanup_error(result, exc)
|
|
403
|
-
|
|
404
|
-
# 释放资源(必须在事务提交之前完成)
|
|
405
|
-
if entry is not None:
|
|
406
|
-
entry.release_resources(self._resources)
|
|
335
|
+
except BaseException:
|
|
336
|
+
if entry is not None and not released:
|
|
337
|
+
entry.release_resources(self._resources)
|
|
338
|
+
raise
|
|
407
339
|
|
|
408
340
|
# 2. 状态分发
|
|
409
341
|
if result.retry_requested:
|
|
@@ -443,6 +375,7 @@ class CompletionMachine:
|
|
|
443
375
|
started_at=started_at,
|
|
444
376
|
active_files=active_files,
|
|
445
377
|
)
|
|
378
|
+
return True
|
|
446
379
|
|
|
447
380
|
# ── 三态分支 ──────────────────────────────────────────────────────
|
|
448
381
|
|
|
@@ -476,7 +409,7 @@ class CompletionMachine:
|
|
|
476
409
|
f"Sent to failure archive."
|
|
477
410
|
)
|
|
478
411
|
failure_metadata: dict[str, Any] = {
|
|
479
|
-
"error_message":
|
|
412
|
+
"error_message": "Retry budget exhausted",
|
|
480
413
|
}
|
|
481
414
|
if result.message is not None:
|
|
482
415
|
failure_metadata["message"] = result.message
|
|
@@ -512,12 +445,12 @@ class CompletionMachine:
|
|
|
512
445
|
started_at=started_at,
|
|
513
446
|
error_message=retry_error,
|
|
514
447
|
files=files_dict,
|
|
515
|
-
result=
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
448
|
+
result=build_result_record(
|
|
449
|
+
message=result.message,
|
|
450
|
+
exception=result.exception,
|
|
451
|
+
metadata=result.result_metadata,
|
|
452
|
+
annotations=result.annotations,
|
|
453
|
+
),
|
|
521
454
|
)
|
|
522
455
|
result.will_retry = True
|
|
523
456
|
self._session.mark_retry_candidate(uid)
|
|
@@ -545,14 +478,12 @@ class CompletionMachine:
|
|
|
545
478
|
if active_files
|
|
546
479
|
else job_snapshot.get("files")
|
|
547
480
|
)
|
|
548
|
-
|
|
481
|
+
effective_outputs: dict[str, FileSpec] = {}
|
|
549
482
|
effective_inputs: list[str] = []
|
|
550
483
|
if active_files:
|
|
551
484
|
for fname, fspec in active_files.items():
|
|
552
485
|
if fspec.role == "output" and fspec.path is not None:
|
|
553
|
-
|
|
554
|
-
ArtifactDeclaration(fspec.path, name=fname)
|
|
555
|
-
)
|
|
486
|
+
effective_outputs[fname] = fspec
|
|
556
487
|
elif fspec.role == "input" and fspec.path is not None:
|
|
557
488
|
effective_inputs.append(fspec.path)
|
|
558
489
|
|
|
@@ -561,19 +492,18 @@ class CompletionMachine:
|
|
|
561
492
|
spawned_job_snapshots=spawned_job_snapshots,
|
|
562
493
|
cursor_updates=result.cursor_updates,
|
|
563
494
|
declared_inputs=effective_inputs,
|
|
564
|
-
declared_artifacts=
|
|
495
|
+
declared_artifacts=effective_outputs,
|
|
565
496
|
run_id=run_id or self._session.run_id,
|
|
566
497
|
job_snapshot=job_snapshot,
|
|
567
498
|
execution_id=execution_id,
|
|
568
499
|
attempt_no=attempt_no,
|
|
569
500
|
started_at=started_at,
|
|
570
501
|
files=files_dict,
|
|
571
|
-
result=
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
},
|
|
502
|
+
result=build_result_record(
|
|
503
|
+
message=result.message,
|
|
504
|
+
metadata=result.result_metadata,
|
|
505
|
+
annotations=result.annotations,
|
|
506
|
+
),
|
|
577
507
|
)
|
|
578
508
|
result.will_retry = False
|
|
579
509
|
self._session.clear_retry_candidate(uid)
|
|
@@ -623,7 +553,7 @@ class CompletionMachine:
|
|
|
623
553
|
accepted: dict[str, dict[str, Any]] = {}
|
|
624
554
|
for spawned_job in spawned_jobs:
|
|
625
555
|
spawned_uid = spawned_job.uid
|
|
626
|
-
snapshot = store.
|
|
556
|
+
snapshot = store.normalize_new_job_snapshot(spawned_job, caller="spawn")
|
|
627
557
|
previous = accepted.get(spawned_uid, busy_snapshots.get(spawned_uid))
|
|
628
558
|
if previous is not None:
|
|
629
559
|
if extract_effective_job_config(previous) != extract_effective_job_config(snapshot):
|
|
@@ -656,10 +586,13 @@ class CompletionMachine:
|
|
|
656
586
|
f"FAIL: {uid} (duration {duration:.2f}s, sent to failure archive). "
|
|
657
587
|
f"Message: {result.message}, Meta: {result.result_metadata}"
|
|
658
588
|
)
|
|
659
|
-
failure_record: dict[str, Any] =
|
|
660
|
-
|
|
661
|
-
if
|
|
662
|
-
|
|
589
|
+
failure_record: dict[str, Any] = (
|
|
590
|
+
copy.deepcopy(failure_record_override)
|
|
591
|
+
if failure_record_override is not None
|
|
592
|
+
else {}
|
|
593
|
+
)
|
|
594
|
+
if failure_record_override is None and result.message is not None:
|
|
595
|
+
failure_record["error_message"] = result.message
|
|
663
596
|
|
|
664
597
|
files_dict = (
|
|
665
598
|
{k: v.to_dict() for k, v in active_files.items()}
|
|
@@ -677,80 +610,65 @@ class CompletionMachine:
|
|
|
677
610
|
run_id=run_id or self._session.run_id,
|
|
678
611
|
started_at=started_at,
|
|
679
612
|
files=files_dict,
|
|
680
|
-
result=
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
613
|
+
result=build_result_record(
|
|
614
|
+
message=result.message,
|
|
615
|
+
exception=result.exception,
|
|
616
|
+
metadata=result.result_metadata,
|
|
617
|
+
annotations=result.annotations,
|
|
618
|
+
),
|
|
686
619
|
)
|
|
687
620
|
result.will_retry = False
|
|
688
621
|
self._session.clear_retry_candidate(uid)
|
|
689
622
|
|
|
690
|
-
# ──
|
|
623
|
+
# ── 批量结算与异常清扫 ────────────────────────────────────────────
|
|
691
624
|
|
|
692
625
|
def settle_reaped(
|
|
693
|
-
self,
|
|
626
|
+
self,
|
|
627
|
+
reaped: Sequence[tuple[JobHandle | InFlightJob, ExecutionResult]],
|
|
694
628
|
) -> int:
|
|
695
629
|
"""统一结算通道结果与未启动 worker 的执行失败。
|
|
696
630
|
|
|
697
631
|
无 worker 的条目直接携带执行身份,不制造句柄或启动超时计时。
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
1. 提取在飞条目;
|
|
701
|
-
2. 经 complete_job 清理产物、释放资源并原子提交;
|
|
702
|
-
3. 提交后注销在途归属;框架异常交由 Run 异常回收;
|
|
703
|
-
返回成功结算的作业数。
|
|
632
|
+
清理授权待决或受阻进程待退出时统一交由 ``AuthorizationCoordinator`` 暂挂,
|
|
633
|
+
待放行或进程退出后仅执行一次收尾提交。
|
|
704
634
|
"""
|
|
705
|
-
store = self._store
|
|
706
635
|
settled_count = 0
|
|
707
|
-
pending_results =
|
|
708
|
-
|
|
636
|
+
pending_results = [
|
|
637
|
+
(item.entry, item.result, item.prepared)
|
|
638
|
+
for item in self._authorization.pending_cleanups()
|
|
639
|
+
]
|
|
640
|
+
candidates: list[tuple[JobHandle | InFlightJob, ExecutionResult, _PreparedCompletion | None]] = [
|
|
641
|
+
*pending_results,
|
|
642
|
+
*((h, r, None) for h, r in reaped),
|
|
643
|
+
]
|
|
709
644
|
for source, result, prepared in candidates:
|
|
710
645
|
entry = self._in_flight.get(source.job_uid)
|
|
711
646
|
if entry is None:
|
|
712
647
|
continue
|
|
713
|
-
handle = entry.handle
|
|
714
648
|
if prepared is None:
|
|
715
649
|
prepared = self._prepare_completion(entry.job_uid, entry.job, result, entry)
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
handle.pause_timeout()
|
|
723
|
-
continue
|
|
724
|
-
elif handle is not None and entry.job_uid in self._pending_cleanup:
|
|
725
|
-
try:
|
|
726
|
-
if handle.process.is_alive():
|
|
727
|
-
continue
|
|
728
|
-
except ValueError:
|
|
729
|
-
# 有效结果交付后,非阻塞回收可能已关闭进程句柄。
|
|
730
|
-
pass
|
|
650
|
+
decision = self._authorization.evaluate_cleanup(
|
|
651
|
+
entry, result, prepared.active_files
|
|
652
|
+
)
|
|
653
|
+
if decision is CleanupAuthorizationDecision.PENDING:
|
|
654
|
+
self._authorization.defer_cleanup(entry, result, prepared)
|
|
655
|
+
continue
|
|
731
656
|
# 框架异常立即上抛,未提交条目保留归属,由异常回收统一处理。
|
|
732
657
|
self.complete_job(entry, result, prepared=prepared)
|
|
733
658
|
settled_count += 1
|
|
734
|
-
self._pending_cleanup.pop(entry.job_uid, None)
|
|
735
|
-
self._in_flight.settle(entry.job_uid, state=store.state)
|
|
736
659
|
return settled_count
|
|
737
660
|
|
|
738
|
-
def
|
|
739
|
-
|
|
740
|
-
cancelled_entries: Sequence[InFlightJob],
|
|
741
|
-
done_entries: Sequence[tuple[InFlightJob, ExecutionResult]],
|
|
742
|
-
) -> None:
|
|
743
|
-
"""异常退出只释放内存归属,未提交的 running 留待下次恢复。"""
|
|
661
|
+
def discard_in_flight(self) -> None:
|
|
662
|
+
"""异常退出单口径清扫:统一释放资源、同步注销在途并清空授权队列,不补提交结果。"""
|
|
744
663
|
cleanup_error: BaseException | None = None
|
|
745
|
-
|
|
746
|
-
for entry in entries:
|
|
664
|
+
for entry in list(self._in_flight.values()):
|
|
747
665
|
try:
|
|
748
666
|
entry.release_resources(self._resources)
|
|
749
667
|
except BaseException as error:
|
|
750
668
|
cleanup_error = cleanup_error or error
|
|
751
669
|
finally:
|
|
752
670
|
self._in_flight.settle(entry.job_uid, state=self._store.state)
|
|
753
|
-
self.
|
|
671
|
+
self._authorization.clear_all()
|
|
754
672
|
if cleanup_error is not None:
|
|
755
673
|
raise cleanup_error
|
|
756
674
|
|