tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,989 @@
|
|
|
1
|
+
"""主进程文件声明与三类授权等待统一协调器。
|
|
2
|
+
|
|
3
|
+
统一收敛主进程侧全部三类文件授权等待生命周期:
|
|
4
|
+
1. 派发前预声明文件授权等待;
|
|
5
|
+
2. 运行期子进程动态文件声明校验、授权等待、前置清理与回复;
|
|
6
|
+
3. 执行收尾清理期(成功/失败/重试清理)授权等待与受阻进程退出等待。
|
|
7
|
+
|
|
8
|
+
核心不变式:
|
|
9
|
+
- ``StopMode != NONE`` → ``AuthorizationStatus.STOPPED`` 在本模块单点收敛;
|
|
10
|
+
- ``has_pending_authorization`` 为事件泵等待决策提供唯一授权等待查询;
|
|
11
|
+
- 动态声明与收尾等待期间的超时暂停/恢复按执行身份追踪,不污染 ``JobHandle`` 值对象;
|
|
12
|
+
- 内部短临界区锁(``self._lock``)仅保护内存状态跃迁与停机态二次复核,严禁持锁执行
|
|
13
|
+
外部 authorizer 回调、文件系统 I/O、进程终止或管道回复,避免跨线程 ``request_stop()`` 死锁。
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import enum
|
|
19
|
+
import logging
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
import traceback
|
|
23
|
+
from collections.abc import Mapping
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from typing import Any, TYPE_CHECKING
|
|
26
|
+
|
|
27
|
+
from ..exceptions import TaskLiteError
|
|
28
|
+
from ..models.artifact import (
|
|
29
|
+
FileSpec,
|
|
30
|
+
merge_named_file_spec,
|
|
31
|
+
validate_file_name,
|
|
32
|
+
)
|
|
33
|
+
from ..models.job import Job
|
|
34
|
+
from .types import AuthorizationOutcome, AuthorizationStatus, JobHandle, StopMode
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from .channel import ExecutionChannel, ExecutionResult
|
|
38
|
+
from .in_flight import InFlightJob, InFlightTracker
|
|
39
|
+
from .session import RunSession
|
|
40
|
+
from .store import StateStore
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger("tasklite")
|
|
43
|
+
|
|
44
|
+
_CLEANUP_ERRORS = (OSError, ValueError, NotImplementedError)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CleanupAuthorizationDecision(str, enum.Enum):
|
|
48
|
+
"""收尾清理授权评估的显式结论(替代控制流异常)。"""
|
|
49
|
+
|
|
50
|
+
READY = "ready"
|
|
51
|
+
PENDING = "pending"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class PendingDeclaration:
|
|
56
|
+
"""一次等待主进程决定的动态文件声明。"""
|
|
57
|
+
|
|
58
|
+
request_id: str
|
|
59
|
+
name: str
|
|
60
|
+
source: FileSpec
|
|
61
|
+
bound: FileSpec
|
|
62
|
+
resolved_path: str
|
|
63
|
+
status: AuthorizationStatus = AuthorizationStatus.PENDING
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class _PendingDispatchAuthorization:
|
|
68
|
+
"""一次派发前待决授权的追踪键。"""
|
|
69
|
+
|
|
70
|
+
run_id: str | None
|
|
71
|
+
job_uid: str
|
|
72
|
+
file_name: str
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class PendingCleanup:
|
|
77
|
+
"""一次收尾清理待决授权(或等待受阻进程退出)的暂挂项。"""
|
|
78
|
+
|
|
79
|
+
entry: "InFlightJob"
|
|
80
|
+
result: "ExecutionResult"
|
|
81
|
+
prepared: Any
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class AuthorizationCoordinator:
|
|
85
|
+
"""统一承接三类主进程文件授权等待的登记、复查、停机解除与超时暂挂。"""
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
*,
|
|
90
|
+
store: "StateStore",
|
|
91
|
+
channel: "ExecutionChannel",
|
|
92
|
+
in_flight: "InFlightTracker",
|
|
93
|
+
session: "RunSession",
|
|
94
|
+
) -> None:
|
|
95
|
+
self._store = store
|
|
96
|
+
self._channel = channel
|
|
97
|
+
self._in_flight = in_flight
|
|
98
|
+
self._session = session
|
|
99
|
+
# 短临界区锁:仅保护状态字段原子跃迁与停机复核,绝不包裹外部回调或 I/O。
|
|
100
|
+
self._lock = threading.RLock()
|
|
101
|
+
|
|
102
|
+
# 三类授权等待与按 execution_id 追踪的运行期授权/超时状态
|
|
103
|
+
self._dispatch_pending: _PendingDispatchAuthorization | None = None
|
|
104
|
+
self._pending_declarations: dict[str, tuple[JobHandle, PendingDeclaration]] = {}
|
|
105
|
+
self._declaration_outcomes: dict[str, AuthorizationOutcome] = {}
|
|
106
|
+
self._paused_remaining: dict[str, float] = {}
|
|
107
|
+
self._pending_cleanups: dict[str, PendingCleanup] = {}
|
|
108
|
+
|
|
109
|
+
# ── 统一查询与状态清理 ────────────────────────────────────────────
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def has_pending_authorization(self) -> bool:
|
|
113
|
+
"""是否存在任意一类等待外部 authorizer 或受阻退出的授权暂挂。"""
|
|
114
|
+
with self._lock:
|
|
115
|
+
return (
|
|
116
|
+
self._dispatch_pending is not None
|
|
117
|
+
or bool(self._pending_declarations)
|
|
118
|
+
or bool(self._pending_cleanups)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def has_in_flight_authorization_wait(self) -> bool:
|
|
123
|
+
"""是否存在在途作业的动态声明或收尾清理授权等待(用于暂停新派发)。"""
|
|
124
|
+
with self._lock:
|
|
125
|
+
return bool(self._pending_declarations) or bool(self._pending_cleanups)
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def has_dispatch_pending(self) -> bool:
|
|
129
|
+
"""是否存在派发前授权等待。"""
|
|
130
|
+
with self._lock:
|
|
131
|
+
return self._dispatch_pending is not None
|
|
132
|
+
|
|
133
|
+
def has_pending_cleanup(self, job_uid: str) -> bool:
|
|
134
|
+
"""指定作业是否已收割并暂挂于收尾清理授权队列。"""
|
|
135
|
+
with self._lock:
|
|
136
|
+
return job_uid in self._pending_cleanups
|
|
137
|
+
|
|
138
|
+
def pending_cleanups(self) -> list[PendingCleanup]:
|
|
139
|
+
"""返回当前全部收尾清理暂挂项的快照列表。"""
|
|
140
|
+
with self._lock:
|
|
141
|
+
return list(self._pending_cleanups.values())
|
|
142
|
+
|
|
143
|
+
def is_waiting_declaration(self, handle: JobHandle) -> bool:
|
|
144
|
+
"""指定在途句柄是否正等待动态文件声明授权。"""
|
|
145
|
+
with self._lock:
|
|
146
|
+
item = self._pending_declarations.get(self._execution_key(handle))
|
|
147
|
+
return item is not None and item[1].status is AuthorizationStatus.PENDING
|
|
148
|
+
|
|
149
|
+
def get_declaration_outcome(self, handle: JobHandle) -> AuthorizationOutcome | None:
|
|
150
|
+
"""查询指定在途句柄是否已产出动态声明终态授权结果。"""
|
|
151
|
+
with self._lock:
|
|
152
|
+
return self._declaration_outcomes.get(self._execution_key(handle))
|
|
153
|
+
|
|
154
|
+
def is_timeout_paused(self, handle: JobHandle) -> bool:
|
|
155
|
+
"""指定在途句柄是否因授权等待而暂停执行超时计时。"""
|
|
156
|
+
with self._lock:
|
|
157
|
+
return self._execution_key(handle) in self._paused_remaining
|
|
158
|
+
|
|
159
|
+
def get_paused_remaining(self, handle: JobHandle) -> float | None:
|
|
160
|
+
"""查询指定在途句柄暂停时保存的剩余执行超时秒数。"""
|
|
161
|
+
with self._lock:
|
|
162
|
+
return self._paused_remaining.get(self._execution_key(handle))
|
|
163
|
+
|
|
164
|
+
def _pause_timeout_locked(self, handle: JobHandle) -> None:
|
|
165
|
+
key = self._execution_key(handle)
|
|
166
|
+
if key not in self._paused_remaining:
|
|
167
|
+
self._paused_remaining[key] = max(
|
|
168
|
+
0.0, handle.timeout_at_monotonic - time.monotonic()
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
def _resume_timeout_locked(self, handle: JobHandle) -> None:
|
|
172
|
+
key = self._execution_key(handle)
|
|
173
|
+
remaining = self._paused_remaining.pop(key, None)
|
|
174
|
+
if remaining is not None:
|
|
175
|
+
handle.timeout_at_monotonic = time.monotonic() + remaining
|
|
176
|
+
|
|
177
|
+
def pause_timeout(self, handle: JobHandle) -> None:
|
|
178
|
+
"""等待授权期间按执行身份暂停超时计时(幂等)。"""
|
|
179
|
+
with self._lock:
|
|
180
|
+
self._pause_timeout_locked(handle)
|
|
181
|
+
|
|
182
|
+
def resume_timeout(self, handle: JobHandle) -> None:
|
|
183
|
+
"""授权解除后恢复执行超时计时(幂等)。"""
|
|
184
|
+
with self._lock:
|
|
185
|
+
self._resume_timeout_locked(handle)
|
|
186
|
+
|
|
187
|
+
def clear_dispatch_pending(self) -> None:
|
|
188
|
+
"""清除派发前授权等待记录。"""
|
|
189
|
+
with self._lock:
|
|
190
|
+
self._dispatch_pending = None
|
|
191
|
+
|
|
192
|
+
def clear_execution(
|
|
193
|
+
self,
|
|
194
|
+
execution_id: str | None,
|
|
195
|
+
*,
|
|
196
|
+
job_uid: str | None = None,
|
|
197
|
+
) -> None:
|
|
198
|
+
"""在途执行结算完成后清理对应授权与超时暂停状态。"""
|
|
199
|
+
with self._lock:
|
|
200
|
+
for key in (execution_id, job_uid):
|
|
201
|
+
if key:
|
|
202
|
+
self._pending_declarations.pop(key, None)
|
|
203
|
+
self._declaration_outcomes.pop(key, None)
|
|
204
|
+
self._paused_remaining.pop(key, None)
|
|
205
|
+
if job_uid:
|
|
206
|
+
self._pending_cleanups.pop(job_uid, None)
|
|
207
|
+
|
|
208
|
+
def clear_all(self) -> None:
|
|
209
|
+
"""清空全部三类授权等待与暂停状态(用于异常中止或新一轮重置)。"""
|
|
210
|
+
with self._lock:
|
|
211
|
+
self._dispatch_pending = None
|
|
212
|
+
self._pending_declarations.clear()
|
|
213
|
+
self._declaration_outcomes.clear()
|
|
214
|
+
self._paused_remaining.clear()
|
|
215
|
+
self._pending_cleanups.clear()
|
|
216
|
+
|
|
217
|
+
def notify_stop(self) -> None:
|
|
218
|
+
"""停机请求到达时在短锁内原子清零派发前等待并收敛动态声明为 STOPPED,锁外终止进程。"""
|
|
219
|
+
to_terminate: list[tuple[JobHandle, str, str]] = []
|
|
220
|
+
with self._lock:
|
|
221
|
+
self._dispatch_pending = None
|
|
222
|
+
for _key, (handle, request) in list(self._pending_declarations.items()):
|
|
223
|
+
entry = self._in_flight.get(handle.job_uid)
|
|
224
|
+
if entry is None:
|
|
225
|
+
continue
|
|
226
|
+
message = self._stopped_message(request.resolved_path)
|
|
227
|
+
if self._transition_declaration_terminal_locked(
|
|
228
|
+
handle,
|
|
229
|
+
entry,
|
|
230
|
+
request,
|
|
231
|
+
AuthorizationStatus.STOPPED,
|
|
232
|
+
message,
|
|
233
|
+
):
|
|
234
|
+
to_terminate.append((handle, request.resolved_path, message))
|
|
235
|
+
for handle, path, message in to_terminate:
|
|
236
|
+
logger.info(
|
|
237
|
+
f"FILE AUTHORIZATION STOPPED: {handle.job_uid} {path} ({message})"
|
|
238
|
+
)
|
|
239
|
+
self._channel.terminate_process(handle)
|
|
240
|
+
|
|
241
|
+
@staticmethod
|
|
242
|
+
def _execution_key(handle: JobHandle) -> str:
|
|
243
|
+
return handle.execution_id if handle.execution_id else handle.job_uid
|
|
244
|
+
|
|
245
|
+
def _is_stopping(self) -> bool:
|
|
246
|
+
"""停机态判定的唯一真相源。"""
|
|
247
|
+
return self._session.stop_mode is not StopMode.NONE
|
|
248
|
+
|
|
249
|
+
@staticmethod
|
|
250
|
+
def _stopped_message(path: str) -> str:
|
|
251
|
+
return f"authorization wait stopped for {path}"
|
|
252
|
+
|
|
253
|
+
@staticmethod
|
|
254
|
+
def _denied_message(path: str) -> str:
|
|
255
|
+
return f"authorization denied for {path}"
|
|
256
|
+
|
|
257
|
+
def _stopped_outcome(self, path: str) -> AuthorizationOutcome:
|
|
258
|
+
return AuthorizationOutcome(
|
|
259
|
+
status=AuthorizationStatus.STOPPED,
|
|
260
|
+
message=self._stopped_message(path),
|
|
261
|
+
path=path,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
def _denied_outcome(self, path: str) -> AuthorizationOutcome:
|
|
265
|
+
return AuthorizationOutcome(
|
|
266
|
+
status=AuthorizationStatus.DENIED,
|
|
267
|
+
message=self._denied_message(path),
|
|
268
|
+
path=path,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
def _set_dispatch_pending_if_allowed(
|
|
272
|
+
self,
|
|
273
|
+
pending: _PendingDispatchAuthorization,
|
|
274
|
+
) -> bool:
|
|
275
|
+
"""短锁内二次复核停机态后再写入派发前等待,杜绝与 notify_stop 竞态产生幽灵等待。"""
|
|
276
|
+
with self._lock:
|
|
277
|
+
if self._is_stopping():
|
|
278
|
+
self._dispatch_pending = None
|
|
279
|
+
return False
|
|
280
|
+
self._dispatch_pending = pending
|
|
281
|
+
return True
|
|
282
|
+
|
|
283
|
+
# ── 类型 1:派发前清理授权等待 ────────────────────────────────────
|
|
284
|
+
|
|
285
|
+
def _resolve_job_files(self, job: Job) -> dict[str, FileSpec]:
|
|
286
|
+
"""按统一 PathPolicy 单次原子解析作业的全部预声明文件。"""
|
|
287
|
+
return {
|
|
288
|
+
name: self._store.resolve_file_spec(spec, depends_on=job.depends_on)
|
|
289
|
+
for name, spec in job.files.items()
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
def poll_dispatch_authorization(self) -> AuthorizationStatus:
|
|
293
|
+
"""复查上拍挂起的派发前授权。
|
|
294
|
+
|
|
295
|
+
返回:
|
|
296
|
+
- ``APPROVED``:无挂起或已放行,可继续扫描派发;
|
|
297
|
+
- ``PENDING``:仍在等待外部决策;
|
|
298
|
+
- ``DENIED``:拒绝并已将作业及其下游临时排除;
|
|
299
|
+
- ``PREPARATION_FAILED``:重校验失败并已拒绝提交;
|
|
300
|
+
- ``STOPPED``:已进入停机态,结束派发。
|
|
301
|
+
"""
|
|
302
|
+
with self._lock:
|
|
303
|
+
if self._is_stopping():
|
|
304
|
+
self._dispatch_pending = None
|
|
305
|
+
return AuthorizationStatus.STOPPED
|
|
306
|
+
pending = self._dispatch_pending
|
|
307
|
+
if pending is None:
|
|
308
|
+
return AuthorizationStatus.APPROVED
|
|
309
|
+
self._dispatch_pending = None
|
|
310
|
+
|
|
311
|
+
store = self._store
|
|
312
|
+
if pending.run_id != self._session.run_id or pending.job_uid not in store.state.queue_uids:
|
|
313
|
+
return AuthorizationStatus.APPROVED
|
|
314
|
+
|
|
315
|
+
pending_idx = next(
|
|
316
|
+
index
|
|
317
|
+
for index, snapshot in enumerate(store.state.queue)
|
|
318
|
+
if Job.from_dict(snapshot).uid == pending.job_uid
|
|
319
|
+
)
|
|
320
|
+
try:
|
|
321
|
+
job = Job.from_dict(store.state.queue[pending_idx])
|
|
322
|
+
pending_files = self._resolve_job_files(job)
|
|
323
|
+
path = pending_files[pending.file_name].path
|
|
324
|
+
assert path is not None
|
|
325
|
+
except Exception as exc:
|
|
326
|
+
snapshot = store.state.pop_job(pending_idx)
|
|
327
|
+
store.apply_failure(
|
|
328
|
+
pending.job_uid,
|
|
329
|
+
{
|
|
330
|
+
"error_message": str(exc),
|
|
331
|
+
"exception": {
|
|
332
|
+
"type": type(exc).__name__,
|
|
333
|
+
"message": str(exc),
|
|
334
|
+
"traceback": traceback.format_exc(),
|
|
335
|
+
},
|
|
336
|
+
},
|
|
337
|
+
job_snapshot=snapshot,
|
|
338
|
+
cascade=True,
|
|
339
|
+
)
|
|
340
|
+
return AuthorizationStatus.PREPARATION_FAILED
|
|
341
|
+
|
|
342
|
+
authorized = store.check_or_request_authorization(path)
|
|
343
|
+
if authorized is None:
|
|
344
|
+
if self._set_dispatch_pending_if_allowed(pending):
|
|
345
|
+
return AuthorizationStatus.PENDING
|
|
346
|
+
return AuthorizationStatus.STOPPED
|
|
347
|
+
if self._is_stopping():
|
|
348
|
+
return AuthorizationStatus.STOPPED
|
|
349
|
+
if authorized is False:
|
|
350
|
+
self._session.temporarily_exclude_chain(pending.job_uid, store.state)
|
|
351
|
+
return AuthorizationStatus.DENIED
|
|
352
|
+
return AuthorizationStatus.APPROVED
|
|
353
|
+
|
|
354
|
+
def authorize_dispatch_files(
|
|
355
|
+
self,
|
|
356
|
+
job_uid: str,
|
|
357
|
+
resolved_files: Mapping[str, FileSpec],
|
|
358
|
+
) -> AuthorizationStatus:
|
|
359
|
+
"""派发前逐项检查预声明文件的删除授权,未决则在短锁停机复核后登记等待槽。"""
|
|
360
|
+
store = self._store
|
|
361
|
+
for fname, fspec in resolved_files.items():
|
|
362
|
+
if not (fspec.delete_existing or fspec.delete_on_success or fspec.delete_on_failure):
|
|
363
|
+
continue
|
|
364
|
+
assert fspec.path is not None
|
|
365
|
+
authorized = store.check_or_request_authorization(fspec.path)
|
|
366
|
+
if authorized is None:
|
|
367
|
+
pending = _PendingDispatchAuthorization(
|
|
368
|
+
run_id=self._session.run_id,
|
|
369
|
+
job_uid=job_uid,
|
|
370
|
+
file_name=fname,
|
|
371
|
+
)
|
|
372
|
+
if self._set_dispatch_pending_if_allowed(pending):
|
|
373
|
+
return AuthorizationStatus.PENDING
|
|
374
|
+
return AuthorizationStatus.STOPPED
|
|
375
|
+
if self._is_stopping():
|
|
376
|
+
return AuthorizationStatus.STOPPED
|
|
377
|
+
if authorized is False:
|
|
378
|
+
self._session.temporarily_exclude_chain(job_uid, store.state)
|
|
379
|
+
return AuthorizationStatus.DENIED
|
|
380
|
+
return AuthorizationStatus.APPROVED
|
|
381
|
+
|
|
382
|
+
# ── 类型 2:运行期动态文件声明授权 ────────────────────────────────
|
|
383
|
+
|
|
384
|
+
def handle_declaration(
|
|
385
|
+
self,
|
|
386
|
+
handle: JobHandle,
|
|
387
|
+
payload: dict[str, Any],
|
|
388
|
+
) -> dict[str, Any] | None:
|
|
389
|
+
"""主进程动态文件声明仲裁出口:验证引用、授权检查、前置清理并回送路径。"""
|
|
390
|
+
raw_req_id = payload.get("request_id")
|
|
391
|
+
if not isinstance(raw_req_id, str) or not raw_req_id:
|
|
392
|
+
return {
|
|
393
|
+
"request_id": "",
|
|
394
|
+
"status": "error",
|
|
395
|
+
"error": "request_id must be a non-empty str",
|
|
396
|
+
}
|
|
397
|
+
req_id = raw_req_id
|
|
398
|
+
name = payload.get("name", "")
|
|
399
|
+
spec_data = payload.get("spec", {})
|
|
400
|
+
entry = self._in_flight.get(handle.job_uid)
|
|
401
|
+
if entry is None or not self._declaration_identity_matches(handle, entry):
|
|
402
|
+
return {
|
|
403
|
+
"request_id": req_id,
|
|
404
|
+
"status": "error",
|
|
405
|
+
"error": f"Job {handle.job_uid} not in flight",
|
|
406
|
+
}
|
|
407
|
+
exec_key = self._execution_key(handle)
|
|
408
|
+
with self._lock:
|
|
409
|
+
if exec_key in self._declaration_outcomes:
|
|
410
|
+
return {
|
|
411
|
+
"request_id": req_id,
|
|
412
|
+
"status": "error",
|
|
413
|
+
"error": f"Job {handle.job_uid} execution is no longer accepting declarations",
|
|
414
|
+
}
|
|
415
|
+
if exec_key in self._pending_declarations:
|
|
416
|
+
return {
|
|
417
|
+
"request_id": req_id,
|
|
418
|
+
"status": "error",
|
|
419
|
+
"error": f"Job {handle.job_uid} already has a pending declaration",
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
try:
|
|
423
|
+
spec = FileSpec.from_dict(spec_data)
|
|
424
|
+
validated_name = validate_file_name(name)
|
|
425
|
+
except Exception as exc:
|
|
426
|
+
return {"request_id": req_id, "status": "error", "error": str(exc)}
|
|
427
|
+
|
|
428
|
+
# 1. 幂等与冲突检查
|
|
429
|
+
try:
|
|
430
|
+
with self._lock:
|
|
431
|
+
existing_sources = dict(entry.active_file_sources)
|
|
432
|
+
existing_files = dict(entry.active_files)
|
|
433
|
+
merged, is_new = merge_named_file_spec(
|
|
434
|
+
existing_sources,
|
|
435
|
+
validated_name,
|
|
436
|
+
spec,
|
|
437
|
+
base_dir=self._store.base_dir,
|
|
438
|
+
depends_on=handle.job.depends_on,
|
|
439
|
+
)
|
|
440
|
+
except Exception as exc:
|
|
441
|
+
return {"request_id": req_id, "status": "error", "error": str(exc)}
|
|
442
|
+
source_spec = merged[validated_name]
|
|
443
|
+
if not is_new:
|
|
444
|
+
existing = existing_files.get(validated_name)
|
|
445
|
+
if existing is None or existing.path is None:
|
|
446
|
+
return {
|
|
447
|
+
"request_id": req_id,
|
|
448
|
+
"status": "error",
|
|
449
|
+
"error": f"File declaration {validated_name!r} has no resolved path",
|
|
450
|
+
}
|
|
451
|
+
return {
|
|
452
|
+
"request_id": req_id,
|
|
453
|
+
"status": "ok",
|
|
454
|
+
"resolved_path": existing.path,
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
# 2. 单次原子解析(上游引用、安全边界、存在性判定与路径绑定)
|
|
458
|
+
needs_authorization = (
|
|
459
|
+
source_spec.delete_existing
|
|
460
|
+
or source_spec.delete_on_success
|
|
461
|
+
or source_spec.delete_on_failure
|
|
462
|
+
)
|
|
463
|
+
if not needs_authorization:
|
|
464
|
+
try:
|
|
465
|
+
bound_spec = self._store.resolve_file_spec(
|
|
466
|
+
source_spec,
|
|
467
|
+
depends_on=handle.job.depends_on,
|
|
468
|
+
enforce_safety=False,
|
|
469
|
+
)
|
|
470
|
+
except Exception as exc:
|
|
471
|
+
return {
|
|
472
|
+
"request_id": req_id,
|
|
473
|
+
"status": "error",
|
|
474
|
+
"error": str(exc),
|
|
475
|
+
}
|
|
476
|
+
assert bound_spec.path is not None
|
|
477
|
+
resolved_path = bound_spec.path
|
|
478
|
+
logger.info(
|
|
479
|
+
f"FILE DECLARATION: {handle.job_uid} requested "
|
|
480
|
+
f"{validated_name!r} at {resolved_path}"
|
|
481
|
+
)
|
|
482
|
+
request = PendingDeclaration(
|
|
483
|
+
request_id=req_id,
|
|
484
|
+
name=validated_name,
|
|
485
|
+
source=source_spec,
|
|
486
|
+
bound=bound_spec,
|
|
487
|
+
resolved_path=resolved_path,
|
|
488
|
+
status=AuthorizationStatus.APPROVED,
|
|
489
|
+
)
|
|
490
|
+
with self._lock:
|
|
491
|
+
self._record_declaration(entry, request)
|
|
492
|
+
logger.info(
|
|
493
|
+
f"FILE DECLARATION APPROVED: {handle.job_uid} {resolved_path}"
|
|
494
|
+
)
|
|
495
|
+
return self._declaration_reply(request)
|
|
496
|
+
|
|
497
|
+
try:
|
|
498
|
+
bound_spec = self._store.resolve_file_spec(
|
|
499
|
+
source_spec,
|
|
500
|
+
depends_on=handle.job.depends_on,
|
|
501
|
+
enforce_safety=True,
|
|
502
|
+
)
|
|
503
|
+
except (TaskLiteError, FileNotFoundError) as exc:
|
|
504
|
+
return {
|
|
505
|
+
"request_id": req_id,
|
|
506
|
+
"status": "error",
|
|
507
|
+
"error": str(exc),
|
|
508
|
+
}
|
|
509
|
+
except _CLEANUP_ERRORS as exc:
|
|
510
|
+
error = {
|
|
511
|
+
"type": type(exc).__name__,
|
|
512
|
+
"message": str(exc),
|
|
513
|
+
"traceback": traceback.format_exc(),
|
|
514
|
+
}
|
|
515
|
+
fallback_path, fallback_bound = self._resolve_fallback_declaration_target(source_spec)
|
|
516
|
+
request = PendingDeclaration(
|
|
517
|
+
request_id=req_id,
|
|
518
|
+
name=validated_name,
|
|
519
|
+
source=source_spec,
|
|
520
|
+
bound=fallback_bound,
|
|
521
|
+
resolved_path=fallback_path,
|
|
522
|
+
)
|
|
523
|
+
with self._lock:
|
|
524
|
+
if exec_key in self._declaration_outcomes or exec_key in self._pending_declarations:
|
|
525
|
+
return None
|
|
526
|
+
self._pending_declarations[exec_key] = (handle, request)
|
|
527
|
+
self._pause_timeout_locked(handle)
|
|
528
|
+
self._finish_declaration(
|
|
529
|
+
handle,
|
|
530
|
+
entry,
|
|
531
|
+
request,
|
|
532
|
+
AuthorizationStatus.PREPARATION_FAILED,
|
|
533
|
+
error["message"],
|
|
534
|
+
error,
|
|
535
|
+
)
|
|
536
|
+
return None
|
|
537
|
+
|
|
538
|
+
assert bound_spec.path is not None
|
|
539
|
+
validated_path = bound_spec.path
|
|
540
|
+
logger.info(
|
|
541
|
+
f"FILE DECLARATION: {handle.job_uid} requested "
|
|
542
|
+
f"{validated_name!r} at {validated_path}"
|
|
543
|
+
)
|
|
544
|
+
request = PendingDeclaration(
|
|
545
|
+
request_id=req_id,
|
|
546
|
+
name=validated_name,
|
|
547
|
+
source=source_spec,
|
|
548
|
+
bound=bound_spec,
|
|
549
|
+
resolved_path=validated_path,
|
|
550
|
+
)
|
|
551
|
+
with self._lock:
|
|
552
|
+
if exec_key in self._declaration_outcomes or exec_key in self._pending_declarations:
|
|
553
|
+
return None
|
|
554
|
+
self._pending_declarations[exec_key] = (handle, request)
|
|
555
|
+
self._pause_timeout_locked(handle)
|
|
556
|
+
stopping = self._is_stopping()
|
|
557
|
+
if stopping:
|
|
558
|
+
self._finish_declaration(
|
|
559
|
+
handle,
|
|
560
|
+
entry,
|
|
561
|
+
request,
|
|
562
|
+
AuthorizationStatus.STOPPED,
|
|
563
|
+
self._stopped_message(validated_path),
|
|
564
|
+
)
|
|
565
|
+
return None
|
|
566
|
+
logger.info(
|
|
567
|
+
f"FILE AUTHORIZATION WAIT: {handle.job_uid} {validated_path}"
|
|
568
|
+
)
|
|
569
|
+
authorized = self._store.check_or_request_authorization(validated_path)
|
|
570
|
+
if authorized is None:
|
|
571
|
+
if self._is_stopping():
|
|
572
|
+
self._finish_declaration(
|
|
573
|
+
handle,
|
|
574
|
+
entry,
|
|
575
|
+
request,
|
|
576
|
+
AuthorizationStatus.STOPPED,
|
|
577
|
+
self._stopped_message(validated_path),
|
|
578
|
+
)
|
|
579
|
+
return None
|
|
580
|
+
return self._apply_declaration_decision(handle, entry, request, authorized)
|
|
581
|
+
|
|
582
|
+
def poll_pending_declarations(self) -> None:
|
|
583
|
+
"""轮询处于等待中的动态文件声明授权;若已决策则恢复执行或终止。"""
|
|
584
|
+
with self._lock:
|
|
585
|
+
snapshot = list(self._pending_declarations.items())
|
|
586
|
+
for exec_key, (handle, request) in snapshot:
|
|
587
|
+
with self._lock:
|
|
588
|
+
current = self._pending_declarations.get(exec_key)
|
|
589
|
+
if (
|
|
590
|
+
current is None
|
|
591
|
+
or current[1] is not request
|
|
592
|
+
or request.status is not AuthorizationStatus.PENDING
|
|
593
|
+
):
|
|
594
|
+
continue
|
|
595
|
+
entry = self._in_flight.get(handle.job_uid)
|
|
596
|
+
if entry is None or not self._declaration_identity_matches(handle, entry):
|
|
597
|
+
self._pending_declarations.pop(exec_key, None)
|
|
598
|
+
continue
|
|
599
|
+
stopping = self._is_stopping()
|
|
600
|
+
current_path = request.resolved_path
|
|
601
|
+
delete_existing = request.bound.delete_existing
|
|
602
|
+
|
|
603
|
+
if stopping:
|
|
604
|
+
self._finish_declaration(
|
|
605
|
+
handle,
|
|
606
|
+
entry,
|
|
607
|
+
request,
|
|
608
|
+
AuthorizationStatus.STOPPED,
|
|
609
|
+
self._stopped_message(current_path),
|
|
610
|
+
)
|
|
611
|
+
continue
|
|
612
|
+
|
|
613
|
+
validated_path, error = self._validate_declaration_target(
|
|
614
|
+
current_path, delete_existing=delete_existing
|
|
615
|
+
)
|
|
616
|
+
if error is not None:
|
|
617
|
+
self._finish_declaration(
|
|
618
|
+
handle,
|
|
619
|
+
entry,
|
|
620
|
+
request,
|
|
621
|
+
AuthorizationStatus.PREPARATION_FAILED,
|
|
622
|
+
error["message"],
|
|
623
|
+
error,
|
|
624
|
+
)
|
|
625
|
+
continue
|
|
626
|
+
if validated_path != current_path:
|
|
627
|
+
rebound = request.bound._bind_output_path(validated_path)
|
|
628
|
+
with self._lock:
|
|
629
|
+
current = self._pending_declarations.get(exec_key)
|
|
630
|
+
if (
|
|
631
|
+
current is None
|
|
632
|
+
or current[1] is not request
|
|
633
|
+
or request.status is not AuthorizationStatus.PENDING
|
|
634
|
+
):
|
|
635
|
+
continue
|
|
636
|
+
request.bound = rebound
|
|
637
|
+
request.resolved_path = validated_path
|
|
638
|
+
|
|
639
|
+
authorized = self._store.check_or_request_authorization(validated_path)
|
|
640
|
+
if authorized is not None:
|
|
641
|
+
reply = self._apply_declaration_decision(
|
|
642
|
+
handle, entry, request, authorized
|
|
643
|
+
)
|
|
644
|
+
if reply is None:
|
|
645
|
+
continue
|
|
646
|
+
reply_sender = self._channel.get_reply_sender(handle.execution_id)
|
|
647
|
+
if reply_sender is not None:
|
|
648
|
+
try:
|
|
649
|
+
reply_sender.send_reply(reply)
|
|
650
|
+
except OSError:
|
|
651
|
+
pass
|
|
652
|
+
elif self._is_stopping():
|
|
653
|
+
self._finish_declaration(
|
|
654
|
+
handle,
|
|
655
|
+
entry,
|
|
656
|
+
request,
|
|
657
|
+
AuthorizationStatus.STOPPED,
|
|
658
|
+
self._stopped_message(validated_path),
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
def _resolve_fallback_declaration_target(
|
|
662
|
+
self,
|
|
663
|
+
source_spec: FileSpec,
|
|
664
|
+
) -> tuple[str, FileSpec]:
|
|
665
|
+
if source_spec.is_output_ref and source_spec.producer_uid and source_spec.output_name:
|
|
666
|
+
desc = (
|
|
667
|
+
self._store.state.successes.get(source_spec.producer_uid, {})
|
|
668
|
+
.get("artifacts", {})
|
|
669
|
+
.get(source_spec.output_name, {})
|
|
670
|
+
)
|
|
671
|
+
fallback_path = str(desc.get("path", "")) if isinstance(desc, dict) else ""
|
|
672
|
+
else:
|
|
673
|
+
fallback_path = source_spec.path or ""
|
|
674
|
+
fallback_bound = source_spec
|
|
675
|
+
if fallback_path:
|
|
676
|
+
try:
|
|
677
|
+
fallback_path = self._store.path_policy.inspect(
|
|
678
|
+
fallback_path, mode="lookup"
|
|
679
|
+
).canonical_path
|
|
680
|
+
except _CLEANUP_ERRORS:
|
|
681
|
+
pass
|
|
682
|
+
try:
|
|
683
|
+
fallback_bound = source_spec._bind_output_path(fallback_path)
|
|
684
|
+
except (TaskLiteError, ValueError):
|
|
685
|
+
fallback_bound = source_spec
|
|
686
|
+
return fallback_path, fallback_bound
|
|
687
|
+
|
|
688
|
+
def _declaration_identity_matches(self, handle: JobHandle, entry: "InFlightJob") -> bool:
|
|
689
|
+
"""请求句柄必须与当前在飞执行使用同一 job、run、attempt 与 execution 身份。"""
|
|
690
|
+
if entry.handle is not handle or entry.job_uid != handle.job_uid:
|
|
691
|
+
return False
|
|
692
|
+
if not handle.execution_id or entry.execution_id != handle.execution_id:
|
|
693
|
+
return False
|
|
694
|
+
if handle.run_id is not None and entry.run_id != handle.run_id:
|
|
695
|
+
return False
|
|
696
|
+
if handle.attempt_no is not None and entry.attempt_no != handle.attempt_no:
|
|
697
|
+
return False
|
|
698
|
+
if self._session.run_id is not None and entry.run_id != self._session.run_id:
|
|
699
|
+
return False
|
|
700
|
+
return True
|
|
701
|
+
|
|
702
|
+
@staticmethod
|
|
703
|
+
def _record_declaration(entry: "InFlightJob", request: PendingDeclaration) -> None:
|
|
704
|
+
entry.active_file_sources[request.name] = request.source
|
|
705
|
+
entry.active_files[request.name] = request.bound
|
|
706
|
+
|
|
707
|
+
def _validate_declaration_target(
|
|
708
|
+
self,
|
|
709
|
+
resolved_path: str,
|
|
710
|
+
*,
|
|
711
|
+
delete_existing: bool,
|
|
712
|
+
) -> tuple[str, dict[str, str] | None]:
|
|
713
|
+
try:
|
|
714
|
+
return (
|
|
715
|
+
self._store.path_policy.inspect(
|
|
716
|
+
resolved_path,
|
|
717
|
+
mode="pre_delete" if delete_existing else "strict",
|
|
718
|
+
).canonical_path,
|
|
719
|
+
None,
|
|
720
|
+
)
|
|
721
|
+
except _CLEANUP_ERRORS as exc:
|
|
722
|
+
return resolved_path, {
|
|
723
|
+
"type": type(exc).__name__,
|
|
724
|
+
"message": str(exc),
|
|
725
|
+
"traceback": traceback.format_exc(),
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
def _apply_declaration_decision(
|
|
729
|
+
self,
|
|
730
|
+
handle: JobHandle,
|
|
731
|
+
entry: "InFlightJob",
|
|
732
|
+
request: PendingDeclaration,
|
|
733
|
+
authorized: bool,
|
|
734
|
+
) -> dict[str, Any] | None:
|
|
735
|
+
exec_key = self._execution_key(handle)
|
|
736
|
+
with self._lock:
|
|
737
|
+
pending = self._pending_declarations.get(exec_key)
|
|
738
|
+
if (
|
|
739
|
+
pending is None
|
|
740
|
+
or pending[1] is not request
|
|
741
|
+
or request.status is not AuthorizationStatus.PENDING
|
|
742
|
+
or exec_key in self._declaration_outcomes
|
|
743
|
+
):
|
|
744
|
+
return None
|
|
745
|
+
stopping = self._is_stopping()
|
|
746
|
+
resolved_path = request.resolved_path
|
|
747
|
+
delete_existing = request.bound.delete_existing
|
|
748
|
+
|
|
749
|
+
if stopping:
|
|
750
|
+
message = self._stopped_message(resolved_path)
|
|
751
|
+
if self._finish_declaration(
|
|
752
|
+
handle, entry, request, AuthorizationStatus.STOPPED, message
|
|
753
|
+
):
|
|
754
|
+
return {
|
|
755
|
+
"request_id": request.request_id,
|
|
756
|
+
"status": "stopped",
|
|
757
|
+
"error": message,
|
|
758
|
+
}
|
|
759
|
+
return None
|
|
760
|
+
|
|
761
|
+
if authorized is False:
|
|
762
|
+
message = self._denied_message(resolved_path)
|
|
763
|
+
if self._finish_declaration(
|
|
764
|
+
handle, entry, request, AuthorizationStatus.DENIED, message
|
|
765
|
+
):
|
|
766
|
+
return {
|
|
767
|
+
"request_id": request.request_id,
|
|
768
|
+
"status": "denied",
|
|
769
|
+
"error": message,
|
|
770
|
+
}
|
|
771
|
+
return None
|
|
772
|
+
|
|
773
|
+
if delete_existing:
|
|
774
|
+
try:
|
|
775
|
+
self._store.path_policy.delete(
|
|
776
|
+
resolved_path,
|
|
777
|
+
allow_broken_leaf_symlink=True,
|
|
778
|
+
authorize=self._store.backend.is_path_authorized,
|
|
779
|
+
)
|
|
780
|
+
except _CLEANUP_ERRORS as exc:
|
|
781
|
+
error = {
|
|
782
|
+
"type": type(exc).__name__,
|
|
783
|
+
"message": str(exc),
|
|
784
|
+
"traceback": traceback.format_exc(),
|
|
785
|
+
}
|
|
786
|
+
message = f"Artifact preparation failed for {resolved_path}: {exc}"
|
|
787
|
+
self._finish_declaration(
|
|
788
|
+
handle,
|
|
789
|
+
entry,
|
|
790
|
+
request,
|
|
791
|
+
AuthorizationStatus.PREPARATION_FAILED,
|
|
792
|
+
message,
|
|
793
|
+
error,
|
|
794
|
+
)
|
|
795
|
+
return None
|
|
796
|
+
|
|
797
|
+
# 短锁内二次复核:若锁外 I/O 期间 notify_stop() 已中止或已进入停机态,绝不覆盖为 APPROVED。
|
|
798
|
+
approved_reply: dict[str, Any] | None = None
|
|
799
|
+
approved_path = ""
|
|
800
|
+
stopped_transition = False
|
|
801
|
+
stopped_path = ""
|
|
802
|
+
stopped_msg = ""
|
|
803
|
+
with self._lock:
|
|
804
|
+
pending = self._pending_declarations.get(exec_key)
|
|
805
|
+
if (
|
|
806
|
+
pending is None
|
|
807
|
+
or pending[1] is not request
|
|
808
|
+
or request.status is not AuthorizationStatus.PENDING
|
|
809
|
+
or exec_key in self._declaration_outcomes
|
|
810
|
+
):
|
|
811
|
+
return None
|
|
812
|
+
if self._is_stopping():
|
|
813
|
+
stopped_path = request.resolved_path
|
|
814
|
+
stopped_msg = self._stopped_message(stopped_path)
|
|
815
|
+
stopped_transition = self._transition_declaration_terminal_locked(
|
|
816
|
+
handle,
|
|
817
|
+
entry,
|
|
818
|
+
request,
|
|
819
|
+
AuthorizationStatus.STOPPED,
|
|
820
|
+
stopped_msg,
|
|
821
|
+
)
|
|
822
|
+
else:
|
|
823
|
+
request.status = AuthorizationStatus.APPROVED
|
|
824
|
+
self._pending_declarations.pop(exec_key, None)
|
|
825
|
+
self._resume_timeout_locked(handle)
|
|
826
|
+
self._record_declaration(entry, request)
|
|
827
|
+
approved_path = request.resolved_path
|
|
828
|
+
approved_reply = self._declaration_reply(request)
|
|
829
|
+
|
|
830
|
+
if approved_reply is not None:
|
|
831
|
+
logger.info(
|
|
832
|
+
f"FILE AUTHORIZATION APPROVED: {handle.job_uid} {approved_path}"
|
|
833
|
+
)
|
|
834
|
+
return approved_reply
|
|
835
|
+
if stopped_transition:
|
|
836
|
+
logger.info(
|
|
837
|
+
f"FILE AUTHORIZATION STOPPED: "
|
|
838
|
+
f"{handle.job_uid} {stopped_path} ({stopped_msg})"
|
|
839
|
+
)
|
|
840
|
+
self._channel.terminate_process(handle)
|
|
841
|
+
return None
|
|
842
|
+
|
|
843
|
+
def _transition_declaration_terminal_locked(
|
|
844
|
+
self,
|
|
845
|
+
handle: JobHandle,
|
|
846
|
+
entry: "InFlightJob",
|
|
847
|
+
request: PendingDeclaration,
|
|
848
|
+
status: AuthorizationStatus,
|
|
849
|
+
message: str,
|
|
850
|
+
exception: dict[str, str] | None = None,
|
|
851
|
+
) -> bool:
|
|
852
|
+
"""短锁内原子完成动态声明向终态的一次性跃迁(防双线程撕裂与双写)。"""
|
|
853
|
+
exec_key = self._execution_key(handle)
|
|
854
|
+
pending = self._pending_declarations.get(exec_key)
|
|
855
|
+
if (
|
|
856
|
+
pending is None
|
|
857
|
+
or pending[1] is not request
|
|
858
|
+
or request.status is not AuthorizationStatus.PENDING
|
|
859
|
+
or exec_key in self._declaration_outcomes
|
|
860
|
+
):
|
|
861
|
+
return False
|
|
862
|
+
request.status = status
|
|
863
|
+
self._pending_declarations.pop(exec_key, None)
|
|
864
|
+
self._declaration_outcomes[exec_key] = AuthorizationOutcome(
|
|
865
|
+
status=status,
|
|
866
|
+
message=message,
|
|
867
|
+
path=request.resolved_path,
|
|
868
|
+
exception=exception,
|
|
869
|
+
)
|
|
870
|
+
self._resume_timeout_locked(handle)
|
|
871
|
+
self._record_declaration(entry, request)
|
|
872
|
+
return True
|
|
873
|
+
|
|
874
|
+
def _finish_declaration(
|
|
875
|
+
self,
|
|
876
|
+
handle: JobHandle,
|
|
877
|
+
entry: "InFlightJob",
|
|
878
|
+
request: PendingDeclaration,
|
|
879
|
+
status: AuthorizationStatus,
|
|
880
|
+
message: str,
|
|
881
|
+
exception: dict[str, str] | None = None,
|
|
882
|
+
) -> bool:
|
|
883
|
+
with self._lock:
|
|
884
|
+
transitioned = self._transition_declaration_terminal_locked(
|
|
885
|
+
handle, entry, request, status, message, exception
|
|
886
|
+
)
|
|
887
|
+
resolved_path = request.resolved_path
|
|
888
|
+
if not transitioned:
|
|
889
|
+
return False
|
|
890
|
+
logger.info(
|
|
891
|
+
f"FILE AUTHORIZATION {status.value.upper()}: "
|
|
892
|
+
f"{handle.job_uid} {resolved_path} ({message})"
|
|
893
|
+
)
|
|
894
|
+
self._channel.terminate_process(handle)
|
|
895
|
+
return True
|
|
896
|
+
|
|
897
|
+
@staticmethod
|
|
898
|
+
def _declaration_reply(request: PendingDeclaration) -> dict[str, Any]:
|
|
899
|
+
return {
|
|
900
|
+
"request_id": request.request_id,
|
|
901
|
+
"status": "ok",
|
|
902
|
+
"resolved_path": request.resolved_path,
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
# ── 类型 3:收尾清理期授权等待 ────────────────────────────────────
|
|
906
|
+
|
|
907
|
+
def evaluate_cleanup(
|
|
908
|
+
self,
|
|
909
|
+
entry: "InFlightJob | None",
|
|
910
|
+
result: "ExecutionResult",
|
|
911
|
+
active_files: Mapping[str, FileSpec],
|
|
912
|
+
) -> CleanupAuthorizationDecision:
|
|
913
|
+
"""评估执行收尾清理所需的全部路径授权;若待决或受阻进程仍在运行则返回 PENDING。"""
|
|
914
|
+
handle = entry.handle if entry is not None else None
|
|
915
|
+
waiting = entry is not None and self.has_pending_cleanup(entry.job_uid)
|
|
916
|
+
if result.authorization_outcome is not None:
|
|
917
|
+
if (
|
|
918
|
+
waiting
|
|
919
|
+
and handle is not None
|
|
920
|
+
and self._channel.is_handle_alive(handle)
|
|
921
|
+
):
|
|
922
|
+
return CleanupAuthorizationDecision.PENDING
|
|
923
|
+
return CleanupAuthorizationDecision.READY
|
|
924
|
+
|
|
925
|
+
policy = self._store.path_policy
|
|
926
|
+
for spec in active_files.values():
|
|
927
|
+
requested = spec.delete_on_success if result.success else spec.delete_on_failure
|
|
928
|
+
if not requested or spec.path is None:
|
|
929
|
+
continue
|
|
930
|
+
try:
|
|
931
|
+
if not policy.inspect(spec.path, mode="strict", verify_unchanged=True).exists:
|
|
932
|
+
continue
|
|
933
|
+
except _CLEANUP_ERRORS:
|
|
934
|
+
continue
|
|
935
|
+
|
|
936
|
+
if self._is_stopping() and waiting:
|
|
937
|
+
result.authorization_outcome = self._stopped_outcome(spec.path)
|
|
938
|
+
else:
|
|
939
|
+
decision = self._store.check_or_request_authorization(spec.path)
|
|
940
|
+
if self._is_stopping() and (waiting or decision is None):
|
|
941
|
+
result.authorization_outcome = self._stopped_outcome(spec.path)
|
|
942
|
+
elif decision is True:
|
|
943
|
+
continue
|
|
944
|
+
elif decision is None:
|
|
945
|
+
return CleanupAuthorizationDecision.PENDING
|
|
946
|
+
else:
|
|
947
|
+
result.authorization_outcome = self._denied_outcome(spec.path)
|
|
948
|
+
|
|
949
|
+
if handle is not None:
|
|
950
|
+
self._channel.terminate_process(handle)
|
|
951
|
+
if self._channel.is_handle_alive(handle):
|
|
952
|
+
return CleanupAuthorizationDecision.PENDING
|
|
953
|
+
return CleanupAuthorizationDecision.READY
|
|
954
|
+
|
|
955
|
+
# 后续询问回调可能改变此前目标,整组复核通过后再交付实际清理。
|
|
956
|
+
for spec in active_files.values():
|
|
957
|
+
requested = spec.delete_on_success if result.success else spec.delete_on_failure
|
|
958
|
+
if requested and spec.path and not self._store.is_path_authorized(spec.path):
|
|
959
|
+
try:
|
|
960
|
+
present = policy.inspect(spec.path, mode="strict", verify_unchanged=True).exists
|
|
961
|
+
except _CLEANUP_ERRORS:
|
|
962
|
+
continue
|
|
963
|
+
if present:
|
|
964
|
+
return CleanupAuthorizationDecision.PENDING
|
|
965
|
+
return CleanupAuthorizationDecision.READY
|
|
966
|
+
|
|
967
|
+
def defer_cleanup(
|
|
968
|
+
self,
|
|
969
|
+
entry: "InFlightJob",
|
|
970
|
+
result: "ExecutionResult",
|
|
971
|
+
prepared: Any,
|
|
972
|
+
) -> None:
|
|
973
|
+
"""将尚未完成授权或待进程退出的收尾项暂挂,并暂停其执行超时计时。"""
|
|
974
|
+
with self._lock:
|
|
975
|
+
self._pending_cleanups[entry.job_uid] = PendingCleanup(
|
|
976
|
+
entry=entry,
|
|
977
|
+
result=result,
|
|
978
|
+
prepared=prepared,
|
|
979
|
+
)
|
|
980
|
+
if entry.handle is not None:
|
|
981
|
+
self._pause_timeout_locked(entry.handle)
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
__all__ = [
|
|
985
|
+
"AuthorizationCoordinator",
|
|
986
|
+
"CleanupAuthorizationDecision",
|
|
987
|
+
"PendingCleanup",
|
|
988
|
+
"PendingDeclaration",
|
|
989
|
+
]
|