tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
tasklite/engine/dispatch.py
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
"""v2
|
|
1
|
+
"""v2 派发机器:预检、两阶段资源租约、阶段回滚守卫与子进程 spawn 编排。
|
|
2
2
|
|
|
3
|
-
预检顺序即契约(顺序即时序约束):dep-failed
|
|
4
|
-
|
|
5
|
-
``store``
|
|
6
|
-
|
|
7
|
-
``session`` 直接使用内部 RunSession;``tasks`` 按具名 Protocol
|
|
8
|
-
``TaskSource`` 消费(TaskRegistry 自动满足)。``output_roots`` 与 config 声明同型
|
|
9
|
-
(``Path | Sequence[Path] | None``,多根输出场景)。
|
|
3
|
+
预检顺序即契约(顺序即时序约束):dep-failed/dep-missing → 预声明解析 →
|
|
4
|
+
admit/dedup → no-handler → 预授权 → 资源重检 → 落 running → 获取资源 →
|
|
5
|
+
前置清理 → 锁外冷启动 → 锁内微秒放行。依赖以显式窄清单注入,经 ``store``
|
|
6
|
+
复用状态仓库深模块,不反向引用门面。
|
|
10
7
|
|
|
11
8
|
进入 running 时分配编号,结算时追加 Attempt:execution_id
|
|
12
9
|
(``run_id.dispatch_no``)在本模块生成并随行落表——dispatch_no 每次派发递增,
|
|
@@ -20,48 +17,32 @@ import logging
|
|
|
20
17
|
import threading
|
|
21
18
|
import time
|
|
22
19
|
import traceback
|
|
20
|
+
from collections.abc import Mapping
|
|
23
21
|
from dataclasses import dataclass, field
|
|
24
|
-
from typing import Any,
|
|
22
|
+
from typing import Any, TYPE_CHECKING
|
|
25
23
|
|
|
26
24
|
if TYPE_CHECKING:
|
|
27
|
-
from collections.abc import Sequence
|
|
28
|
-
from pathlib import Path
|
|
29
|
-
|
|
30
25
|
from ..models.state import PipelineState
|
|
31
|
-
from ..models.task import
|
|
26
|
+
from ..models.task import TaskRegistry
|
|
32
27
|
from .admission import TriggerRegistry
|
|
33
|
-
from .channel import ExecutionChannel
|
|
28
|
+
from .channel import ExecutionChannel, PreparedLaunch
|
|
34
29
|
from .completion import CompletionMachine
|
|
35
|
-
from .files import FileSpecResolver
|
|
36
30
|
from .in_flight import InFlightTracker
|
|
37
|
-
from .recovery import RecoveryOrchestrator
|
|
38
31
|
from .resource import ResourceManager
|
|
39
32
|
from .scheduler import JobScheduler
|
|
40
|
-
from .store import StateStore
|
|
41
33
|
from .session import RunSession
|
|
34
|
+
from .store import StateStore
|
|
42
35
|
|
|
43
|
-
|
|
44
|
-
"""派发机器消费的任务规格来源契约(TaskRegistry 自动满足)。"""
|
|
45
|
-
|
|
46
|
-
def lookup(self, task_type: str) -> "Task": ...
|
|
47
|
-
|
|
48
|
-
def __contains__(self, task_type: object) -> bool: ...
|
|
49
|
-
|
|
50
|
-
import os
|
|
51
|
-
from collections.abc import Mapping, Sequence
|
|
52
|
-
from pathlib import Path
|
|
53
|
-
|
|
54
|
-
from ..exceptions import TaskLiteError
|
|
55
|
-
from ..models.artifact import ArtifactDeclaration, FileSpec
|
|
36
|
+
from ..models.artifact import FileSpec
|
|
56
37
|
from ..models.context import JobContext
|
|
57
38
|
from ..models.job import Job
|
|
58
|
-
from ..utils.
|
|
59
|
-
from .
|
|
39
|
+
from ..utils.clock import utc_now_iso
|
|
40
|
+
from .authorization import AuthorizationCoordinator
|
|
41
|
+
from .channel import ExecutionResult, WorkerLaunchSpec
|
|
60
42
|
from .in_flight import InFlightJob
|
|
43
|
+
from .launcher import _WorkerLaunchCancelled
|
|
61
44
|
from .scheduler import ScheduleResult, StandstillFacts
|
|
62
|
-
from .types import StopMode
|
|
63
|
-
from .wait import AUTHORIZATION_RECHECK_SECONDS
|
|
64
|
-
from ..utils.clock import utc_now_iso
|
|
45
|
+
from .types import AuthorizationStatus, JobHandle, StopMode
|
|
65
46
|
|
|
66
47
|
logger = logging.getLogger("tasklite")
|
|
67
48
|
|
|
@@ -69,11 +50,11 @@ logger = logging.getLogger("tasklite")
|
|
|
69
50
|
class DispatchKind(enum.Enum):
|
|
70
51
|
"""一次调度派发周期的结局分类(单一事实,无交叠布尔)。"""
|
|
71
52
|
|
|
72
|
-
SPAWNED = "spawned"
|
|
73
|
-
HANDLED_NO_SUBPROCESS = "handled_no_subprocess" #
|
|
74
|
-
AUTHORIZATION_PENDING = "authorization_pending"
|
|
75
|
-
WORKER_SATURATED = "worker_saturated"
|
|
76
|
-
NO_CANDIDATE = "no_candidate"
|
|
53
|
+
SPAWNED = "spawned" # 子进程已启动并登记在飞
|
|
54
|
+
HANDLED_NO_SUBPROCESS = "handled_no_subprocess" # 预检/校验拦截,无需子进程即终结
|
|
55
|
+
AUTHORIZATION_PENDING = "authorization_pending" # 派发前路径授权待决
|
|
56
|
+
WORKER_SATURATED = "worker_saturated" # 工人槽位耗尽,退出填池等待
|
|
57
|
+
NO_CANDIDATE = "no_candidate" # 队列无可运行候选(停摆事实随附)
|
|
77
58
|
|
|
78
59
|
|
|
79
60
|
@dataclass(frozen=True)
|
|
@@ -91,6 +72,88 @@ class DispatchOutcome:
|
|
|
91
72
|
standstill: StandstillFacts = field(default_factory=StandstillFacts)
|
|
92
73
|
|
|
93
74
|
|
|
75
|
+
class _DispatchStopped(Exception):
|
|
76
|
+
"""派发阶段内检测到停机请求,触发未启动状态回退的内部信号。"""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class _DispatchStageGuard:
|
|
80
|
+
"""派发阶段回滚守卫:按已越过阶段精确回退。
|
|
81
|
+
|
|
82
|
+
- 停机路径(``rollback_for_stop``):中止未放行启动、释放资源,若已落 running
|
|
83
|
+
则回退 pending 且不写 Attempt,否则原位插回队首;
|
|
84
|
+
- 故障路径(``cleanup_on_fault``):回收未放行启动/已提交句柄/已占资源;未落
|
|
85
|
+
running 前回队,已落 running 后保留磁盘 running 供下次启动恢复。
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(
|
|
89
|
+
self,
|
|
90
|
+
machine: "DispatchMachine",
|
|
91
|
+
*,
|
|
92
|
+
job_uid: str,
|
|
93
|
+
job_snapshot: dict[str, Any],
|
|
94
|
+
) -> None:
|
|
95
|
+
self._machine = machine
|
|
96
|
+
self.job_uid = job_uid
|
|
97
|
+
self.job_snapshot = job_snapshot
|
|
98
|
+
self.execution_id: str | None = None
|
|
99
|
+
self.attempt_no: int | None = None
|
|
100
|
+
self.acquired: list[tuple[str, float]] = []
|
|
101
|
+
self.prepared: "PreparedLaunch | None" = None
|
|
102
|
+
self.committed_handle: JobHandle | None = None
|
|
103
|
+
self._finalized: bool = False
|
|
104
|
+
|
|
105
|
+
def ensure_allowed(self) -> None:
|
|
106
|
+
"""停机语义屏障:若已请求停机则立即退出本次派发并触发阶段回退。"""
|
|
107
|
+
if not self._machine._dispatch_allowed():
|
|
108
|
+
raise _DispatchStopped
|
|
109
|
+
|
|
110
|
+
def commit(self) -> None:
|
|
111
|
+
"""移交在飞追踪或完成机器接管,后续退出不再触发回滚。"""
|
|
112
|
+
self._finalized = True
|
|
113
|
+
|
|
114
|
+
def rollback_for_stop(self) -> None:
|
|
115
|
+
"""停机回退:按已越过阶段逆序清理并将未启动作业退回 pending(不写 Attempt)。"""
|
|
116
|
+
if self._finalized:
|
|
117
|
+
return
|
|
118
|
+
self._finalized = True
|
|
119
|
+
if self.prepared is not None:
|
|
120
|
+
self._machine._channel.abort_spawn(self.prepared)
|
|
121
|
+
self.prepared = None
|
|
122
|
+
if self.acquired:
|
|
123
|
+
acquired, self.acquired = self.acquired, []
|
|
124
|
+
self._machine._resources.release_all(acquired, job_uid=self.job_uid)
|
|
125
|
+
if self.execution_id is not None and self.attempt_no is not None:
|
|
126
|
+
self._machine._store.apply_pending(
|
|
127
|
+
self.job_uid,
|
|
128
|
+
self.job_snapshot,
|
|
129
|
+
execution_id=self.execution_id,
|
|
130
|
+
attempt_no=self.attempt_no,
|
|
131
|
+
run_id=self._machine._session.run_id or "run",
|
|
132
|
+
started_at=None,
|
|
133
|
+
error_message="stop requested before worker launch",
|
|
134
|
+
record_attempt=False,
|
|
135
|
+
)
|
|
136
|
+
else:
|
|
137
|
+
self._machine._store.state.requeue_jobs([self.job_snapshot], front=True)
|
|
138
|
+
|
|
139
|
+
def cleanup_on_fault(self) -> None:
|
|
140
|
+
"""框架故障清理:回收启动/通道/资源;未入 running 前回队,已入 running 留待恢复。"""
|
|
141
|
+
if self._finalized:
|
|
142
|
+
return
|
|
143
|
+
self._finalized = True
|
|
144
|
+
if self.prepared is not None:
|
|
145
|
+
self._machine._channel.abort_spawn(self.prepared)
|
|
146
|
+
self.prepared = None
|
|
147
|
+
if self.committed_handle is not None:
|
|
148
|
+
self._machine._channel.cleanup_in_flight([self.committed_handle])
|
|
149
|
+
self.committed_handle = None
|
|
150
|
+
if self.acquired:
|
|
151
|
+
acquired, self.acquired = self.acquired, []
|
|
152
|
+
self._machine._resources.release_all(acquired, job_uid=self.job_uid)
|
|
153
|
+
if self.execution_id is None:
|
|
154
|
+
self._machine._store.state.requeue_jobs([self.job_snapshot], front=True)
|
|
155
|
+
|
|
156
|
+
|
|
94
157
|
class DispatchMachine:
|
|
95
158
|
"""派发预检 + 资源租约 + running 编号 + 子进程 spawn 的编排器。"""
|
|
96
159
|
|
|
@@ -104,13 +167,10 @@ class DispatchMachine:
|
|
|
104
167
|
channel: "ExecutionChannel",
|
|
105
168
|
in_flight: "InFlightTracker",
|
|
106
169
|
session: "RunSession",
|
|
107
|
-
recovery: "RecoveryOrchestrator",
|
|
108
170
|
completion: "CompletionMachine",
|
|
109
|
-
tasks: "
|
|
110
|
-
|
|
111
|
-
protected_roots: tuple[Path, ...] = (),
|
|
171
|
+
tasks: "TaskRegistry",
|
|
172
|
+
authorization: AuthorizationCoordinator | None = None,
|
|
112
173
|
dispatch_start_lock: Any | None = None,
|
|
113
|
-
file_resolver: "FileSpecResolver | None" = None,
|
|
114
174
|
) -> None:
|
|
115
175
|
self._store = store
|
|
116
176
|
self._scheduler = scheduler
|
|
@@ -119,21 +179,18 @@ class DispatchMachine:
|
|
|
119
179
|
self._channel = channel
|
|
120
180
|
self._in_flight = in_flight
|
|
121
181
|
self._session = session
|
|
122
|
-
self._recovery = recovery
|
|
123
182
|
self._completion = completion
|
|
124
183
|
self._tasks = tasks
|
|
125
|
-
self.
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
184
|
+
self._authorization = (
|
|
185
|
+
authorization
|
|
186
|
+
if authorization is not None
|
|
187
|
+
else AuthorizationCoordinator(
|
|
188
|
+
store=store,
|
|
189
|
+
channel=channel,
|
|
190
|
+
in_flight=in_flight,
|
|
191
|
+
session=session,
|
|
192
|
+
)
|
|
131
193
|
)
|
|
132
|
-
if file_resolver is None:
|
|
133
|
-
from .files import FileSpecResolver
|
|
134
|
-
file_resolver = FileSpecResolver(store)
|
|
135
|
-
self._file_resolver = file_resolver
|
|
136
|
-
self._authorization_pending: tuple[str | None, str, str] | None = None
|
|
137
194
|
self._dispatch_start_lock = dispatch_start_lock or threading.RLock()
|
|
138
195
|
|
|
139
196
|
# ── 扫描与派发接缝 ────────────────────────────────────────────────
|
|
@@ -143,63 +200,27 @@ class DispatchMachine:
|
|
|
143
200
|
return self._session.stop_mode is StopMode.NONE
|
|
144
201
|
|
|
145
202
|
def dispatch_next(self) -> DispatchOutcome:
|
|
146
|
-
"""
|
|
147
|
-
资源租约 → 子进程派发。
|
|
203
|
+
"""统一扫描与派发接缝:派发前授权复查 → 队列扫描 → 工人槽位预检 →
|
|
204
|
+
预检关卡 → 资源租约 → 子进程派发。
|
|
148
205
|
|
|
149
206
|
返回 DispatchOutcome(kind 单一判据):
|
|
150
207
|
- SPAWNED:entry 携带在飞条目,调用方继续收割;
|
|
151
|
-
- HANDLED_NO_SUBPROCESS
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
- WORKER_SATURATED:工人槽位耗尽(worker_wait 为建议等待),
|
|
155
|
-
调用方应退出填池循环;
|
|
208
|
+
- HANDLED_NO_SUBPROCESS:作业被预检/校验拦截并已终结,调用方可继续填池;
|
|
209
|
+
- AUTHORIZATION_PENDING:派发前授权待决,由 LoopFacts.authorization_wait 驱动复查;
|
|
210
|
+
- WORKER_SATURATED:工人槽位耗尽(worker_wait 为建议等待),调用方退出填池;
|
|
156
211
|
- NO_CANDIDATE:无可运行作业,停摆事实随 standstill 供仲裁。
|
|
157
212
|
"""
|
|
158
213
|
store = self._store
|
|
159
|
-
|
|
160
|
-
|
|
214
|
+
auth_status = self._authorization.poll_dispatch_authorization()
|
|
215
|
+
if auth_status is AuthorizationStatus.STOPPED:
|
|
216
|
+
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
217
|
+
if auth_status is AuthorizationStatus.PENDING:
|
|
218
|
+
return DispatchOutcome(kind=DispatchKind.AUTHORIZATION_PENDING)
|
|
219
|
+
if auth_status in (
|
|
220
|
+
AuthorizationStatus.DENIED,
|
|
221
|
+
AuthorizationStatus.PREPARATION_FAILED,
|
|
222
|
+
):
|
|
161
223
|
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
162
|
-
|
|
163
|
-
pending = self._authorization_pending
|
|
164
|
-
if pending is not None:
|
|
165
|
-
run_id, pending_uid, name = pending
|
|
166
|
-
self._authorization_pending = None
|
|
167
|
-
if run_id == self._session.run_id and pending_uid in store.state.queue_uids:
|
|
168
|
-
pending_idx = next(
|
|
169
|
-
index for index, snapshot in enumerate(store.state.queue)
|
|
170
|
-
if Job.from_dict(snapshot).uid == pending_uid
|
|
171
|
-
)
|
|
172
|
-
try:
|
|
173
|
-
pending_files = self._resolve_job_files(
|
|
174
|
-
Job.from_dict(store.state.queue[pending_idx])
|
|
175
|
-
)
|
|
176
|
-
path = pending_files[name].path
|
|
177
|
-
assert path is not None
|
|
178
|
-
except Exception as exc:
|
|
179
|
-
snapshot = store.state.pop_job(pending_idx)
|
|
180
|
-
self._reject_and_commit(
|
|
181
|
-
pending_uid, snapshot, {
|
|
182
|
-
"error_message": str(exc),
|
|
183
|
-
"exception": {
|
|
184
|
-
"type": type(exc).__name__, "message": str(exc),
|
|
185
|
-
"traceback": traceback.format_exc(),
|
|
186
|
-
},
|
|
187
|
-
},
|
|
188
|
-
count_as="failed",
|
|
189
|
-
)
|
|
190
|
-
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
191
|
-
authorized = store.check_or_request_authorization(path)
|
|
192
|
-
if not self._dispatch_allowed():
|
|
193
|
-
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
194
|
-
if authorized is None:
|
|
195
|
-
self._authorization_pending = pending
|
|
196
|
-
return DispatchOutcome(
|
|
197
|
-
kind=DispatchKind.AUTHORIZATION_PENDING,
|
|
198
|
-
worker_wait=AUTHORIZATION_RECHECK_SECONDS,
|
|
199
|
-
)
|
|
200
|
-
if authorized is False:
|
|
201
|
-
self._session.temporarily_exclude_chain(pending_uid)
|
|
202
|
-
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
203
224
|
|
|
204
225
|
sched = self._scheduler.scan_next_runnable(store.state, store.state.in_flight_uids)
|
|
205
226
|
if sched.runnable_idx is None:
|
|
@@ -220,18 +241,15 @@ class DispatchMachine:
|
|
|
220
241
|
if not self._dispatch_allowed():
|
|
221
242
|
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
222
243
|
entry = self.dispatch_job(sched)
|
|
223
|
-
if self.
|
|
224
|
-
return DispatchOutcome(
|
|
225
|
-
kind=DispatchKind.AUTHORIZATION_PENDING,
|
|
226
|
-
worker_wait=AUTHORIZATION_RECHECK_SECONDS,
|
|
227
|
-
)
|
|
244
|
+
if self._authorization.has_dispatch_pending:
|
|
245
|
+
return DispatchOutcome(kind=DispatchKind.AUTHORIZATION_PENDING)
|
|
228
246
|
if entry is None:
|
|
229
247
|
return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
|
|
230
248
|
return DispatchOutcome(kind=DispatchKind.SPAWNED, entry=entry)
|
|
231
249
|
|
|
232
250
|
def dispatch_job(self, sched: ScheduleResult) -> InFlightJob | None:
|
|
233
251
|
"""统一派发单个作业:依赖失败处理 / 统一准入求值 → 出队 → running 编号 →
|
|
234
|
-
两阶段资源租约 →
|
|
252
|
+
两阶段资源租约 → 前置清理 → 锁外冷启动 → 锁内放行与在飞原子登记。
|
|
235
253
|
|
|
236
254
|
返回 InFlightJob 条目;被预检拦截(已终结/跳过/已回队)时返回 None。
|
|
237
255
|
"""
|
|
@@ -241,32 +259,24 @@ class DispatchMachine:
|
|
|
241
259
|
|
|
242
260
|
# 1. 依赖终态分支:优先于去重/准入求值(失败或缺失阻断下游)。
|
|
243
261
|
if sched.kind == "dep_failed":
|
|
244
|
-
pending_dep_failure = sched.pending_dep_failure
|
|
245
262
|
job_snapshot = store.state.pop_job(sched.runnable_idx)
|
|
246
|
-
|
|
247
|
-
job_uid
|
|
248
|
-
self.dispatch_dep_failed(
|
|
249
|
-
job_uid, job_snapshot, pending_dep_failure
|
|
250
|
-
)
|
|
263
|
+
job_uid = Job.from_dict(job_snapshot).uid
|
|
264
|
+
self.dispatch_dep_failed(job_uid, job_snapshot, sched.pending_dep_failure)
|
|
251
265
|
return None
|
|
252
266
|
if sched.kind == "dep_missing":
|
|
253
267
|
job_snapshot = store.state.pop_job(sched.runnable_idx)
|
|
254
268
|
job_uid = Job.from_dict(job_snapshot).uid
|
|
255
|
-
self.dispatch_dep_missing(
|
|
256
|
-
job_uid,
|
|
257
|
-
job_snapshot,
|
|
258
|
-
sched.pending_dep_missing,
|
|
259
|
-
)
|
|
269
|
+
self.dispatch_dep_missing(job_uid, job_snapshot, sched.pending_dep_missing)
|
|
260
270
|
return None
|
|
261
271
|
|
|
262
272
|
# 2. 候选就绪:求值前不 pop 丢队,异常原位留队
|
|
263
273
|
peek_snapshot = store.state.queue[sched.runnable_idx]
|
|
264
274
|
if peek_snapshot.get("trigger") is None:
|
|
265
|
-
|
|
275
|
+
store.finalize_intake_snapshot(peek_snapshot, peek_snapshot["task_type"])
|
|
266
276
|
job = Job.from_dict(peek_snapshot)
|
|
267
277
|
job_uid = job.uid
|
|
268
278
|
|
|
269
|
-
# 依赖就绪后、Trigger判断前解析预声明文件
|
|
279
|
+
# 依赖就绪后、Trigger 判断前解析预声明文件
|
|
270
280
|
resolved_files: dict[str, FileSpec] = {}
|
|
271
281
|
if job.files:
|
|
272
282
|
try:
|
|
@@ -279,46 +289,32 @@ class DispatchMachine:
|
|
|
279
289
|
{
|
|
280
290
|
"error_message": str(err),
|
|
281
291
|
"exception": {
|
|
282
|
-
"type": type(err).__name__,
|
|
292
|
+
"type": type(err).__name__,
|
|
293
|
+
"message": str(err),
|
|
283
294
|
"traceback": traceback.format_exc(),
|
|
284
295
|
},
|
|
285
296
|
},
|
|
286
|
-
count_as="failed",
|
|
287
297
|
)
|
|
288
298
|
return None
|
|
289
299
|
|
|
290
300
|
# 准入求值(仅显式 retry 登记的同 Run 候选跳过触发器)
|
|
291
|
-
|
|
292
|
-
if not is_session_retry:
|
|
301
|
+
if not self._session.is_retry_candidate(job_uid):
|
|
293
302
|
should_run = self._trigger_registry.admit(
|
|
294
303
|
peek_snapshot, store.state, baseline_lookup=store.trusted_last_succeeded_at_us
|
|
295
304
|
)
|
|
296
305
|
if not should_run:
|
|
297
|
-
job_snapshot
|
|
298
|
-
store.apply_skip(job_uid, job_snapshot=job_snapshot)
|
|
299
|
-
store.state.mark_trigger_skipped(job_uid)
|
|
306
|
+
store.apply_skip(job_uid, job_snapshot=peek_snapshot)
|
|
300
307
|
return None
|
|
301
308
|
|
|
302
309
|
if job.task_type not in self._tasks:
|
|
303
310
|
job_snapshot = store.state.pop_job(sched.runnable_idx)
|
|
304
|
-
if self.dispatch_no_handler(
|
|
305
|
-
job_uid, job_snapshot, job.task_type
|
|
306
|
-
):
|
|
311
|
+
if self.dispatch_no_handler(job_uid, job_snapshot, job.task_type):
|
|
307
312
|
return None
|
|
308
313
|
|
|
309
314
|
# 必要路径授权(未 running 不造 Attempt,授权拒绝本 Run 临时排除该作业及下游)
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
authorized = store.check_or_request_authorization(fspec.path)
|
|
314
|
-
if not self._dispatch_allowed():
|
|
315
|
-
return None
|
|
316
|
-
if authorized is None:
|
|
317
|
-
self._authorization_pending = (self._session.run_id, job_uid, fname)
|
|
318
|
-
return None
|
|
319
|
-
if authorized is False:
|
|
320
|
-
self._session.temporarily_exclude_chain(job_uid)
|
|
321
|
-
return None
|
|
315
|
+
auth_status = self._authorization.authorize_dispatch_files(job_uid, resolved_files)
|
|
316
|
+
if auth_status is not AuthorizationStatus.APPROVED:
|
|
317
|
+
return None
|
|
322
318
|
|
|
323
319
|
# 再次检查全部资源(授权后的全部资源检查到连续获取完成之间,不插入其他分配或回收)
|
|
324
320
|
eval_res = self._resources.evaluate(job.task_type, job.resources)
|
|
@@ -329,8 +325,10 @@ class DispatchMachine:
|
|
|
329
325
|
self._reject_and_commit(
|
|
330
326
|
job_uid,
|
|
331
327
|
job_snapshot,
|
|
332
|
-
{
|
|
333
|
-
|
|
328
|
+
{
|
|
329
|
+
"error_message": "Resource requirements cannot be satisfied",
|
|
330
|
+
"error_details": eval_res.reason,
|
|
331
|
+
},
|
|
334
332
|
)
|
|
335
333
|
return None
|
|
336
334
|
if not eval_res.is_available:
|
|
@@ -339,133 +337,101 @@ class DispatchMachine:
|
|
|
339
337
|
if not self._dispatch_allowed():
|
|
340
338
|
return None
|
|
341
339
|
job_snapshot = store.state.pop_job(sched.runnable_idx)
|
|
342
|
-
|
|
343
|
-
store.state.requeue_jobs([job_snapshot], front=True)
|
|
344
|
-
return None
|
|
345
|
-
running_at = utc_now_iso()
|
|
346
|
-
attempt_no = 1
|
|
347
|
-
|
|
348
|
-
effective_run_id = self._session.run_id or "run"
|
|
349
|
-
execution_id = f"{effective_run_id}.{self._session.next_dispatch_no()}"
|
|
350
|
-
if not self._dispatch_allowed():
|
|
351
|
-
store.state.requeue_jobs([job_snapshot], front=True)
|
|
352
|
-
return None
|
|
340
|
+
guard = _DispatchStageGuard(self, job_uid=job_uid, job_snapshot=job_snapshot)
|
|
353
341
|
try:
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
)
|
|
342
|
+
# 屏障 1:出队后、落 running 前
|
|
343
|
+
guard.ensure_allowed()
|
|
344
|
+
|
|
345
|
+
running_at = utc_now_iso()
|
|
346
|
+
effective_run_id = self._session.run_id or "run"
|
|
347
|
+
execution_id = f"{effective_run_id}.{self._session.next_dispatch_no()}"
|
|
348
|
+
try:
|
|
349
|
+
attempt_no = store.mark_job_running(
|
|
350
|
+
job_uid,
|
|
351
|
+
execution_id=execution_id,
|
|
352
|
+
run_id=effective_run_id,
|
|
353
|
+
started_at=running_at,
|
|
354
|
+
)
|
|
355
|
+
except Exception as e:
|
|
356
|
+
logger.critical(
|
|
357
|
+
f"Failed to mark job running for {job_uid}: {e}; "
|
|
358
|
+
f"requeueing and aborting dispatch without starting handler."
|
|
359
|
+
)
|
|
360
|
+
raise
|
|
361
|
+
guard.execution_id = execution_id
|
|
362
|
+
guard.attempt_no = attempt_no
|
|
360
363
|
job._attempt_no = attempt_no
|
|
361
364
|
job_snapshot["attempt_no"] = attempt_no
|
|
362
365
|
job_snapshot["run_id"] = effective_run_id
|
|
363
366
|
job_snapshot["started_at"] = running_at
|
|
364
|
-
except Exception as e:
|
|
365
|
-
logger.critical(
|
|
366
|
-
f"Failed to mark job running for {job_uid}: {e}; "
|
|
367
|
-
f"requeueing and aborting dispatch without starting handler."
|
|
368
|
-
)
|
|
369
|
-
store.state.requeue_jobs([job_snapshot], front=True)
|
|
370
|
-
raise
|
|
371
367
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
job_uid=job_uid,
|
|
375
|
-
job_snapshot=job_snapshot,
|
|
376
|
-
execution_id=execution_id,
|
|
377
|
-
attempt_no=attempt_no,
|
|
378
|
-
)
|
|
379
|
-
return None
|
|
368
|
+
# 屏障 2:落 running 后、获取资源前
|
|
369
|
+
guard.ensure_allowed()
|
|
380
370
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
self._resources.release_all(acquired)
|
|
385
|
-
self._rollback_unstarted_running(
|
|
386
|
-
job_uid=job_uid,
|
|
387
|
-
job_snapshot=job_snapshot,
|
|
388
|
-
execution_id=execution_id,
|
|
389
|
-
attempt_no=attempt_no,
|
|
371
|
+
# 逐项获取资源(串行连续获取,异常属 Run 级框架故障直接上抛)
|
|
372
|
+
guard.acquired = self._resources.acquire_all(
|
|
373
|
+
job.task_type, job.resources, job_uid=job_uid
|
|
390
374
|
)
|
|
391
|
-
return None
|
|
392
375
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
for fspec in resolved_files.values()
|
|
396
|
-
if fspec.delete_existing and fspec.path
|
|
397
|
-
]
|
|
398
|
-
try:
|
|
399
|
-
_execute_pre_deletes(self._path_policy, pre_deletes)
|
|
400
|
-
except (OSError, ValueError, NotImplementedError) as exc:
|
|
401
|
-
entry = InFlightJob(
|
|
402
|
-
job_uid=job_uid,
|
|
403
|
-
job_snapshot=job_snapshot,
|
|
404
|
-
job=job,
|
|
405
|
-
acquired=acquired,
|
|
406
|
-
execution_id=execution_id,
|
|
407
|
-
attempt_no=attempt_no,
|
|
408
|
-
run_id=self._session.run_id or "run",
|
|
409
|
-
active_files=dict(resolved_files),
|
|
410
|
-
active_file_sources=dict(job.files),
|
|
411
|
-
)
|
|
412
|
-
self._in_flight.track(entry, state=store.state)
|
|
413
|
-
self._completion.settle_reaped([(entry, ExecutionResult(
|
|
414
|
-
message=f"Pre-deletion failed: {exc}",
|
|
415
|
-
exception={
|
|
416
|
-
"type": type(exc).__name__, "message": str(exc),
|
|
417
|
-
"traceback": traceback.format_exc(),
|
|
418
|
-
},
|
|
419
|
-
))])
|
|
420
|
-
return None
|
|
421
|
-
except BaseException:
|
|
422
|
-
self._resources.release_all(acquired)
|
|
423
|
-
raise
|
|
376
|
+
# 屏障 3:获取资源后、前置删除与启动准备前
|
|
377
|
+
guard.ensure_allowed()
|
|
424
378
|
|
|
425
|
-
|
|
426
|
-
self.
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
379
|
+
policy = self._store.path_policy
|
|
380
|
+
authorize_cb = self._store.backend.is_path_authorized
|
|
381
|
+
try:
|
|
382
|
+
for fspec in resolved_files.values():
|
|
383
|
+
if fspec.delete_existing and fspec.path:
|
|
384
|
+
policy.delete(
|
|
385
|
+
fspec.path,
|
|
386
|
+
allow_broken_leaf_symlink=True,
|
|
387
|
+
authorize=authorize_cb,
|
|
388
|
+
)
|
|
389
|
+
except (OSError, ValueError, NotImplementedError) as exc:
|
|
390
|
+
acquired = guard.acquired
|
|
391
|
+
guard.acquired = []
|
|
392
|
+
guard.commit()
|
|
393
|
+
entry = InFlightJob(
|
|
394
|
+
job_uid=job_uid,
|
|
395
|
+
job_snapshot=job_snapshot,
|
|
396
|
+
job=job,
|
|
397
|
+
acquired=acquired,
|
|
398
|
+
execution_id=execution_id,
|
|
399
|
+
attempt_no=attempt_no,
|
|
400
|
+
run_id=effective_run_id,
|
|
401
|
+
active_files=dict(resolved_files),
|
|
402
|
+
active_file_sources=dict(job.files),
|
|
403
|
+
)
|
|
404
|
+
self._in_flight.track(entry, state=store.state)
|
|
405
|
+
self._completion.settle_reaped(
|
|
406
|
+
[
|
|
407
|
+
(
|
|
408
|
+
entry,
|
|
409
|
+
ExecutionResult(
|
|
410
|
+
message=f"Pre-deletion failed: {exc}",
|
|
411
|
+
exception={
|
|
412
|
+
"type": type(exc).__name__,
|
|
413
|
+
"message": str(exc),
|
|
414
|
+
"traceback": traceback.format_exc(),
|
|
415
|
+
},
|
|
416
|
+
),
|
|
417
|
+
)
|
|
418
|
+
]
|
|
419
|
+
)
|
|
420
|
+
return None
|
|
434
421
|
|
|
435
|
-
try:
|
|
436
422
|
return self._spawn_entry(
|
|
437
|
-
|
|
438
|
-
job,
|
|
439
|
-
|
|
440
|
-
execution_id,
|
|
441
|
-
acquired,
|
|
442
|
-
attempt_no=attempt_no,
|
|
423
|
+
guard,
|
|
424
|
+
job=job,
|
|
425
|
+
effective_run_id=effective_run_id,
|
|
443
426
|
resolved_files=resolved_files,
|
|
444
427
|
)
|
|
428
|
+
except (_DispatchStopped, _WorkerLaunchCancelled):
|
|
429
|
+
guard.rollback_for_stop()
|
|
430
|
+
return None
|
|
445
431
|
except BaseException:
|
|
446
|
-
|
|
432
|
+
guard.cleanup_on_fault()
|
|
447
433
|
raise
|
|
448
434
|
|
|
449
|
-
def _rollback_unstarted_running(
|
|
450
|
-
self,
|
|
451
|
-
*,
|
|
452
|
-
job_uid: str,
|
|
453
|
-
job_snapshot: dict,
|
|
454
|
-
execution_id: str,
|
|
455
|
-
attempt_no: int,
|
|
456
|
-
) -> None:
|
|
457
|
-
"""停止门越过 running 提交时,将未启动执行回退为 pending 且不写 Attempt。"""
|
|
458
|
-
self._store.apply_pending(
|
|
459
|
-
job_uid,
|
|
460
|
-
job_snapshot,
|
|
461
|
-
execution_id=execution_id,
|
|
462
|
-
attempt_no=attempt_no,
|
|
463
|
-
run_id=self._session.run_id or "run",
|
|
464
|
-
started_at=None,
|
|
465
|
-
error_message="stop requested before worker launch",
|
|
466
|
-
record_attempt=False,
|
|
467
|
-
)
|
|
468
|
-
|
|
469
435
|
# ── 预检关卡 ──────────────────────────────────────────────────────
|
|
470
436
|
|
|
471
437
|
def dispatch_dep_failed(
|
|
@@ -476,21 +442,17 @@ class DispatchMachine:
|
|
|
476
442
|
) -> bool:
|
|
477
443
|
"""派发预检关 2——依赖失败(父任务已进失败档案 → 本任务依赖失败)。
|
|
478
444
|
|
|
479
|
-
返回 True = 已处理(依赖失败直接 commit
|
|
445
|
+
返回 True = 已处理(依赖失败直接 commit,含级联下游);
|
|
480
446
|
仅当 pending_dep_failure 非空时调用。事务异常原样传播。
|
|
481
447
|
"""
|
|
482
448
|
if pending_dep_failure is None:
|
|
483
449
|
return False
|
|
484
450
|
logger.warning(f"SKIP: {job_uid} (Dependency {pending_dep_failure} failed)")
|
|
485
451
|
failure_metadata: dict[str, Any] = {
|
|
486
|
-
"error_message":
|
|
452
|
+
"error_message": "Dependency failed or has no record",
|
|
487
453
|
"failed_dependency": pending_dep_failure,
|
|
488
454
|
}
|
|
489
|
-
|
|
490
|
-
# 「真实业务失败率」统计不被级联稀释。
|
|
491
|
-
self._reject_and_commit(
|
|
492
|
-
job_uid, job_snapshot, failure_metadata, count_as="cascade_failed"
|
|
493
|
-
)
|
|
455
|
+
self._reject_and_commit(job_uid, job_snapshot, failure_metadata)
|
|
494
456
|
return True
|
|
495
457
|
|
|
496
458
|
def dispatch_dep_missing(
|
|
@@ -506,16 +468,11 @@ class DispatchMachine:
|
|
|
506
468
|
f"SKIP: {job_uid} (Dependency {pending_dep_missing} has no record)"
|
|
507
469
|
)
|
|
508
470
|
failure_metadata: dict[str, Any] = {
|
|
509
|
-
"error_message":
|
|
471
|
+
"error_message": "Dependency failed or has no record",
|
|
510
472
|
"failed_dependency": pending_dep_missing,
|
|
511
473
|
"missing_dependency": pending_dep_missing,
|
|
512
474
|
}
|
|
513
|
-
self._reject_and_commit(
|
|
514
|
-
job_uid,
|
|
515
|
-
job_snapshot,
|
|
516
|
-
failure_metadata,
|
|
517
|
-
count_as="failed",
|
|
518
|
-
)
|
|
475
|
+
self._reject_and_commit(job_uid, job_snapshot, failure_metadata)
|
|
519
476
|
return True
|
|
520
477
|
|
|
521
478
|
def dispatch_no_handler(
|
|
@@ -530,7 +487,7 @@ class DispatchMachine:
|
|
|
530
487
|
task_type not in tasks 时调用。事务异常原样传播。
|
|
531
488
|
"""
|
|
532
489
|
logger.error(f"SKIP: {job_uid} (No handler for task_type '{task_type}')")
|
|
533
|
-
failure_metadata = {"error_message":
|
|
490
|
+
failure_metadata = {"error_message": "No registered handler for task type"}
|
|
534
491
|
self._reject_and_commit(job_uid, job_snapshot, failure_metadata)
|
|
535
492
|
return True
|
|
536
493
|
|
|
@@ -538,99 +495,65 @@ class DispatchMachine:
|
|
|
538
495
|
|
|
539
496
|
def _spawn_entry(
|
|
540
497
|
self,
|
|
541
|
-
|
|
542
|
-
job: Job,
|
|
543
|
-
job_snapshot: dict,
|
|
544
|
-
execution_id: str,
|
|
545
|
-
acquired: list[tuple[str, float]],
|
|
498
|
+
guard: _DispatchStageGuard,
|
|
546
499
|
*,
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
spawn 已启动而后续步骤异常时(entry 尚未登记在飞),此处清理
|
|
553
|
-
子进程句柄后原样上抛——崩溃信号与断言信号不清理(交由运行循环
|
|
554
|
-
崩溃网/断言网处置)。
|
|
555
|
-
"""
|
|
500
|
+
job: Job,
|
|
501
|
+
effective_run_id: str,
|
|
502
|
+
resolved_files: dict[str, FileSpec],
|
|
503
|
+
) -> InFlightJob:
|
|
504
|
+
"""锁外冷启动准备(停在放行门前)→ 微秒级锁内放行与在飞原子登记。"""
|
|
556
505
|
store = self._store
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
self._resources.release_all(acquired)
|
|
575
|
-
self._rollback_unstarted_running(
|
|
576
|
-
job_uid=job_uid,
|
|
577
|
-
job_snapshot=job_snapshot,
|
|
578
|
-
execution_id=execution_id,
|
|
579
|
-
attempt_no=attempt_no,
|
|
580
|
-
)
|
|
581
|
-
return None
|
|
582
|
-
started_monotonic = time.monotonic()
|
|
583
|
-
handle = self._channel.spawn(WorkerLaunchSpec(
|
|
584
|
-
handler=task.handler,
|
|
585
|
-
job=job,
|
|
586
|
-
job_context=ctx,
|
|
587
|
-
execution_id=execution_id,
|
|
588
|
-
timeout_seconds=float(
|
|
589
|
-
job.timeout_seconds if job.timeout_seconds is not None else 3600.0
|
|
590
|
-
),
|
|
591
|
-
run_id=self._session.run_id or "run",
|
|
592
|
-
attempt_no=attempt_no,
|
|
593
|
-
launch_guard=self._dispatch_allowed,
|
|
594
|
-
))
|
|
506
|
+
assert guard.execution_id is not None and guard.attempt_no is not None
|
|
507
|
+
task = self._tasks.lookup(job.task_type)
|
|
508
|
+
ctx = self._build_context(job, store.state, files=resolved_files)
|
|
509
|
+
logger.info(f"RUN: {guard.job_uid}")
|
|
510
|
+
guard.ensure_allowed()
|
|
511
|
+
|
|
512
|
+
spec = WorkerLaunchSpec(
|
|
513
|
+
handler=task.handler,
|
|
514
|
+
job=job,
|
|
515
|
+
job_context=ctx,
|
|
516
|
+
execution_id=guard.execution_id,
|
|
517
|
+
timeout_seconds=float(
|
|
518
|
+
job.timeout_seconds if job.timeout_seconds is not None else 3600.0
|
|
519
|
+
),
|
|
520
|
+
run_id=effective_run_id,
|
|
521
|
+
attempt_no=guard.attempt_no,
|
|
522
|
+
)
|
|
595
523
|
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
attempt_no=attempt_no,
|
|
524
|
+
# 阶段一(锁外):冷启动子进程、握手与进程树绑定,停在 b"GO" 栅栏前
|
|
525
|
+
guard.prepared = self._channel.prepare_spawn(spec)
|
|
526
|
+
|
|
527
|
+
# 屏障 4(微秒级短持锁):与 stop() 同步二次校验,放行并原子登记在飞
|
|
528
|
+
with self._dispatch_start_lock:
|
|
529
|
+
guard.ensure_allowed()
|
|
530
|
+
started_monotonic = time.monotonic()
|
|
531
|
+
handle = self._channel.release_spawn(guard.prepared)
|
|
532
|
+
guard.prepared = None
|
|
533
|
+
guard.committed_handle = handle
|
|
534
|
+
entry = InFlightJob(
|
|
535
|
+
job_uid=guard.job_uid,
|
|
536
|
+
job_snapshot=guard.job_snapshot,
|
|
537
|
+
job=job,
|
|
538
|
+
handle=handle,
|
|
539
|
+
started_monotonic=started_monotonic,
|
|
540
|
+
acquired=guard.acquired,
|
|
541
|
+
execution_id=guard.execution_id,
|
|
542
|
+
attempt_no=guard.attempt_no,
|
|
543
|
+
run_id=effective_run_id,
|
|
544
|
+
started_at=handle.started_at,
|
|
545
|
+
known_job_uids=frozenset(
|
|
546
|
+
store.state.queue_uids
|
|
547
|
+
| store.state.in_flight_uids
|
|
548
|
+
| store.state.success_uids
|
|
549
|
+
| store.state.failed_uids
|
|
550
|
+
),
|
|
551
|
+
active_files=dict(resolved_files),
|
|
552
|
+
active_file_sources=dict(job.files),
|
|
626
553
|
)
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
except BaseException:
|
|
631
|
-
if handle is not None:
|
|
632
|
-
self._channel.cleanup_in_flight([handle])
|
|
633
|
-
raise
|
|
554
|
+
self._in_flight.track(entry, state=store.state)
|
|
555
|
+
guard.commit()
|
|
556
|
+
return entry
|
|
634
557
|
|
|
635
558
|
def _build_context(
|
|
636
559
|
self,
|
|
@@ -646,9 +569,8 @@ class DispatchMachine:
|
|
|
646
569
|
frozenset(state.success_uids),
|
|
647
570
|
frozenset(state.failed_uids),
|
|
648
571
|
dict(state.cursors),
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
base_dir=self._trigger_registry.base_dir,
|
|
572
|
+
protected_roots=self._store.path_policy.protected_paths,
|
|
573
|
+
base_dir=self._store.base_dir,
|
|
652
574
|
known_job_uids=frozenset(
|
|
653
575
|
state.queue_uids
|
|
654
576
|
| state.in_flight_uids
|
|
@@ -657,25 +579,18 @@ class DispatchMachine:
|
|
|
657
579
|
),
|
|
658
580
|
artifact_descriptions={
|
|
659
581
|
uid: state.successes[uid].get("artifacts", {})
|
|
660
|
-
for uid in job.depends_on
|
|
582
|
+
for uid in job.depends_on
|
|
583
|
+
if uid in state.success_uids
|
|
661
584
|
},
|
|
662
585
|
files=files,
|
|
663
586
|
)
|
|
664
587
|
|
|
665
588
|
def _resolve_job_files(self, job: Job) -> dict[str, FileSpec]:
|
|
666
|
-
"""
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
resolved_path = bound_spec.path
|
|
672
|
-
if bound_spec.delete_existing or bound_spec.delete_on_success or bound_spec.delete_on_failure:
|
|
673
|
-
validated_path = self._path_policy.resolve(
|
|
674
|
-
resolved_path, delete_existing=bound_spec.delete_existing,
|
|
675
|
-
)
|
|
676
|
-
bound_spec = bound_spec.bind_output_path(validated_path)
|
|
677
|
-
resolved[name] = bound_spec
|
|
678
|
-
return resolved
|
|
589
|
+
"""准入前基于统一 PathPolicy 单次原子解析预声明文件。"""
|
|
590
|
+
return {
|
|
591
|
+
name: self._store.resolve_file_spec(spec, depends_on=job.depends_on)
|
|
592
|
+
for name, spec in job.files.items()
|
|
593
|
+
}
|
|
679
594
|
|
|
680
595
|
# ── 拒绝与异常分支 ────────────────────────────────────────────────
|
|
681
596
|
|
|
@@ -684,12 +599,12 @@ class DispatchMachine:
|
|
|
684
599
|
job_uid: str,
|
|
685
600
|
job_snapshot: dict,
|
|
686
601
|
meta: dict,
|
|
687
|
-
*,
|
|
688
|
-
count_as: str = "failed",
|
|
689
602
|
) -> None:
|
|
690
603
|
"""执行前拒绝统一登记失败;事务异常原样传播。"""
|
|
691
604
|
self._store.apply_failure(
|
|
692
|
-
job_uid,
|
|
605
|
+
job_uid,
|
|
606
|
+
meta,
|
|
607
|
+
job_snapshot=job_snapshot,
|
|
693
608
|
cascade=True,
|
|
694
609
|
)
|
|
695
610
|
|