tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. tasklite/__init__.py +42 -18
  2. tasklite/__main__.py +205 -0
  3. tasklite/_report.py +562 -0
  4. tasklite/backend/__init__.py +19 -2
  5. tasklite/backend/base.py +327 -156
  6. tasklite/backend/memory.py +261 -339
  7. tasklite/backend/sqlite_backend.py +416 -488
  8. tasklite/edit.py +1276 -0
  9. tasklite/engine/__init__.py +6 -7
  10. tasklite/engine/_worker_bootstrap.py +221 -0
  11. tasklite/engine/_worker_runtime.py +229 -0
  12. tasklite/engine/admission.py +20 -114
  13. tasklite/engine/authorization.py +989 -0
  14. tasklite/engine/channel.py +531 -468
  15. tasklite/engine/completion.py +193 -275
  16. tasklite/engine/config.py +5 -11
  17. tasklite/engine/dispatch.py +299 -384
  18. tasklite/engine/governor.py +134 -69
  19. tasklite/engine/in_flight.py +20 -88
  20. tasklite/engine/launcher.py +1173 -0
  21. tasklite/engine/recovery.py +147 -100
  22. tasklite/engine/resource.py +50 -103
  23. tasklite/engine/runtime.py +158 -525
  24. tasklite/engine/scheduler.py +9 -46
  25. tasklite/engine/session.py +60 -15
  26. tasklite/engine/store.py +235 -462
  27. tasklite/engine/types.py +29 -40
  28. tasklite/engine/wait.py +8 -0
  29. tasklite/exceptions.py +0 -1
  30. tasklite/models/artifact.py +84 -62
  31. tasklite/models/context.py +37 -30
  32. tasklite/models/graph.py +114 -0
  33. tasklite/models/job.py +3 -5
  34. tasklite/models/records.py +482 -0
  35. tasklite/models/state.py +19 -125
  36. tasklite/models/trigger.py +63 -3
  37. tasklite/pipeline.py +92 -100
  38. tasklite/resource.py +79 -2
  39. tasklite/snapshot.py +608 -405
  40. tasklite/testing.py +424 -11
  41. tasklite/utils/artifact.py +1091 -179
  42. tasklite/utils/encoding.py +21 -40
  43. tasklite/utils/ipc.py +49 -1
  44. tasklite/utils/lockfile.py +50 -19
  45. tasklite/wrappers/__init__.py +1 -1
  46. tasklite/wrappers/discovery.py +13 -16
  47. tasklite/wrappers/http.py +77 -43
  48. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
  49. tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
  50. tasklite/contrib/__init__.py +0 -7
  51. tasklite/engine/execution_access.py +0 -39
  52. tasklite/engine/files.py +0 -52
  53. tasklite/engine/ops.py +0 -25
  54. tasklite/engine/selection.py +0 -186
  55. tasklite/hooks.py +0 -49
  56. tasklite/triggers.py +0 -47
  57. tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
  58. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
  59. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
  60. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,9 @@
1
- """v2 派发机器:四关预检、两阶段资源租约与子进程 spawn 编排。
1
+ """v2 派发机器:预检、两阶段资源租约、阶段回滚守卫与子进程 spawn 编排。
2
2
 
3
- 预检顺序即契约(顺序即时序约束):dep-failed → admit/dedup → no-handler →
4
- pre-submit-cleanup。依赖以显式窄清单注入(无共享袋),经
5
- ``store`` 复用状态仓库深模块,不反向引用门面。
6
-
7
- ``session`` 直接使用内部 RunSession;``tasks`` 按具名 Protocol
8
- ``TaskSource`` 消费(TaskRegistry 自动满足)。``output_roots`` 与 config 声明同型
9
- (``Path | Sequence[Path] | None``,多根输出场景)。
3
+ 预检顺序即契约(顺序即时序约束):dep-failed/dep-missing → 预声明解析 →
4
+ admit/dedup → no-handler → 预授权 → 资源重检 → 落 running → 获取资源 →
5
+ 前置清理 → 锁外冷启动 → 锁内微秒放行。依赖以显式窄清单注入,经 ``store``
6
+ 复用状态仓库深模块,不反向引用门面。
10
7
 
11
8
  进入 running 时分配编号,结算时追加 Attempt:execution_id
12
9
  (``run_id.dispatch_no``)在本模块生成并随行落表——dispatch_no 每次派发递增,
@@ -20,48 +17,32 @@ import logging
20
17
  import threading
21
18
  import time
22
19
  import traceback
20
+ from collections.abc import Mapping
23
21
  from dataclasses import dataclass, field
24
- from typing import Any, Protocol, TYPE_CHECKING
22
+ from typing import Any, TYPE_CHECKING
25
23
 
26
24
  if TYPE_CHECKING:
27
- from collections.abc import Sequence
28
- from pathlib import Path
29
-
30
25
  from ..models.state import PipelineState
31
- from ..models.task import Task
26
+ from ..models.task import TaskRegistry
32
27
  from .admission import TriggerRegistry
33
- from .channel import ExecutionChannel
28
+ from .channel import ExecutionChannel, PreparedLaunch
34
29
  from .completion import CompletionMachine
35
- from .files import FileSpecResolver
36
30
  from .in_flight import InFlightTracker
37
- from .recovery import RecoveryOrchestrator
38
31
  from .resource import ResourceManager
39
32
  from .scheduler import JobScheduler
40
- from .store import StateStore
41
33
  from .session import RunSession
34
+ from .store import StateStore
42
35
 
43
- class TaskSource(Protocol):
44
- """派发机器消费的任务规格来源契约(TaskRegistry 自动满足)。"""
45
-
46
- def lookup(self, task_type: str) -> "Task": ...
47
-
48
- def __contains__(self, task_type: object) -> bool: ...
49
-
50
- import os
51
- from collections.abc import Mapping, Sequence
52
- from pathlib import Path
53
-
54
- from ..exceptions import TaskLiteError
55
- from ..models.artifact import ArtifactDeclaration, FileSpec
36
+ from ..models.artifact import FileSpec
56
37
  from ..models.context import JobContext
57
38
  from ..models.job import Job
58
- from ..utils.artifact import ArtifactPathPolicy, _execute_pre_deletes
59
- from .channel import ExecutionResult, WorkerLaunchSpec, _WorkerLaunchCancelled
39
+ from ..utils.clock import utc_now_iso
40
+ from .authorization import AuthorizationCoordinator
41
+ from .channel import ExecutionResult, WorkerLaunchSpec
60
42
  from .in_flight import InFlightJob
43
+ from .launcher import _WorkerLaunchCancelled
61
44
  from .scheduler import ScheduleResult, StandstillFacts
62
- from .types import StopMode
63
- from .wait import AUTHORIZATION_RECHECK_SECONDS
64
- from ..utils.clock import utc_now_iso
45
+ from .types import AuthorizationStatus, JobHandle, StopMode
65
46
 
66
47
  logger = logging.getLogger("tasklite")
67
48
 
@@ -69,11 +50,11 @@ logger = logging.getLogger("tasklite")
69
50
  class DispatchKind(enum.Enum):
70
51
  """一次调度派发周期的结局分类(单一事实,无交叠布尔)。"""
71
52
 
72
- SPAWNED = "spawned" # 子进程已启动并登记在飞
73
- HANDLED_NO_SUBPROCESS = "handled_no_subprocess" # 五关/校验拦截,无需子进程即终结
74
- AUTHORIZATION_PENDING = "authorization_pending"
75
- WORKER_SATURATED = "worker_saturated" # 工人槽位耗尽,退出填池等待
76
- NO_CANDIDATE = "no_candidate" # 队列无可运行候选(停摆事实随附)
53
+ SPAWNED = "spawned" # 子进程已启动并登记在飞
54
+ HANDLED_NO_SUBPROCESS = "handled_no_subprocess" # 预检/校验拦截,无需子进程即终结
55
+ AUTHORIZATION_PENDING = "authorization_pending" # 派发前路径授权待决
56
+ WORKER_SATURATED = "worker_saturated" # 工人槽位耗尽,退出填池等待
57
+ NO_CANDIDATE = "no_candidate" # 队列无可运行候选(停摆事实随附)
77
58
 
78
59
 
79
60
  @dataclass(frozen=True)
@@ -91,6 +72,88 @@ class DispatchOutcome:
91
72
  standstill: StandstillFacts = field(default_factory=StandstillFacts)
92
73
 
93
74
 
75
+ class _DispatchStopped(Exception):
76
+ """派发阶段内检测到停机请求,触发未启动状态回退的内部信号。"""
77
+
78
+
79
+ class _DispatchStageGuard:
80
+ """派发阶段回滚守卫:按已越过阶段精确回退。
81
+
82
+ - 停机路径(``rollback_for_stop``):中止未放行启动、释放资源,若已落 running
83
+ 则回退 pending 且不写 Attempt,否则原位插回队首;
84
+ - 故障路径(``cleanup_on_fault``):回收未放行启动/已提交句柄/已占资源;未落
85
+ running 前回队,已落 running 后保留磁盘 running 供下次启动恢复。
86
+ """
87
+
88
+ def __init__(
89
+ self,
90
+ machine: "DispatchMachine",
91
+ *,
92
+ job_uid: str,
93
+ job_snapshot: dict[str, Any],
94
+ ) -> None:
95
+ self._machine = machine
96
+ self.job_uid = job_uid
97
+ self.job_snapshot = job_snapshot
98
+ self.execution_id: str | None = None
99
+ self.attempt_no: int | None = None
100
+ self.acquired: list[tuple[str, float]] = []
101
+ self.prepared: "PreparedLaunch | None" = None
102
+ self.committed_handle: JobHandle | None = None
103
+ self._finalized: bool = False
104
+
105
+ def ensure_allowed(self) -> None:
106
+ """停机语义屏障:若已请求停机则立即退出本次派发并触发阶段回退。"""
107
+ if not self._machine._dispatch_allowed():
108
+ raise _DispatchStopped
109
+
110
+ def commit(self) -> None:
111
+ """移交在飞追踪或完成机器接管,后续退出不再触发回滚。"""
112
+ self._finalized = True
113
+
114
+ def rollback_for_stop(self) -> None:
115
+ """停机回退:按已越过阶段逆序清理并将未启动作业退回 pending(不写 Attempt)。"""
116
+ if self._finalized:
117
+ return
118
+ self._finalized = True
119
+ if self.prepared is not None:
120
+ self._machine._channel.abort_spawn(self.prepared)
121
+ self.prepared = None
122
+ if self.acquired:
123
+ acquired, self.acquired = self.acquired, []
124
+ self._machine._resources.release_all(acquired, job_uid=self.job_uid)
125
+ if self.execution_id is not None and self.attempt_no is not None:
126
+ self._machine._store.apply_pending(
127
+ self.job_uid,
128
+ self.job_snapshot,
129
+ execution_id=self.execution_id,
130
+ attempt_no=self.attempt_no,
131
+ run_id=self._machine._session.run_id or "run",
132
+ started_at=None,
133
+ error_message="stop requested before worker launch",
134
+ record_attempt=False,
135
+ )
136
+ else:
137
+ self._machine._store.state.requeue_jobs([self.job_snapshot], front=True)
138
+
139
+ def cleanup_on_fault(self) -> None:
140
+ """框架故障清理:回收启动/通道/资源;未入 running 前回队,已入 running 留待恢复。"""
141
+ if self._finalized:
142
+ return
143
+ self._finalized = True
144
+ if self.prepared is not None:
145
+ self._machine._channel.abort_spawn(self.prepared)
146
+ self.prepared = None
147
+ if self.committed_handle is not None:
148
+ self._machine._channel.cleanup_in_flight([self.committed_handle])
149
+ self.committed_handle = None
150
+ if self.acquired:
151
+ acquired, self.acquired = self.acquired, []
152
+ self._machine._resources.release_all(acquired, job_uid=self.job_uid)
153
+ if self.execution_id is None:
154
+ self._machine._store.state.requeue_jobs([self.job_snapshot], front=True)
155
+
156
+
94
157
  class DispatchMachine:
95
158
  """派发预检 + 资源租约 + running 编号 + 子进程 spawn 的编排器。"""
96
159
 
@@ -104,13 +167,10 @@ class DispatchMachine:
104
167
  channel: "ExecutionChannel",
105
168
  in_flight: "InFlightTracker",
106
169
  session: "RunSession",
107
- recovery: "RecoveryOrchestrator",
108
170
  completion: "CompletionMachine",
109
- tasks: "TaskSource",
110
- output_roots: "Path | Sequence[Path] | None",
111
- protected_roots: tuple[Path, ...] = (),
171
+ tasks: "TaskRegistry",
172
+ authorization: AuthorizationCoordinator | None = None,
112
173
  dispatch_start_lock: Any | None = None,
113
- file_resolver: "FileSpecResolver | None" = None,
114
174
  ) -> None:
115
175
  self._store = store
116
176
  self._scheduler = scheduler
@@ -119,21 +179,18 @@ class DispatchMachine:
119
179
  self._channel = channel
120
180
  self._in_flight = in_flight
121
181
  self._session = session
122
- self._recovery = recovery
123
182
  self._completion = completion
124
183
  self._tasks = tasks
125
- self._output_roots = output_roots
126
- self._protected_roots = protected_roots
127
- self._path_policy = ArtifactPathPolicy(
128
- output_roots=output_roots,
129
- protected_roots=protected_roots,
130
- authorization_check=lambda path, kind: self._store.backend.is_path_authorized(path, kind),
184
+ self._authorization = (
185
+ authorization
186
+ if authorization is not None
187
+ else AuthorizationCoordinator(
188
+ store=store,
189
+ channel=channel,
190
+ in_flight=in_flight,
191
+ session=session,
192
+ )
131
193
  )
132
- if file_resolver is None:
133
- from .files import FileSpecResolver
134
- file_resolver = FileSpecResolver(store)
135
- self._file_resolver = file_resolver
136
- self._authorization_pending: tuple[str | None, str, str] | None = None
137
194
  self._dispatch_start_lock = dispatch_start_lock or threading.RLock()
138
195
 
139
196
  # ── 扫描与派发接缝 ────────────────────────────────────────────────
@@ -143,63 +200,27 @@ class DispatchMachine:
143
200
  return self._session.stop_mode is StopMode.NONE
144
201
 
145
202
  def dispatch_next(self) -> DispatchOutcome:
146
- """统一扫描与派发接缝:工人槽位预检 → 队列扫描 → 五关预检 →
147
- 资源租约 → 子进程派发。
203
+ """统一扫描与派发接缝:派发前授权复查 → 队列扫描 → 工人槽位预检 →
204
+ 预检关卡 → 资源租约 → 子进程派发。
148
205
 
149
206
  返回 DispatchOutcome(kind 单一判据):
150
207
  - SPAWNED:entry 携带在飞条目,调用方继续收割;
151
- - HANDLED_NO_SUBPROCESS:作业被五关/校验拦截并已终结(跳过/
152
- 依赖失败/无 handler/校验失败/孤儿延迟/残留恢复),调用方可
153
- 继续填池;
154
- - WORKER_SATURATED:工人槽位耗尽(worker_wait 为建议等待),
155
- 调用方应退出填池循环;
208
+ - HANDLED_NO_SUBPROCESS:作业被预检/校验拦截并已终结,调用方可继续填池;
209
+ - AUTHORIZATION_PENDING:派发前授权待决,由 LoopFacts.authorization_wait 驱动复查;
210
+ - WORKER_SATURATED:工人槽位耗尽(worker_wait 为建议等待),调用方退出填池;
156
211
  - NO_CANDIDATE:无可运行作业,停摆事实随 standstill 供仲裁。
157
212
  """
158
213
  store = self._store
159
- if not self._dispatch_allowed():
160
- self._authorization_pending = None
214
+ auth_status = self._authorization.poll_dispatch_authorization()
215
+ if auth_status is AuthorizationStatus.STOPPED:
216
+ return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
217
+ if auth_status is AuthorizationStatus.PENDING:
218
+ return DispatchOutcome(kind=DispatchKind.AUTHORIZATION_PENDING)
219
+ if auth_status in (
220
+ AuthorizationStatus.DENIED,
221
+ AuthorizationStatus.PREPARATION_FAILED,
222
+ ):
161
223
  return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
162
-
163
- pending = self._authorization_pending
164
- if pending is not None:
165
- run_id, pending_uid, name = pending
166
- self._authorization_pending = None
167
- if run_id == self._session.run_id and pending_uid in store.state.queue_uids:
168
- pending_idx = next(
169
- index for index, snapshot in enumerate(store.state.queue)
170
- if Job.from_dict(snapshot).uid == pending_uid
171
- )
172
- try:
173
- pending_files = self._resolve_job_files(
174
- Job.from_dict(store.state.queue[pending_idx])
175
- )
176
- path = pending_files[name].path
177
- assert path is not None
178
- except Exception as exc:
179
- snapshot = store.state.pop_job(pending_idx)
180
- self._reject_and_commit(
181
- pending_uid, snapshot, {
182
- "error_message": str(exc),
183
- "exception": {
184
- "type": type(exc).__name__, "message": str(exc),
185
- "traceback": traceback.format_exc(),
186
- },
187
- },
188
- count_as="failed",
189
- )
190
- return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
191
- authorized = store.check_or_request_authorization(path)
192
- if not self._dispatch_allowed():
193
- return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
194
- if authorized is None:
195
- self._authorization_pending = pending
196
- return DispatchOutcome(
197
- kind=DispatchKind.AUTHORIZATION_PENDING,
198
- worker_wait=AUTHORIZATION_RECHECK_SECONDS,
199
- )
200
- if authorized is False:
201
- self._session.temporarily_exclude_chain(pending_uid)
202
- return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
203
224
 
204
225
  sched = self._scheduler.scan_next_runnable(store.state, store.state.in_flight_uids)
205
226
  if sched.runnable_idx is None:
@@ -220,18 +241,15 @@ class DispatchMachine:
220
241
  if not self._dispatch_allowed():
221
242
  return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
222
243
  entry = self.dispatch_job(sched)
223
- if self._authorization_pending:
224
- return DispatchOutcome(
225
- kind=DispatchKind.AUTHORIZATION_PENDING,
226
- worker_wait=AUTHORIZATION_RECHECK_SECONDS,
227
- )
244
+ if self._authorization.has_dispatch_pending:
245
+ return DispatchOutcome(kind=DispatchKind.AUTHORIZATION_PENDING)
228
246
  if entry is None:
229
247
  return DispatchOutcome(kind=DispatchKind.HANDLED_NO_SUBPROCESS)
230
248
  return DispatchOutcome(kind=DispatchKind.SPAWNED, entry=entry)
231
249
 
232
250
  def dispatch_job(self, sched: ScheduleResult) -> InFlightJob | None:
233
251
  """统一派发单个作业:依赖失败处理 / 统一准入求值 → 出队 → running 编号 →
234
- 两阶段资源租约 → 子进程启动 → 在飞原子登记。
252
+ 两阶段资源租约 → 前置清理 → 锁外冷启动 → 锁内放行与在飞原子登记。
235
253
 
236
254
  返回 InFlightJob 条目;被预检拦截(已终结/跳过/已回队)时返回 None。
237
255
  """
@@ -241,32 +259,24 @@ class DispatchMachine:
241
259
 
242
260
  # 1. 依赖终态分支:优先于去重/准入求值(失败或缺失阻断下游)。
243
261
  if sched.kind == "dep_failed":
244
- pending_dep_failure = sched.pending_dep_failure
245
262
  job_snapshot = store.state.pop_job(sched.runnable_idx)
246
- job = Job.from_dict(job_snapshot)
247
- job_uid = job.uid
248
- self.dispatch_dep_failed(
249
- job_uid, job_snapshot, pending_dep_failure
250
- )
263
+ job_uid = Job.from_dict(job_snapshot).uid
264
+ self.dispatch_dep_failed(job_uid, job_snapshot, sched.pending_dep_failure)
251
265
  return None
252
266
  if sched.kind == "dep_missing":
253
267
  job_snapshot = store.state.pop_job(sched.runnable_idx)
254
268
  job_uid = Job.from_dict(job_snapshot).uid
255
- self.dispatch_dep_missing(
256
- job_uid,
257
- job_snapshot,
258
- sched.pending_dep_missing,
259
- )
269
+ self.dispatch_dep_missing(job_uid, job_snapshot, sched.pending_dep_missing)
260
270
  return None
261
271
 
262
272
  # 2. 候选就绪:求值前不 pop 丢队,异常原位留队
263
273
  peek_snapshot = store.state.queue[sched.runnable_idx]
264
274
  if peek_snapshot.get("trigger") is None:
265
- self._trigger_registry.normalize_job_snapshot(peek_snapshot, peek_snapshot["task_type"])
275
+ store.finalize_intake_snapshot(peek_snapshot, peek_snapshot["task_type"])
266
276
  job = Job.from_dict(peek_snapshot)
267
277
  job_uid = job.uid
268
278
 
269
- # 依赖就绪后、Trigger判断前解析预声明文件
279
+ # 依赖就绪后、Trigger 判断前解析预声明文件
270
280
  resolved_files: dict[str, FileSpec] = {}
271
281
  if job.files:
272
282
  try:
@@ -279,46 +289,32 @@ class DispatchMachine:
279
289
  {
280
290
  "error_message": str(err),
281
291
  "exception": {
282
- "type": type(err).__name__, "message": str(err),
292
+ "type": type(err).__name__,
293
+ "message": str(err),
283
294
  "traceback": traceback.format_exc(),
284
295
  },
285
296
  },
286
- count_as="failed",
287
297
  )
288
298
  return None
289
299
 
290
300
  # 准入求值(仅显式 retry 登记的同 Run 候选跳过触发器)
291
- is_session_retry = self._session.is_retry_candidate(job_uid)
292
- if not is_session_retry:
301
+ if not self._session.is_retry_candidate(job_uid):
293
302
  should_run = self._trigger_registry.admit(
294
303
  peek_snapshot, store.state, baseline_lookup=store.trusted_last_succeeded_at_us
295
304
  )
296
305
  if not should_run:
297
- job_snapshot = peek_snapshot
298
- store.apply_skip(job_uid, job_snapshot=job_snapshot)
299
- store.state.mark_trigger_skipped(job_uid)
306
+ store.apply_skip(job_uid, job_snapshot=peek_snapshot)
300
307
  return None
301
308
 
302
309
  if job.task_type not in self._tasks:
303
310
  job_snapshot = store.state.pop_job(sched.runnable_idx)
304
- if self.dispatch_no_handler(
305
- job_uid, job_snapshot, job.task_type
306
- ):
311
+ if self.dispatch_no_handler(job_uid, job_snapshot, job.task_type):
307
312
  return None
308
313
 
309
314
  # 必要路径授权(未 running 不造 Attempt,授权拒绝本 Run 临时排除该作业及下游)
310
- for fname, fspec in resolved_files.items():
311
- if fspec.delete_existing or fspec.delete_on_success or fspec.delete_on_failure:
312
- assert fspec.path is not None
313
- authorized = store.check_or_request_authorization(fspec.path)
314
- if not self._dispatch_allowed():
315
- return None
316
- if authorized is None:
317
- self._authorization_pending = (self._session.run_id, job_uid, fname)
318
- return None
319
- if authorized is False:
320
- self._session.temporarily_exclude_chain(job_uid)
321
- return None
315
+ auth_status = self._authorization.authorize_dispatch_files(job_uid, resolved_files)
316
+ if auth_status is not AuthorizationStatus.APPROVED:
317
+ return None
322
318
 
323
319
  # 再次检查全部资源(授权后的全部资源检查到连续获取完成之间,不插入其他分配或回收)
324
320
  eval_res = self._resources.evaluate(job.task_type, job.resources)
@@ -329,8 +325,10 @@ class DispatchMachine:
329
325
  self._reject_and_commit(
330
326
  job_uid,
331
327
  job_snapshot,
332
- {"error_message": 'Resource requirements cannot be satisfied', "error_details": eval_res.reason},
333
- count_as="failed",
328
+ {
329
+ "error_message": "Resource requirements cannot be satisfied",
330
+ "error_details": eval_res.reason,
331
+ },
334
332
  )
335
333
  return None
336
334
  if not eval_res.is_available:
@@ -339,133 +337,101 @@ class DispatchMachine:
339
337
  if not self._dispatch_allowed():
340
338
  return None
341
339
  job_snapshot = store.state.pop_job(sched.runnable_idx)
342
- if not self._dispatch_allowed():
343
- store.state.requeue_jobs([job_snapshot], front=True)
344
- return None
345
- running_at = utc_now_iso()
346
- attempt_no = 1
347
-
348
- effective_run_id = self._session.run_id or "run"
349
- execution_id = f"{effective_run_id}.{self._session.next_dispatch_no()}"
350
- if not self._dispatch_allowed():
351
- store.state.requeue_jobs([job_snapshot], front=True)
352
- return None
340
+ guard = _DispatchStageGuard(self, job_uid=job_uid, job_snapshot=job_snapshot)
353
341
  try:
354
- attempt_no = store.mark_job_running(
355
- job_uid,
356
- execution_id=execution_id,
357
- run_id=effective_run_id,
358
- started_at=running_at,
359
- )
342
+ # 屏障 1:出队后、落 running 前
343
+ guard.ensure_allowed()
344
+
345
+ running_at = utc_now_iso()
346
+ effective_run_id = self._session.run_id or "run"
347
+ execution_id = f"{effective_run_id}.{self._session.next_dispatch_no()}"
348
+ try:
349
+ attempt_no = store.mark_job_running(
350
+ job_uid,
351
+ execution_id=execution_id,
352
+ run_id=effective_run_id,
353
+ started_at=running_at,
354
+ )
355
+ except Exception as e:
356
+ logger.critical(
357
+ f"Failed to mark job running for {job_uid}: {e}; "
358
+ f"requeueing and aborting dispatch without starting handler."
359
+ )
360
+ raise
361
+ guard.execution_id = execution_id
362
+ guard.attempt_no = attempt_no
360
363
  job._attempt_no = attempt_no
361
364
  job_snapshot["attempt_no"] = attempt_no
362
365
  job_snapshot["run_id"] = effective_run_id
363
366
  job_snapshot["started_at"] = running_at
364
- except Exception as e:
365
- logger.critical(
366
- f"Failed to mark job running for {job_uid}: {e}; "
367
- f"requeueing and aborting dispatch without starting handler."
368
- )
369
- store.state.requeue_jobs([job_snapshot], front=True)
370
- raise
371
367
 
372
- if not self._dispatch_allowed():
373
- self._rollback_unstarted_running(
374
- job_uid=job_uid,
375
- job_snapshot=job_snapshot,
376
- execution_id=execution_id,
377
- attempt_no=attempt_no,
378
- )
379
- return None
368
+ # 屏障 2:落 running 后、获取资源前
369
+ guard.ensure_allowed()
380
370
 
381
- # 逐项获取资源(串行连续获取,异常属 Run 级框架故障直接上抛)
382
- acquired = self._resources.acquire_all(job.task_type, job.resources, job_uid=job_uid)
383
- if not self._dispatch_allowed():
384
- self._resources.release_all(acquired)
385
- self._rollback_unstarted_running(
386
- job_uid=job_uid,
387
- job_snapshot=job_snapshot,
388
- execution_id=execution_id,
389
- attempt_no=attempt_no,
371
+ # 逐项获取资源(串行连续获取,异常属 Run 级框架故障直接上抛)
372
+ guard.acquired = self._resources.acquire_all(
373
+ job.task_type, job.resources, job_uid=job_uid
390
374
  )
391
- return None
392
375
 
393
- pre_deletes = [
394
- ArtifactDeclaration(fspec.path, delete_existing=True)
395
- for fspec in resolved_files.values()
396
- if fspec.delete_existing and fspec.path
397
- ]
398
- try:
399
- _execute_pre_deletes(self._path_policy, pre_deletes)
400
- except (OSError, ValueError, NotImplementedError) as exc:
401
- entry = InFlightJob(
402
- job_uid=job_uid,
403
- job_snapshot=job_snapshot,
404
- job=job,
405
- acquired=acquired,
406
- execution_id=execution_id,
407
- attempt_no=attempt_no,
408
- run_id=self._session.run_id or "run",
409
- active_files=dict(resolved_files),
410
- active_file_sources=dict(job.files),
411
- )
412
- self._in_flight.track(entry, state=store.state)
413
- self._completion.settle_reaped([(entry, ExecutionResult(
414
- message=f"Pre-deletion failed: {exc}",
415
- exception={
416
- "type": type(exc).__name__, "message": str(exc),
417
- "traceback": traceback.format_exc(),
418
- },
419
- ))])
420
- return None
421
- except BaseException:
422
- self._resources.release_all(acquired)
423
- raise
376
+ # 屏障 3:获取资源后、前置删除与启动准备前
377
+ guard.ensure_allowed()
424
378
 
425
- if not self._dispatch_allowed():
426
- self._resources.release_all(acquired)
427
- self._rollback_unstarted_running(
428
- job_uid=job_uid,
429
- job_snapshot=job_snapshot,
430
- execution_id=execution_id,
431
- attempt_no=attempt_no,
432
- )
433
- return None
379
+ policy = self._store.path_policy
380
+ authorize_cb = self._store.backend.is_path_authorized
381
+ try:
382
+ for fspec in resolved_files.values():
383
+ if fspec.delete_existing and fspec.path:
384
+ policy.delete(
385
+ fspec.path,
386
+ allow_broken_leaf_symlink=True,
387
+ authorize=authorize_cb,
388
+ )
389
+ except (OSError, ValueError, NotImplementedError) as exc:
390
+ acquired = guard.acquired
391
+ guard.acquired = []
392
+ guard.commit()
393
+ entry = InFlightJob(
394
+ job_uid=job_uid,
395
+ job_snapshot=job_snapshot,
396
+ job=job,
397
+ acquired=acquired,
398
+ execution_id=execution_id,
399
+ attempt_no=attempt_no,
400
+ run_id=effective_run_id,
401
+ active_files=dict(resolved_files),
402
+ active_file_sources=dict(job.files),
403
+ )
404
+ self._in_flight.track(entry, state=store.state)
405
+ self._completion.settle_reaped(
406
+ [
407
+ (
408
+ entry,
409
+ ExecutionResult(
410
+ message=f"Pre-deletion failed: {exc}",
411
+ exception={
412
+ "type": type(exc).__name__,
413
+ "message": str(exc),
414
+ "traceback": traceback.format_exc(),
415
+ },
416
+ ),
417
+ )
418
+ ]
419
+ )
420
+ return None
434
421
 
435
- try:
436
422
  return self._spawn_entry(
437
- job_uid,
438
- job,
439
- job_snapshot,
440
- execution_id,
441
- acquired,
442
- attempt_no=attempt_no,
423
+ guard,
424
+ job=job,
425
+ effective_run_id=effective_run_id,
443
426
  resolved_files=resolved_files,
444
427
  )
428
+ except (_DispatchStopped, _WorkerLaunchCancelled):
429
+ guard.rollback_for_stop()
430
+ return None
445
431
  except BaseException:
446
- self._resources.release_all(acquired)
432
+ guard.cleanup_on_fault()
447
433
  raise
448
434
 
449
- def _rollback_unstarted_running(
450
- self,
451
- *,
452
- job_uid: str,
453
- job_snapshot: dict,
454
- execution_id: str,
455
- attempt_no: int,
456
- ) -> None:
457
- """停止门越过 running 提交时,将未启动执行回退为 pending 且不写 Attempt。"""
458
- self._store.apply_pending(
459
- job_uid,
460
- job_snapshot,
461
- execution_id=execution_id,
462
- attempt_no=attempt_no,
463
- run_id=self._session.run_id or "run",
464
- started_at=None,
465
- error_message="stop requested before worker launch",
466
- record_attempt=False,
467
- )
468
-
469
435
  # ── 预检关卡 ──────────────────────────────────────────────────────
470
436
 
471
437
  def dispatch_dep_failed(
@@ -476,21 +442,17 @@ class DispatchMachine:
476
442
  ) -> bool:
477
443
  """派发预检关 2——依赖失败(父任务已进失败档案 → 本任务依赖失败)。
478
444
 
479
- 返回 True = 已处理(依赖失败直接 commit,含级联下游 + 钩子);
445
+ 返回 True = 已处理(依赖失败直接 commit,含级联下游);
480
446
  仅当 pending_dep_failure 非空时调用。事务异常原样传播。
481
447
  """
482
448
  if pending_dep_failure is None:
483
449
  return False
484
450
  logger.warning(f"SKIP: {job_uid} (Dependency {pending_dep_failure} failed)")
485
451
  failure_metadata: dict[str, Any] = {
486
- "error_message": 'Dependency failed or has no record',
452
+ "error_message": "Dependency failed or has no record",
487
453
  "failed_dependency": pending_dep_failure,
488
454
  }
489
- # 依赖父失败的级联下游计入 cascade_failed 而非 failed——
490
- # 「真实业务失败率」统计不被级联稀释。
491
- self._reject_and_commit(
492
- job_uid, job_snapshot, failure_metadata, count_as="cascade_failed"
493
- )
455
+ self._reject_and_commit(job_uid, job_snapshot, failure_metadata)
494
456
  return True
495
457
 
496
458
  def dispatch_dep_missing(
@@ -506,16 +468,11 @@ class DispatchMachine:
506
468
  f"SKIP: {job_uid} (Dependency {pending_dep_missing} has no record)"
507
469
  )
508
470
  failure_metadata: dict[str, Any] = {
509
- "error_message": 'Dependency failed or has no record',
471
+ "error_message": "Dependency failed or has no record",
510
472
  "failed_dependency": pending_dep_missing,
511
473
  "missing_dependency": pending_dep_missing,
512
474
  }
513
- self._reject_and_commit(
514
- job_uid,
515
- job_snapshot,
516
- failure_metadata,
517
- count_as="failed",
518
- )
475
+ self._reject_and_commit(job_uid, job_snapshot, failure_metadata)
519
476
  return True
520
477
 
521
478
  def dispatch_no_handler(
@@ -530,7 +487,7 @@ class DispatchMachine:
530
487
  task_type not in tasks 时调用。事务异常原样传播。
531
488
  """
532
489
  logger.error(f"SKIP: {job_uid} (No handler for task_type '{task_type}')")
533
- failure_metadata = {"error_message": 'No registered handler for task type'}
490
+ failure_metadata = {"error_message": "No registered handler for task type"}
534
491
  self._reject_and_commit(job_uid, job_snapshot, failure_metadata)
535
492
  return True
536
493
 
@@ -538,99 +495,65 @@ class DispatchMachine:
538
495
 
539
496
  def _spawn_entry(
540
497
  self,
541
- job_uid: str,
542
- job: Job,
543
- job_snapshot: dict,
544
- execution_id: str,
545
- acquired: list[tuple[str, float]],
498
+ guard: _DispatchStageGuard,
546
499
  *,
547
- attempt_no: int = 1,
548
- resolved_files: dict[str, FileSpec] | None = None,
549
- ) -> InFlightJob | None:
550
- """载荷校验 → 执行上下文组装 → spawn → 在飞登记。
551
-
552
- spawn 已启动而后续步骤异常时(entry 尚未登记在飞),此处清理
553
- 子进程句柄后原样上抛——崩溃信号与断言信号不清理(交由运行循环
554
- 崩溃网/断言网处置)。
555
- """
500
+ job: Job,
501
+ effective_run_id: str,
502
+ resolved_files: dict[str, FileSpec],
503
+ ) -> InFlightJob:
504
+ """锁外冷启动准备(停在放行门前)→ 微秒级锁内放行与在飞原子登记。"""
556
505
  store = self._store
557
- handle = None
558
- try:
559
- task = self._tasks.lookup(job.task_type)
560
- ctx = self._build_context(job, store.state, files=resolved_files)
561
- # 不变式:停止状态转移与 worker 启动、在飞登记共用同一同步门。
562
- with self._dispatch_start_lock:
563
- if not self._dispatch_allowed():
564
- self._resources.release_all(acquired)
565
- self._rollback_unstarted_running(
566
- job_uid=job_uid,
567
- job_snapshot=job_snapshot,
568
- execution_id=execution_id,
569
- attempt_no=attempt_no,
570
- )
571
- return None
572
- logger.info(f"RUN: {job_uid}")
573
- if not self._dispatch_allowed():
574
- self._resources.release_all(acquired)
575
- self._rollback_unstarted_running(
576
- job_uid=job_uid,
577
- job_snapshot=job_snapshot,
578
- execution_id=execution_id,
579
- attempt_no=attempt_no,
580
- )
581
- return None
582
- started_monotonic = time.monotonic()
583
- handle = self._channel.spawn(WorkerLaunchSpec(
584
- handler=task.handler,
585
- job=job,
586
- job_context=ctx,
587
- execution_id=execution_id,
588
- timeout_seconds=float(
589
- job.timeout_seconds if job.timeout_seconds is not None else 3600.0
590
- ),
591
- run_id=self._session.run_id or "run",
592
- attempt_no=attempt_no,
593
- launch_guard=self._dispatch_allowed,
594
- ))
506
+ assert guard.execution_id is not None and guard.attempt_no is not None
507
+ task = self._tasks.lookup(job.task_type)
508
+ ctx = self._build_context(job, store.state, files=resolved_files)
509
+ logger.info(f"RUN: {guard.job_uid}")
510
+ guard.ensure_allowed()
511
+
512
+ spec = WorkerLaunchSpec(
513
+ handler=task.handler,
514
+ job=job,
515
+ job_context=ctx,
516
+ execution_id=guard.execution_id,
517
+ timeout_seconds=float(
518
+ job.timeout_seconds if job.timeout_seconds is not None else 3600.0
519
+ ),
520
+ run_id=effective_run_id,
521
+ attempt_no=guard.attempt_no,
522
+ )
595
523
 
596
- entry = InFlightJob(
597
- job_uid=job_uid,
598
- job_snapshot=job_snapshot,
599
- job=job,
600
- handle=handle,
601
- started_monotonic=started_monotonic,
602
- acquired=acquired,
603
- execution_id=execution_id,
604
- attempt_no=attempt_no,
605
- run_id=self._session.run_id or "run",
606
- started_at=handle.started_at,
607
- known_job_uids=frozenset(
608
- store.state.queue_uids
609
- | store.state.in_flight_uids
610
- | store.state.success_uids
611
- | store.state.failed_uids
612
- ),
613
- active_files=dict(resolved_files or {}),
614
- active_file_sources=dict(job.files),
615
- )
616
- # 在 return 前原子登记到 in_flight 与 state 索引,避免时序真空
617
- self._in_flight.track(entry, state=store.state)
618
- return entry
619
- except _WorkerLaunchCancelled:
620
- self._resources.release_all(acquired)
621
- self._rollback_unstarted_running(
622
- job_uid=job_uid,
623
- job_snapshot=job_snapshot,
624
- execution_id=execution_id,
625
- attempt_no=attempt_no,
524
+ # 阶段一(锁外):冷启动子进程、握手与进程树绑定,停在 b"GO" 栅栏前
525
+ guard.prepared = self._channel.prepare_spawn(spec)
526
+
527
+ # 屏障 4(微秒级短持锁):与 stop() 同步二次校验,放行并原子登记在飞
528
+ with self._dispatch_start_lock:
529
+ guard.ensure_allowed()
530
+ started_monotonic = time.monotonic()
531
+ handle = self._channel.release_spawn(guard.prepared)
532
+ guard.prepared = None
533
+ guard.committed_handle = handle
534
+ entry = InFlightJob(
535
+ job_uid=guard.job_uid,
536
+ job_snapshot=guard.job_snapshot,
537
+ job=job,
538
+ handle=handle,
539
+ started_monotonic=started_monotonic,
540
+ acquired=guard.acquired,
541
+ execution_id=guard.execution_id,
542
+ attempt_no=guard.attempt_no,
543
+ run_id=effective_run_id,
544
+ started_at=handle.started_at,
545
+ known_job_uids=frozenset(
546
+ store.state.queue_uids
547
+ | store.state.in_flight_uids
548
+ | store.state.success_uids
549
+ | store.state.failed_uids
550
+ ),
551
+ active_files=dict(resolved_files),
552
+ active_file_sources=dict(job.files),
626
553
  )
627
- return None
628
- except AssertionError:
629
- raise
630
- except BaseException:
631
- if handle is not None:
632
- self._channel.cleanup_in_flight([handle])
633
- raise
554
+ self._in_flight.track(entry, state=store.state)
555
+ guard.commit()
556
+ return entry
634
557
 
635
558
  def _build_context(
636
559
  self,
@@ -646,9 +569,8 @@ class DispatchMachine:
646
569
  frozenset(state.success_uids),
647
570
  frozenset(state.failed_uids),
648
571
  dict(state.cursors),
649
- output_roots=self._output_roots,
650
- protected_roots=self._protected_roots,
651
- base_dir=self._trigger_registry.base_dir,
572
+ protected_roots=self._store.path_policy.protected_paths,
573
+ base_dir=self._store.base_dir,
652
574
  known_job_uids=frozenset(
653
575
  state.queue_uids
654
576
  | state.in_flight_uids
@@ -657,25 +579,18 @@ class DispatchMachine:
657
579
  ),
658
580
  artifact_descriptions={
659
581
  uid: state.successes[uid].get("artifacts", {})
660
- for uid in job.depends_on if uid in state.success_uids
582
+ for uid in job.depends_on
583
+ if uid in state.success_uids
661
584
  },
662
585
  files=files,
663
586
  )
664
587
 
665
588
  def _resolve_job_files(self, job: Job) -> dict[str, FileSpec]:
666
- """准入前解析预声明文件(普通路径绝对化、上游产物引用解析与输入存在性校验)。"""
667
- resolved: dict[str, FileSpec] = {}
668
- for name, spec in job.files.items():
669
- bound_spec = self._file_resolver.resolve(spec, depends_on=job.depends_on)
670
- assert bound_spec.path is not None
671
- resolved_path = bound_spec.path
672
- if bound_spec.delete_existing or bound_spec.delete_on_success or bound_spec.delete_on_failure:
673
- validated_path = self._path_policy.resolve(
674
- resolved_path, delete_existing=bound_spec.delete_existing,
675
- )
676
- bound_spec = bound_spec.bind_output_path(validated_path)
677
- resolved[name] = bound_spec
678
- return resolved
589
+ """准入前基于统一 PathPolicy 单次原子解析预声明文件。"""
590
+ return {
591
+ name: self._store.resolve_file_spec(spec, depends_on=job.depends_on)
592
+ for name, spec in job.files.items()
593
+ }
679
594
 
680
595
  # ── 拒绝与异常分支 ────────────────────────────────────────────────
681
596
 
@@ -684,12 +599,12 @@ class DispatchMachine:
684
599
  job_uid: str,
685
600
  job_snapshot: dict,
686
601
  meta: dict,
687
- *,
688
- count_as: str = "failed",
689
602
  ) -> None:
690
603
  """执行前拒绝统一登记失败;事务异常原样传播。"""
691
604
  self._store.apply_failure(
692
- job_uid, meta, job_snapshot=job_snapshot, count_as=count_as,
605
+ job_uid,
606
+ meta,
607
+ job_snapshot=job_snapshot,
693
608
  cascade=True,
694
609
  )
695
610