tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
tasklite/engine/runtime.py
CHANGED
|
@@ -1,64 +1,45 @@
|
|
|
1
|
-
"""EngineRuntime:v2 核心运行期深模块(装配 +
|
|
1
|
+
"""EngineRuntime:v2 核心运行期深模块(装配 + 薄四相事件泵)。
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
统一聚合主循环事件泵(四相:回收结算 → 填池派发 → 死锁仲裁 → 等待决策)、
|
|
4
|
+
机器群装配、停机信号管理与异常承重网。主进程文件声明与三类授权等待统一
|
|
5
|
+
下沉至 ``AuthorizationCoordinator``;一次 run 的生命周期与排除状态归属
|
|
6
|
+
``RunSession``。
|
|
7
7
|
"""
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
|
-
import os
|
|
12
11
|
import pickle
|
|
13
12
|
import signal
|
|
14
13
|
import threading
|
|
15
14
|
import time
|
|
16
15
|
import traceback
|
|
17
16
|
import uuid
|
|
17
|
+
from collections.abc import Iterator
|
|
18
|
+
from contextlib import contextmanager, nullcontext
|
|
18
19
|
from dataclasses import dataclass
|
|
19
20
|
from typing import Any
|
|
20
21
|
|
|
21
|
-
from .
|
|
22
|
-
from
|
|
23
|
-
from .
|
|
24
|
-
from .
|
|
22
|
+
from ..backend.base import AbstractStateBackend
|
|
23
|
+
from ..exceptions import TaskLiteError
|
|
24
|
+
from ..models.state import PipelineState
|
|
25
|
+
from ._worker_runtime import stop_signals
|
|
26
|
+
from .authorization import AuthorizationCoordinator
|
|
25
27
|
from .channel import ExecutionChannel
|
|
26
28
|
from .completion import CompletionMachine
|
|
27
|
-
from .
|
|
28
|
-
from .
|
|
29
|
+
from .config import RunConfig
|
|
30
|
+
from .dispatch import DispatchKind, DispatchMachine, DispatchOutcome
|
|
31
|
+
from .governor import SelectionStandstill
|
|
29
32
|
from .in_flight import InFlightTracker
|
|
30
33
|
from .recovery import RecoveryOrchestrator
|
|
31
|
-
from .
|
|
34
|
+
from .resource import RunNotifier
|
|
35
|
+
from .scheduler import JobScheduler
|
|
32
36
|
from .session import RunSession
|
|
33
37
|
from .store import StateStore
|
|
34
|
-
from .types import
|
|
35
|
-
AuthorizationOutcome,
|
|
36
|
-
AuthorizationStatus,
|
|
37
|
-
ExecutionOptions,
|
|
38
|
-
ExitReason,
|
|
39
|
-
JobHandle,
|
|
40
|
-
PendingDeclaration,
|
|
41
|
-
StepOutcome,
|
|
42
|
-
StopMode,
|
|
43
|
-
)
|
|
38
|
+
from .types import ExecutionOptions, StepOutcome, StopMode
|
|
44
39
|
from .wait import LoopFacts, decide_wait
|
|
45
|
-
from ..backend.base import AbstractStateBackend
|
|
46
|
-
from ..exceptions import TaskLiteError
|
|
47
|
-
from ..models.artifact import (
|
|
48
|
-
ArtifactDeclaration,
|
|
49
|
-
FileSpec,
|
|
50
|
-
merge_named_file_spec,
|
|
51
|
-
validate_file_name,
|
|
52
|
-
)
|
|
53
|
-
from ..models.job import Job
|
|
54
|
-
from ..models.state import PipelineState, job_uid_from_snapshot
|
|
55
|
-
from ..selection import Exclude
|
|
56
|
-
from ..utils.artifact import ArtifactPathPolicy, _execute_pre_deletes
|
|
57
40
|
|
|
58
41
|
logger = logging.getLogger("tasklite")
|
|
59
42
|
|
|
60
|
-
_REAL_SLEEP = time.sleep
|
|
61
|
-
|
|
62
43
|
# step() 单步派发上限缺省(显式上限由调用方传入)
|
|
63
44
|
_UNBOUNDED_DISPATCH_LIMIT = 1_000_000
|
|
64
45
|
|
|
@@ -83,46 +64,53 @@ class EngineRuntime:
|
|
|
83
64
|
def __init__(self, config: RunConfig) -> None:
|
|
84
65
|
self.config = config
|
|
85
66
|
self.tasks = config.tasks
|
|
86
|
-
self.scheduler = JobScheduler(
|
|
87
|
-
resources=config.resources, ordering_policy=config.ordering_policy
|
|
88
|
-
)
|
|
89
67
|
self.governor = config.governor
|
|
90
68
|
|
|
91
69
|
# 持久会话:execute() 经 begin() 复位(语义等价于每次新建会话,
|
|
92
70
|
# 同时保证机器持有的 session 引用跨 run 稳定)。
|
|
93
71
|
self._session = RunSession()
|
|
72
|
+
self.scheduler = JobScheduler(
|
|
73
|
+
resources=config.resources,
|
|
74
|
+
ordering_policy=config.ordering_policy,
|
|
75
|
+
session=self._session,
|
|
76
|
+
)
|
|
94
77
|
|
|
95
78
|
# 构造期即建 StateStore(enqueue 与管理写操作在 run 前可用)。
|
|
96
|
-
# backend 所有权唯一锚定 StateStore
|
|
97
|
-
# 期装配快照的搬运通道),运行期一切读写经 store.backend 派生。
|
|
79
|
+
# backend 所有权唯一锚定 StateStore,运行期一切读写经 store.backend 派生。
|
|
98
80
|
self.store: StateStore = StateStore(
|
|
99
81
|
config.backend,
|
|
82
|
+
tasks=config.tasks,
|
|
100
83
|
trigger_registry=config.trigger_registry,
|
|
101
84
|
requeue_policy=config.requeue_policy,
|
|
102
|
-
|
|
85
|
+
path_policy=config.path_policy,
|
|
86
|
+
session=self._session,
|
|
103
87
|
)
|
|
104
88
|
|
|
105
89
|
self.channel: ExecutionChannel = config.channel
|
|
106
90
|
self._in_flight: InFlightTracker = InFlightTracker()
|
|
107
91
|
self._notifier: RunNotifier = RunNotifier()
|
|
92
|
+
self._authorization = AuthorizationCoordinator(
|
|
93
|
+
store=self.store,
|
|
94
|
+
channel=self.channel,
|
|
95
|
+
in_flight=self._in_flight,
|
|
96
|
+
session=self._session,
|
|
97
|
+
)
|
|
108
98
|
|
|
109
|
-
# 构建机器依赖拓扑(recovery 依赖 completion
|
|
99
|
+
# 构建机器依赖拓扑(recovery 依赖 completion,三者共享授权协调器)
|
|
110
100
|
self._completion = CompletionMachine(
|
|
111
101
|
store=self.store,
|
|
112
|
-
trigger_registry=config.trigger_registry,
|
|
113
102
|
channel=self.channel,
|
|
114
103
|
resources=config.resources,
|
|
115
104
|
in_flight=self._in_flight,
|
|
116
105
|
session=self._session,
|
|
106
|
+
authorization=self._authorization,
|
|
117
107
|
)
|
|
118
108
|
self._recovery = RecoveryOrchestrator(
|
|
119
109
|
store=self.store,
|
|
120
110
|
channel=self.channel,
|
|
121
|
-
resources=config.resources,
|
|
122
111
|
in_flight=self._in_flight,
|
|
123
112
|
completion=self._completion,
|
|
124
113
|
)
|
|
125
|
-
self.file_resolver = FileSpecResolver(self.store)
|
|
126
114
|
self._dispatch_start_lock = threading.RLock()
|
|
127
115
|
self._dispatch = DispatchMachine(
|
|
128
116
|
store=self.store,
|
|
@@ -132,334 +120,32 @@ class EngineRuntime:
|
|
|
132
120
|
channel=self.channel,
|
|
133
121
|
in_flight=self._in_flight,
|
|
134
122
|
session=self._session,
|
|
135
|
-
recovery=self._recovery,
|
|
136
123
|
completion=self._completion,
|
|
137
124
|
tasks=config.tasks,
|
|
138
|
-
|
|
139
|
-
protected_roots=config.protected_roots,
|
|
125
|
+
authorization=self._authorization,
|
|
140
126
|
dispatch_start_lock=self._dispatch_start_lock,
|
|
141
|
-
file_resolver=self.file_resolver,
|
|
142
127
|
)
|
|
143
128
|
|
|
144
129
|
self._is_running: bool = False
|
|
145
|
-
self.
|
|
146
|
-
self.
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
self.channel.set_wakeup_callback(self._notifier.notify)
|
|
150
|
-
self._session.bind_exclude_chain_handler(self.temporarily_exclude_chain)
|
|
151
|
-
|
|
152
|
-
def temporarily_exclude(self, job_uid: str) -> None:
|
|
153
|
-
"""内部排除入口:登记单个临时排除 UID(供授权拒绝等复用)。"""
|
|
154
|
-
self._temporary_excluded_uids.add(job_uid)
|
|
155
|
-
self.scheduler.temporarily_exclude(job_uid)
|
|
156
|
-
self.store.temporarily_exclude(job_uid)
|
|
157
|
-
self.selection_machine.temporarily_exclude(job_uid)
|
|
158
|
-
|
|
159
|
-
def temporarily_exclude_chain(self, job_uid: str) -> None:
|
|
160
|
-
"""临时排除指定作业及其所有直接和间接下游作业。"""
|
|
161
|
-
self.temporarily_exclude(job_uid)
|
|
162
|
-
for down_uid in self.store.state.find_all_downstream(job_uid):
|
|
163
|
-
self.temporarily_exclude(down_uid)
|
|
164
|
-
|
|
165
|
-
def _handle_declaration(
|
|
166
|
-
self, handle: JobHandle, payload: dict[str, Any]
|
|
167
|
-
) -> dict[str, Any] | None:
|
|
168
|
-
"""主进程动态文件声明仲裁出口:验证引用、授权检查、前置清理并回送路径。"""
|
|
169
|
-
raw_req_id = payload.get("request_id")
|
|
170
|
-
if not isinstance(raw_req_id, str) or not raw_req_id:
|
|
171
|
-
return {
|
|
172
|
-
"request_id": "",
|
|
173
|
-
"status": "error",
|
|
174
|
-
"error": "request_id must be a non-empty str",
|
|
175
|
-
}
|
|
176
|
-
req_id = raw_req_id
|
|
177
|
-
name = payload.get("name", "")
|
|
178
|
-
spec_data = payload.get("spec", {})
|
|
179
|
-
entry = self._in_flight.get(handle.job_uid)
|
|
180
|
-
if entry is None or not self._declaration_identity_matches(handle, entry):
|
|
181
|
-
return {
|
|
182
|
-
"request_id": req_id,
|
|
183
|
-
"status": "error",
|
|
184
|
-
"error": f"Job {handle.job_uid} not in flight",
|
|
185
|
-
}
|
|
186
|
-
if handle.authorization_outcome is not None:
|
|
187
|
-
return {
|
|
188
|
-
"request_id": req_id,
|
|
189
|
-
"status": "error",
|
|
190
|
-
"error": f"Job {handle.job_uid} execution is no longer accepting declarations",
|
|
191
|
-
}
|
|
192
|
-
if handle.pending_declaration is not None:
|
|
193
|
-
return {
|
|
194
|
-
"request_id": req_id,
|
|
195
|
-
"status": "error",
|
|
196
|
-
"error": f"Job {handle.job_uid} already has a pending declaration",
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
try:
|
|
200
|
-
spec = FileSpec.from_dict(spec_data)
|
|
201
|
-
validated_name = validate_file_name(name)
|
|
202
|
-
except Exception as exc:
|
|
203
|
-
return {"request_id": req_id, "status": "error", "error": str(exc)}
|
|
204
|
-
|
|
205
|
-
# 1. 幂等与冲突检查
|
|
206
|
-
try:
|
|
207
|
-
merged, is_new = merge_named_file_spec(
|
|
208
|
-
entry.active_file_sources,
|
|
209
|
-
validated_name,
|
|
210
|
-
spec,
|
|
211
|
-
base_dir=self.store.base_dir,
|
|
212
|
-
depends_on=handle.job.depends_on,
|
|
213
|
-
)
|
|
214
|
-
except Exception as exc:
|
|
215
|
-
return {"request_id": req_id, "status": "error", "error": str(exc)}
|
|
216
|
-
source_spec = merged[validated_name]
|
|
217
|
-
if not is_new:
|
|
218
|
-
existing = entry.active_files.get(validated_name)
|
|
219
|
-
if existing is None or existing.path is None:
|
|
220
|
-
return {
|
|
221
|
-
"request_id": req_id,
|
|
222
|
-
"status": "error",
|
|
223
|
-
"error": f"File declaration {validated_name!r} has no resolved path",
|
|
224
|
-
}
|
|
225
|
-
return {
|
|
226
|
-
"request_id": req_id,
|
|
227
|
-
"status": "ok",
|
|
228
|
-
"resolved_path": existing.path,
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
# 2. 上游产物引用解析
|
|
232
|
-
try:
|
|
233
|
-
bound_spec = self.file_resolver.resolve(source_spec, depends_on=handle.job.depends_on)
|
|
234
|
-
except Exception as exc:
|
|
235
|
-
return {
|
|
236
|
-
"request_id": req_id,
|
|
237
|
-
"status": "error",
|
|
238
|
-
"error": str(exc),
|
|
239
|
-
}
|
|
240
|
-
assert bound_spec.path is not None
|
|
241
|
-
resolved_path = bound_spec.path
|
|
242
|
-
|
|
243
|
-
needs_authorization = (
|
|
244
|
-
bound_spec.delete_existing
|
|
245
|
-
or bound_spec.delete_on_success
|
|
246
|
-
or bound_spec.delete_on_failure
|
|
247
|
-
)
|
|
248
|
-
logger.info(
|
|
249
|
-
f"FILE DECLARATION: {handle.job_uid} requested "
|
|
250
|
-
f"{validated_name!r} at {resolved_path}"
|
|
251
|
-
)
|
|
252
|
-
request = PendingDeclaration(
|
|
253
|
-
request_id=req_id,
|
|
254
|
-
name=validated_name,
|
|
255
|
-
source=source_spec,
|
|
256
|
-
bound=bound_spec,
|
|
257
|
-
resolved_path=resolved_path,
|
|
258
|
-
)
|
|
259
|
-
if not needs_authorization:
|
|
260
|
-
self._record_declaration(entry, request)
|
|
261
|
-
request.status = AuthorizationStatus.APPROVED
|
|
262
|
-
logger.info(
|
|
263
|
-
f"FILE DECLARATION APPROVED: {handle.job_uid} {resolved_path}"
|
|
264
|
-
)
|
|
265
|
-
return self._declaration_reply(request)
|
|
266
|
-
|
|
267
|
-
handle.pending_declaration = request
|
|
268
|
-
handle.pause_timeout()
|
|
269
|
-
validated_path, error = self._validate_declaration_target(request)
|
|
270
|
-
if error is not None:
|
|
271
|
-
return self._finish_authorization(
|
|
272
|
-
handle,
|
|
273
|
-
entry,
|
|
274
|
-
request,
|
|
275
|
-
AuthorizationStatus.PREPARATION_FAILED,
|
|
276
|
-
error["message"],
|
|
277
|
-
error,
|
|
278
|
-
)
|
|
279
|
-
request.bound = request.bound.bind_output_path(validated_path)
|
|
280
|
-
request.resolved_path = validated_path
|
|
281
|
-
if self._session.stop_mode is not StopMode.NONE:
|
|
282
|
-
self._finish_authorization(
|
|
283
|
-
handle,
|
|
284
|
-
entry,
|
|
285
|
-
request,
|
|
286
|
-
AuthorizationStatus.STOPPED,
|
|
287
|
-
f"authorization wait stopped for {validated_path}",
|
|
288
|
-
)
|
|
289
|
-
return None
|
|
290
|
-
logger.info(
|
|
291
|
-
f"FILE AUTHORIZATION WAIT: {handle.job_uid} {validated_path}"
|
|
130
|
+
self.channel.set_declaration_handler(self._authorization.handle_declaration)
|
|
131
|
+
self.channel.set_authorization_queries(
|
|
132
|
+
is_timeout_paused=self._authorization.is_timeout_paused,
|
|
133
|
+
get_outcome=self._authorization.get_declaration_outcome,
|
|
292
134
|
)
|
|
293
|
-
|
|
294
|
-
if authorized is None:
|
|
295
|
-
return None
|
|
296
|
-
return self._apply_authorization_decision(handle, entry, request, authorized)
|
|
297
|
-
|
|
298
|
-
def _declaration_identity_matches(self, handle: JobHandle, entry: Any) -> bool:
|
|
299
|
-
"""请求句柄必须与当前在飞执行使用同一 job、run、attempt 与 execution 身份。"""
|
|
300
|
-
if entry.handle is not handle or entry.job_uid != handle.job_uid:
|
|
301
|
-
return False
|
|
302
|
-
if not handle.execution_id or entry.execution_id != handle.execution_id:
|
|
303
|
-
return False
|
|
304
|
-
if handle.run_id is not None and entry.run_id != handle.run_id:
|
|
305
|
-
return False
|
|
306
|
-
if handle.attempt_no is not None and entry.attempt_no != handle.attempt_no:
|
|
307
|
-
return False
|
|
308
|
-
if self._session.run_id is not None and entry.run_id != self._session.run_id:
|
|
309
|
-
return False
|
|
310
|
-
return True
|
|
311
|
-
|
|
312
|
-
def _record_declaration(self, entry: Any, request: PendingDeclaration) -> None:
|
|
313
|
-
entry.active_file_sources[request.name] = request.source
|
|
314
|
-
entry.active_files[request.name] = request.bound
|
|
315
|
-
|
|
316
|
-
def _validate_declaration_target(
|
|
317
|
-
self, request: PendingDeclaration
|
|
318
|
-
) -> tuple[str, dict[str, str] | None]:
|
|
319
|
-
policy = ArtifactPathPolicy(
|
|
320
|
-
output_roots=self.config.output_roots,
|
|
321
|
-
protected_roots=self.config.protected_roots,
|
|
322
|
-
)
|
|
323
|
-
try:
|
|
324
|
-
return policy.resolve(
|
|
325
|
-
request.resolved_path, delete_existing=request.bound.delete_existing,
|
|
326
|
-
), None
|
|
327
|
-
except (OSError, ValueError, NotImplementedError) as exc:
|
|
328
|
-
return request.resolved_path, {
|
|
329
|
-
"type": type(exc).__name__,
|
|
330
|
-
"message": str(exc),
|
|
331
|
-
"traceback": traceback.format_exc(),
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
def _apply_authorization_decision(
|
|
335
|
-
self,
|
|
336
|
-
handle: JobHandle,
|
|
337
|
-
entry: Any,
|
|
338
|
-
request: PendingDeclaration,
|
|
339
|
-
authorized: bool,
|
|
340
|
-
) -> dict[str, Any] | None:
|
|
341
|
-
if (
|
|
342
|
-
handle.pending_declaration is not request
|
|
343
|
-
or request.status is not AuthorizationStatus.PENDING
|
|
344
|
-
or handle.authorization_outcome is not None
|
|
345
|
-
):
|
|
346
|
-
return None
|
|
347
|
-
if self._session.stop_mode is not StopMode.NONE:
|
|
348
|
-
message = f"authorization wait stopped for {request.resolved_path}"
|
|
349
|
-
self._finish_authorization(
|
|
350
|
-
handle, entry, request, AuthorizationStatus.STOPPED, message
|
|
351
|
-
)
|
|
352
|
-
return {
|
|
353
|
-
"request_id": request.request_id,
|
|
354
|
-
"status": "stopped",
|
|
355
|
-
"error": message,
|
|
356
|
-
}
|
|
357
|
-
if authorized is False:
|
|
358
|
-
message = f"authorization denied for {request.resolved_path}"
|
|
359
|
-
self._finish_authorization(
|
|
360
|
-
handle, entry, request, AuthorizationStatus.DENIED, message
|
|
361
|
-
)
|
|
362
|
-
return {
|
|
363
|
-
"request_id": request.request_id,
|
|
364
|
-
"status": "denied",
|
|
365
|
-
"error": message,
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
if request.bound.delete_existing:
|
|
369
|
-
policy = ArtifactPathPolicy(
|
|
370
|
-
output_roots=self.config.output_roots,
|
|
371
|
-
protected_roots=self.config.protected_roots,
|
|
372
|
-
authorization_check=lambda path, kind: self.store.backend.is_path_authorized(path, kind),
|
|
373
|
-
)
|
|
374
|
-
try:
|
|
375
|
-
_execute_pre_deletes(
|
|
376
|
-
policy,
|
|
377
|
-
[ArtifactDeclaration(request.resolved_path, delete_existing=True)],
|
|
378
|
-
)
|
|
379
|
-
except (OSError, ValueError, NotImplementedError) as exc:
|
|
380
|
-
error = {
|
|
381
|
-
"type": type(exc).__name__,
|
|
382
|
-
"message": str(exc),
|
|
383
|
-
"traceback": traceback.format_exc(),
|
|
384
|
-
}
|
|
385
|
-
message = f"Artifact preparation failed for {request.resolved_path}: {exc}"
|
|
386
|
-
self._finish_authorization(
|
|
387
|
-
handle,
|
|
388
|
-
entry,
|
|
389
|
-
request,
|
|
390
|
-
AuthorizationStatus.PREPARATION_FAILED,
|
|
391
|
-
message,
|
|
392
|
-
error,
|
|
393
|
-
)
|
|
394
|
-
return None
|
|
395
|
-
|
|
396
|
-
request.status = AuthorizationStatus.APPROVED
|
|
397
|
-
handle.pending_declaration = None
|
|
398
|
-
handle.resume_timeout()
|
|
399
|
-
self._record_declaration(entry, request)
|
|
400
|
-
logger.info(
|
|
401
|
-
f"FILE AUTHORIZATION APPROVED: {handle.job_uid} {request.resolved_path}"
|
|
402
|
-
)
|
|
403
|
-
return self._declaration_reply(request)
|
|
404
|
-
|
|
405
|
-
def _finish_authorization(
|
|
406
|
-
self,
|
|
407
|
-
handle: JobHandle,
|
|
408
|
-
entry: Any,
|
|
409
|
-
request: PendingDeclaration,
|
|
410
|
-
status: AuthorizationStatus,
|
|
411
|
-
message: str,
|
|
412
|
-
exception: dict[str, str] | None = None,
|
|
413
|
-
) -> dict[str, Any] | None:
|
|
414
|
-
if (
|
|
415
|
-
handle.pending_declaration is not request
|
|
416
|
-
or handle.authorization_outcome is not None
|
|
417
|
-
):
|
|
418
|
-
return None
|
|
419
|
-
request.status = status
|
|
420
|
-
handle.pending_declaration = None
|
|
421
|
-
handle.authorization_outcome = AuthorizationOutcome(
|
|
422
|
-
status=status,
|
|
423
|
-
message=message,
|
|
424
|
-
path=request.resolved_path,
|
|
425
|
-
exception=exception,
|
|
426
|
-
)
|
|
427
|
-
handle.resume_timeout()
|
|
428
|
-
self._record_declaration(entry, request)
|
|
429
|
-
logger.info(
|
|
430
|
-
f"FILE AUTHORIZATION {status.value.upper()}: "
|
|
431
|
-
f"{handle.job_uid} {request.resolved_path} ({message})"
|
|
432
|
-
)
|
|
433
|
-
self.channel.terminate_process(handle)
|
|
434
|
-
return None
|
|
435
|
-
|
|
436
|
-
@staticmethod
|
|
437
|
-
def _declaration_reply(request: PendingDeclaration) -> dict[str, Any]:
|
|
438
|
-
return {
|
|
439
|
-
"request_id": request.request_id,
|
|
440
|
-
"status": "ok",
|
|
441
|
-
"resolved_path": request.resolved_path,
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
def is_job_excluded(self, job_uid: str, task_type: str) -> bool:
|
|
445
|
-
"""统一判断作业是否被排除(用户 exclude 规则或运行期临时排除)。"""
|
|
446
|
-
if job_uid in self._temporary_excluded_uids:
|
|
447
|
-
return True
|
|
448
|
-
if self._exclude is not None and self._exclude.matches_task_type(task_type):
|
|
449
|
-
return True
|
|
450
|
-
return False
|
|
135
|
+
self.channel.set_wakeup_callback(self._notifier.notify)
|
|
451
136
|
|
|
452
137
|
@property
|
|
453
138
|
def session(self) -> RunSession:
|
|
454
139
|
return self._session
|
|
455
140
|
|
|
456
141
|
@property
|
|
457
|
-
def
|
|
458
|
-
"""
|
|
142
|
+
def authorization(self) -> AuthorizationCoordinator:
|
|
143
|
+
"""主进程文件声明与三类授权等待的统一协调器。"""
|
|
144
|
+
return self._authorization
|
|
459
145
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
"""
|
|
146
|
+
@property
|
|
147
|
+
def backend(self) -> AbstractStateBackend:
|
|
148
|
+
"""持久化后端只读派生引用(所有权唯一锚定 StateStore)。"""
|
|
463
149
|
return self.store.backend
|
|
464
150
|
|
|
465
151
|
@property
|
|
@@ -478,23 +164,51 @@ class EngineRuntime:
|
|
|
478
164
|
def state(self) -> PipelineState:
|
|
479
165
|
return self.store.state
|
|
480
166
|
|
|
167
|
+
@staticmethod
|
|
168
|
+
def _stop_signals() -> tuple[signal.Signals, ...]:
|
|
169
|
+
"""返回当前平台需安装的优雅停机信号列表(委托唯一信号表)。"""
|
|
170
|
+
return stop_signals(signal)
|
|
171
|
+
|
|
172
|
+
@contextmanager
|
|
173
|
+
def install_stop_signals(self) -> Iterator[None]:
|
|
174
|
+
"""安装平台停机信号并按 LIFO 逆序恢复;安装中途失败自动回滚已装信号。"""
|
|
175
|
+
installed: list[tuple[signal.Signals, Any]] = []
|
|
176
|
+
try:
|
|
177
|
+
for sig in self._stop_signals():
|
|
178
|
+
previous = signal.signal(sig, self._handle_signal)
|
|
179
|
+
installed.append((sig, previous))
|
|
180
|
+
except BaseException:
|
|
181
|
+
for sig, previous in reversed(installed):
|
|
182
|
+
if previous is None:
|
|
183
|
+
continue
|
|
184
|
+
try:
|
|
185
|
+
signal.signal(sig, previous)
|
|
186
|
+
except Exception:
|
|
187
|
+
pass
|
|
188
|
+
raise
|
|
189
|
+
try:
|
|
190
|
+
yield
|
|
191
|
+
finally:
|
|
192
|
+
first_error: BaseException | None = None
|
|
193
|
+
for sig, previous in reversed(installed):
|
|
194
|
+
if previous is None:
|
|
195
|
+
continue
|
|
196
|
+
try:
|
|
197
|
+
signal.signal(sig, previous)
|
|
198
|
+
except BaseException as exc:
|
|
199
|
+
if first_error is None:
|
|
200
|
+
first_error = exc
|
|
201
|
+
if first_error is not None:
|
|
202
|
+
raise first_error
|
|
203
|
+
|
|
481
204
|
def request_stop(self) -> StopMode:
|
|
482
|
-
"""停机请求接口(单调状态转移,委托 RunSession
|
|
205
|
+
"""停机请求接口(单调状态转移,委托 RunSession,取消未提交启动并唤醒运行循环)。"""
|
|
483
206
|
with self._dispatch_start_lock:
|
|
484
207
|
if not self._is_running:
|
|
485
208
|
return self._session.stop_mode
|
|
486
209
|
mode = self._session.request_stop()
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
entry = self._in_flight.get(handle.job_uid)
|
|
490
|
-
if request is not None and entry is not None:
|
|
491
|
-
self._finish_authorization(
|
|
492
|
-
handle,
|
|
493
|
-
entry,
|
|
494
|
-
request,
|
|
495
|
-
AuthorizationStatus.STOPPED,
|
|
496
|
-
f"authorization wait stopped for {request.resolved_path}",
|
|
497
|
-
)
|
|
210
|
+
self.channel.cancel_pending_launches()
|
|
211
|
+
self._authorization.notify_stop()
|
|
498
212
|
self._notifier.notify()
|
|
499
213
|
return mode
|
|
500
214
|
|
|
@@ -506,47 +220,34 @@ class EngineRuntime:
|
|
|
506
220
|
with self._dispatch_start_lock:
|
|
507
221
|
if self._is_running:
|
|
508
222
|
raise TaskLiteError("Pipeline run() already in progress on this instance.")
|
|
509
|
-
self._session.begin()
|
|
223
|
+
self._session.begin(exclude=opts.exclude)
|
|
510
224
|
self._is_running = True
|
|
511
225
|
|
|
512
|
-
old_sigterm = None
|
|
513
|
-
old_sigint = None
|
|
514
226
|
try:
|
|
515
|
-
self.
|
|
516
|
-
self.
|
|
517
|
-
self.scheduler.reset_run(opts.exclude)
|
|
518
|
-
self.store.set_run_exclusions(opts.exclude, self._temporary_excluded_uids)
|
|
519
|
-
self.selection_machine.set_exclusions(opts.exclude, self._temporary_excluded_uids)
|
|
227
|
+
self._authorization.clear_all()
|
|
228
|
+
self.scheduler.begin_round()
|
|
520
229
|
|
|
521
|
-
with
|
|
230
|
+
with self.store.exclusive_access():
|
|
522
231
|
try:
|
|
523
|
-
#
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
232
|
+
# 主线程安装信号处理器;安装失败自动回滚,退出按 LIFO 逆序恢复。
|
|
233
|
+
with (
|
|
234
|
+
self.install_stop_signals()
|
|
235
|
+
if opts.install_signals
|
|
236
|
+
else nullcontext()
|
|
237
|
+
):
|
|
238
|
+
self._preflight_picklable_handlers()
|
|
239
|
+
self.governor.reset()
|
|
240
|
+
self._run_body()
|
|
531
241
|
finally:
|
|
532
242
|
self.config.resources.unbind_notifier()
|
|
533
243
|
finally:
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
if old_sigterm is not None:
|
|
537
|
-
signal.signal(signal.SIGTERM, old_sigterm)
|
|
538
|
-
finally:
|
|
539
|
-
if old_sigint is not None:
|
|
540
|
-
signal.signal(signal.SIGINT, old_sigint)
|
|
541
|
-
finally:
|
|
542
|
-
with self._dispatch_start_lock:
|
|
543
|
-
self._is_running = False
|
|
244
|
+
with self._dispatch_start_lock:
|
|
245
|
+
self._is_running = False
|
|
544
246
|
|
|
545
247
|
# ── 四相事件泵 ────────────────────────────────────────────────────
|
|
546
248
|
|
|
547
249
|
def _terminal_outcome(self) -> StepOutcome:
|
|
548
|
-
"""
|
|
549
|
-
全零计数 + should_terminate=True + 当前停机态与退出原因透传。"""
|
|
250
|
+
"""终态早退(排空完毕 / 空闲完成)的统一形状。"""
|
|
550
251
|
return StepOutcome(
|
|
551
252
|
dispatched_count=0,
|
|
552
253
|
settled_count=0,
|
|
@@ -562,21 +263,12 @@ class EngineRuntime:
|
|
|
562
263
|
def _fill_dispatch_pool(
|
|
563
264
|
self, limit: int, draining: bool
|
|
564
265
|
) -> tuple[DispatchOutcome | None, float, int]:
|
|
565
|
-
"""
|
|
566
|
-
SPAWNED / HANDLED_NO_SUBPROCESS 继续填池,WORKER_SATURATED /
|
|
567
|
-
NO_CANDIDATE 断流即停;worker_wait 聚合取 min(多次资源挂起恢复
|
|
568
|
-
取最早者)。
|
|
569
|
-
|
|
570
|
-
Returns:
|
|
571
|
-
(最后一次派发结果或 None, worker_wait 最小值或 0, 实派发计数)
|
|
572
|
-
"""
|
|
266
|
+
"""相二·填池派发(仅非 DRAINING、无在途授权等待且未超单步上限)。"""
|
|
573
267
|
last_outcome: DispatchOutcome | None = None
|
|
574
268
|
worker_wait = 0.0
|
|
575
269
|
dispatched = 0
|
|
576
270
|
if not draining:
|
|
577
|
-
if self.
|
|
578
|
-
h.waiting_auth for h in self._in_flight.active_handles()
|
|
579
|
-
):
|
|
271
|
+
if self._authorization.has_in_flight_authorization_wait:
|
|
580
272
|
return last_outcome, worker_wait, dispatched
|
|
581
273
|
while dispatched < limit and self._session.stop_mode is StopMode.NONE:
|
|
582
274
|
outcome = self._dispatch.dispatch_next()
|
|
@@ -598,7 +290,7 @@ class EngineRuntime:
|
|
|
598
290
|
def _arbitrate_deadlock(
|
|
599
291
|
self, last_outcome: DispatchOutcome | None, store: StateStore
|
|
600
292
|
) -> DeadlockArbitration:
|
|
601
|
-
"""
|
|
293
|
+
"""相三·无可运行候选时的死锁归因(委托 DeadlockGovernor 自闭环仲裁)。"""
|
|
602
294
|
deadlock_detected = False
|
|
603
295
|
should_terminate = False
|
|
604
296
|
wait_seconds = 0.0
|
|
@@ -606,23 +298,10 @@ class EngineRuntime:
|
|
|
606
298
|
if store.state.is_empty and not self._in_flight:
|
|
607
299
|
should_terminate = True
|
|
608
300
|
elif not self._in_flight:
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
store=self.store,
|
|
614
|
-
selection_machine=self.selection_machine,
|
|
615
|
-
)
|
|
616
|
-
except TypeError:
|
|
617
|
-
decision = self.governor.arbitrate(
|
|
618
|
-
last_outcome.standstill,
|
|
619
|
-
store=self.store,
|
|
620
|
-
)
|
|
621
|
-
else:
|
|
622
|
-
decision = self.governor.arbitrate(
|
|
623
|
-
last_outcome.standstill,
|
|
624
|
-
store=self.store,
|
|
625
|
-
)
|
|
301
|
+
decision = self.governor.arbitrate(
|
|
302
|
+
last_outcome.standstill,
|
|
303
|
+
store=self.store,
|
|
304
|
+
)
|
|
626
305
|
if decision.action in ("resolved", "selection_blocked"):
|
|
627
306
|
if decision.action == "selection_blocked":
|
|
628
307
|
if decision.standstill is not None:
|
|
@@ -631,7 +310,7 @@ class EngineRuntime:
|
|
|
631
310
|
deadlock_detected = True
|
|
632
311
|
if decision.should_terminate:
|
|
633
312
|
should_terminate = True
|
|
634
|
-
elif decision.action
|
|
313
|
+
elif decision.action == "gap_retrying":
|
|
635
314
|
if decision.wait_time > 0:
|
|
636
315
|
wait_seconds = decision.wait_time
|
|
637
316
|
return DeadlockArbitration(
|
|
@@ -654,62 +333,14 @@ class EngineRuntime:
|
|
|
654
333
|
"completed with the queue preserved."
|
|
655
334
|
)
|
|
656
335
|
|
|
657
|
-
def _poll_pending_authorizations(self) -> None:
|
|
658
|
-
"""轮询处于等待中的文件声明授权;若已决策则恢复执行或终止。"""
|
|
659
|
-
is_stopping = self._session.stop_mode is not StopMode.NONE
|
|
660
|
-
for handle in self._in_flight.active_handles():
|
|
661
|
-
request = handle.pending_declaration
|
|
662
|
-
if request is None or not handle.waiting_auth:
|
|
663
|
-
continue
|
|
664
|
-
entry = self._in_flight.get(handle.job_uid)
|
|
665
|
-
if entry is None or not self._declaration_identity_matches(handle, entry):
|
|
666
|
-
continue
|
|
667
|
-
|
|
668
|
-
if is_stopping:
|
|
669
|
-
self._finish_authorization(
|
|
670
|
-
handle,
|
|
671
|
-
entry,
|
|
672
|
-
request,
|
|
673
|
-
AuthorizationStatus.STOPPED,
|
|
674
|
-
f"authorization wait stopped for {request.resolved_path}",
|
|
675
|
-
)
|
|
676
|
-
continue
|
|
677
|
-
|
|
678
|
-
validated_path, error = self._validate_declaration_target(request)
|
|
679
|
-
if error is not None:
|
|
680
|
-
self._finish_authorization(
|
|
681
|
-
handle,
|
|
682
|
-
entry,
|
|
683
|
-
request,
|
|
684
|
-
AuthorizationStatus.PREPARATION_FAILED,
|
|
685
|
-
error["message"],
|
|
686
|
-
error,
|
|
687
|
-
)
|
|
688
|
-
continue
|
|
689
|
-
request.bound = request.bound.bind_output_path(validated_path)
|
|
690
|
-
request.resolved_path = validated_path
|
|
691
|
-
|
|
692
|
-
authorized = self.store.check_or_request_authorization(validated_path)
|
|
693
|
-
if authorized is not None:
|
|
694
|
-
reply = self._apply_authorization_decision(
|
|
695
|
-
handle, entry, request, authorized
|
|
696
|
-
)
|
|
697
|
-
if reply is None:
|
|
698
|
-
continue
|
|
699
|
-
reply_sender = self.channel.get_reply_sender(handle.execution_id)
|
|
700
|
-
if reply_sender is not None:
|
|
701
|
-
try:
|
|
702
|
-
reply_sender.send_reply(reply)
|
|
703
|
-
except OSError:
|
|
704
|
-
pass
|
|
705
|
-
|
|
706
336
|
def _drain_and_settle(self) -> int:
|
|
707
|
-
"""
|
|
337
|
+
"""相一·回收在途结果并统一结算,返回本轮完成计数(无在途为 0)。"""
|
|
708
338
|
if not self._in_flight:
|
|
709
339
|
return 0
|
|
710
340
|
handles = [
|
|
711
|
-
h
|
|
712
|
-
|
|
341
|
+
h
|
|
342
|
+
for h in self._in_flight.active_handles()
|
|
343
|
+
if not self._authorization.has_pending_cleanup(h.job_uid)
|
|
713
344
|
]
|
|
714
345
|
completed = self.channel.reap_completed(handles)
|
|
715
346
|
return self._completion.settle_reaped(completed)
|
|
@@ -719,19 +350,15 @@ class EngineRuntime:
|
|
|
719
350
|
deadlines = [
|
|
720
351
|
handle.timeout_at_monotonic
|
|
721
352
|
for handle in self._in_flight.active_handles()
|
|
722
|
-
if not
|
|
353
|
+
if not self._authorization.is_timeout_paused(handle)
|
|
354
|
+
and not self._authorization.has_pending_cleanup(handle.job_uid)
|
|
723
355
|
]
|
|
724
356
|
if not deadlines:
|
|
725
357
|
return None
|
|
726
358
|
return max(0.0, min(deadlines) - time.monotonic())
|
|
727
359
|
|
|
728
360
|
def step(self, max_dispatch: int | None = None) -> StepOutcome:
|
|
729
|
-
"""非阻塞单步推进事件泵(主循环与单步测试共用的统一事件泵)。
|
|
730
|
-
|
|
731
|
-
run 身份(run_id/空态装载)唯一负责者是 prepare_run_state(execute()
|
|
732
|
-
启动屏障)——store.state 恒非 None(构造与 set_state 双点归一化),
|
|
733
|
-
经 run() 外直达 step 的路径由测试显式 set_state 装载,无需惰性引导。
|
|
734
|
-
"""
|
|
361
|
+
"""非阻塞单步推进事件泵(主循环与单步测试共用的统一事件泵)。"""
|
|
735
362
|
store = self.store
|
|
736
363
|
|
|
737
364
|
# 0. 停机门:DRAINING 排空完毕与空闲完成早退
|
|
@@ -743,7 +370,7 @@ class EngineRuntime:
|
|
|
743
370
|
logger.info("Pipeline drained.")
|
|
744
371
|
return self._terminal_outcome()
|
|
745
372
|
|
|
746
|
-
self.
|
|
373
|
+
self._authorization.poll_pending_declarations()
|
|
747
374
|
|
|
748
375
|
if store.state.is_empty and not self._in_flight:
|
|
749
376
|
return self._terminal_outcome()
|
|
@@ -764,32 +391,43 @@ class EngineRuntime:
|
|
|
764
391
|
|
|
765
392
|
# 4. 等待/空闲决策(唯一实现见 wait.decide_wait)
|
|
766
393
|
is_idle = store.state.is_empty and not self._in_flight
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
394
|
+
decision = decide_wait(
|
|
395
|
+
LoopFacts(
|
|
396
|
+
stop_mode=self._session.stop_mode,
|
|
397
|
+
has_in_flight=bool(self._in_flight),
|
|
398
|
+
store_empty=store.state.is_empty,
|
|
399
|
+
dispatched=dispatched,
|
|
400
|
+
settled=settled_count,
|
|
401
|
+
has_runnable=(
|
|
402
|
+
last_outcome.kind is not DispatchKind.NO_CANDIDATE
|
|
403
|
+
if last_outcome is not None
|
|
404
|
+
else True
|
|
405
|
+
),
|
|
406
|
+
min_wait=(
|
|
407
|
+
last_outcome.standstill.min_wait
|
|
408
|
+
if last_outcome is not None
|
|
409
|
+
else float("inf")
|
|
410
|
+
),
|
|
411
|
+
in_flight_wait=self._next_in_flight_wait(),
|
|
412
|
+
authorization_wait=self._authorization.has_pending_authorization,
|
|
413
|
+
worker_wait=worker_wait,
|
|
414
|
+
deadlock_wait=arbitration.wait_seconds,
|
|
415
|
+
should_terminate=arbitration.should_terminate,
|
|
416
|
+
deadlock_detected=arbitration.deadlock_detected,
|
|
417
|
+
)
|
|
770
418
|
)
|
|
771
|
-
decision = decide_wait(LoopFacts(
|
|
772
|
-
stop_mode=self._session.stop_mode, has_in_flight=bool(self._in_flight),
|
|
773
|
-
store_empty=store.state.is_empty, dispatched=dispatched, settled=settled_count,
|
|
774
|
-
has_runnable=(last_outcome.kind is not DispatchKind.NO_CANDIDATE
|
|
775
|
-
if last_outcome is not None else True),
|
|
776
|
-
min_wait=(last_outcome.standstill.min_wait
|
|
777
|
-
if last_outcome is not None else float("inf")),
|
|
778
|
-
in_flight_wait=self._next_in_flight_wait(),
|
|
779
|
-
authorization_wait=authorization_wait,
|
|
780
|
-
worker_wait=worker_wait, deadlock_wait=arbitration.wait_seconds,
|
|
781
|
-
should_terminate=arbitration.should_terminate,
|
|
782
|
-
deadlock_detected=arbitration.deadlock_detected,
|
|
783
|
-
))
|
|
784
419
|
|
|
785
420
|
exit_reason = (
|
|
786
|
-
self._session.exit_reason().value
|
|
421
|
+
self._session.exit_reason().value
|
|
422
|
+
if (is_idle or arbitration.should_terminate)
|
|
787
423
|
else None
|
|
788
424
|
)
|
|
789
425
|
|
|
790
426
|
return StepOutcome(
|
|
791
|
-
dispatched_count=dispatched,
|
|
792
|
-
|
|
427
|
+
dispatched_count=dispatched,
|
|
428
|
+
settled_count=settled_count,
|
|
429
|
+
is_idle=is_idle,
|
|
430
|
+
should_wait=decision.should_wait,
|
|
793
431
|
wait_time=decision.wait_time,
|
|
794
432
|
deadlock_detected=arbitration.deadlock_detected,
|
|
795
433
|
stop_mode=self._session.stop_mode,
|
|
@@ -800,19 +438,15 @@ class EngineRuntime:
|
|
|
800
438
|
# ── 主循环与承重网 ────────────────────────────────────────────────
|
|
801
439
|
|
|
802
440
|
def _wait(self, timeout: float | None, last_seq: int | None = None) -> None:
|
|
803
|
-
"""
|
|
804
|
-
if last_seq is
|
|
805
|
-
|
|
806
|
-
else:
|
|
807
|
-
if time.sleep is not _REAL_SLEEP and timeout is not None and timeout > 0:
|
|
808
|
-
time.sleep(timeout)
|
|
809
|
-
self._notifier._event.wait(timeout=timeout)
|
|
810
|
-
self._notifier._event.clear()
|
|
441
|
+
"""运行循环调度等待:统一经 _notifier 实现跨线程与 check-to-wait 唤醒。"""
|
|
442
|
+
seq = self._notifier.snapshot() if last_seq is None else last_seq
|
|
443
|
+
self._notifier.wait(seq, timeout=timeout)
|
|
811
444
|
|
|
812
445
|
def run_loop(self) -> None:
|
|
813
446
|
"""事件驱动主循环:以 step() 统一驱动填池、回收与等待(裸循环)。"""
|
|
814
447
|
store = self.store
|
|
815
|
-
self._in_flight.
|
|
448
|
+
self._in_flight.clear_all(state=store.state)
|
|
449
|
+
self._authorization.clear_all()
|
|
816
450
|
self._notifier.clear()
|
|
817
451
|
while not store.state.is_empty or self._in_flight:
|
|
818
452
|
last_seq = self._notifier.snapshot()
|
|
@@ -828,9 +462,10 @@ class EngineRuntime:
|
|
|
828
462
|
elif store.state.is_empty and not self._in_flight:
|
|
829
463
|
break
|
|
830
464
|
|
|
831
|
-
logger.info(
|
|
465
|
+
logger.info("Pipeline finished.")
|
|
832
466
|
self.channel.wait_for_lingering_workers()
|
|
833
|
-
self._in_flight.
|
|
467
|
+
self._in_flight.clear_all(state=store.state)
|
|
468
|
+
self._authorization.clear_all()
|
|
834
469
|
|
|
835
470
|
def prepare_run_state(self) -> PipelineState:
|
|
836
471
|
"""恢复持久化状态,再登记本次 Run 身份。"""
|
|
@@ -854,8 +489,7 @@ class EngineRuntime:
|
|
|
854
489
|
self.run_loop()
|
|
855
490
|
except BaseException as e:
|
|
856
491
|
# 单点崩溃网:所有异常同构处理(在途清扫并原样上抛);
|
|
857
|
-
# KI 与非 Exception(如 SystemExit
|
|
858
|
-
# 不附 traceback,其余附。
|
|
492
|
+
# KI 与非 Exception(如 SystemExit)不附 traceback,其余附。
|
|
859
493
|
self._crash_log(e)
|
|
860
494
|
try:
|
|
861
495
|
self._recovery.abort_in_flight()
|
|
@@ -879,7 +513,6 @@ class EngineRuntime:
|
|
|
879
513
|
def _run_body(self) -> None:
|
|
880
514
|
"""主执行体:启动屏障(装载 + fencing)→ 带承重网的主循环。"""
|
|
881
515
|
self.prepare_run_state()
|
|
882
|
-
self._skip_dirty = True
|
|
883
516
|
self._run_with_crash_net()
|
|
884
517
|
|
|
885
518
|
def _handle_signal(self, signum: int, frame: Any) -> None:
|