tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
tasklite/engine/channel.py
CHANGED
|
@@ -1,46 +1,40 @@
|
|
|
1
|
-
"""v2
|
|
1
|
+
"""v2 执行通道:线性两阶段启动、单一会话状态机、阶梯看门狗、瞬时 Pipe 收割与 TOCTOU 中止。
|
|
2
2
|
|
|
3
|
-
ExecutionChannel
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
状态/线程/连接池半成品复制进子进程(引擎侧任何锁的持有状态都会在
|
|
8
|
-
子进程镜像里失真),spawn 注册表经 WorkerLaunchSpec 显式下发、不经
|
|
9
|
-
继承,进程 seam 的全部载荷具名可审计。
|
|
3
|
+
ExecutionChannel 以每执行单一会话对象(``_ExecutionSession``)统一管理
|
|
4
|
+
``prepared -> active -> lingering -> closed`` 全生命周期状态,通过
|
|
5
|
+
``prepare_spawn -> release_spawn / abort_spawn`` 线性两阶段协议驱动底层
|
|
6
|
+
``WorkerLauncher``,对真实子进程与进程内替身完全透明无特判。
|
|
10
7
|
"""
|
|
11
8
|
|
|
12
9
|
from __future__ import annotations
|
|
13
10
|
|
|
14
|
-
import
|
|
11
|
+
import enum
|
|
15
12
|
import logging
|
|
16
|
-
import multiprocessing as mp
|
|
17
|
-
import os
|
|
18
13
|
import signal
|
|
19
14
|
import threading
|
|
20
15
|
import time
|
|
21
|
-
import traceback
|
|
22
16
|
from collections.abc import Callable, Sequence
|
|
23
17
|
from dataclasses import dataclass, field
|
|
24
|
-
from pathlib import Path
|
|
25
18
|
from typing import Any
|
|
26
19
|
|
|
27
|
-
from ..exceptions import TaskLiteError
|
|
28
|
-
from ..models.context import JobContext
|
|
20
|
+
from ..exceptions import TaskLiteError
|
|
29
21
|
from ..models.job import Job
|
|
30
|
-
from ..models.result import
|
|
31
|
-
from ..utils.clock import utc_now_iso
|
|
22
|
+
from ..models.result import validate_strict_json_object
|
|
32
23
|
from ..utils.ipc import (
|
|
33
24
|
PIPE_MSG_DECLARE,
|
|
34
25
|
PIPE_MSG_RESULT,
|
|
35
26
|
WorkerPipeMessage,
|
|
36
27
|
WorkerPipeReceiver,
|
|
37
|
-
WorkerPipeSender,
|
|
38
|
-
WorkerReplyReceiver,
|
|
39
28
|
WorkerReplySender,
|
|
40
|
-
create_worker_pipe,
|
|
41
|
-
create_worker_reply_pipe,
|
|
42
29
|
)
|
|
43
|
-
from .
|
|
30
|
+
from ._worker_runtime import WorkerLaunchSpec
|
|
31
|
+
from .launcher import (
|
|
32
|
+
PreparedWorker,
|
|
33
|
+
SubprocessWorkerLauncher,
|
|
34
|
+
WorkerLauncher,
|
|
35
|
+
_WorkerLaunchCancelled,
|
|
36
|
+
)
|
|
37
|
+
from .types import AuthorizationOutcome, JobHandle, is_importing_mp_main
|
|
44
38
|
|
|
45
39
|
logger = logging.getLogger("tasklite")
|
|
46
40
|
|
|
@@ -254,113 +248,6 @@ def _decode_ipc_result(
|
|
|
254
248
|
return _corrupt_result(job, f"unknown status {status!r}")
|
|
255
249
|
|
|
256
250
|
|
|
257
|
-
def _isolate_worker_process_signals() -> None:
|
|
258
|
-
"""让进程组停止信号只由主进程处理。"""
|
|
259
|
-
if os.name != "posix":
|
|
260
|
-
return
|
|
261
|
-
setsid = getattr(os, "setsid", None)
|
|
262
|
-
if setsid is not None:
|
|
263
|
-
try:
|
|
264
|
-
setsid()
|
|
265
|
-
except OSError:
|
|
266
|
-
pass
|
|
267
|
-
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
268
|
-
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
def _worker_process_entry(spec: "WorkerLaunchSpec") -> None:
|
|
272
|
-
"""spawn 子进程执行体入口:执行 handler 并经瞬时管道发送结果。
|
|
273
|
-
|
|
274
|
-
不变式:结果通过管道单向回传;执行体任何出口(含未捕获异常)
|
|
275
|
-
均在 finally 统一关闭管道写端。
|
|
276
|
-
"""
|
|
277
|
-
_isolate_worker_process_signals()
|
|
278
|
-
handler_func = spec.handler
|
|
279
|
-
job = spec.job
|
|
280
|
-
if getattr(spec, "attempt_no", None) is not None:
|
|
281
|
-
job._attempt_no = spec.attempt_no
|
|
282
|
-
ctx = spec.job_context
|
|
283
|
-
job_uid = job.uid
|
|
284
|
-
execution_id = spec.execution_id
|
|
285
|
-
if not execution_id:
|
|
286
|
-
raise RuntimeError(
|
|
287
|
-
f"worker started without execution_id for {job_uid}: "
|
|
288
|
-
f"fencing requires WorkerLaunchSpec.execution_id"
|
|
289
|
-
)
|
|
290
|
-
pipe_sender = spec.pipe_sender
|
|
291
|
-
reply_receiver = spec.reply_receiver
|
|
292
|
-
if pipe_sender is not None or reply_receiver is not None:
|
|
293
|
-
ctx._bind_pipe_sender(pipe_sender, reply_receiver)
|
|
294
|
-
|
|
295
|
-
def _write_result(payload: dict[str, Any]) -> None:
|
|
296
|
-
if pipe_sender is None:
|
|
297
|
-
return
|
|
298
|
-
try:
|
|
299
|
-
pipe_sender.send_result(payload)
|
|
300
|
-
except OSError as exc:
|
|
301
|
-
logger.error(
|
|
302
|
-
f"worker pipe result send failed for {job_uid} ({execution_id}): {exc}"
|
|
303
|
-
)
|
|
304
|
-
|
|
305
|
-
try:
|
|
306
|
-
try:
|
|
307
|
-
raw_result = handler_func(job, ctx)
|
|
308
|
-
if isinstance(raw_result, TaskResult):
|
|
309
|
-
if raw_result.status == "success":
|
|
310
|
-
_write_result({
|
|
311
|
-
"status": "success",
|
|
312
|
-
"message": raw_result.message,
|
|
313
|
-
"metadata": raw_result.metadata,
|
|
314
|
-
"spawned_jobs": [j.to_dict() for j in ctx._spawned_jobs],
|
|
315
|
-
"cursor_updates": dict(ctx._cursor_updates),
|
|
316
|
-
})
|
|
317
|
-
elif raw_result.status == "failure":
|
|
318
|
-
_write_result({
|
|
319
|
-
"status": "failure",
|
|
320
|
-
"message": raw_result.message,
|
|
321
|
-
"metadata": raw_result.metadata,
|
|
322
|
-
"exception": None,
|
|
323
|
-
})
|
|
324
|
-
else:
|
|
325
|
-
_write_result({
|
|
326
|
-
"status": "retry",
|
|
327
|
-
"message": raw_result.message,
|
|
328
|
-
"metadata": raw_result.metadata,
|
|
329
|
-
"consume_budget": raw_result.consume_budget,
|
|
330
|
-
})
|
|
331
|
-
else:
|
|
332
|
-
_write_result({
|
|
333
|
-
"status": "failure",
|
|
334
|
-
"message": (
|
|
335
|
-
f"invalid handler return type: expected TaskResult, "
|
|
336
|
-
f"got {type(raw_result).__name__}"
|
|
337
|
-
),
|
|
338
|
-
"metadata": {},
|
|
339
|
-
"exception": None,
|
|
340
|
-
})
|
|
341
|
-
except _ArtifactIOError as e:
|
|
342
|
-
_write_result({"status": "artifact_error", "error_message": str(e)})
|
|
343
|
-
except BaseException as e:
|
|
344
|
-
_write_result({
|
|
345
|
-
"status": "failure",
|
|
346
|
-
"message": (
|
|
347
|
-
type(e).__name__ if not str(e)
|
|
348
|
-
else f"{type(e).__name__}: {e}"
|
|
349
|
-
),
|
|
350
|
-
"metadata": {},
|
|
351
|
-
"exception": {
|
|
352
|
-
"type": type(e).__name__,
|
|
353
|
-
"message": str(e),
|
|
354
|
-
"traceback": traceback.format_exc(),
|
|
355
|
-
},
|
|
356
|
-
})
|
|
357
|
-
finally:
|
|
358
|
-
if pipe_sender is not None:
|
|
359
|
-
pipe_sender.close()
|
|
360
|
-
if reply_receiver is not None:
|
|
361
|
-
reply_receiver.close()
|
|
362
|
-
|
|
363
|
-
|
|
364
251
|
@dataclass
|
|
365
252
|
class ExecutionResult:
|
|
366
253
|
"""单次子进程执行的收割产物(收割/中止路径共形)。"""
|
|
@@ -397,35 +284,37 @@ class ExecutionResult:
|
|
|
397
284
|
|
|
398
285
|
|
|
399
286
|
@dataclass(frozen=True)
|
|
400
|
-
class
|
|
401
|
-
"""
|
|
287
|
+
class PreparedLaunch:
|
|
288
|
+
"""已完成冷启动准备、等待主进程放行或中止的执行令牌。"""
|
|
402
289
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
"""
|
|
406
|
-
|
|
407
|
-
handler: Callable[[Job, JobContext], Any]
|
|
408
|
-
job: Job
|
|
409
|
-
job_context: JobContext
|
|
410
|
-
execution_id: str
|
|
411
|
-
timeout_seconds: float
|
|
412
|
-
pipe_sender: WorkerPipeSender | None = None
|
|
413
|
-
reply_receiver: WorkerReplyReceiver | None = None
|
|
414
|
-
run_id: str = ""
|
|
415
|
-
attempt_no: int = 1
|
|
416
|
-
launch_guard: Callable[[], bool] | None = None
|
|
290
|
+
spec: WorkerLaunchSpec
|
|
291
|
+
session_key: str
|
|
417
292
|
|
|
418
293
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
"""强制终止/异常停机时的收尾结果。"""
|
|
294
|
+
class _SessionState(str, enum.Enum):
|
|
295
|
+
"""单次 worker 执行会话的显式状态机(单向推进至 closed)。"""
|
|
422
296
|
|
|
423
|
-
|
|
424
|
-
|
|
297
|
+
PREPARED = "prepared"
|
|
298
|
+
ACTIVE = "active"
|
|
299
|
+
LINGERING = "lingering"
|
|
300
|
+
CLOSED = "closed"
|
|
425
301
|
|
|
426
302
|
|
|
427
|
-
|
|
428
|
-
|
|
303
|
+
@dataclass
|
|
304
|
+
class _ExecutionSession:
|
|
305
|
+
"""单次 worker 执行的唯一会话状态对象(取代多份并行字典与残留列表)。"""
|
|
306
|
+
|
|
307
|
+
session_key: str
|
|
308
|
+
state: _SessionState = _SessionState.PREPARED
|
|
309
|
+
cancel_event: threading.Event = field(default_factory=threading.Event)
|
|
310
|
+
prepared_worker: PreparedWorker | None = None
|
|
311
|
+
handle: JobHandle | None = None
|
|
312
|
+
process: Any = None
|
|
313
|
+
receiver: WorkerPipeReceiver | None = None
|
|
314
|
+
reply_sender: WorkerReplySender | None = None
|
|
315
|
+
health_stop: threading.Event | None = None
|
|
316
|
+
result: dict[str, Any] | None = None
|
|
317
|
+
error: Exception | None = None
|
|
429
318
|
|
|
430
319
|
|
|
431
320
|
class ExecutionChannel:
|
|
@@ -436,23 +325,96 @@ class ExecutionChannel:
|
|
|
436
325
|
def __init__(
|
|
437
326
|
self,
|
|
438
327
|
*,
|
|
439
|
-
|
|
440
|
-
output_roots: str | Path | Sequence[str | Path] | None = None,
|
|
441
|
-
protected_roots: Sequence[str | Path] = (),
|
|
328
|
+
launcher: WorkerLauncher | None = None,
|
|
442
329
|
) -> None:
|
|
443
|
-
self.
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
self.
|
|
447
|
-
self.
|
|
448
|
-
self._reply_senders: dict[str, WorkerReplySender] = {}
|
|
330
|
+
self._launcher: WorkerLauncher = (
|
|
331
|
+
launcher if launcher is not None else SubprocessWorkerLauncher()
|
|
332
|
+
)
|
|
333
|
+
self._sessions_lock = threading.RLock()
|
|
334
|
+
self._sessions: dict[str, _ExecutionSession] = {}
|
|
449
335
|
self._declaration_handler: Callable[[JobHandle, dict[str, Any]], dict[str, Any] | None] | None = None
|
|
336
|
+
self._is_timeout_paused_cb: Callable[[JobHandle], bool] | None = None
|
|
337
|
+
self._get_authorization_outcome_cb: Callable[[JobHandle], AuthorizationOutcome | None] | None = None
|
|
450
338
|
self._wakeup_callback: Callable[[], None] | None = None
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
self.
|
|
339
|
+
|
|
340
|
+
@property
|
|
341
|
+
def launcher(self) -> WorkerLauncher:
|
|
342
|
+
"""返回当前绑定的 worker 启动器(供内部与测试 seam 只读观察)。"""
|
|
343
|
+
return self._launcher
|
|
344
|
+
|
|
345
|
+
def set_launcher(self, launcher: WorkerLauncher) -> WorkerLauncher:
|
|
346
|
+
"""受控替换底层 worker 启动器并返回先前启动器(供测试 seam 使用)。"""
|
|
347
|
+
if not isinstance(launcher, WorkerLauncher):
|
|
348
|
+
raise TypeError(
|
|
349
|
+
f"launcher must be a WorkerLauncher instance, got {type(launcher).__name__}"
|
|
350
|
+
)
|
|
351
|
+
previous = self._launcher
|
|
352
|
+
self._launcher = launcher
|
|
353
|
+
return previous
|
|
354
|
+
|
|
355
|
+
def _find_session(self, handle: JobHandle) -> _ExecutionSession | None:
|
|
356
|
+
exec_id = handle.execution_id
|
|
357
|
+
with self._sessions_lock:
|
|
358
|
+
if exec_id:
|
|
359
|
+
session = self._sessions.get(exec_id)
|
|
360
|
+
if session is not None:
|
|
361
|
+
return session
|
|
362
|
+
for session in self._sessions.values():
|
|
363
|
+
if session.handle is handle:
|
|
364
|
+
return session
|
|
365
|
+
return None
|
|
366
|
+
|
|
367
|
+
def _get_session_by_id(self, execution_id: str | None) -> _ExecutionSession | None:
|
|
368
|
+
if not execution_id:
|
|
369
|
+
return None
|
|
370
|
+
with self._sessions_lock:
|
|
371
|
+
return self._sessions.get(execution_id)
|
|
372
|
+
|
|
373
|
+
@property
|
|
374
|
+
def has_active_workers(self) -> bool:
|
|
375
|
+
"""通道是否存在准备中、在途或残留的 worker 会话。"""
|
|
376
|
+
self._reap_lingering_workers()
|
|
377
|
+
with self._sessions_lock:
|
|
378
|
+
return any(
|
|
379
|
+
session.state is not _SessionState.CLOSED
|
|
380
|
+
for session in self._sessions.values()
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
def active_handles(self) -> list[JobHandle]:
|
|
384
|
+
"""返回当前处于 ``active`` 状态的全部在途执行句柄快照。"""
|
|
385
|
+
with self._sessions_lock:
|
|
386
|
+
return [
|
|
387
|
+
session.handle
|
|
388
|
+
for session in self._sessions.values()
|
|
389
|
+
if session.state is _SessionState.ACTIVE and session.handle is not None
|
|
390
|
+
]
|
|
391
|
+
|
|
392
|
+
def has_received_result(self, execution_id: str | None) -> bool:
|
|
393
|
+
"""查询指定执行会话是否已通过管道收到结果帧(未消费)。"""
|
|
394
|
+
session = self._get_session_by_id(execution_id)
|
|
395
|
+
return session is not None and session.result is not None
|
|
396
|
+
|
|
397
|
+
def cancel_pending_launches(self) -> None:
|
|
398
|
+
"""立即唤醒并中止全部尚在准备/握手阶段的未放行启动。"""
|
|
399
|
+
with self._sessions_lock:
|
|
400
|
+
events = [
|
|
401
|
+
session.cancel_event
|
|
402
|
+
for session in self._sessions.values()
|
|
403
|
+
if session.state is _SessionState.PREPARED
|
|
404
|
+
]
|
|
405
|
+
for ev in events:
|
|
406
|
+
ev.set()
|
|
407
|
+
|
|
408
|
+
@staticmethod
|
|
409
|
+
def _kill_process(p: Any) -> None:
|
|
410
|
+
"""优先调用进程树原子终止接口,兼容测试进程桩的 kill()。"""
|
|
411
|
+
kill_tree = getattr(p, "kill_tree", None)
|
|
412
|
+
if callable(kill_tree):
|
|
413
|
+
kill_tree()
|
|
414
|
+
return
|
|
415
|
+
kill = getattr(p, "kill", None)
|
|
416
|
+
if callable(kill):
|
|
417
|
+
kill()
|
|
456
418
|
|
|
457
419
|
@staticmethod
|
|
458
420
|
def _terminate_and_join_process(p: Any) -> None:
|
|
@@ -460,97 +422,254 @@ class ExecutionChannel:
|
|
|
460
422
|
if getattr(p, "_closed", False):
|
|
461
423
|
return
|
|
462
424
|
try:
|
|
463
|
-
|
|
464
|
-
|
|
425
|
+
is_alive_fn = getattr(p, "is_alive", None)
|
|
426
|
+
if callable(is_alive_fn) and is_alive_fn():
|
|
427
|
+
ExecutionChannel._kill_process(p)
|
|
465
428
|
except Exception as e:
|
|
466
429
|
logger.warning(f"Process kill failed during finalize: {e}")
|
|
467
430
|
try:
|
|
468
|
-
|
|
469
|
-
|
|
431
|
+
join_fn = getattr(p, "join", None)
|
|
432
|
+
if callable(join_fn):
|
|
433
|
+
join_fn(timeout=ExecutionChannel._JOIN_REAP_TIMEOUT)
|
|
470
434
|
except Exception as e:
|
|
471
435
|
logger.warning(f"Process join failed during finalize: {e}")
|
|
472
436
|
|
|
473
437
|
@staticmethod
|
|
474
438
|
def _finalize_process(p: Any) -> None:
|
|
475
|
-
"""清理单个子进程:
|
|
439
|
+
"""清理单个子进程:kill_tree + join + close(各步失败降级但留痕,不阻断收割)。"""
|
|
476
440
|
if getattr(p, "_closed", False):
|
|
477
441
|
return
|
|
478
442
|
ExecutionChannel._terminate_and_join_process(p)
|
|
479
443
|
try:
|
|
480
444
|
process_close = getattr(p, "close", None)
|
|
481
|
-
if process_close
|
|
445
|
+
if callable(process_close):
|
|
482
446
|
process_close()
|
|
483
447
|
except Exception as e:
|
|
484
448
|
logger.warning(f"Process close failed during finalize: {e}")
|
|
485
449
|
|
|
486
|
-
def
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
450
|
+
def _stop_health_watch(self, session: _ExecutionSession) -> None:
|
|
451
|
+
with self._sessions_lock:
|
|
452
|
+
stop = session.health_stop
|
|
453
|
+
session.health_stop = None
|
|
454
|
+
if stop is not None:
|
|
455
|
+
stop.set()
|
|
456
|
+
|
|
457
|
+
def _close_reply_sender(self, session: _ExecutionSession) -> None:
|
|
458
|
+
with self._sessions_lock:
|
|
459
|
+
reply_sender = session.reply_sender
|
|
460
|
+
session.reply_sender = None
|
|
461
|
+
if reply_sender is not None:
|
|
462
|
+
try:
|
|
463
|
+
reply_sender.close()
|
|
464
|
+
except Exception as exc:
|
|
465
|
+
logger.debug(f"Reply sender close warning for {session.session_key}: {exc}")
|
|
466
|
+
|
|
467
|
+
def _close_receiver(
|
|
468
|
+
self,
|
|
469
|
+
session: _ExecutionSession,
|
|
470
|
+
*,
|
|
471
|
+
timeout: float | None = None,
|
|
472
|
+
) -> None:
|
|
473
|
+
with self._sessions_lock:
|
|
474
|
+
receiver = session.receiver
|
|
475
|
+
session.receiver = None
|
|
476
|
+
if receiver is not None:
|
|
477
|
+
try:
|
|
478
|
+
receiver.close(
|
|
479
|
+
timeout=self._JOIN_REAP_TIMEOUT if timeout is None else timeout
|
|
480
|
+
)
|
|
481
|
+
except Exception as exc:
|
|
482
|
+
logger.debug(f"Receiver close warning for {session.session_key}: {exc}")
|
|
483
|
+
|
|
484
|
+
def _transition_to_lingering(self, session: _ExecutionSession) -> None:
|
|
485
|
+
"""将已产出结果但进程尚未退出的会话转入 lingering(保活 process 与 receiver)。"""
|
|
486
|
+
self._stop_health_watch(session)
|
|
487
|
+
self._close_reply_sender(session)
|
|
488
|
+
with self._sessions_lock:
|
|
489
|
+
session.handle = None
|
|
490
|
+
session.result = None
|
|
491
|
+
session.error = None
|
|
492
|
+
session.state = _SessionState.LINGERING
|
|
493
|
+
|
|
494
|
+
def _close_session(
|
|
495
|
+
self,
|
|
496
|
+
session: _ExecutionSession | None,
|
|
497
|
+
*,
|
|
498
|
+
finalize_process: bool = True,
|
|
499
|
+
receiver_timeout: float | None = None,
|
|
500
|
+
) -> None:
|
|
501
|
+
"""幂等关闭单个会话:停止健康探测、关闭全部管道、按需回收进程并移出状态表。"""
|
|
502
|
+
if session is None:
|
|
503
|
+
return
|
|
504
|
+
with self._sessions_lock:
|
|
505
|
+
health_stop = session.health_stop
|
|
506
|
+
session.health_stop = None
|
|
507
|
+
process = session.process
|
|
508
|
+
prepared_worker = session.prepared_worker
|
|
509
|
+
reply_sender = session.reply_sender
|
|
510
|
+
receiver = session.receiver
|
|
511
|
+
session.prepared_worker = None
|
|
512
|
+
session.handle = None
|
|
513
|
+
session.process = None
|
|
514
|
+
session.reply_sender = None
|
|
515
|
+
session.receiver = None
|
|
516
|
+
session.result = None
|
|
517
|
+
session.error = None
|
|
518
|
+
session.state = _SessionState.CLOSED
|
|
519
|
+
self._sessions.pop(session.session_key, None)
|
|
520
|
+
|
|
521
|
+
if health_stop is not None:
|
|
522
|
+
health_stop.set()
|
|
523
|
+
if prepared_worker is not None:
|
|
524
|
+
try:
|
|
525
|
+
prepared_worker.abort()
|
|
526
|
+
except Exception as exc:
|
|
527
|
+
logger.error(f"Prepared worker abort warning for {session.session_key}: {exc}")
|
|
528
|
+
if finalize_process and process is not None:
|
|
529
|
+
self._finalize_process(process)
|
|
530
|
+
if reply_sender is not None:
|
|
531
|
+
try:
|
|
532
|
+
reply_sender.close()
|
|
533
|
+
except Exception as exc:
|
|
534
|
+
logger.debug(f"Reply sender close warning for {session.session_key}: {exc}")
|
|
535
|
+
if receiver is not None:
|
|
536
|
+
try:
|
|
537
|
+
receiver.close(
|
|
538
|
+
timeout=self._JOIN_REAP_TIMEOUT if receiver_timeout is None else receiver_timeout
|
|
539
|
+
)
|
|
540
|
+
except Exception as exc:
|
|
541
|
+
logger.debug(f"Receiver close warning for {session.session_key}: {exc}")
|
|
542
|
+
|
|
543
|
+
# ── 线性两阶段启动协议(prepare_spawn -> release_spawn / abort_spawn)──
|
|
544
|
+
|
|
545
|
+
def prepare_spawn(self, spec: WorkerLaunchSpec) -> PreparedLaunch:
|
|
546
|
+
"""阶段一:冷启动 worker 并完成握手与进程树绑定,停在放行门前。"""
|
|
547
|
+
if is_importing_mp_main():
|
|
548
|
+
raise RuntimeError(
|
|
549
|
+
"ExecutionChannel.spawn() cannot be called while bootstrapping a worker process. "
|
|
550
|
+
"Guard top-level execution with `if __name__ == '__main__':`."
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
cancel_event = threading.Event()
|
|
554
|
+
session_key = spec.execution_id if spec.execution_id else f"_pending_{id(cancel_event)}"
|
|
555
|
+
session = _ExecutionSession(
|
|
556
|
+
session_key=session_key,
|
|
557
|
+
state=_SessionState.PREPARED,
|
|
558
|
+
cancel_event=cancel_event,
|
|
509
559
|
)
|
|
510
|
-
|
|
560
|
+
with self._sessions_lock:
|
|
561
|
+
self._sessions[session_key] = session
|
|
562
|
+
|
|
511
563
|
try:
|
|
512
|
-
|
|
513
|
-
|
|
564
|
+
prepared_worker = self._launcher.prepare(
|
|
565
|
+
spec,
|
|
566
|
+
on_message=self._notify_wakeup,
|
|
567
|
+
cancel_event=cancel_event,
|
|
568
|
+
)
|
|
569
|
+
with self._sessions_lock:
|
|
570
|
+
if session.state is _SessionState.CLOSED or cancel_event.is_set():
|
|
571
|
+
session.state = _SessionState.CLOSED
|
|
572
|
+
self._sessions.pop(session_key, None)
|
|
573
|
+
should_abort = True
|
|
574
|
+
else:
|
|
575
|
+
session.prepared_worker = prepared_worker
|
|
576
|
+
should_abort = False
|
|
577
|
+
if should_abort:
|
|
578
|
+
prepared_worker.abort()
|
|
514
579
|
raise _WorkerLaunchCancelled(
|
|
515
|
-
f"stop requested before worker
|
|
580
|
+
f"stop requested before worker release for {spec.job.uid}"
|
|
516
581
|
)
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
reply_sender.close()
|
|
524
|
-
if p is not None:
|
|
525
|
-
self._finalize_process(p)
|
|
582
|
+
return PreparedLaunch(spec=spec, session_key=session_key)
|
|
583
|
+
except BaseException:
|
|
584
|
+
with self._sessions_lock:
|
|
585
|
+
session.prepared_worker = None
|
|
586
|
+
session.state = _SessionState.CLOSED
|
|
587
|
+
self._sessions.pop(session_key, None)
|
|
526
588
|
raise
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
589
|
+
|
|
590
|
+
def release_spawn(self, prepared: PreparedLaunch) -> JobHandle:
|
|
591
|
+
"""阶段二(正常路径):放行 worker(发送 b"GO"),转入 active 并返回句柄。"""
|
|
592
|
+
spec = prepared.spec
|
|
593
|
+
with self._sessions_lock:
|
|
594
|
+
session = self._sessions.get(prepared.session_key)
|
|
595
|
+
if session is None or session.state is not _SessionState.PREPARED:
|
|
596
|
+
raise RuntimeError(
|
|
597
|
+
f"Cannot release execution session {prepared.session_key!r}: not prepared"
|
|
598
|
+
)
|
|
599
|
+
prepared_worker = session.prepared_worker
|
|
600
|
+
if prepared_worker is None:
|
|
601
|
+
raise RuntimeError(
|
|
602
|
+
f"Cannot release execution session {prepared.session_key!r}: missing prepared worker"
|
|
603
|
+
)
|
|
604
|
+
cancelled = session.cancel_event.is_set()
|
|
605
|
+
|
|
606
|
+
if cancelled:
|
|
607
|
+
self.abort_spawn(prepared)
|
|
608
|
+
raise _WorkerLaunchCancelled(
|
|
609
|
+
f"stop requested before worker release for {spec.job.uid}"
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
try:
|
|
613
|
+
launched = prepared_worker.release()
|
|
614
|
+
except BaseException:
|
|
615
|
+
with self._sessions_lock:
|
|
616
|
+
session.prepared_worker = None
|
|
617
|
+
session.state = _SessionState.CLOSED
|
|
618
|
+
self._sessions.pop(prepared.session_key, None)
|
|
619
|
+
raise
|
|
620
|
+
|
|
533
621
|
timeout_at_monotonic = time.monotonic() + spec.timeout_seconds
|
|
534
622
|
handle = JobHandle(
|
|
535
623
|
job_uid=spec.job.uid,
|
|
536
|
-
process=
|
|
624
|
+
process=launched.handle,
|
|
537
625
|
timeout_at_monotonic=timeout_at_monotonic,
|
|
538
626
|
timeout_seconds=spec.timeout_seconds,
|
|
539
|
-
started_at=started_at,
|
|
627
|
+
started_at=launched.started_at,
|
|
540
628
|
job=spec.job,
|
|
541
629
|
execution_id=spec.execution_id,
|
|
542
630
|
run_id=spec.run_id or None,
|
|
543
631
|
attempt_no=spec.attempt_no,
|
|
544
632
|
)
|
|
545
|
-
self.
|
|
546
|
-
|
|
547
|
-
|
|
633
|
+
with self._sessions_lock:
|
|
634
|
+
session.prepared_worker = None
|
|
635
|
+
session.handle = handle
|
|
636
|
+
session.process = launched.handle
|
|
637
|
+
session.receiver = launched.receiver
|
|
638
|
+
session.reply_sender = launched.reply_sender
|
|
639
|
+
session.state = _SessionState.ACTIVE
|
|
640
|
+
self._watch_process_health(session)
|
|
548
641
|
return handle
|
|
549
642
|
|
|
550
|
-
def
|
|
643
|
+
def abort_spawn(self, prepared: PreparedLaunch) -> None:
|
|
644
|
+
"""阶段二(中止路径):幂等终止并回收尚未放行的 worker 与会话。"""
|
|
645
|
+
with self._sessions_lock:
|
|
646
|
+
session = self._sessions.get(prepared.session_key)
|
|
647
|
+
if session is None:
|
|
648
|
+
return
|
|
649
|
+
session.cancel_event.set()
|
|
650
|
+
if session.state is not _SessionState.PREPARED:
|
|
651
|
+
return
|
|
652
|
+
prepared_worker = session.prepared_worker
|
|
653
|
+
session.prepared_worker = None
|
|
654
|
+
session.state = _SessionState.CLOSED
|
|
655
|
+
self._sessions.pop(prepared.session_key, None)
|
|
656
|
+
if prepared_worker is not None:
|
|
657
|
+
prepared_worker.abort()
|
|
658
|
+
|
|
659
|
+
def spawn(self, spec: WorkerLaunchSpec) -> JobHandle:
|
|
660
|
+
"""单步便捷启动接口:连续执行 prepare_spawn 与 release_spawn。"""
|
|
661
|
+
prepared = self.prepare_spawn(spec)
|
|
662
|
+
try:
|
|
663
|
+
return self.release_spawn(prepared)
|
|
664
|
+
except BaseException:
|
|
665
|
+
self.abort_spawn(prepared)
|
|
666
|
+
raise
|
|
667
|
+
|
|
668
|
+
def _watch_process_health(self, session: _ExecutionSession) -> None:
|
|
551
669
|
"""独立探测 worker 存活,仅在状态变化时唤醒主循环。"""
|
|
552
670
|
stop = threading.Event()
|
|
553
|
-
|
|
671
|
+
process = session.process
|
|
672
|
+
session.health_stop = stop
|
|
554
673
|
|
|
555
674
|
def watch() -> None:
|
|
556
675
|
try:
|
|
@@ -563,20 +682,22 @@ class ExecutionChannel:
|
|
|
563
682
|
self._notify_wakeup()
|
|
564
683
|
break
|
|
565
684
|
finally:
|
|
566
|
-
|
|
685
|
+
if session.health_stop is stop:
|
|
686
|
+
session.health_stop = None
|
|
567
687
|
|
|
568
688
|
threading.Thread(
|
|
569
689
|
target=watch,
|
|
570
|
-
name=f"tasklite-worker-health-{
|
|
690
|
+
name=f"tasklite-worker-health-{session.session_key}",
|
|
571
691
|
daemon=True,
|
|
572
692
|
).start()
|
|
573
693
|
|
|
574
694
|
def _accept_message(self, handle: JobHandle, msg: WorkerPipeMessage) -> None:
|
|
575
|
-
"""
|
|
695
|
+
"""消费一条已校验信封的管道消息并更新会话状态。"""
|
|
696
|
+
session = self._find_session(handle)
|
|
576
697
|
if msg.kind == PIPE_MSG_DECLARE:
|
|
577
698
|
if not isinstance(msg.payload, dict):
|
|
578
699
|
raise ValueError(f"invalid declare payload for {handle.job_uid}")
|
|
579
|
-
reply_sender =
|
|
700
|
+
reply_sender = session.reply_sender if session is not None else None
|
|
580
701
|
if self._declaration_handler is not None and reply_sender is not None:
|
|
581
702
|
reply = self._declaration_handler(handle, msg.payload)
|
|
582
703
|
if reply is not None:
|
|
@@ -587,13 +708,13 @@ class ExecutionChannel:
|
|
|
587
708
|
elif msg.kind == PIPE_MSG_RESULT:
|
|
588
709
|
if not isinstance(msg.payload, dict):
|
|
589
710
|
raise ValueError(f"invalid result payload for {handle.job_uid}")
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
self._results.setdefault(exec_id, msg.payload)
|
|
711
|
+
if session is not None and session.error is None and session.result is None:
|
|
712
|
+
session.result = msg.payload
|
|
593
713
|
|
|
594
714
|
def _process_pipe_item(
|
|
595
715
|
self, handle: JobHandle, item: WorkerPipeMessage | Exception
|
|
596
716
|
) -> None:
|
|
717
|
+
session = self._find_session(handle)
|
|
597
718
|
exec_id = handle.execution_id or ""
|
|
598
719
|
if isinstance(item, OSError):
|
|
599
720
|
logger.debug(
|
|
@@ -601,38 +722,29 @@ class ExecutionChannel:
|
|
|
601
722
|
)
|
|
602
723
|
return
|
|
603
724
|
if isinstance(item, Exception):
|
|
604
|
-
if
|
|
605
|
-
|
|
725
|
+
if session is not None and session.result is None and session.error is None:
|
|
726
|
+
session.error = item
|
|
606
727
|
return
|
|
607
728
|
try:
|
|
608
729
|
self._accept_message(handle, item)
|
|
609
730
|
except Exception as exc:
|
|
610
|
-
if
|
|
611
|
-
|
|
731
|
+
if session is not None and session.result is None and session.error is None:
|
|
732
|
+
session.error = exc
|
|
612
733
|
|
|
613
734
|
def _receive_available(self, handle: JobHandle) -> None:
|
|
614
735
|
"""非阻塞拉取 handle 对应接收端当前已入队的全部消息。"""
|
|
615
|
-
|
|
616
|
-
if
|
|
617
|
-
return
|
|
618
|
-
receiver = self._receivers.get(exec_id)
|
|
619
|
-
if receiver is None:
|
|
736
|
+
session = self._find_session(handle)
|
|
737
|
+
if session is None or session.receiver is None:
|
|
620
738
|
return
|
|
621
|
-
for item in receiver.take_messages():
|
|
739
|
+
for item in session.receiver.take_messages():
|
|
622
740
|
self._process_pipe_item(handle, item)
|
|
623
741
|
|
|
624
742
|
def _drain_finished(self, handle: JobHandle) -> None:
|
|
625
|
-
"""
|
|
626
|
-
|
|
627
|
-
if
|
|
743
|
+
"""在进程终止后排空接收端直至 EOF。"""
|
|
744
|
+
session = self._find_session(handle)
|
|
745
|
+
if session is None or session.receiver is None:
|
|
628
746
|
return
|
|
629
|
-
|
|
630
|
-
if sender is not None:
|
|
631
|
-
sender.close()
|
|
632
|
-
receiver = self._receivers.get(exec_id)
|
|
633
|
-
if receiver is None:
|
|
634
|
-
return
|
|
635
|
-
for item in receiver.drain_to_eof(timeout=self._JOIN_REAP_TIMEOUT):
|
|
747
|
+
for item in session.receiver.drain_to_eof(timeout=self._JOIN_REAP_TIMEOUT):
|
|
636
748
|
self._process_pipe_item(handle, item)
|
|
637
749
|
|
|
638
750
|
def set_declaration_handler(
|
|
@@ -641,6 +753,24 @@ class ExecutionChannel:
|
|
|
641
753
|
) -> None:
|
|
642
754
|
self._declaration_handler = handler
|
|
643
755
|
|
|
756
|
+
def set_authorization_queries(
|
|
757
|
+
self,
|
|
758
|
+
*,
|
|
759
|
+
is_timeout_paused: Callable[[JobHandle], bool] | None = None,
|
|
760
|
+
get_outcome: Callable[[JobHandle], AuthorizationOutcome | None] | None = None,
|
|
761
|
+
) -> None:
|
|
762
|
+
"""绑定运行期授权状态只读查询回调(超时暂停判定与终态授权结果)。"""
|
|
763
|
+
self._is_timeout_paused_cb = is_timeout_paused
|
|
764
|
+
self._get_authorization_outcome_cb = get_outcome
|
|
765
|
+
|
|
766
|
+
def _is_timeout_paused(self, handle: JobHandle) -> bool:
|
|
767
|
+
cb = self._is_timeout_paused_cb
|
|
768
|
+
return bool(cb(handle)) if cb is not None else False
|
|
769
|
+
|
|
770
|
+
def _get_authorization_outcome(self, handle: JobHandle) -> AuthorizationOutcome | None:
|
|
771
|
+
cb = self._get_authorization_outcome_cb
|
|
772
|
+
return cb(handle) if cb is not None else None
|
|
773
|
+
|
|
644
774
|
def set_wakeup_callback(self, callback: Callable[[], None] | None) -> None:
|
|
645
775
|
"""绑定 IPC 消息到达时的主循环唤醒入口。"""
|
|
646
776
|
self._wakeup_callback = callback
|
|
@@ -650,102 +780,111 @@ class ExecutionChannel:
|
|
|
650
780
|
if callback is not None:
|
|
651
781
|
callback()
|
|
652
782
|
|
|
783
|
+
def is_handle_alive(self, handle: JobHandle) -> bool:
|
|
784
|
+
"""受控判定在途句柄对应子进程是否仍在运行(屏蔽已关闭句柄的 ValueError)。"""
|
|
785
|
+
p = handle.process
|
|
786
|
+
if p is None or getattr(p, "_closed", False):
|
|
787
|
+
return False
|
|
788
|
+
is_alive_fn = getattr(p, "is_alive", None)
|
|
789
|
+
if not callable(is_alive_fn):
|
|
790
|
+
return False
|
|
791
|
+
try:
|
|
792
|
+
return bool(is_alive_fn())
|
|
793
|
+
except (ValueError, OSError):
|
|
794
|
+
return False
|
|
795
|
+
|
|
653
796
|
def terminate_process(self, handle: JobHandle) -> None:
|
|
654
|
-
"""
|
|
797
|
+
"""终止指定作业的子进程树。"""
|
|
655
798
|
p = handle.process
|
|
656
799
|
if p is not None:
|
|
657
800
|
try:
|
|
658
|
-
|
|
659
|
-
|
|
801
|
+
is_alive_fn = getattr(p, "is_alive", None)
|
|
802
|
+
if callable(is_alive_fn) and is_alive_fn():
|
|
803
|
+
self._kill_process(p)
|
|
660
804
|
except Exception:
|
|
661
805
|
pass
|
|
662
806
|
|
|
663
807
|
def get_reply_sender(self, execution_id: str | None) -> WorkerReplySender | None:
|
|
664
|
-
"""
|
|
665
|
-
|
|
808
|
+
"""获取指定活跃会话的回复发送端。"""
|
|
809
|
+
session = self._get_session_by_id(execution_id)
|
|
810
|
+
if session is None or session.state is not _SessionState.ACTIVE:
|
|
666
811
|
return None
|
|
667
|
-
return
|
|
668
|
-
|
|
669
|
-
def _close_handle_pipe(self, execution_id: str | None) -> None:
|
|
670
|
-
if not execution_id:
|
|
671
|
-
return
|
|
672
|
-
stop = self._health_stops.pop(execution_id, None)
|
|
673
|
-
if stop is not None:
|
|
674
|
-
stop.set()
|
|
675
|
-
sender = self._in_process_senders.pop(execution_id, None)
|
|
676
|
-
if sender is not None:
|
|
677
|
-
sender.close()
|
|
678
|
-
receiver = self._receivers.pop(execution_id, None)
|
|
679
|
-
if receiver is not None:
|
|
680
|
-
receiver.close(timeout=self._JOIN_REAP_TIMEOUT)
|
|
681
|
-
reply_sender = self._reply_senders.pop(execution_id, None)
|
|
682
|
-
if reply_sender is not None:
|
|
683
|
-
reply_sender.close()
|
|
812
|
+
return session.reply_sender
|
|
684
813
|
|
|
685
814
|
def _consume_result_if_present(self, handle: JobHandle) -> ExecutionResult | None:
|
|
686
|
-
"""读取已收到的结果负载 → 守卫过滤 → 解码。
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
"""
|
|
690
|
-
exec_id = getattr(handle, "execution_id", None)
|
|
691
|
-
if not exec_id or exec_id in self._errors:
|
|
815
|
+
"""读取已收到的结果负载 → 守卫过滤 → 解码。"""
|
|
816
|
+
session = self._find_session(handle)
|
|
817
|
+
if session is None or session.error is not None:
|
|
692
818
|
return None
|
|
693
|
-
raw_res =
|
|
819
|
+
raw_res = session.result
|
|
694
820
|
if not isinstance(raw_res, dict):
|
|
695
821
|
return None
|
|
696
|
-
|
|
822
|
+
session.result = None
|
|
697
823
|
return _decode_ipc_result(raw_res, handle.job)
|
|
698
824
|
|
|
825
|
+
def _lingering_sessions(self) -> list[_ExecutionSession]:
|
|
826
|
+
"""返回当前处于 ``lingering`` 状态的全部会话快照。"""
|
|
827
|
+
with self._sessions_lock:
|
|
828
|
+
return [
|
|
829
|
+
s for s in self._sessions.values()
|
|
830
|
+
if s.state is _SessionState.LINGERING
|
|
831
|
+
]
|
|
832
|
+
|
|
833
|
+
def _cleanup_lingering_sessions(
|
|
834
|
+
self,
|
|
835
|
+
*,
|
|
836
|
+
receiver_timeout: float = 0.5,
|
|
837
|
+
log_errors: bool = True,
|
|
838
|
+
) -> None:
|
|
839
|
+
"""回收全部处于 ``lingering`` 状态的会话,异常收敛不阻断后续清理。"""
|
|
840
|
+
for session in self._lingering_sessions():
|
|
841
|
+
try:
|
|
842
|
+
self._close_session(
|
|
843
|
+
session,
|
|
844
|
+
finalize_process=True,
|
|
845
|
+
receiver_timeout=receiver_timeout,
|
|
846
|
+
)
|
|
847
|
+
except Exception as e:
|
|
848
|
+
if log_errors:
|
|
849
|
+
logger.error(f"Error cleaning up lingering worker {session.session_key}: {e}")
|
|
850
|
+
|
|
699
851
|
def _reap_lingering_workers(self) -> None:
|
|
700
|
-
"""
|
|
701
|
-
|
|
702
|
-
|
|
852
|
+
"""非阻塞轮询已结算但残留的 lingering 会话,进程死亡后回收句柄与接收端。"""
|
|
853
|
+
for session in self._lingering_sessions():
|
|
854
|
+
p = session.process
|
|
703
855
|
try:
|
|
704
|
-
alive = p.is_alive()
|
|
856
|
+
alive = p is not None and p.is_alive()
|
|
705
857
|
except Exception:
|
|
706
858
|
alive = False
|
|
707
859
|
if not alive:
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
except Exception as e:
|
|
712
|
-
logger.debug(f"Lingering receiver close warning for {exec_id}: {e}")
|
|
713
|
-
self._finalize_process(p)
|
|
714
|
-
else:
|
|
715
|
-
if receiver is not None and receiver.eof:
|
|
716
|
-
try:
|
|
717
|
-
receiver.close(timeout=0.1)
|
|
718
|
-
receiver = None
|
|
719
|
-
except Exception:
|
|
720
|
-
pass
|
|
721
|
-
still_lingering.append((exec_id, p, receiver))
|
|
722
|
-
self._lingering_workers = still_lingering
|
|
860
|
+
self._close_session(session, finalize_process=True, receiver_timeout=0.2)
|
|
861
|
+
elif session.receiver is not None and session.receiver.eof:
|
|
862
|
+
self._close_receiver(session, timeout=0.1)
|
|
723
863
|
|
|
724
864
|
def wait_for_lingering_workers(self) -> None:
|
|
725
|
-
"""正常 Run 收口前等待已结算
|
|
726
|
-
|
|
865
|
+
"""正常 Run 收口前等待已结算 lingering 会话退出,超时后回收残留进程。"""
|
|
866
|
+
lingering = self._lingering_sessions()
|
|
867
|
+
if not lingering:
|
|
727
868
|
return
|
|
728
869
|
deadline = time.monotonic() + self._JOIN_REAP_TIMEOUT
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
try:
|
|
734
|
-
while process.is_alive() and remaining > 0:
|
|
735
|
-
process.join(timeout=min(0.05, remaining))
|
|
736
|
-
remaining = deadline - time.monotonic()
|
|
737
|
-
if process.is_alive() and remaining > 0:
|
|
738
|
-
time.sleep(min(0.01, remaining))
|
|
739
|
-
except Exception as exc:
|
|
740
|
-
logger.warning(f"Settled worker wait failed for {exec_id}: {exc}")
|
|
741
|
-
if process.is_alive():
|
|
742
|
-
logger.warning(f"Settled worker {exec_id} did not exit before Run end")
|
|
743
|
-
self._finalize_process(process)
|
|
744
|
-
if receiver is not None:
|
|
870
|
+
for session in lingering:
|
|
871
|
+
process = session.process
|
|
872
|
+
if process is not None:
|
|
873
|
+
remaining = deadline - time.monotonic()
|
|
745
874
|
try:
|
|
746
|
-
|
|
875
|
+
while process.is_alive() and remaining > 0:
|
|
876
|
+
process.join(timeout=min(0.05, remaining))
|
|
877
|
+
remaining = deadline - time.monotonic()
|
|
878
|
+
if process.is_alive() and remaining > 0:
|
|
879
|
+
time.sleep(min(0.01, remaining))
|
|
747
880
|
except Exception as exc:
|
|
748
|
-
logger.
|
|
881
|
+
logger.warning(f"Settled worker wait failed for {session.session_key}: {exc}")
|
|
882
|
+
try:
|
|
883
|
+
if process.is_alive():
|
|
884
|
+
logger.warning(f"Settled worker {session.session_key} did not exit before Run end")
|
|
885
|
+
except Exception:
|
|
886
|
+
pass
|
|
887
|
+
self._close_session(session, finalize_process=True, receiver_timeout=0.2)
|
|
749
888
|
|
|
750
889
|
def reap_completed(
|
|
751
890
|
self, handles: Sequence[JobHandle]
|
|
@@ -754,8 +893,8 @@ class ExecutionChannel:
|
|
|
754
893
|
|
|
755
894
|
有效结果到达即结算:收到完整有效 TaskResult 即认为本次执行已返回并
|
|
756
895
|
交付结算,不将 worker 进程退出作为额外成功条件,不以前置 join 阻塞
|
|
757
|
-
|
|
758
|
-
派发或结算,亦不因之后的进程退出状态改写有效结果。
|
|
896
|
+
主循环。结果返回后仍在运行的子进程转入 lingering 后台非阻塞回收,
|
|
897
|
+
不阻塞其他 Job 派发或结算,亦不因之后的进程退出状态改写有效结果。
|
|
759
898
|
"""
|
|
760
899
|
self._reap_lingering_workers()
|
|
761
900
|
completed: list[tuple[JobHandle, ExecutionResult]] = []
|
|
@@ -763,28 +902,16 @@ class ExecutionChannel:
|
|
|
763
902
|
for handle in handles:
|
|
764
903
|
now = time.monotonic()
|
|
765
904
|
p = handle.process
|
|
766
|
-
|
|
905
|
+
session = self._find_session(handle)
|
|
767
906
|
self._receive_available(handle)
|
|
768
907
|
|
|
769
|
-
authorization_outcome = handle
|
|
908
|
+
authorization_outcome = self._get_authorization_outcome(handle)
|
|
770
909
|
if authorization_outcome is not None:
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
if not is_alive:
|
|
777
|
-
self._handles.pop(exec_id, None)
|
|
778
|
-
receiver = self._receivers.pop(exec_id, None)
|
|
779
|
-
if receiver is not None:
|
|
780
|
-
try:
|
|
781
|
-
receiver.close(timeout=0.2)
|
|
782
|
-
except Exception:
|
|
783
|
-
pass
|
|
784
|
-
self._close_handle_pipe(exec_id)
|
|
785
|
-
self._results.pop(exec_id, None)
|
|
786
|
-
self._errors.pop(exec_id, None)
|
|
787
|
-
self._finalize_process(p)
|
|
910
|
+
if not self.is_handle_alive(handle):
|
|
911
|
+
if session is not None:
|
|
912
|
+
self._close_session(session, finalize_process=True, receiver_timeout=0.2)
|
|
913
|
+
else:
|
|
914
|
+
self._finalize_process(p)
|
|
788
915
|
decoded = ExecutionResult(
|
|
789
916
|
success=False,
|
|
790
917
|
message=authorization_outcome.message,
|
|
@@ -795,36 +922,22 @@ class ExecutionChannel:
|
|
|
795
922
|
continue
|
|
796
923
|
|
|
797
924
|
# 1. 有效结果已到达:立即收割结果,不阻塞等待子进程退出
|
|
798
|
-
if
|
|
799
|
-
raw_res =
|
|
925
|
+
if session is not None and session.result is not None:
|
|
926
|
+
raw_res = session.result
|
|
927
|
+
session.result = None
|
|
800
928
|
decoded = _decode_ipc_result(raw_res, handle.job)
|
|
801
|
-
self._handles.pop(exec_id, None)
|
|
802
|
-
receiver = self._receivers.pop(exec_id, None)
|
|
803
|
-
self._errors.pop(exec_id, None)
|
|
804
|
-
# 结果后不再回复声明;接收端独立移交,不等待 worker 退出。
|
|
805
|
-
self._close_handle_pipe(exec_id)
|
|
806
929
|
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
is_alive = p.is_alive()
|
|
810
|
-
except Exception:
|
|
811
|
-
pass
|
|
812
|
-
|
|
813
|
-
if not is_alive:
|
|
814
|
-
if receiver is not None:
|
|
815
|
-
try:
|
|
816
|
-
receiver.close(timeout=0.2)
|
|
817
|
-
except Exception:
|
|
818
|
-
pass
|
|
819
|
-
self._finalize_process(p)
|
|
930
|
+
if not self.is_handle_alive(handle):
|
|
931
|
+
self._close_session(session, finalize_process=True, receiver_timeout=0.2)
|
|
820
932
|
else:
|
|
821
|
-
|
|
933
|
+
# 结果后不再回复声明;会话转入 lingering,保活进程句柄与接收端。
|
|
934
|
+
self._transition_to_lingering(session)
|
|
822
935
|
|
|
823
936
|
completed.append((handle, decoded))
|
|
824
937
|
continue
|
|
825
938
|
|
|
826
939
|
# 2. 管道异常分支
|
|
827
|
-
if
|
|
940
|
+
if session is not None and session.error is not None:
|
|
828
941
|
completed.append((handle, self._collect_outcome(handle)))
|
|
829
942
|
continue
|
|
830
943
|
|
|
@@ -834,11 +947,11 @@ class ExecutionChannel:
|
|
|
834
947
|
continue
|
|
835
948
|
|
|
836
949
|
# 4. 超时:从启动到结果到达计单次超时(等待授权期间暂停不计)
|
|
837
|
-
if not handle
|
|
950
|
+
if not self._is_timeout_paused(handle) and now > handle.timeout_at_monotonic:
|
|
838
951
|
p.join(timeout=0.5)
|
|
839
952
|
if p.is_alive():
|
|
840
953
|
try:
|
|
841
|
-
|
|
954
|
+
self._kill_process(p)
|
|
842
955
|
except Exception:
|
|
843
956
|
pass
|
|
844
957
|
completed.append(
|
|
@@ -850,68 +963,83 @@ class ExecutionChannel:
|
|
|
850
963
|
def _collect_outcome(
|
|
851
964
|
self, handle: JobHandle, res: dict | None = None, *, is_timeout: bool = False
|
|
852
965
|
) -> ExecutionResult:
|
|
853
|
-
"""收敛收割尾部:终止并收割子进程 → 排空管道至 EOF → 解码结果 →
|
|
854
|
-
|
|
966
|
+
"""收敛收割尾部:终止并收割子进程 → 排空管道至 EOF → 解码结果 → 关闭会话。"""
|
|
967
|
+
p = handle.process
|
|
968
|
+
session = self._find_session(handle)
|
|
969
|
+
authorization_outcome = self._get_authorization_outcome(handle)
|
|
855
970
|
if authorization_outcome is not None:
|
|
856
|
-
p = handle.process
|
|
857
|
-
exec_id = handle.execution_id or ""
|
|
858
971
|
try:
|
|
859
972
|
self._terminate_and_join_process(p)
|
|
860
973
|
finally:
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
974
|
+
if session is not None:
|
|
975
|
+
self._close_session(session, finalize_process=True)
|
|
976
|
+
else:
|
|
977
|
+
self._finalize_process(p)
|
|
865
978
|
return ExecutionResult(
|
|
866
979
|
success=False,
|
|
867
980
|
message=authorization_outcome.message,
|
|
868
981
|
exception=authorization_outcome.exception,
|
|
869
982
|
authorization_outcome=authorization_outcome,
|
|
870
983
|
)
|
|
871
|
-
|
|
872
|
-
exec_id = handle.execution_id or ""
|
|
984
|
+
|
|
873
985
|
result: ExecutionResult | None = None
|
|
874
986
|
try:
|
|
875
987
|
self._terminate_and_join_process(p)
|
|
876
988
|
self._drain_finished(handle)
|
|
877
|
-
if
|
|
878
|
-
err =
|
|
879
|
-
|
|
989
|
+
if session is not None and session.error is not None:
|
|
990
|
+
err = session.error
|
|
991
|
+
session.error = None
|
|
992
|
+
session.result = None
|
|
880
993
|
raise err
|
|
881
|
-
payload =
|
|
994
|
+
payload = (
|
|
995
|
+
res
|
|
996
|
+
if res is not None
|
|
997
|
+
else (session.result if session is not None else None)
|
|
998
|
+
)
|
|
999
|
+
if session is not None:
|
|
1000
|
+
session.result = None
|
|
882
1001
|
if payload is not None and not is_timeout:
|
|
883
1002
|
result = _decode_ipc_result(payload, handle.job)
|
|
884
1003
|
else:
|
|
885
1004
|
result = self._build_terminal_failure(p, handle, is_timeout=is_timeout)
|
|
886
1005
|
finally:
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1006
|
+
if session is not None:
|
|
1007
|
+
self._close_session(session, finalize_process=True)
|
|
1008
|
+
else:
|
|
1009
|
+
self._finalize_process(p)
|
|
891
1010
|
return result
|
|
892
1011
|
|
|
893
1012
|
@staticmethod
|
|
894
1013
|
def _build_terminal_failure(
|
|
895
1014
|
p: Any, handle: JobHandle, *, is_timeout: bool
|
|
896
1015
|
) -> ExecutionResult:
|
|
897
|
-
"""
|
|
898
|
-
|
|
1016
|
+
"""构造进程终止(崩溃/超时/强杀)但无有效结果时的失败证据。"""
|
|
1017
|
+
raw_exitcode = getattr(p, "exitcode", None)
|
|
1018
|
+
exitcode: int | None = (
|
|
1019
|
+
raw_exitcode
|
|
1020
|
+
if isinstance(raw_exitcode, int) and not isinstance(raw_exitcode, bool)
|
|
1021
|
+
else None
|
|
1022
|
+
)
|
|
1023
|
+
was_force_killed = bool(getattr(p, "was_force_killed", False))
|
|
899
1024
|
annotations: dict[str, Any] = {}
|
|
900
1025
|
if is_timeout:
|
|
901
1026
|
message = f"Worker timed out after {handle.timeout_seconds}s"
|
|
902
1027
|
annotations["timed_out"] = True
|
|
903
|
-
elif
|
|
1028
|
+
elif exitcode is not None and exitcode < 0:
|
|
904
1029
|
try:
|
|
905
1030
|
signal_name = signal.Signals(-exitcode).name
|
|
906
1031
|
except (ValueError, AttributeError):
|
|
907
1032
|
signal_name = f"SIGNO{-exitcode}"
|
|
908
1033
|
message = f"Worker exited on signal {signal_name} ({exitcode}) without a result"
|
|
909
1034
|
annotations["signal"] = signal_name
|
|
910
|
-
elif
|
|
1035
|
+
elif was_force_killed:
|
|
1036
|
+
message = "Worker was forcefully terminated without a result"
|
|
1037
|
+
annotations["terminated"] = True
|
|
1038
|
+
elif exitcode is not None and exitcode > 0:
|
|
911
1039
|
message = f"Worker exited with code {exitcode} without a result"
|
|
912
1040
|
else:
|
|
913
1041
|
message = "Worker exited without a result frame"
|
|
914
|
-
if
|
|
1042
|
+
if exitcode is not None:
|
|
915
1043
|
annotations["exitcode"] = exitcode
|
|
916
1044
|
return ExecutionResult(
|
|
917
1045
|
success=False,
|
|
@@ -919,103 +1047,38 @@ class ExecutionChannel:
|
|
|
919
1047
|
annotations=annotations,
|
|
920
1048
|
)
|
|
921
1049
|
|
|
922
|
-
def abort_in_flight(self, handles: Sequence[JobHandle]) -> AbortOutcome:
|
|
923
|
-
"""TOCTOU 闭环中止:初查分类 → 进程终止 → 排空管道至 EOF → 重探测闭环 → 关闭管道。"""
|
|
924
|
-
if not handles:
|
|
925
|
-
return AbortOutcome(completed=[], cancelled=[])
|
|
926
|
-
|
|
927
|
-
done_pairs: list[tuple[JobHandle, ExecutionResult]] = []
|
|
928
|
-
pending_handles: list[JobHandle] = []
|
|
929
|
-
|
|
930
|
-
for handle in handles:
|
|
931
|
-
self._receive_available(handle)
|
|
932
|
-
decoded = self._consume_result_if_present(handle)
|
|
933
|
-
if decoded is not None:
|
|
934
|
-
done_pairs.append((handle, decoded))
|
|
935
|
-
continue
|
|
936
|
-
pending_handles.append(handle)
|
|
937
|
-
|
|
938
|
-
for handle, _ in done_pairs:
|
|
939
|
-
try:
|
|
940
|
-
handle.process.join(timeout=self._JOIN_REAP_TIMEOUT)
|
|
941
|
-
except Exception as error:
|
|
942
|
-
logger.warning(f"Completed process join failed: {error}")
|
|
943
|
-
self.finalize_processes(handles)
|
|
944
|
-
|
|
945
|
-
for handle, decoded in done_pairs:
|
|
946
|
-
self._drain_finished(handle)
|
|
947
|
-
exec_id = handle.execution_id or ""
|
|
948
|
-
self._close_handle_pipe(exec_id)
|
|
949
|
-
self._handles.pop(exec_id, None)
|
|
950
|
-
self._results.pop(exec_id, None)
|
|
951
|
-
self._errors.pop(exec_id, None)
|
|
952
|
-
|
|
953
|
-
truly_cancelled: list[JobHandle] = []
|
|
954
|
-
for handle in pending_handles:
|
|
955
|
-
self._drain_finished(handle)
|
|
956
|
-
decoded = self._consume_result_if_present(handle)
|
|
957
|
-
if decoded is not None:
|
|
958
|
-
done_pairs.append((handle, decoded))
|
|
959
|
-
else:
|
|
960
|
-
truly_cancelled.append(handle)
|
|
961
|
-
exec_id = handle.execution_id or ""
|
|
962
|
-
self._close_handle_pipe(exec_id)
|
|
963
|
-
self._handles.pop(exec_id, None)
|
|
964
|
-
self._results.pop(exec_id, None)
|
|
965
|
-
self._errors.pop(exec_id, None)
|
|
966
|
-
|
|
967
|
-
return AbortOutcome(completed=done_pairs, cancelled=truly_cancelled)
|
|
968
|
-
|
|
969
1050
|
def cleanup_in_flight(self, handles: Sequence[JobHandle]) -> None:
|
|
970
|
-
"""
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
self._finalize_process(p)
|
|
974
|
-
if receiver is not None:
|
|
975
|
-
receiver.close(timeout=0.5)
|
|
976
|
-
except Exception as e:
|
|
977
|
-
logger.error(f"Error cleaning up lingering worker {exec_id}: {e}")
|
|
978
|
-
self._lingering_workers.clear()
|
|
1051
|
+
"""主循环异常退出时取消未放行启动、终止并回收所有残留与在途会话。"""
|
|
1052
|
+
self.cancel_pending_launches()
|
|
1053
|
+
self._cleanup_lingering_sessions(receiver_timeout=0.5, log_errors=True)
|
|
979
1054
|
|
|
980
1055
|
for handle in handles:
|
|
981
1056
|
try:
|
|
982
|
-
self.
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
self._errors.pop(exec_id, None)
|
|
1057
|
+
handle_session = self._find_session(handle)
|
|
1058
|
+
if handle_session is not None:
|
|
1059
|
+
self._close_session(handle_session, finalize_process=True)
|
|
1060
|
+
else:
|
|
1061
|
+
self._finalize_process(handle.process)
|
|
988
1062
|
except Exception as e:
|
|
989
1063
|
logger.error(f"Error cleaning up in-flight job {handle.job_uid}: {e}")
|
|
990
1064
|
|
|
991
|
-
def
|
|
992
|
-
"""
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
except Exception:
|
|
999
|
-
pass
|
|
1000
|
-
self._lingering_workers.clear()
|
|
1001
|
-
|
|
1002
|
-
for handle in handles:
|
|
1065
|
+
def close(self) -> None:
|
|
1066
|
+
"""关闭通道并回收所有准备中、在途与残留会话的进程和管道资源。"""
|
|
1067
|
+
self.cancel_pending_launches()
|
|
1068
|
+
self.cleanup_in_flight(self.active_handles())
|
|
1069
|
+
with self._sessions_lock:
|
|
1070
|
+
remaining = list(self._sessions.values())
|
|
1071
|
+
for session in remaining:
|
|
1003
1072
|
try:
|
|
1004
|
-
self.
|
|
1073
|
+
self._close_session(session, finalize_process=True, receiver_timeout=0.5)
|
|
1005
1074
|
except Exception as e:
|
|
1006
|
-
logger.error(f"Error
|
|
1075
|
+
logger.error(f"Error closing remaining session {session.session_key}: {e}")
|
|
1007
1076
|
|
|
1008
|
-
def close(self) -> None:
|
|
1009
|
-
"""关闭通道并回收所有在途与残留子进程和管道资源。"""
|
|
1010
|
-
self.cleanup_in_flight(list(self._handles.values()))
|
|
1011
1077
|
|
|
1012
1078
|
__all__ = [
|
|
1013
|
-
"AbortOutcome",
|
|
1014
1079
|
"ExecutionChannel",
|
|
1015
1080
|
"ExecutionResult",
|
|
1016
1081
|
"JobHandle",
|
|
1082
|
+
"PreparedLaunch",
|
|
1017
1083
|
"WorkerLaunchSpec",
|
|
1018
|
-
"_WorkerLaunchCancelled",
|
|
1019
|
-
"_decode_ipc_result",
|
|
1020
|
-
"_worker_process_entry",
|
|
1021
1084
|
]
|