tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. tasklite/__init__.py +42 -18
  2. tasklite/__main__.py +205 -0
  3. tasklite/_report.py +562 -0
  4. tasklite/backend/__init__.py +19 -2
  5. tasklite/backend/base.py +327 -156
  6. tasklite/backend/memory.py +261 -339
  7. tasklite/backend/sqlite_backend.py +416 -488
  8. tasklite/edit.py +1276 -0
  9. tasklite/engine/__init__.py +6 -7
  10. tasklite/engine/_worker_bootstrap.py +221 -0
  11. tasklite/engine/_worker_runtime.py +229 -0
  12. tasklite/engine/admission.py +20 -114
  13. tasklite/engine/authorization.py +989 -0
  14. tasklite/engine/channel.py +531 -468
  15. tasklite/engine/completion.py +193 -275
  16. tasklite/engine/config.py +5 -11
  17. tasklite/engine/dispatch.py +299 -384
  18. tasklite/engine/governor.py +134 -69
  19. tasklite/engine/in_flight.py +20 -88
  20. tasklite/engine/launcher.py +1173 -0
  21. tasklite/engine/recovery.py +147 -100
  22. tasklite/engine/resource.py +50 -103
  23. tasklite/engine/runtime.py +158 -525
  24. tasklite/engine/scheduler.py +9 -46
  25. tasklite/engine/session.py +60 -15
  26. tasklite/engine/store.py +235 -462
  27. tasklite/engine/types.py +29 -40
  28. tasklite/engine/wait.py +8 -0
  29. tasklite/exceptions.py +0 -1
  30. tasklite/models/artifact.py +84 -62
  31. tasklite/models/context.py +37 -30
  32. tasklite/models/graph.py +114 -0
  33. tasklite/models/job.py +3 -5
  34. tasklite/models/records.py +482 -0
  35. tasklite/models/state.py +19 -125
  36. tasklite/models/trigger.py +63 -3
  37. tasklite/pipeline.py +92 -100
  38. tasklite/resource.py +79 -2
  39. tasklite/snapshot.py +608 -405
  40. tasklite/testing.py +424 -11
  41. tasklite/utils/artifact.py +1091 -179
  42. tasklite/utils/encoding.py +21 -40
  43. tasklite/utils/ipc.py +49 -1
  44. tasklite/utils/lockfile.py +50 -19
  45. tasklite/wrappers/__init__.py +1 -1
  46. tasklite/wrappers/discovery.py +13 -16
  47. tasklite/wrappers/http.py +77 -43
  48. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
  49. tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
  50. tasklite/contrib/__init__.py +0 -7
  51. tasklite/engine/execution_access.py +0 -39
  52. tasklite/engine/files.py +0 -52
  53. tasklite/engine/ops.py +0 -25
  54. tasklite/engine/selection.py +0 -186
  55. tasklite/hooks.py +0 -49
  56. tasklite/triggers.py +0 -47
  57. tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
  58. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
  59. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
  60. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
@@ -1,46 +1,40 @@
1
- """v2 执行通道:子进程 spawn、阶梯看门狗、瞬时 Pipe 收割、fencing 与 TOCTOU 中止。
1
+ """v2 执行通道:线性两阶段启动、单一会话状态机、阶梯看门狗、瞬时 Pipe 收割与 TOCTOU 中止。
2
2
 
3
- ExecutionChannel 统一内敛子进程派发、看门狗阶梯终止、单向 Pipe 消息收割、
4
- 跨平台排他文件锁探测与 TOCTOU 闭环清理的底层复杂性。
5
-
6
- spawn 上下文显式取 ``mp.get_context("spawn")``:fork 会把父进程的锁
7
- 状态/线程/连接池半成品复制进子进程(引擎侧任何锁的持有状态都会在
8
- 子进程镜像里失真),spawn 注册表经 WorkerLaunchSpec 显式下发、不经
9
- 继承,进程 seam 的全部载荷具名可审计。
3
+ ExecutionChannel 以每执行单一会话对象(``_ExecutionSession``)统一管理
4
+ ``prepared -> active -> lingering -> closed`` 全生命周期状态,通过
5
+ ``prepare_spawn -> release_spawn / abort_spawn`` 线性两阶段协议驱动底层
6
+ ``WorkerLauncher``,对真实子进程与进程内替身完全透明无特判。
10
7
  """
11
8
 
12
9
  from __future__ import annotations
13
10
 
14
- import dataclasses
11
+ import enum
15
12
  import logging
16
- import multiprocessing as mp
17
- import os
18
13
  import signal
19
14
  import threading
20
15
  import time
21
- import traceback
22
16
  from collections.abc import Callable, Sequence
23
17
  from dataclasses import dataclass, field
24
- from pathlib import Path
25
18
  from typing import Any
26
19
 
27
- from ..exceptions import TaskLiteError, _ArtifactIOError
28
- from ..models.context import JobContext
20
+ from ..exceptions import TaskLiteError
29
21
  from ..models.job import Job
30
- from ..models.result import TaskResult, validate_strict_json_object
31
- from ..utils.clock import utc_now_iso
22
+ from ..models.result import validate_strict_json_object
32
23
  from ..utils.ipc import (
33
24
  PIPE_MSG_DECLARE,
34
25
  PIPE_MSG_RESULT,
35
26
  WorkerPipeMessage,
36
27
  WorkerPipeReceiver,
37
- WorkerPipeSender,
38
- WorkerReplyReceiver,
39
28
  WorkerReplySender,
40
- create_worker_pipe,
41
- create_worker_reply_pipe,
42
29
  )
43
- from .types import AuthorizationOutcome, JobHandle
30
+ from ._worker_runtime import WorkerLaunchSpec
31
+ from .launcher import (
32
+ PreparedWorker,
33
+ SubprocessWorkerLauncher,
34
+ WorkerLauncher,
35
+ _WorkerLaunchCancelled,
36
+ )
37
+ from .types import AuthorizationOutcome, JobHandle, is_importing_mp_main
44
38
 
45
39
  logger = logging.getLogger("tasklite")
46
40
 
@@ -254,113 +248,6 @@ def _decode_ipc_result(
254
248
  return _corrupt_result(job, f"unknown status {status!r}")
255
249
 
256
250
 
257
- def _isolate_worker_process_signals() -> None:
258
- """让进程组停止信号只由主进程处理。"""
259
- if os.name != "posix":
260
- return
261
- setsid = getattr(os, "setsid", None)
262
- if setsid is not None:
263
- try:
264
- setsid()
265
- except OSError:
266
- pass
267
- signal.signal(signal.SIGINT, signal.SIG_IGN)
268
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
269
-
270
-
271
- def _worker_process_entry(spec: "WorkerLaunchSpec") -> None:
272
- """spawn 子进程执行体入口:执行 handler 并经瞬时管道发送结果。
273
-
274
- 不变式:结果通过管道单向回传;执行体任何出口(含未捕获异常)
275
- 均在 finally 统一关闭管道写端。
276
- """
277
- _isolate_worker_process_signals()
278
- handler_func = spec.handler
279
- job = spec.job
280
- if getattr(spec, "attempt_no", None) is not None:
281
- job._attempt_no = spec.attempt_no
282
- ctx = spec.job_context
283
- job_uid = job.uid
284
- execution_id = spec.execution_id
285
- if not execution_id:
286
- raise RuntimeError(
287
- f"worker started without execution_id for {job_uid}: "
288
- f"fencing requires WorkerLaunchSpec.execution_id"
289
- )
290
- pipe_sender = spec.pipe_sender
291
- reply_receiver = spec.reply_receiver
292
- if pipe_sender is not None or reply_receiver is not None:
293
- ctx._bind_pipe_sender(pipe_sender, reply_receiver)
294
-
295
- def _write_result(payload: dict[str, Any]) -> None:
296
- if pipe_sender is None:
297
- return
298
- try:
299
- pipe_sender.send_result(payload)
300
- except OSError as exc:
301
- logger.error(
302
- f"worker pipe result send failed for {job_uid} ({execution_id}): {exc}"
303
- )
304
-
305
- try:
306
- try:
307
- raw_result = handler_func(job, ctx)
308
- if isinstance(raw_result, TaskResult):
309
- if raw_result.status == "success":
310
- _write_result({
311
- "status": "success",
312
- "message": raw_result.message,
313
- "metadata": raw_result.metadata,
314
- "spawned_jobs": [j.to_dict() for j in ctx._spawned_jobs],
315
- "cursor_updates": dict(ctx._cursor_updates),
316
- })
317
- elif raw_result.status == "failure":
318
- _write_result({
319
- "status": "failure",
320
- "message": raw_result.message,
321
- "metadata": raw_result.metadata,
322
- "exception": None,
323
- })
324
- else:
325
- _write_result({
326
- "status": "retry",
327
- "message": raw_result.message,
328
- "metadata": raw_result.metadata,
329
- "consume_budget": raw_result.consume_budget,
330
- })
331
- else:
332
- _write_result({
333
- "status": "failure",
334
- "message": (
335
- f"invalid handler return type: expected TaskResult, "
336
- f"got {type(raw_result).__name__}"
337
- ),
338
- "metadata": {},
339
- "exception": None,
340
- })
341
- except _ArtifactIOError as e:
342
- _write_result({"status": "artifact_error", "error_message": str(e)})
343
- except BaseException as e:
344
- _write_result({
345
- "status": "failure",
346
- "message": (
347
- type(e).__name__ if not str(e)
348
- else f"{type(e).__name__}: {e}"
349
- ),
350
- "metadata": {},
351
- "exception": {
352
- "type": type(e).__name__,
353
- "message": str(e),
354
- "traceback": traceback.format_exc(),
355
- },
356
- })
357
- finally:
358
- if pipe_sender is not None:
359
- pipe_sender.close()
360
- if reply_receiver is not None:
361
- reply_receiver.close()
362
-
363
-
364
251
  @dataclass
365
252
  class ExecutionResult:
366
253
  """单次子进程执行的收割产物(收割/中止路径共形)。"""
@@ -397,35 +284,37 @@ class ExecutionResult:
397
284
 
398
285
 
399
286
  @dataclass(frozen=True)
400
- class WorkerLaunchSpec:
401
- """进程 seam 具名契约——spawn 下发子进程执行体的全部载荷。
287
+ class PreparedLaunch:
288
+ """已完成冷启动准备、等待主进程放行或中止的执行令牌。"""
402
289
 
403
- execution_id(``{run_id}.{dispatch_no}`` 执行身份)与管道发送端归
404
- 本 spec,不借道 JobContext 公开属性穿透进程边界。
405
- """
406
-
407
- handler: Callable[[Job, JobContext], Any]
408
- job: Job
409
- job_context: JobContext
410
- execution_id: str
411
- timeout_seconds: float
412
- pipe_sender: WorkerPipeSender | None = None
413
- reply_receiver: WorkerReplyReceiver | None = None
414
- run_id: str = ""
415
- attempt_no: int = 1
416
- launch_guard: Callable[[], bool] | None = None
290
+ spec: WorkerLaunchSpec
291
+ session_key: str
417
292
 
418
293
 
419
- @dataclass(frozen=True)
420
- class AbortOutcome:
421
- """强制终止/异常停机时的收尾结果。"""
294
+ class _SessionState(str, enum.Enum):
295
+ """单次 worker 执行会话的显式状态机(单向推进至 closed)。"""
422
296
 
423
- completed: list[tuple[JobHandle, ExecutionResult]]
424
- cancelled: list[JobHandle]
297
+ PREPARED = "prepared"
298
+ ACTIVE = "active"
299
+ LINGERING = "lingering"
300
+ CLOSED = "closed"
425
301
 
426
302
 
427
- class _WorkerLaunchCancelled(Exception):
428
- """最终启动门前观察到停止请求,子进程不得启动。"""
303
+ @dataclass
304
+ class _ExecutionSession:
305
+ """单次 worker 执行的唯一会话状态对象(取代多份并行字典与残留列表)。"""
306
+
307
+ session_key: str
308
+ state: _SessionState = _SessionState.PREPARED
309
+ cancel_event: threading.Event = field(default_factory=threading.Event)
310
+ prepared_worker: PreparedWorker | None = None
311
+ handle: JobHandle | None = None
312
+ process: Any = None
313
+ receiver: WorkerPipeReceiver | None = None
314
+ reply_sender: WorkerReplySender | None = None
315
+ health_stop: threading.Event | None = None
316
+ result: dict[str, Any] | None = None
317
+ error: Exception | None = None
429
318
 
430
319
 
431
320
  class ExecutionChannel:
@@ -436,23 +325,96 @@ class ExecutionChannel:
436
325
  def __init__(
437
326
  self,
438
327
  *,
439
- process_context: Any | None = None,
440
- output_roots: str | Path | Sequence[str | Path] | None = None,
441
- protected_roots: Sequence[str | Path] = (),
328
+ launcher: WorkerLauncher | None = None,
442
329
  ) -> None:
443
- self._process_context = process_context or mp.get_context("spawn")
444
- self.output_roots = output_roots
445
- self.protected_roots = tuple(protected_roots)
446
- self._receivers: dict[str, WorkerPipeReceiver] = {}
447
- self._in_process_senders: dict[str, WorkerPipeSender] = {}
448
- self._reply_senders: dict[str, WorkerReplySender] = {}
330
+ self._launcher: WorkerLauncher = (
331
+ launcher if launcher is not None else SubprocessWorkerLauncher()
332
+ )
333
+ self._sessions_lock = threading.RLock()
334
+ self._sessions: dict[str, _ExecutionSession] = {}
449
335
  self._declaration_handler: Callable[[JobHandle, dict[str, Any]], dict[str, Any] | None] | None = None
336
+ self._is_timeout_paused_cb: Callable[[JobHandle], bool] | None = None
337
+ self._get_authorization_outcome_cb: Callable[[JobHandle], AuthorizationOutcome | None] | None = None
450
338
  self._wakeup_callback: Callable[[], None] | None = None
451
- self._handles: dict[str, JobHandle] = {}
452
- self._results: dict[str, dict[str, Any]] = {}
453
- self._errors: dict[str, Exception] = {}
454
- self._lingering_workers: list[tuple[str, Any, WorkerPipeReceiver | None]] = []
455
- self._health_stops: dict[str, threading.Event] = {}
339
+
340
+ @property
341
+ def launcher(self) -> WorkerLauncher:
342
+ """返回当前绑定的 worker 启动器(供内部与测试 seam 只读观察)。"""
343
+ return self._launcher
344
+
345
+ def set_launcher(self, launcher: WorkerLauncher) -> WorkerLauncher:
346
+ """受控替换底层 worker 启动器并返回先前启动器(供测试 seam 使用)。"""
347
+ if not isinstance(launcher, WorkerLauncher):
348
+ raise TypeError(
349
+ f"launcher must be a WorkerLauncher instance, got {type(launcher).__name__}"
350
+ )
351
+ previous = self._launcher
352
+ self._launcher = launcher
353
+ return previous
354
+
355
+ def _find_session(self, handle: JobHandle) -> _ExecutionSession | None:
356
+ exec_id = handle.execution_id
357
+ with self._sessions_lock:
358
+ if exec_id:
359
+ session = self._sessions.get(exec_id)
360
+ if session is not None:
361
+ return session
362
+ for session in self._sessions.values():
363
+ if session.handle is handle:
364
+ return session
365
+ return None
366
+
367
+ def _get_session_by_id(self, execution_id: str | None) -> _ExecutionSession | None:
368
+ if not execution_id:
369
+ return None
370
+ with self._sessions_lock:
371
+ return self._sessions.get(execution_id)
372
+
373
+ @property
374
+ def has_active_workers(self) -> bool:
375
+ """通道是否存在准备中、在途或残留的 worker 会话。"""
376
+ self._reap_lingering_workers()
377
+ with self._sessions_lock:
378
+ return any(
379
+ session.state is not _SessionState.CLOSED
380
+ for session in self._sessions.values()
381
+ )
382
+
383
+ def active_handles(self) -> list[JobHandle]:
384
+ """返回当前处于 ``active`` 状态的全部在途执行句柄快照。"""
385
+ with self._sessions_lock:
386
+ return [
387
+ session.handle
388
+ for session in self._sessions.values()
389
+ if session.state is _SessionState.ACTIVE and session.handle is not None
390
+ ]
391
+
392
+ def has_received_result(self, execution_id: str | None) -> bool:
393
+ """查询指定执行会话是否已通过管道收到结果帧(未消费)。"""
394
+ session = self._get_session_by_id(execution_id)
395
+ return session is not None and session.result is not None
396
+
397
+ def cancel_pending_launches(self) -> None:
398
+ """立即唤醒并中止全部尚在准备/握手阶段的未放行启动。"""
399
+ with self._sessions_lock:
400
+ events = [
401
+ session.cancel_event
402
+ for session in self._sessions.values()
403
+ if session.state is _SessionState.PREPARED
404
+ ]
405
+ for ev in events:
406
+ ev.set()
407
+
408
+ @staticmethod
409
+ def _kill_process(p: Any) -> None:
410
+ """优先调用进程树原子终止接口,兼容测试进程桩的 kill()。"""
411
+ kill_tree = getattr(p, "kill_tree", None)
412
+ if callable(kill_tree):
413
+ kill_tree()
414
+ return
415
+ kill = getattr(p, "kill", None)
416
+ if callable(kill):
417
+ kill()
456
418
 
457
419
  @staticmethod
458
420
  def _terminate_and_join_process(p: Any) -> None:
@@ -460,97 +422,254 @@ class ExecutionChannel:
460
422
  if getattr(p, "_closed", False):
461
423
  return
462
424
  try:
463
- if p.is_alive():
464
- p.kill()
425
+ is_alive_fn = getattr(p, "is_alive", None)
426
+ if callable(is_alive_fn) and is_alive_fn():
427
+ ExecutionChannel._kill_process(p)
465
428
  except Exception as e:
466
429
  logger.warning(f"Process kill failed during finalize: {e}")
467
430
  try:
468
- if not isinstance(p, mp.process.BaseProcess) or getattr(p, "_popen", None) is not None:
469
- p.join(timeout=ExecutionChannel._JOIN_REAP_TIMEOUT)
431
+ join_fn = getattr(p, "join", None)
432
+ if callable(join_fn):
433
+ join_fn(timeout=ExecutionChannel._JOIN_REAP_TIMEOUT)
470
434
  except Exception as e:
471
435
  logger.warning(f"Process join failed during finalize: {e}")
472
436
 
473
437
  @staticmethod
474
438
  def _finalize_process(p: Any) -> None:
475
- """清理单个子进程:kill + join + close(各步失败降级但留痕,不阻断收割)。"""
439
+ """清理单个子进程:kill_tree + join + close(各步失败降级但留痕,不阻断收割)。"""
476
440
  if getattr(p, "_closed", False):
477
441
  return
478
442
  ExecutionChannel._terminate_and_join_process(p)
479
443
  try:
480
444
  process_close = getattr(p, "close", None)
481
- if process_close is not None:
445
+ if callable(process_close):
482
446
  process_close()
483
447
  except Exception as e:
484
448
  logger.warning(f"Process close failed during finalize: {e}")
485
449
 
486
- def spawn(self, spec: WorkerLaunchSpec) -> JobHandle:
487
- """启动隔离子进程执行 handler,建立瞬时管道与 execution_id 绑定,立即返回句柄。"""
488
- receiver, sender = create_worker_pipe(
489
- self._process_context,
490
- job_uid=spec.job.uid,
491
- execution_id=spec.execution_id,
492
- run_id=spec.run_id or None,
493
- attempt_no=spec.attempt_no,
494
- on_message=self._notify_wakeup,
495
- )
496
- reply_receiver, reply_sender = create_worker_reply_pipe(
497
- self._process_context,
498
- job_uid=spec.job.uid,
499
- execution_id=spec.execution_id,
500
- run_id=spec.run_id or None,
501
- attempt_no=spec.attempt_no,
502
- )
503
- spec.job_context._bind_pipe_sender(sender, reply_receiver)
504
- launch_spec = dataclasses.replace(
505
- spec,
506
- pipe_sender=sender,
507
- reply_receiver=reply_receiver,
508
- launch_guard=None,
450
+ def _stop_health_watch(self, session: _ExecutionSession) -> None:
451
+ with self._sessions_lock:
452
+ stop = session.health_stop
453
+ session.health_stop = None
454
+ if stop is not None:
455
+ stop.set()
456
+
457
+ def _close_reply_sender(self, session: _ExecutionSession) -> None:
458
+ with self._sessions_lock:
459
+ reply_sender = session.reply_sender
460
+ session.reply_sender = None
461
+ if reply_sender is not None:
462
+ try:
463
+ reply_sender.close()
464
+ except Exception as exc:
465
+ logger.debug(f"Reply sender close warning for {session.session_key}: {exc}")
466
+
467
+ def _close_receiver(
468
+ self,
469
+ session: _ExecutionSession,
470
+ *,
471
+ timeout: float | None = None,
472
+ ) -> None:
473
+ with self._sessions_lock:
474
+ receiver = session.receiver
475
+ session.receiver = None
476
+ if receiver is not None:
477
+ try:
478
+ receiver.close(
479
+ timeout=self._JOIN_REAP_TIMEOUT if timeout is None else timeout
480
+ )
481
+ except Exception as exc:
482
+ logger.debug(f"Receiver close warning for {session.session_key}: {exc}")
483
+
484
+ def _transition_to_lingering(self, session: _ExecutionSession) -> None:
485
+ """将已产出结果但进程尚未退出的会话转入 lingering(保活 process 与 receiver)。"""
486
+ self._stop_health_watch(session)
487
+ self._close_reply_sender(session)
488
+ with self._sessions_lock:
489
+ session.handle = None
490
+ session.result = None
491
+ session.error = None
492
+ session.state = _SessionState.LINGERING
493
+
494
+ def _close_session(
495
+ self,
496
+ session: _ExecutionSession | None,
497
+ *,
498
+ finalize_process: bool = True,
499
+ receiver_timeout: float | None = None,
500
+ ) -> None:
501
+ """幂等关闭单个会话:停止健康探测、关闭全部管道、按需回收进程并移出状态表。"""
502
+ if session is None:
503
+ return
504
+ with self._sessions_lock:
505
+ health_stop = session.health_stop
506
+ session.health_stop = None
507
+ process = session.process
508
+ prepared_worker = session.prepared_worker
509
+ reply_sender = session.reply_sender
510
+ receiver = session.receiver
511
+ session.prepared_worker = None
512
+ session.handle = None
513
+ session.process = None
514
+ session.reply_sender = None
515
+ session.receiver = None
516
+ session.result = None
517
+ session.error = None
518
+ session.state = _SessionState.CLOSED
519
+ self._sessions.pop(session.session_key, None)
520
+
521
+ if health_stop is not None:
522
+ health_stop.set()
523
+ if prepared_worker is not None:
524
+ try:
525
+ prepared_worker.abort()
526
+ except Exception as exc:
527
+ logger.error(f"Prepared worker abort warning for {session.session_key}: {exc}")
528
+ if finalize_process and process is not None:
529
+ self._finalize_process(process)
530
+ if reply_sender is not None:
531
+ try:
532
+ reply_sender.close()
533
+ except Exception as exc:
534
+ logger.debug(f"Reply sender close warning for {session.session_key}: {exc}")
535
+ if receiver is not None:
536
+ try:
537
+ receiver.close(
538
+ timeout=self._JOIN_REAP_TIMEOUT if receiver_timeout is None else receiver_timeout
539
+ )
540
+ except Exception as exc:
541
+ logger.debug(f"Receiver close warning for {session.session_key}: {exc}")
542
+
543
+ # ── 线性两阶段启动协议(prepare_spawn -> release_spawn / abort_spawn)──
544
+
545
+ def prepare_spawn(self, spec: WorkerLaunchSpec) -> PreparedLaunch:
546
+ """阶段一:冷启动 worker 并完成握手与进程树绑定,停在放行门前。"""
547
+ if is_importing_mp_main():
548
+ raise RuntimeError(
549
+ "ExecutionChannel.spawn() cannot be called while bootstrapping a worker process. "
550
+ "Guard top-level execution with `if __name__ == '__main__':`."
551
+ )
552
+
553
+ cancel_event = threading.Event()
554
+ session_key = spec.execution_id if spec.execution_id else f"_pending_{id(cancel_event)}"
555
+ session = _ExecutionSession(
556
+ session_key=session_key,
557
+ state=_SessionState.PREPARED,
558
+ cancel_event=cancel_event,
509
559
  )
510
- p = None
560
+ with self._sessions_lock:
561
+ self._sessions[session_key] = session
562
+
511
563
  try:
512
- p = self._process_context.Process(target=_worker_process_entry, args=(launch_spec,))
513
- if spec.launch_guard is not None and not spec.launch_guard():
564
+ prepared_worker = self._launcher.prepare(
565
+ spec,
566
+ on_message=self._notify_wakeup,
567
+ cancel_event=cancel_event,
568
+ )
569
+ with self._sessions_lock:
570
+ if session.state is _SessionState.CLOSED or cancel_event.is_set():
571
+ session.state = _SessionState.CLOSED
572
+ self._sessions.pop(session_key, None)
573
+ should_abort = True
574
+ else:
575
+ session.prepared_worker = prepared_worker
576
+ should_abort = False
577
+ if should_abort:
578
+ prepared_worker.abort()
514
579
  raise _WorkerLaunchCancelled(
515
- f"stop requested before worker process start for {spec.job.uid}"
580
+ f"stop requested before worker release for {spec.job.uid}"
516
581
  )
517
- started_at = utc_now_iso()
518
- p.start()
519
- except Exception:
520
- sender.close()
521
- receiver.close()
522
- reply_receiver.close()
523
- reply_sender.close()
524
- if p is not None:
525
- self._finalize_process(p)
582
+ return PreparedLaunch(spec=spec, session_key=session_key)
583
+ except BaseException:
584
+ with self._sessions_lock:
585
+ session.prepared_worker = None
586
+ session.state = _SessionState.CLOSED
587
+ self._sessions.pop(session_key, None)
526
588
  raise
527
- if isinstance(p, mp.process.BaseProcess):
528
- sender.close()
529
- reply_receiver.close()
530
- else:
531
- self._in_process_senders[spec.execution_id] = sender
532
- self._reply_senders[spec.execution_id] = reply_sender
589
+
590
+ def release_spawn(self, prepared: PreparedLaunch) -> JobHandle:
591
+ """阶段二(正常路径):放行 worker(发送 b"GO"),转入 active 并返回句柄。"""
592
+ spec = prepared.spec
593
+ with self._sessions_lock:
594
+ session = self._sessions.get(prepared.session_key)
595
+ if session is None or session.state is not _SessionState.PREPARED:
596
+ raise RuntimeError(
597
+ f"Cannot release execution session {prepared.session_key!r}: not prepared"
598
+ )
599
+ prepared_worker = session.prepared_worker
600
+ if prepared_worker is None:
601
+ raise RuntimeError(
602
+ f"Cannot release execution session {prepared.session_key!r}: missing prepared worker"
603
+ )
604
+ cancelled = session.cancel_event.is_set()
605
+
606
+ if cancelled:
607
+ self.abort_spawn(prepared)
608
+ raise _WorkerLaunchCancelled(
609
+ f"stop requested before worker release for {spec.job.uid}"
610
+ )
611
+
612
+ try:
613
+ launched = prepared_worker.release()
614
+ except BaseException:
615
+ with self._sessions_lock:
616
+ session.prepared_worker = None
617
+ session.state = _SessionState.CLOSED
618
+ self._sessions.pop(prepared.session_key, None)
619
+ raise
620
+
533
621
  timeout_at_monotonic = time.monotonic() + spec.timeout_seconds
534
622
  handle = JobHandle(
535
623
  job_uid=spec.job.uid,
536
- process=p,
624
+ process=launched.handle,
537
625
  timeout_at_monotonic=timeout_at_monotonic,
538
626
  timeout_seconds=spec.timeout_seconds,
539
- started_at=started_at,
627
+ started_at=launched.started_at,
540
628
  job=spec.job,
541
629
  execution_id=spec.execution_id,
542
630
  run_id=spec.run_id or None,
543
631
  attempt_no=spec.attempt_no,
544
632
  )
545
- self._receivers[spec.execution_id] = receiver
546
- self._handles[spec.execution_id] = handle
547
- self._watch_process_health(spec.execution_id, p)
633
+ with self._sessions_lock:
634
+ session.prepared_worker = None
635
+ session.handle = handle
636
+ session.process = launched.handle
637
+ session.receiver = launched.receiver
638
+ session.reply_sender = launched.reply_sender
639
+ session.state = _SessionState.ACTIVE
640
+ self._watch_process_health(session)
548
641
  return handle
549
642
 
550
- def _watch_process_health(self, execution_id: str, process: Any) -> None:
643
+ def abort_spawn(self, prepared: PreparedLaunch) -> None:
644
+ """阶段二(中止路径):幂等终止并回收尚未放行的 worker 与会话。"""
645
+ with self._sessions_lock:
646
+ session = self._sessions.get(prepared.session_key)
647
+ if session is None:
648
+ return
649
+ session.cancel_event.set()
650
+ if session.state is not _SessionState.PREPARED:
651
+ return
652
+ prepared_worker = session.prepared_worker
653
+ session.prepared_worker = None
654
+ session.state = _SessionState.CLOSED
655
+ self._sessions.pop(prepared.session_key, None)
656
+ if prepared_worker is not None:
657
+ prepared_worker.abort()
658
+
659
+ def spawn(self, spec: WorkerLaunchSpec) -> JobHandle:
660
+ """单步便捷启动接口:连续执行 prepare_spawn 与 release_spawn。"""
661
+ prepared = self.prepare_spawn(spec)
662
+ try:
663
+ return self.release_spawn(prepared)
664
+ except BaseException:
665
+ self.abort_spawn(prepared)
666
+ raise
667
+
668
+ def _watch_process_health(self, session: _ExecutionSession) -> None:
551
669
  """独立探测 worker 存活,仅在状态变化时唤醒主循环。"""
552
670
  stop = threading.Event()
553
- self._health_stops[execution_id] = stop
671
+ process = session.process
672
+ session.health_stop = stop
554
673
 
555
674
  def watch() -> None:
556
675
  try:
@@ -563,20 +682,22 @@ class ExecutionChannel:
563
682
  self._notify_wakeup()
564
683
  break
565
684
  finally:
566
- self._health_stops.pop(execution_id, None)
685
+ if session.health_stop is stop:
686
+ session.health_stop = None
567
687
 
568
688
  threading.Thread(
569
689
  target=watch,
570
- name=f"tasklite-worker-health-{execution_id}",
690
+ name=f"tasklite-worker-health-{session.session_key}",
571
691
  daemon=True,
572
692
  ).start()
573
693
 
574
694
  def _accept_message(self, handle: JobHandle, msg: WorkerPipeMessage) -> None:
575
- """消费一条已校验信封的管道消息并更新通道内存状态。"""
695
+ """消费一条已校验信封的管道消息并更新会话状态。"""
696
+ session = self._find_session(handle)
576
697
  if msg.kind == PIPE_MSG_DECLARE:
577
698
  if not isinstance(msg.payload, dict):
578
699
  raise ValueError(f"invalid declare payload for {handle.job_uid}")
579
- reply_sender = self._reply_senders.get(handle.execution_id or "")
700
+ reply_sender = session.reply_sender if session is not None else None
580
701
  if self._declaration_handler is not None and reply_sender is not None:
581
702
  reply = self._declaration_handler(handle, msg.payload)
582
703
  if reply is not None:
@@ -587,13 +708,13 @@ class ExecutionChannel:
587
708
  elif msg.kind == PIPE_MSG_RESULT:
588
709
  if not isinstance(msg.payload, dict):
589
710
  raise ValueError(f"invalid result payload for {handle.job_uid}")
590
- exec_id = handle.execution_id or ""
591
- if exec_id not in self._errors:
592
- self._results.setdefault(exec_id, msg.payload)
711
+ if session is not None and session.error is None and session.result is None:
712
+ session.result = msg.payload
593
713
 
594
714
  def _process_pipe_item(
595
715
  self, handle: JobHandle, item: WorkerPipeMessage | Exception
596
716
  ) -> None:
717
+ session = self._find_session(handle)
597
718
  exec_id = handle.execution_id or ""
598
719
  if isinstance(item, OSError):
599
720
  logger.debug(
@@ -601,38 +722,29 @@ class ExecutionChannel:
601
722
  )
602
723
  return
603
724
  if isinstance(item, Exception):
604
- if exec_id not in self._results:
605
- self._errors.setdefault(exec_id, item)
725
+ if session is not None and session.result is None and session.error is None:
726
+ session.error = item
606
727
  return
607
728
  try:
608
729
  self._accept_message(handle, item)
609
730
  except Exception as exc:
610
- if exec_id not in self._results:
611
- self._errors.setdefault(exec_id, exc)
731
+ if session is not None and session.result is None and session.error is None:
732
+ session.error = exc
612
733
 
613
734
  def _receive_available(self, handle: JobHandle) -> None:
614
735
  """非阻塞拉取 handle 对应接收端当前已入队的全部消息。"""
615
- exec_id = handle.execution_id
616
- if not exec_id:
617
- return
618
- receiver = self._receivers.get(exec_id)
619
- if receiver is None:
736
+ session = self._find_session(handle)
737
+ if session is None or session.receiver is None:
620
738
  return
621
- for item in receiver.take_messages():
739
+ for item in session.receiver.take_messages():
622
740
  self._process_pipe_item(handle, item)
623
741
 
624
742
  def _drain_finished(self, handle: JobHandle) -> None:
625
- """在进程终止后关闭写端并排空接收端直至 EOF。"""
626
- exec_id = handle.execution_id
627
- if not exec_id:
743
+ """在进程终止后排空接收端直至 EOF。"""
744
+ session = self._find_session(handle)
745
+ if session is None or session.receiver is None:
628
746
  return
629
- sender = self._in_process_senders.pop(exec_id, None)
630
- if sender is not None:
631
- sender.close()
632
- receiver = self._receivers.get(exec_id)
633
- if receiver is None:
634
- return
635
- for item in receiver.drain_to_eof(timeout=self._JOIN_REAP_TIMEOUT):
747
+ for item in session.receiver.drain_to_eof(timeout=self._JOIN_REAP_TIMEOUT):
636
748
  self._process_pipe_item(handle, item)
637
749
 
638
750
  def set_declaration_handler(
@@ -641,6 +753,24 @@ class ExecutionChannel:
641
753
  ) -> None:
642
754
  self._declaration_handler = handler
643
755
 
756
+ def set_authorization_queries(
757
+ self,
758
+ *,
759
+ is_timeout_paused: Callable[[JobHandle], bool] | None = None,
760
+ get_outcome: Callable[[JobHandle], AuthorizationOutcome | None] | None = None,
761
+ ) -> None:
762
+ """绑定运行期授权状态只读查询回调(超时暂停判定与终态授权结果)。"""
763
+ self._is_timeout_paused_cb = is_timeout_paused
764
+ self._get_authorization_outcome_cb = get_outcome
765
+
766
+ def _is_timeout_paused(self, handle: JobHandle) -> bool:
767
+ cb = self._is_timeout_paused_cb
768
+ return bool(cb(handle)) if cb is not None else False
769
+
770
+ def _get_authorization_outcome(self, handle: JobHandle) -> AuthorizationOutcome | None:
771
+ cb = self._get_authorization_outcome_cb
772
+ return cb(handle) if cb is not None else None
773
+
644
774
  def set_wakeup_callback(self, callback: Callable[[], None] | None) -> None:
645
775
  """绑定 IPC 消息到达时的主循环唤醒入口。"""
646
776
  self._wakeup_callback = callback
@@ -650,102 +780,111 @@ class ExecutionChannel:
650
780
  if callback is not None:
651
781
  callback()
652
782
 
783
+ def is_handle_alive(self, handle: JobHandle) -> bool:
784
+ """受控判定在途句柄对应子进程是否仍在运行(屏蔽已关闭句柄的 ValueError)。"""
785
+ p = handle.process
786
+ if p is None or getattr(p, "_closed", False):
787
+ return False
788
+ is_alive_fn = getattr(p, "is_alive", None)
789
+ if not callable(is_alive_fn):
790
+ return False
791
+ try:
792
+ return bool(is_alive_fn())
793
+ except (ValueError, OSError):
794
+ return False
795
+
653
796
  def terminate_process(self, handle: JobHandle) -> None:
654
- """终止指定作业的子进程。"""
797
+ """终止指定作业的子进程树。"""
655
798
  p = handle.process
656
799
  if p is not None:
657
800
  try:
658
- if p.is_alive():
659
- p.kill()
801
+ is_alive_fn = getattr(p, "is_alive", None)
802
+ if callable(is_alive_fn) and is_alive_fn():
803
+ self._kill_process(p)
660
804
  except Exception:
661
805
  pass
662
806
 
663
807
  def get_reply_sender(self, execution_id: str | None) -> WorkerReplySender | None:
664
- """获取指定 execution_id 的回复发送端。"""
665
- if not execution_id:
808
+ """获取指定活跃会话的回复发送端。"""
809
+ session = self._get_session_by_id(execution_id)
810
+ if session is None or session.state is not _SessionState.ACTIVE:
666
811
  return None
667
- return self._reply_senders.get(execution_id)
668
-
669
- def _close_handle_pipe(self, execution_id: str | None) -> None:
670
- if not execution_id:
671
- return
672
- stop = self._health_stops.pop(execution_id, None)
673
- if stop is not None:
674
- stop.set()
675
- sender = self._in_process_senders.pop(execution_id, None)
676
- if sender is not None:
677
- sender.close()
678
- receiver = self._receivers.pop(execution_id, None)
679
- if receiver is not None:
680
- receiver.close(timeout=self._JOIN_REAP_TIMEOUT)
681
- reply_sender = self._reply_senders.pop(execution_id, None)
682
- if reply_sender is not None:
683
- reply_sender.close()
812
+ return session.reply_sender
684
813
 
685
814
  def _consume_result_if_present(self, handle: JobHandle) -> ExecutionResult | None:
686
- """读取已收到的结果负载 → 守卫过滤 → 解码。
687
-
688
- 守卫契约:任何字典结果都是一次尝试结果;内容损坏由解码器收敛为失败。
689
- """
690
- exec_id = getattr(handle, "execution_id", None)
691
- if not exec_id or exec_id in self._errors:
815
+ """读取已收到的结果负载 → 守卫过滤 → 解码。"""
816
+ session = self._find_session(handle)
817
+ if session is None or session.error is not None:
692
818
  return None
693
- raw_res = self._results.get(exec_id)
819
+ raw_res = session.result
694
820
  if not isinstance(raw_res, dict):
695
821
  return None
696
- self._results.pop(exec_id, None)
822
+ session.result = None
697
823
  return _decode_ipc_result(raw_res, handle.job)
698
824
 
825
+ def _lingering_sessions(self) -> list[_ExecutionSession]:
826
+ """返回当前处于 ``lingering`` 状态的全部会话快照。"""
827
+ with self._sessions_lock:
828
+ return [
829
+ s for s in self._sessions.values()
830
+ if s.state is _SessionState.LINGERING
831
+ ]
832
+
833
+ def _cleanup_lingering_sessions(
834
+ self,
835
+ *,
836
+ receiver_timeout: float = 0.5,
837
+ log_errors: bool = True,
838
+ ) -> None:
839
+ """回收全部处于 ``lingering`` 状态的会话,异常收敛不阻断后续清理。"""
840
+ for session in self._lingering_sessions():
841
+ try:
842
+ self._close_session(
843
+ session,
844
+ finalize_process=True,
845
+ receiver_timeout=receiver_timeout,
846
+ )
847
+ except Exception as e:
848
+ if log_errors:
849
+ logger.error(f"Error cleaning up lingering worker {session.session_key}: {e}")
850
+
699
851
  def _reap_lingering_workers(self) -> None:
700
- """非阻塞轮询已结算但残留的子进程与管道接收端,进程死亡后回收句柄。"""
701
- still_lingering: list[tuple[str, Any, WorkerPipeReceiver | None]] = []
702
- for exec_id, p, receiver in self._lingering_workers:
852
+ """非阻塞轮询已结算但残留的 lingering 会话,进程死亡后回收句柄与接收端。"""
853
+ for session in self._lingering_sessions():
854
+ p = session.process
703
855
  try:
704
- alive = p.is_alive()
856
+ alive = p is not None and p.is_alive()
705
857
  except Exception:
706
858
  alive = False
707
859
  if not alive:
708
- if receiver is not None:
709
- try:
710
- receiver.close(timeout=0.2)
711
- except Exception as e:
712
- logger.debug(f"Lingering receiver close warning for {exec_id}: {e}")
713
- self._finalize_process(p)
714
- else:
715
- if receiver is not None and receiver.eof:
716
- try:
717
- receiver.close(timeout=0.1)
718
- receiver = None
719
- except Exception:
720
- pass
721
- still_lingering.append((exec_id, p, receiver))
722
- self._lingering_workers = still_lingering
860
+ self._close_session(session, finalize_process=True, receiver_timeout=0.2)
861
+ elif session.receiver is not None and session.receiver.eof:
862
+ self._close_receiver(session, timeout=0.1)
723
863
 
724
864
  def wait_for_lingering_workers(self) -> None:
725
- """正常 Run 收口前等待已结算 worker 退出,超时后回收残留进程。"""
726
- if not self._lingering_workers:
865
+ """正常 Run 收口前等待已结算 lingering 会话退出,超时后回收残留进程。"""
866
+ lingering = self._lingering_sessions()
867
+ if not lingering:
727
868
  return
728
869
  deadline = time.monotonic() + self._JOIN_REAP_TIMEOUT
729
- lingering = self._lingering_workers
730
- self._lingering_workers = []
731
- for exec_id, process, receiver in lingering:
732
- remaining = deadline - time.monotonic()
733
- try:
734
- while process.is_alive() and remaining > 0:
735
- process.join(timeout=min(0.05, remaining))
736
- remaining = deadline - time.monotonic()
737
- if process.is_alive() and remaining > 0:
738
- time.sleep(min(0.01, remaining))
739
- except Exception as exc:
740
- logger.warning(f"Settled worker wait failed for {exec_id}: {exc}")
741
- if process.is_alive():
742
- logger.warning(f"Settled worker {exec_id} did not exit before Run end")
743
- self._finalize_process(process)
744
- if receiver is not None:
870
+ for session in lingering:
871
+ process = session.process
872
+ if process is not None:
873
+ remaining = deadline - time.monotonic()
745
874
  try:
746
- receiver.close(timeout=0.2)
875
+ while process.is_alive() and remaining > 0:
876
+ process.join(timeout=min(0.05, remaining))
877
+ remaining = deadline - time.monotonic()
878
+ if process.is_alive() and remaining > 0:
879
+ time.sleep(min(0.01, remaining))
747
880
  except Exception as exc:
748
- logger.debug(f"Settled receiver close warning for {exec_id}: {exc}")
881
+ logger.warning(f"Settled worker wait failed for {session.session_key}: {exc}")
882
+ try:
883
+ if process.is_alive():
884
+ logger.warning(f"Settled worker {session.session_key} did not exit before Run end")
885
+ except Exception:
886
+ pass
887
+ self._close_session(session, finalize_process=True, receiver_timeout=0.2)
749
888
 
750
889
  def reap_completed(
751
890
  self, handles: Sequence[JobHandle]
@@ -754,8 +893,8 @@ class ExecutionChannel:
754
893
 
755
894
  有效结果到达即结算:收到完整有效 TaskResult 即认为本次执行已返回并
756
895
  交付结算,不将 worker 进程退出作为额外成功条件,不以前置 join 阻塞
757
- 主循环。结果返回后仍在运行的子进程移交后台非阻塞回收,不阻塞其他 Job
758
- 派发或结算,亦不因之后的进程退出状态改写有效结果。
896
+ 主循环。结果返回后仍在运行的子进程转入 lingering 后台非阻塞回收,
897
+ 不阻塞其他 Job 派发或结算,亦不因之后的进程退出状态改写有效结果。
759
898
  """
760
899
  self._reap_lingering_workers()
761
900
  completed: list[tuple[JobHandle, ExecutionResult]] = []
@@ -763,28 +902,16 @@ class ExecutionChannel:
763
902
  for handle in handles:
764
903
  now = time.monotonic()
765
904
  p = handle.process
766
- exec_id = handle.execution_id or ""
905
+ session = self._find_session(handle)
767
906
  self._receive_available(handle)
768
907
 
769
- authorization_outcome = handle.authorization_outcome
908
+ authorization_outcome = self._get_authorization_outcome(handle)
770
909
  if authorization_outcome is not None:
771
- is_alive = False
772
- try:
773
- is_alive = p.is_alive()
774
- except Exception:
775
- pass
776
- if not is_alive:
777
- self._handles.pop(exec_id, None)
778
- receiver = self._receivers.pop(exec_id, None)
779
- if receiver is not None:
780
- try:
781
- receiver.close(timeout=0.2)
782
- except Exception:
783
- pass
784
- self._close_handle_pipe(exec_id)
785
- self._results.pop(exec_id, None)
786
- self._errors.pop(exec_id, None)
787
- self._finalize_process(p)
910
+ if not self.is_handle_alive(handle):
911
+ if session is not None:
912
+ self._close_session(session, finalize_process=True, receiver_timeout=0.2)
913
+ else:
914
+ self._finalize_process(p)
788
915
  decoded = ExecutionResult(
789
916
  success=False,
790
917
  message=authorization_outcome.message,
@@ -795,36 +922,22 @@ class ExecutionChannel:
795
922
  continue
796
923
 
797
924
  # 1. 有效结果已到达:立即收割结果,不阻塞等待子进程退出
798
- if exec_id in self._results:
799
- raw_res = self._results.pop(exec_id)
925
+ if session is not None and session.result is not None:
926
+ raw_res = session.result
927
+ session.result = None
800
928
  decoded = _decode_ipc_result(raw_res, handle.job)
801
- self._handles.pop(exec_id, None)
802
- receiver = self._receivers.pop(exec_id, None)
803
- self._errors.pop(exec_id, None)
804
- # 结果后不再回复声明;接收端独立移交,不等待 worker 退出。
805
- self._close_handle_pipe(exec_id)
806
929
 
807
- is_alive = False
808
- try:
809
- is_alive = p.is_alive()
810
- except Exception:
811
- pass
812
-
813
- if not is_alive:
814
- if receiver is not None:
815
- try:
816
- receiver.close(timeout=0.2)
817
- except Exception:
818
- pass
819
- self._finalize_process(p)
930
+ if not self.is_handle_alive(handle):
931
+ self._close_session(session, finalize_process=True, receiver_timeout=0.2)
820
932
  else:
821
- self._lingering_workers.append((exec_id, p, receiver))
933
+ # 结果后不再回复声明;会话转入 lingering,保活进程句柄与接收端。
934
+ self._transition_to_lingering(session)
822
935
 
823
936
  completed.append((handle, decoded))
824
937
  continue
825
938
 
826
939
  # 2. 管道异常分支
827
- if exec_id in self._errors:
940
+ if session is not None and session.error is not None:
828
941
  completed.append((handle, self._collect_outcome(handle)))
829
942
  continue
830
943
 
@@ -834,11 +947,11 @@ class ExecutionChannel:
834
947
  continue
835
948
 
836
949
  # 4. 超时:从启动到结果到达计单次超时(等待授权期间暂停不计)
837
- if not handle.waiting_auth and now > handle.timeout_at_monotonic:
950
+ if not self._is_timeout_paused(handle) and now > handle.timeout_at_monotonic:
838
951
  p.join(timeout=0.5)
839
952
  if p.is_alive():
840
953
  try:
841
- p.kill()
954
+ self._kill_process(p)
842
955
  except Exception:
843
956
  pass
844
957
  completed.append(
@@ -850,68 +963,83 @@ class ExecutionChannel:
850
963
  def _collect_outcome(
851
964
  self, handle: JobHandle, res: dict | None = None, *, is_timeout: bool = False
852
965
  ) -> ExecutionResult:
853
- """收敛收割尾部:终止并收割子进程 → 排空管道至 EOF → 解码结果 → 关闭管道。"""
854
- authorization_outcome = handle.authorization_outcome
966
+ """收敛收割尾部:终止并收割子进程 → 排空管道至 EOF → 解码结果 → 关闭会话。"""
967
+ p = handle.process
968
+ session = self._find_session(handle)
969
+ authorization_outcome = self._get_authorization_outcome(handle)
855
970
  if authorization_outcome is not None:
856
- p = handle.process
857
- exec_id = handle.execution_id or ""
858
971
  try:
859
972
  self._terminate_and_join_process(p)
860
973
  finally:
861
- self._finalize_process(p)
862
- self._close_handle_pipe(exec_id)
863
- self._results.pop(exec_id, None)
864
- self._errors.pop(exec_id, None)
974
+ if session is not None:
975
+ self._close_session(session, finalize_process=True)
976
+ else:
977
+ self._finalize_process(p)
865
978
  return ExecutionResult(
866
979
  success=False,
867
980
  message=authorization_outcome.message,
868
981
  exception=authorization_outcome.exception,
869
982
  authorization_outcome=authorization_outcome,
870
983
  )
871
- p = handle.process
872
- exec_id = handle.execution_id or ""
984
+
873
985
  result: ExecutionResult | None = None
874
986
  try:
875
987
  self._terminate_and_join_process(p)
876
988
  self._drain_finished(handle)
877
- if exec_id in self._errors:
878
- err = self._errors.pop(exec_id)
879
- self._results.pop(exec_id, None)
989
+ if session is not None and session.error is not None:
990
+ err = session.error
991
+ session.error = None
992
+ session.result = None
880
993
  raise err
881
- payload = res if res is not None else self._results.pop(exec_id, None)
994
+ payload = (
995
+ res
996
+ if res is not None
997
+ else (session.result if session is not None else None)
998
+ )
999
+ if session is not None:
1000
+ session.result = None
882
1001
  if payload is not None and not is_timeout:
883
1002
  result = _decode_ipc_result(payload, handle.job)
884
1003
  else:
885
1004
  result = self._build_terminal_failure(p, handle, is_timeout=is_timeout)
886
1005
  finally:
887
- self._finalize_process(p)
888
- self._close_handle_pipe(exec_id)
889
- self._handles.pop(exec_id, None)
890
- self._results.pop(exec_id, None)
1006
+ if session is not None:
1007
+ self._close_session(session, finalize_process=True)
1008
+ else:
1009
+ self._finalize_process(p)
891
1010
  return result
892
1011
 
893
1012
  @staticmethod
894
1013
  def _build_terminal_failure(
895
1014
  p: Any, handle: JobHandle, *, is_timeout: bool
896
1015
  ) -> ExecutionResult:
897
- """构造进程终止(崩溃/超时)但无有效结果时的失败证据。"""
898
- exitcode = getattr(p, "exitcode", None)
1016
+ """构造进程终止(崩溃/超时/强杀)但无有效结果时的失败证据。"""
1017
+ raw_exitcode = getattr(p, "exitcode", None)
1018
+ exitcode: int | None = (
1019
+ raw_exitcode
1020
+ if isinstance(raw_exitcode, int) and not isinstance(raw_exitcode, bool)
1021
+ else None
1022
+ )
1023
+ was_force_killed = bool(getattr(p, "was_force_killed", False))
899
1024
  annotations: dict[str, Any] = {}
900
1025
  if is_timeout:
901
1026
  message = f"Worker timed out after {handle.timeout_seconds}s"
902
1027
  annotations["timed_out"] = True
903
- elif isinstance(exitcode, int) and not isinstance(exitcode, bool) and exitcode < 0:
1028
+ elif exitcode is not None and exitcode < 0:
904
1029
  try:
905
1030
  signal_name = signal.Signals(-exitcode).name
906
1031
  except (ValueError, AttributeError):
907
1032
  signal_name = f"SIGNO{-exitcode}"
908
1033
  message = f"Worker exited on signal {signal_name} ({exitcode}) without a result"
909
1034
  annotations["signal"] = signal_name
910
- elif isinstance(exitcode, int) and not isinstance(exitcode, bool) and exitcode > 0:
1035
+ elif was_force_killed:
1036
+ message = "Worker was forcefully terminated without a result"
1037
+ annotations["terminated"] = True
1038
+ elif exitcode is not None and exitcode > 0:
911
1039
  message = f"Worker exited with code {exitcode} without a result"
912
1040
  else:
913
1041
  message = "Worker exited without a result frame"
914
- if isinstance(exitcode, int) and not isinstance(exitcode, bool):
1042
+ if exitcode is not None:
915
1043
  annotations["exitcode"] = exitcode
916
1044
  return ExecutionResult(
917
1045
  success=False,
@@ -919,103 +1047,38 @@ class ExecutionChannel:
919
1047
  annotations=annotations,
920
1048
  )
921
1049
 
922
- def abort_in_flight(self, handles: Sequence[JobHandle]) -> AbortOutcome:
923
- """TOCTOU 闭环中止:初查分类 → 进程终止 → 排空管道至 EOF → 重探测闭环 → 关闭管道。"""
924
- if not handles:
925
- return AbortOutcome(completed=[], cancelled=[])
926
-
927
- done_pairs: list[tuple[JobHandle, ExecutionResult]] = []
928
- pending_handles: list[JobHandle] = []
929
-
930
- for handle in handles:
931
- self._receive_available(handle)
932
- decoded = self._consume_result_if_present(handle)
933
- if decoded is not None:
934
- done_pairs.append((handle, decoded))
935
- continue
936
- pending_handles.append(handle)
937
-
938
- for handle, _ in done_pairs:
939
- try:
940
- handle.process.join(timeout=self._JOIN_REAP_TIMEOUT)
941
- except Exception as error:
942
- logger.warning(f"Completed process join failed: {error}")
943
- self.finalize_processes(handles)
944
-
945
- for handle, decoded in done_pairs:
946
- self._drain_finished(handle)
947
- exec_id = handle.execution_id or ""
948
- self._close_handle_pipe(exec_id)
949
- self._handles.pop(exec_id, None)
950
- self._results.pop(exec_id, None)
951
- self._errors.pop(exec_id, None)
952
-
953
- truly_cancelled: list[JobHandle] = []
954
- for handle in pending_handles:
955
- self._drain_finished(handle)
956
- decoded = self._consume_result_if_present(handle)
957
- if decoded is not None:
958
- done_pairs.append((handle, decoded))
959
- else:
960
- truly_cancelled.append(handle)
961
- exec_id = handle.execution_id or ""
962
- self._close_handle_pipe(exec_id)
963
- self._handles.pop(exec_id, None)
964
- self._results.pop(exec_id, None)
965
- self._errors.pop(exec_id, None)
966
-
967
- return AbortOutcome(completed=done_pairs, cancelled=truly_cancelled)
968
-
969
1050
  def cleanup_in_flight(self, handles: Sequence[JobHandle]) -> None:
970
- """主循环异常退出时 kill + join 所有残留 in-flight 子进程并关闭管道与声明。"""
971
- for exec_id, p, receiver in self._lingering_workers:
972
- try:
973
- self._finalize_process(p)
974
- if receiver is not None:
975
- receiver.close(timeout=0.5)
976
- except Exception as e:
977
- logger.error(f"Error cleaning up lingering worker {exec_id}: {e}")
978
- self._lingering_workers.clear()
1051
+ """主循环异常退出时取消未放行启动、终止并回收所有残留与在途会话。"""
1052
+ self.cancel_pending_launches()
1053
+ self._cleanup_lingering_sessions(receiver_timeout=0.5, log_errors=True)
979
1054
 
980
1055
  for handle in handles:
981
1056
  try:
982
- self._finalize_process(handle.process)
983
- exec_id = handle.execution_id or ""
984
- self._close_handle_pipe(exec_id)
985
- self._handles.pop(exec_id, None)
986
- self._results.pop(exec_id, None)
987
- self._errors.pop(exec_id, None)
1057
+ handle_session = self._find_session(handle)
1058
+ if handle_session is not None:
1059
+ self._close_session(handle_session, finalize_process=True)
1060
+ else:
1061
+ self._finalize_process(handle.process)
988
1062
  except Exception as e:
989
1063
  logger.error(f"Error cleaning up in-flight job {handle.job_uid}: {e}")
990
1064
 
991
- def finalize_processes(self, handles: Sequence[JobHandle]) -> None:
992
- """只 kill + join 残留子进程,不删声明。"""
993
- for exec_id, p, receiver in self._lingering_workers:
994
- try:
995
- self._finalize_process(p)
996
- if receiver is not None:
997
- receiver.close(timeout=0.5)
998
- except Exception:
999
- pass
1000
- self._lingering_workers.clear()
1001
-
1002
- for handle in handles:
1065
+ def close(self) -> None:
1066
+ """关闭通道并回收所有准备中、在途与残留会话的进程和管道资源。"""
1067
+ self.cancel_pending_launches()
1068
+ self.cleanup_in_flight(self.active_handles())
1069
+ with self._sessions_lock:
1070
+ remaining = list(self._sessions.values())
1071
+ for session in remaining:
1003
1072
  try:
1004
- self._finalize_process(handle.process)
1073
+ self._close_session(session, finalize_process=True, receiver_timeout=0.5)
1005
1074
  except Exception as e:
1006
- logger.error(f"Error finalizing in-flight job {handle.job_uid}: {e}")
1075
+ logger.error(f"Error closing remaining session {session.session_key}: {e}")
1007
1076
 
1008
- def close(self) -> None:
1009
- """关闭通道并回收所有在途与残留子进程和管道资源。"""
1010
- self.cleanup_in_flight(list(self._handles.values()))
1011
1077
 
1012
1078
  __all__ = [
1013
- "AbortOutcome",
1014
1079
  "ExecutionChannel",
1015
1080
  "ExecutionResult",
1016
1081
  "JobHandle",
1082
+ "PreparedLaunch",
1017
1083
  "WorkerLaunchSpec",
1018
- "_WorkerLaunchCancelled",
1019
- "_decode_ipc_result",
1020
- "_worker_process_entry",
1021
1084
  ]