tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,1173 @@
|
|
|
1
|
+
"""Worker 启动器与跨平台进程树内部接缝(纯生产子进程实现)。
|
|
2
|
+
|
|
3
|
+
收拢两层内部接缝:
|
|
4
|
+
1. WorkerLauncher + PreparedWorker + ManagedWorkerHandle + LaunchedWorker:
|
|
5
|
+
以线性两阶段协议(``prepare()`` 冷启动与握手停在放行门前 → ``release()`` 发送
|
|
6
|
+
``b"GO"`` 或 ``abort()`` 零泄漏回收)隔离真实子进程生命周期;
|
|
7
|
+
2. ProcessTreeAdapter + _BasePopenWorkerHandle:
|
|
8
|
+
抽取 Windows / POSIX 共用的 ``Popen`` 等待、退出码缓存与幂等关闭骨架,仅在
|
|
9
|
+
``WindowsManagedWorkerHandle``(无窗口独立控制台 + 匿名 Job Object + 跳板
|
|
10
|
+
嵌套绑定 + lingering 保活 ``hJob``)与 ``PosixManagedWorkerHandle``(独立会话
|
|
11
|
+
进程组 + 先 ``killpg`` 后 ``wait``)中保留真实平台差异。
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import ctypes
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import pickle
|
|
20
|
+
import secrets
|
|
21
|
+
import signal
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import tempfile
|
|
25
|
+
import threading
|
|
26
|
+
import time
|
|
27
|
+
from abc import ABC, abstractmethod
|
|
28
|
+
from collections.abc import Callable, Sequence
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from multiprocessing.connection import Client, Listener
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any, Protocol
|
|
33
|
+
|
|
34
|
+
from ..utils.clock import utc_now_iso
|
|
35
|
+
from ..utils.ipc import (
|
|
36
|
+
HANDSHAKE_ROLE_CANCEL,
|
|
37
|
+
HANDSHAKE_ROLE_DOWNLINK,
|
|
38
|
+
HANDSHAKE_ROLE_UPLINK,
|
|
39
|
+
WorkerPipeReceiver,
|
|
40
|
+
WorkerReplySender,
|
|
41
|
+
decode_handshake_frame,
|
|
42
|
+
encode_handshake_frame,
|
|
43
|
+
encode_json_frame,
|
|
44
|
+
)
|
|
45
|
+
from ._worker_runtime import WorkerLaunchSpec
|
|
46
|
+
from .types import (
|
|
47
|
+
ENV_IPC_ADDR,
|
|
48
|
+
ENV_IPC_AUTHKEY,
|
|
49
|
+
ENV_IPC_FAMILY,
|
|
50
|
+
LAUNCH_GO_TOKEN,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
logger = logging.getLogger("tasklite")
|
|
54
|
+
|
|
55
|
+
# 记录主进程启动时的原始工作目录,供子进程展开 sys.path 中的空字符串条目。
|
|
56
|
+
_ORIGINAL_DIR: str = os.path.abspath(os.getcwd())
|
|
57
|
+
|
|
58
|
+
# 仓库根或已安装包父目录,确保子进程在任意 cwd 下都能找到 tasklite 包。
|
|
59
|
+
_PACKAGE_ROOT: str = str(Path(__file__).resolve().parents[2])
|
|
60
|
+
|
|
61
|
+
# POSIX AF_UNIX 路径最大安全长度阈值(留足余量防 sun_path 溢出)。
|
|
62
|
+
_POSIX_SOCKET_PATH_LIMIT: int = 95
|
|
63
|
+
|
|
64
|
+
# Windows 强杀 Job Object 时的退出码(如实暴露给 OS 退出码字段,不伪造 -9)。
|
|
65
|
+
_WIN_TERMINATE_EXIT_CODE: int = 1
|
|
66
|
+
|
|
67
|
+
# Windows Job Object 常量:仅设 KILL_ON_JOB_CLOSE,严禁 BREAKAWAY_OK。
|
|
68
|
+
_JOB_OBJECT_EXTENDED_LIMIT_INFORMATION_CLASS: int = 9
|
|
69
|
+
_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: int = 0x2000
|
|
70
|
+
# OpenProcess 最小无特权权限(PROCESS_SET_QUOTA | PROCESS_TERMINATE),供嵌套 Job 绑定 venv 真实子进程。
|
|
71
|
+
_WIN_PROCESS_JOB_ASSIGN_ACCESS: int = 0x0100 | 0x0001
|
|
72
|
+
|
|
73
|
+
_DEFAULT_HANDSHAKE_TIMEOUT: float = 15.0
|
|
74
|
+
_HANDSHAKE_POLL_SLICE: float = 0.05
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class _WorkerLaunchCancelled(Exception):
|
|
78
|
+
"""最终启动门前观察到停止请求,子进程不得放行执行。"""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class ManagedWorkerHandle(Protocol):
|
|
82
|
+
"""屏蔽 Popen / hJob / pgid / 进程内替身差异的统一句柄协议。"""
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def pid(self) -> int | None: ...
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def exitcode(self) -> int | None: ...
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def was_force_killed(self) -> bool: ...
|
|
92
|
+
|
|
93
|
+
def is_alive(self) -> bool: ...
|
|
94
|
+
|
|
95
|
+
def join(self, timeout: float | None = None) -> None: ...
|
|
96
|
+
|
|
97
|
+
def kill_tree(self) -> None: ...
|
|
98
|
+
|
|
99
|
+
def close(self) -> None: ...
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class LaunchedWorker:
|
|
104
|
+
"""一次已放行的 worker 启动产物。"""
|
|
105
|
+
|
|
106
|
+
handle: ManagedWorkerHandle
|
|
107
|
+
receiver: WorkerPipeReceiver
|
|
108
|
+
reply_sender: WorkerReplySender
|
|
109
|
+
started_at: str
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class PreparedWorker:
|
|
113
|
+
"""已完成冷启动、握手与进程树绑定,停在放行门前的待放行 worker。
|
|
114
|
+
|
|
115
|
+
两阶段线性契约:
|
|
116
|
+
- ``release()``:记录启动时间戳并放行 worker(如发送 ``b"GO"``),返回 ``LaunchedWorker``;
|
|
117
|
+
若放行本身失败则自动执行 ``abort()`` 后原样上抛;
|
|
118
|
+
- ``abort()``:幂等终止并回收进程树、句柄与全部双工连接,确保任意未放行路径零泄漏。
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def __init__(
|
|
122
|
+
self,
|
|
123
|
+
*,
|
|
124
|
+
handle: ManagedWorkerHandle,
|
|
125
|
+
receiver: WorkerPipeReceiver,
|
|
126
|
+
reply_sender: WorkerReplySender,
|
|
127
|
+
on_release: Callable[[], None],
|
|
128
|
+
on_abort: Callable[[], None] | None = None,
|
|
129
|
+
) -> None:
|
|
130
|
+
self.handle = handle
|
|
131
|
+
self.receiver = receiver
|
|
132
|
+
self.reply_sender = reply_sender
|
|
133
|
+
self._on_release = on_release
|
|
134
|
+
self._on_abort = on_abort
|
|
135
|
+
self._lock = threading.Lock()
|
|
136
|
+
self._released = False
|
|
137
|
+
self._aborted = False
|
|
138
|
+
|
|
139
|
+
def release(self) -> LaunchedWorker:
|
|
140
|
+
"""放行 worker 开始执行 handler 并返回已启动产物。"""
|
|
141
|
+
with self._lock:
|
|
142
|
+
if self._aborted:
|
|
143
|
+
raise RuntimeError("PreparedWorker already aborted")
|
|
144
|
+
if self._released:
|
|
145
|
+
raise RuntimeError("PreparedWorker already released")
|
|
146
|
+
self._released = True
|
|
147
|
+
started_at = utc_now_iso()
|
|
148
|
+
try:
|
|
149
|
+
self._on_release()
|
|
150
|
+
except BaseException:
|
|
151
|
+
self.abort()
|
|
152
|
+
raise
|
|
153
|
+
return LaunchedWorker(
|
|
154
|
+
handle=self.handle,
|
|
155
|
+
receiver=self.receiver,
|
|
156
|
+
reply_sender=self.reply_sender,
|
|
157
|
+
started_at=started_at,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
def abort(self) -> None:
|
|
161
|
+
"""幂等中止未放行的 worker 并回收进程树、句柄与双工连接。
|
|
162
|
+
|
|
163
|
+
内置标准 5 步清理(kill_tree -> join -> reply_sender.close ->
|
|
164
|
+
receiver.close -> handle.close),若提供可选 ``on_abort`` 额外钩子则在其后执行。
|
|
165
|
+
"""
|
|
166
|
+
with self._lock:
|
|
167
|
+
if self._aborted:
|
|
168
|
+
return
|
|
169
|
+
self._aborted = True
|
|
170
|
+
try:
|
|
171
|
+
self.handle.kill_tree()
|
|
172
|
+
except Exception:
|
|
173
|
+
pass
|
|
174
|
+
try:
|
|
175
|
+
self.handle.join(timeout=5.0)
|
|
176
|
+
except Exception:
|
|
177
|
+
pass
|
|
178
|
+
try:
|
|
179
|
+
self.reply_sender.close()
|
|
180
|
+
except Exception:
|
|
181
|
+
pass
|
|
182
|
+
try:
|
|
183
|
+
self.receiver.close(timeout=2.0)
|
|
184
|
+
except Exception:
|
|
185
|
+
pass
|
|
186
|
+
try:
|
|
187
|
+
self.handle.close()
|
|
188
|
+
except Exception:
|
|
189
|
+
pass
|
|
190
|
+
if self._on_abort is not None:
|
|
191
|
+
self._on_abort()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class WorkerLauncher(ABC):
|
|
195
|
+
"""Worker 启动器抽象接缝。"""
|
|
196
|
+
|
|
197
|
+
@abstractmethod
|
|
198
|
+
def prepare(
|
|
199
|
+
self,
|
|
200
|
+
spec: WorkerLaunchSpec,
|
|
201
|
+
*,
|
|
202
|
+
on_message: Callable[[], None] | None,
|
|
203
|
+
cancel_event: threading.Event,
|
|
204
|
+
) -> PreparedWorker:
|
|
205
|
+
"""冷启动 worker 并完成握手与进程树绑定,停在放行门前返回 PreparedWorker。
|
|
206
|
+
|
|
207
|
+
准备阶段任何失败或取消必须零泄漏清理后再抛出异常。
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class ProcessTreeAdapter(ABC):
|
|
212
|
+
"""跨平台进程树隔离与回收适配器接缝。"""
|
|
213
|
+
|
|
214
|
+
@abstractmethod
|
|
215
|
+
def popen_kwargs(self) -> dict[str, Any]:
|
|
216
|
+
"""返回传给 ``subprocess.Popen`` 的平台隔离参数。"""
|
|
217
|
+
|
|
218
|
+
@abstractmethod
|
|
219
|
+
def attach_and_wrap(
|
|
220
|
+
self,
|
|
221
|
+
proc: subprocess.Popen[Any],
|
|
222
|
+
*,
|
|
223
|
+
worker_pid: int | None = None,
|
|
224
|
+
) -> ManagedWorkerHandle:
|
|
225
|
+
"""将已启动但尚未执行用户代码的子进程纳入进程树管理并返回句柄。"""
|
|
226
|
+
|
|
227
|
+
@abstractmethod
|
|
228
|
+
def kill_unattached(self, proc: subprocess.Popen[Any]) -> None:
|
|
229
|
+
"""强杀并回收尚未完成进程树绑定的空壳子进程。"""
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
# ── Popen 公共句柄基类 ───────────────────────────────────────────────────────
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class _BasePopenWorkerHandle(ABC):
|
|
236
|
+
"""Windows 与 POSIX 共用的 ``Popen`` 句柄生命周期骨架。
|
|
237
|
+
|
|
238
|
+
统一承载线程安全的 ``pid`` / ``exitcode`` / ``was_force_killed`` /
|
|
239
|
+
``is_alive`` / ``join`` / ``kill`` / ``close`` 骨架;平台子类仅实现
|
|
240
|
+
``kill_tree()`` 与可选的退出/关闭钩子。
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
def __init__(self, proc: subprocess.Popen[Any]) -> None:
|
|
244
|
+
self._proc = proc
|
|
245
|
+
self._lock = threading.RLock()
|
|
246
|
+
self._was_force_killed = False
|
|
247
|
+
self._closed = False
|
|
248
|
+
self._cached_exitcode: int | None = None
|
|
249
|
+
|
|
250
|
+
def _on_exited_locked(self) -> None:
|
|
251
|
+
"""子类钩子(在 ``self._lock`` 内调用):子进程被观测到已退出时触发。"""
|
|
252
|
+
|
|
253
|
+
def _take_close_resources_locked(self) -> Any:
|
|
254
|
+
"""子类钩子(在 ``self._lock`` 内调用):取出待释放的平台原生句柄。"""
|
|
255
|
+
return None
|
|
256
|
+
|
|
257
|
+
def _release_close_resources(self, resources: Any) -> None:
|
|
258
|
+
"""子类钩子(在锁外调用):释放由 ``_take_close_resources_locked`` 取出的资源。"""
|
|
259
|
+
|
|
260
|
+
@property
|
|
261
|
+
def pid(self) -> int | None:
|
|
262
|
+
return self._proc.pid
|
|
263
|
+
|
|
264
|
+
@property
|
|
265
|
+
def exitcode(self) -> int | None:
|
|
266
|
+
with self._lock:
|
|
267
|
+
if self._cached_exitcode is not None:
|
|
268
|
+
return self._cached_exitcode
|
|
269
|
+
if self._closed:
|
|
270
|
+
return self._proc.returncode
|
|
271
|
+
code = self._proc.poll()
|
|
272
|
+
if code is not None:
|
|
273
|
+
self._cached_exitcode = code
|
|
274
|
+
self._on_exited_locked()
|
|
275
|
+
return code
|
|
276
|
+
|
|
277
|
+
@property
|
|
278
|
+
def was_force_killed(self) -> bool:
|
|
279
|
+
with self._lock:
|
|
280
|
+
return self._was_force_killed
|
|
281
|
+
|
|
282
|
+
def is_alive(self) -> bool:
|
|
283
|
+
with self._lock:
|
|
284
|
+
if self._closed:
|
|
285
|
+
return False
|
|
286
|
+
code = self._proc.poll()
|
|
287
|
+
if code is not None:
|
|
288
|
+
self._cached_exitcode = code
|
|
289
|
+
self._on_exited_locked()
|
|
290
|
+
return False
|
|
291
|
+
return True
|
|
292
|
+
|
|
293
|
+
def join(self, timeout: float | None = None) -> None:
|
|
294
|
+
with self._lock:
|
|
295
|
+
if self._closed:
|
|
296
|
+
return
|
|
297
|
+
try:
|
|
298
|
+
code = self._proc.wait(timeout=timeout)
|
|
299
|
+
except subprocess.TimeoutExpired:
|
|
300
|
+
return
|
|
301
|
+
except OSError:
|
|
302
|
+
return
|
|
303
|
+
with self._lock:
|
|
304
|
+
self._cached_exitcode = code
|
|
305
|
+
self._on_exited_locked()
|
|
306
|
+
|
|
307
|
+
@abstractmethod
|
|
308
|
+
def kill_tree(self) -> None:
|
|
309
|
+
"""按当前平台语义原子终止整棵进程树。"""
|
|
310
|
+
|
|
311
|
+
def kill(self) -> None:
|
|
312
|
+
"""兼容测试调用接口,直接委托进程树终止。"""
|
|
313
|
+
self.kill_tree()
|
|
314
|
+
|
|
315
|
+
def close(self) -> None:
|
|
316
|
+
"""幂等关闭句柄并释放平台原生资源。"""
|
|
317
|
+
with self._lock:
|
|
318
|
+
if self._closed:
|
|
319
|
+
return
|
|
320
|
+
self._closed = True
|
|
321
|
+
if self._cached_exitcode is None:
|
|
322
|
+
self._cached_exitcode = self._proc.returncode
|
|
323
|
+
resources = self._take_close_resources_locked()
|
|
324
|
+
self._release_close_resources(resources)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
# ── Windows 进程树适配(无窗口独立控制台 + Job Object)────────────────────────
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _build_win_job_structures() -> tuple[Any, Any]:
|
|
331
|
+
"""按 64 位安全对齐构造 Windows Job Object ctypes 结构体并绑定 kernel32。"""
|
|
332
|
+
from ctypes import wintypes
|
|
333
|
+
|
|
334
|
+
class _BasicLimitInfo(ctypes.Structure):
|
|
335
|
+
_fields_ = [
|
|
336
|
+
("PerProcessUserTimeLimit", ctypes.c_int64),
|
|
337
|
+
("PerJobUserTimeLimit", ctypes.c_int64),
|
|
338
|
+
("LimitFlags", wintypes.DWORD),
|
|
339
|
+
("MinimumWorkingSetSize", ctypes.c_size_t),
|
|
340
|
+
("MaximumWorkingSetSize", ctypes.c_size_t),
|
|
341
|
+
("ActiveProcessLimit", wintypes.DWORD),
|
|
342
|
+
("Affinity", ctypes.c_size_t),
|
|
343
|
+
("PriorityClass", wintypes.DWORD),
|
|
344
|
+
("SchedulingClass", wintypes.DWORD),
|
|
345
|
+
]
|
|
346
|
+
|
|
347
|
+
class _IoCounters(ctypes.Structure):
|
|
348
|
+
_fields_ = [
|
|
349
|
+
("ReadOperationCount", ctypes.c_uint64),
|
|
350
|
+
("WriteOperationCount", ctypes.c_uint64),
|
|
351
|
+
("OtherOperationCount", ctypes.c_uint64),
|
|
352
|
+
("ReadTransferCount", ctypes.c_uint64),
|
|
353
|
+
("WriteTransferCount", ctypes.c_uint64),
|
|
354
|
+
("OtherTransferCount", ctypes.c_uint64),
|
|
355
|
+
]
|
|
356
|
+
|
|
357
|
+
class _ExtendedLimitInfo(ctypes.Structure):
|
|
358
|
+
_fields_ = [
|
|
359
|
+
("BasicLimitInformation", _BasicLimitInfo),
|
|
360
|
+
("IoInfo", _IoCounters),
|
|
361
|
+
("ProcessMemoryLimit", ctypes.c_size_t),
|
|
362
|
+
("JobMemoryLimit", ctypes.c_size_t),
|
|
363
|
+
("PeakProcessMemoryUsed", ctypes.c_size_t),
|
|
364
|
+
("PeakJobMemoryUsed", ctypes.c_size_t),
|
|
365
|
+
]
|
|
366
|
+
|
|
367
|
+
kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
368
|
+
|
|
369
|
+
kernel.CreateJobObjectW.argtypes = [wintypes.LPVOID, wintypes.LPCWSTR]
|
|
370
|
+
kernel.CreateJobObjectW.restype = wintypes.HANDLE
|
|
371
|
+
|
|
372
|
+
kernel.SetInformationJobObject.argtypes = [
|
|
373
|
+
wintypes.HANDLE,
|
|
374
|
+
ctypes.c_int,
|
|
375
|
+
wintypes.LPVOID,
|
|
376
|
+
wintypes.DWORD,
|
|
377
|
+
]
|
|
378
|
+
kernel.SetInformationJobObject.restype = wintypes.BOOL
|
|
379
|
+
|
|
380
|
+
kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
|
|
381
|
+
kernel.AssignProcessToJobObject.restype = wintypes.BOOL
|
|
382
|
+
|
|
383
|
+
kernel.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT]
|
|
384
|
+
kernel.TerminateJobObject.restype = wintypes.BOOL
|
|
385
|
+
|
|
386
|
+
kernel.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
387
|
+
kernel.CloseHandle.restype = wintypes.BOOL
|
|
388
|
+
|
|
389
|
+
kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
390
|
+
kernel.OpenProcess.restype = wintypes.HANDLE
|
|
391
|
+
|
|
392
|
+
kernel.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT]
|
|
393
|
+
kernel.TerminateProcess.restype = wintypes.BOOL
|
|
394
|
+
|
|
395
|
+
return kernel, _ExtendedLimitInfo
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
_WIN_KERNEL_CACHE: tuple[Any, Any] | None = None
|
|
399
|
+
_WIN_KERNEL_LOCK = threading.Lock()
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _get_win_kernel() -> tuple[Any, Any]:
|
|
403
|
+
"""惰性初始化并缓存 64 位安全 kernel32 绑定与 Job 结构体。"""
|
|
404
|
+
global _WIN_KERNEL_CACHE
|
|
405
|
+
if _WIN_KERNEL_CACHE is None:
|
|
406
|
+
with _WIN_KERNEL_LOCK:
|
|
407
|
+
if _WIN_KERNEL_CACHE is None:
|
|
408
|
+
_WIN_KERNEL_CACHE = _build_win_job_structures()
|
|
409
|
+
return _WIN_KERNEL_CACHE
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _win_os_error(action: str, err_code: int) -> OSError:
|
|
413
|
+
message = ctypes.FormatError(err_code) if hasattr(ctypes, "FormatError") else f"error {err_code}"
|
|
414
|
+
return OSError(err_code, f"{action} failed: {message}")
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
class WindowsManagedWorkerHandle(_BasePopenWorkerHandle):
|
|
418
|
+
"""绑定 Windows 匿名 Job Object 的线程安全进程树句柄。
|
|
419
|
+
|
|
420
|
+
不变式:``hJob`` 在句柄存活及 lingering 期间始终保活,``kill_tree()`` 仅调用
|
|
421
|
+
``TerminateJobObject`` 终止进程树,仅在 ``close()`` 时才幂等释放 ``hJob``,
|
|
422
|
+
避免提前关闭 Job 句柄误杀仍在收尾的进程。
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
def __init__(
|
|
426
|
+
self,
|
|
427
|
+
proc: subprocess.Popen[Any],
|
|
428
|
+
h_jobs: int | Sequence[int],
|
|
429
|
+
kernel: Any,
|
|
430
|
+
) -> None:
|
|
431
|
+
super().__init__(proc)
|
|
432
|
+
self._h_jobs: tuple[int, ...] = (
|
|
433
|
+
(int(h_jobs),)
|
|
434
|
+
if isinstance(h_jobs, int)
|
|
435
|
+
else tuple(int(h) for h in h_jobs)
|
|
436
|
+
)
|
|
437
|
+
self._kernel = kernel
|
|
438
|
+
|
|
439
|
+
def kill_tree(self) -> None:
|
|
440
|
+
"""通过 TerminateJobObject 原子终止整棵进程树(不提前关闭 hJob)。"""
|
|
441
|
+
with self._lock:
|
|
442
|
+
self._was_force_killed = True
|
|
443
|
+
h_jobs = () if self._closed else self._h_jobs
|
|
444
|
+
for h_job in h_jobs:
|
|
445
|
+
try:
|
|
446
|
+
self._kernel.TerminateJobObject(h_job, _WIN_TERMINATE_EXIT_CODE)
|
|
447
|
+
except Exception as exc:
|
|
448
|
+
logger.warning(f"TerminateJobObject failed for pid={self._proc.pid}: {exc}")
|
|
449
|
+
try:
|
|
450
|
+
if self._proc.poll() is None:
|
|
451
|
+
self._proc.kill()
|
|
452
|
+
except OSError:
|
|
453
|
+
pass
|
|
454
|
+
|
|
455
|
+
def _take_close_resources_locked(self) -> tuple[int, ...]:
|
|
456
|
+
h_jobs = self._h_jobs
|
|
457
|
+
self._h_jobs = ()
|
|
458
|
+
return h_jobs
|
|
459
|
+
|
|
460
|
+
def _release_close_resources(self, resources: Any) -> None:
|
|
461
|
+
for h_job in resources:
|
|
462
|
+
try:
|
|
463
|
+
self._kernel.CloseHandle(h_job)
|
|
464
|
+
except Exception as exc:
|
|
465
|
+
logger.warning(f"CloseHandle(hJob) failed for pid={self._proc.pid}: {exc}")
|
|
466
|
+
|
|
467
|
+
def __del__(self) -> None:
|
|
468
|
+
try:
|
|
469
|
+
self.close()
|
|
470
|
+
except Exception:
|
|
471
|
+
pass
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
class WindowsProcessTreeAdapter(ProcessTreeAdapter):
|
|
475
|
+
"""Windows 进程树隔离适配器:独立无窗口控制台 + 匿名 Job Object。"""
|
|
476
|
+
|
|
477
|
+
def __init__(
|
|
478
|
+
self,
|
|
479
|
+
*,
|
|
480
|
+
kernel: Any | None = None,
|
|
481
|
+
extended_limit_cls: Any | None = None,
|
|
482
|
+
) -> None:
|
|
483
|
+
self._kernel = kernel
|
|
484
|
+
self._extended_limit_cls = extended_limit_cls
|
|
485
|
+
|
|
486
|
+
def _resolve_kernel(self) -> tuple[Any, Any]:
|
|
487
|
+
if self._kernel is not None and self._extended_limit_cls is not None:
|
|
488
|
+
return self._kernel, self._extended_limit_cls
|
|
489
|
+
return _get_win_kernel()
|
|
490
|
+
|
|
491
|
+
@staticmethod
|
|
492
|
+
def _create_kill_on_close_job(kernel: Any, extended_limit_cls: Any) -> int:
|
|
493
|
+
"""创建仅启用 KILL_ON_JOB_CLOSE(严禁 BREAKAWAY_OK)的匿名 Job Object。"""
|
|
494
|
+
h_job = kernel.CreateJobObjectW(None, None)
|
|
495
|
+
if not h_job:
|
|
496
|
+
err = ctypes.get_last_error()
|
|
497
|
+
raise _win_os_error("CreateJobObjectW", err)
|
|
498
|
+
try:
|
|
499
|
+
info = extended_limit_cls()
|
|
500
|
+
info.BasicLimitInformation.LimitFlags = _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
501
|
+
ok = kernel.SetInformationJobObject(
|
|
502
|
+
h_job,
|
|
503
|
+
_JOB_OBJECT_EXTENDED_LIMIT_INFORMATION_CLASS,
|
|
504
|
+
ctypes.byref(info),
|
|
505
|
+
ctypes.sizeof(info),
|
|
506
|
+
)
|
|
507
|
+
if not ok:
|
|
508
|
+
err = ctypes.get_last_error()
|
|
509
|
+
raise _win_os_error("SetInformationJobObject", err)
|
|
510
|
+
except BaseException:
|
|
511
|
+
try:
|
|
512
|
+
kernel.CloseHandle(h_job)
|
|
513
|
+
except Exception:
|
|
514
|
+
pass
|
|
515
|
+
raise
|
|
516
|
+
return int(h_job)
|
|
517
|
+
|
|
518
|
+
def popen_kwargs(self) -> dict[str, Any]:
|
|
519
|
+
create_no_window = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
|
|
520
|
+
create_new_group = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200)
|
|
521
|
+
return {
|
|
522
|
+
"creationflags": create_no_window | create_new_group,
|
|
523
|
+
"stdin": subprocess.DEVNULL,
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
def attach_and_wrap(
|
|
527
|
+
self,
|
|
528
|
+
proc: subprocess.Popen[Any],
|
|
529
|
+
*,
|
|
530
|
+
worker_pid: int | None = None,
|
|
531
|
+
) -> ManagedWorkerHandle:
|
|
532
|
+
"""绑定进程树:在 venvlauncher 跳板场景下同时以嵌套 Job 绑定真实 worker 进程。
|
|
533
|
+
|
|
534
|
+
Windows 下虚拟环境 ``python.exe`` 为跳板壳进程,在握手前已派生出真正执行
|
|
535
|
+
``_worker_bootstrap`` 的解释器子进程;仅绑定跳板壳不会覆盖其此前派生的真实
|
|
536
|
+
worker,因此须为真实 ``worker_pid`` 建立独立嵌套 Job Object(此时其阻塞于
|
|
537
|
+
握手屏障、尚未执行任何用户代码,后续派生的多层孙进程自动继承该 Job)。
|
|
538
|
+
"""
|
|
539
|
+
kernel, extended_limit_cls = self._resolve_kernel()
|
|
540
|
+
created_jobs: list[int] = []
|
|
541
|
+
try:
|
|
542
|
+
# 1. 若存在 venvlauncher 跳板(worker_pid != proc.pid),先为真实 worker 建嵌套 Job
|
|
543
|
+
if worker_pid is not None and worker_pid != proc.pid:
|
|
544
|
+
h_worker_proc = kernel.OpenProcess(
|
|
545
|
+
_WIN_PROCESS_JOB_ASSIGN_ACCESS,
|
|
546
|
+
False,
|
|
547
|
+
int(worker_pid),
|
|
548
|
+
)
|
|
549
|
+
if not h_worker_proc:
|
|
550
|
+
err = ctypes.get_last_error()
|
|
551
|
+
raise _win_os_error("OpenProcess", err)
|
|
552
|
+
worker_assigned = False
|
|
553
|
+
try:
|
|
554
|
+
h_worker_job = self._create_kill_on_close_job(kernel, extended_limit_cls)
|
|
555
|
+
created_jobs.append(h_worker_job)
|
|
556
|
+
ok = kernel.AssignProcessToJobObject(h_worker_job, int(h_worker_proc))
|
|
557
|
+
if not ok:
|
|
558
|
+
err = ctypes.get_last_error()
|
|
559
|
+
raise _win_os_error("AssignProcessToJobObject", err)
|
|
560
|
+
worker_assigned = True
|
|
561
|
+
except BaseException:
|
|
562
|
+
if not worker_assigned:
|
|
563
|
+
try:
|
|
564
|
+
kernel.TerminateProcess(
|
|
565
|
+
int(h_worker_proc), _WIN_TERMINATE_EXIT_CODE
|
|
566
|
+
)
|
|
567
|
+
except Exception:
|
|
568
|
+
pass
|
|
569
|
+
raise
|
|
570
|
+
finally:
|
|
571
|
+
try:
|
|
572
|
+
kernel.CloseHandle(h_worker_proc)
|
|
573
|
+
except Exception:
|
|
574
|
+
pass
|
|
575
|
+
|
|
576
|
+
# 2. 为 Popen 直接句柄绑定 Job Object
|
|
577
|
+
h_proc_job = self._create_kill_on_close_job(kernel, extended_limit_cls)
|
|
578
|
+
created_jobs.append(h_proc_job)
|
|
579
|
+
raw_handle = getattr(proc, "_handle", None)
|
|
580
|
+
if raw_handle is None:
|
|
581
|
+
raise OSError("subprocess.Popen missing native _handle on Windows")
|
|
582
|
+
ok = kernel.AssignProcessToJobObject(h_proc_job, int(raw_handle))
|
|
583
|
+
if not ok:
|
|
584
|
+
err = ctypes.get_last_error()
|
|
585
|
+
raise _win_os_error("AssignProcessToJobObject", err)
|
|
586
|
+
except BaseException:
|
|
587
|
+
for h_job in created_jobs:
|
|
588
|
+
try:
|
|
589
|
+
kernel.TerminateJobObject(h_job, _WIN_TERMINATE_EXIT_CODE)
|
|
590
|
+
except Exception:
|
|
591
|
+
pass
|
|
592
|
+
try:
|
|
593
|
+
kernel.CloseHandle(h_job)
|
|
594
|
+
except Exception:
|
|
595
|
+
pass
|
|
596
|
+
self.kill_unattached(proc)
|
|
597
|
+
raise
|
|
598
|
+
|
|
599
|
+
return WindowsManagedWorkerHandle(proc, created_jobs, kernel)
|
|
600
|
+
|
|
601
|
+
def kill_unattached(self, proc: subprocess.Popen[Any]) -> None:
|
|
602
|
+
try:
|
|
603
|
+
if proc.poll() is None:
|
|
604
|
+
proc.kill()
|
|
605
|
+
except OSError:
|
|
606
|
+
pass
|
|
607
|
+
try:
|
|
608
|
+
proc.wait(timeout=5.0)
|
|
609
|
+
except Exception:
|
|
610
|
+
pass
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
# ── POSIX 进程树适配(独立会话 + 先 killpg 后 wait)──────────────────────────
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
class PosixManagedWorkerHandle(_BasePopenWorkerHandle):
|
|
617
|
+
"""POSIX 独立会话进程组句柄。
|
|
618
|
+
|
|
619
|
+
不变式:``kill_tree()`` 严格在组长进程尚未被 ``wait()`` 回收前向整个进程组
|
|
620
|
+
发送 ``SIGKILL``,杜绝先回收组长 PID 引起的 PID 复用误杀。
|
|
621
|
+
"""
|
|
622
|
+
|
|
623
|
+
def __init__(self, proc: subprocess.Popen[Any]) -> None:
|
|
624
|
+
super().__init__(proc)
|
|
625
|
+
self._pgid = proc.pid
|
|
626
|
+
self._reaped = False
|
|
627
|
+
|
|
628
|
+
def _on_exited_locked(self) -> None:
|
|
629
|
+
self._reaped = True
|
|
630
|
+
|
|
631
|
+
def kill_tree(self) -> None:
|
|
632
|
+
"""先对进程组发送 SIGKILL(不提前 wait 组长),再由 join() 回收。"""
|
|
633
|
+
with self._lock:
|
|
634
|
+
self._was_force_killed = True
|
|
635
|
+
if self._reaped or self._closed:
|
|
636
|
+
return
|
|
637
|
+
pgid = self._pgid
|
|
638
|
+
killpg = getattr(os, "killpg", None)
|
|
639
|
+
sigkill = getattr(signal, "SIGKILL", signal.SIGTERM)
|
|
640
|
+
if killpg is not None:
|
|
641
|
+
try:
|
|
642
|
+
killpg(pgid, sigkill)
|
|
643
|
+
except ProcessLookupError:
|
|
644
|
+
pass
|
|
645
|
+
except PermissionError:
|
|
646
|
+
try:
|
|
647
|
+
self._proc.kill()
|
|
648
|
+
except OSError:
|
|
649
|
+
pass
|
|
650
|
+
else:
|
|
651
|
+
try:
|
|
652
|
+
self._proc.kill()
|
|
653
|
+
except OSError:
|
|
654
|
+
pass
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
class PosixProcessTreeAdapter(ProcessTreeAdapter):
|
|
658
|
+
"""POSIX 进程树隔离适配器:start_new_session 建新进程组。"""
|
|
659
|
+
|
|
660
|
+
def popen_kwargs(self) -> dict[str, Any]:
|
|
661
|
+
return {
|
|
662
|
+
"start_new_session": True,
|
|
663
|
+
"stdin": subprocess.DEVNULL,
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
def attach_and_wrap(
|
|
667
|
+
self,
|
|
668
|
+
proc: subprocess.Popen[Any],
|
|
669
|
+
*,
|
|
670
|
+
worker_pid: int | None = None,
|
|
671
|
+
) -> ManagedWorkerHandle:
|
|
672
|
+
return PosixManagedWorkerHandle(proc)
|
|
673
|
+
|
|
674
|
+
def kill_unattached(self, proc: subprocess.Popen[Any]) -> None:
|
|
675
|
+
killpg = getattr(os, "killpg", None)
|
|
676
|
+
sigkill = getattr(signal, "SIGKILL", signal.SIGTERM)
|
|
677
|
+
if killpg is not None:
|
|
678
|
+
try:
|
|
679
|
+
killpg(proc.pid, sigkill)
|
|
680
|
+
except (ProcessLookupError, PermissionError):
|
|
681
|
+
pass
|
|
682
|
+
else:
|
|
683
|
+
try:
|
|
684
|
+
proc.kill()
|
|
685
|
+
except OSError:
|
|
686
|
+
pass
|
|
687
|
+
try:
|
|
688
|
+
proc.wait(timeout=5.0)
|
|
689
|
+
except Exception:
|
|
690
|
+
pass
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def default_process_tree_adapter() -> ProcessTreeAdapter:
|
|
694
|
+
"""按当前操作系统返回默认进程树适配器。"""
|
|
695
|
+
if os.name == "nt":
|
|
696
|
+
return WindowsProcessTreeAdapter()
|
|
697
|
+
return PosixProcessTreeAdapter()
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
# ── 单 Listener 双物理连接握手接收器 ──────────────────────────────────────────
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def _make_ipc_address() -> tuple[str, str]:
|
|
704
|
+
"""按平台生成单次握手监听地址与 family。"""
|
|
705
|
+
token = secrets.token_hex(12)
|
|
706
|
+
if os.name == "nt":
|
|
707
|
+
return rf"\\.\pipe\tasklite-{os.getpid()}-{token}", "AF_PIPE"
|
|
708
|
+
sock_name = f"tl-{os.getpid()}-{token[:16]}.sock"
|
|
709
|
+
candidate = os.path.join(tempfile.gettempdir(), sock_name)
|
|
710
|
+
if len(candidate.encode("utf-8")) > _POSIX_SOCKET_PATH_LIMIT:
|
|
711
|
+
candidate = os.path.join("/tmp", sock_name)
|
|
712
|
+
return candidate, "AF_UNIX"
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
class _HandshakeAcceptor:
|
|
716
|
+
"""守护线程接收 worker 的 uplink / downlink 双物理连接,支持 cancel 自连唤醒。"""
|
|
717
|
+
|
|
718
|
+
def __init__(
|
|
719
|
+
self,
|
|
720
|
+
listener: Listener,
|
|
721
|
+
*,
|
|
722
|
+
address: str,
|
|
723
|
+
family: str,
|
|
724
|
+
authkey: bytes,
|
|
725
|
+
) -> None:
|
|
726
|
+
self._listener = listener
|
|
727
|
+
self._address = address
|
|
728
|
+
self._family = family
|
|
729
|
+
self._authkey = authkey
|
|
730
|
+
self._lock = threading.Lock()
|
|
731
|
+
self._cancelled = threading.Event()
|
|
732
|
+
self._done = threading.Event()
|
|
733
|
+
self.uplink: Any = None
|
|
734
|
+
self.downlink: Any = None
|
|
735
|
+
self.worker_pid: int | None = None
|
|
736
|
+
self.error: BaseException | None = None
|
|
737
|
+
self._thread = threading.Thread(
|
|
738
|
+
target=self._accept_loop,
|
|
739
|
+
name="tasklite-ipc-handshake",
|
|
740
|
+
daemon=True,
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
def start(self) -> None:
|
|
744
|
+
self._thread.start()
|
|
745
|
+
|
|
746
|
+
def wait_done(self, timeout: float) -> bool:
|
|
747
|
+
return self._done.wait(timeout=timeout)
|
|
748
|
+
|
|
749
|
+
def _accept_loop(self) -> None:
|
|
750
|
+
try:
|
|
751
|
+
while not self._cancelled.is_set():
|
|
752
|
+
try:
|
|
753
|
+
conn = self._listener.accept()
|
|
754
|
+
except BaseException as exc:
|
|
755
|
+
if not self._cancelled.is_set():
|
|
756
|
+
self.error = exc
|
|
757
|
+
return
|
|
758
|
+
|
|
759
|
+
try:
|
|
760
|
+
raw = conn.recv_bytes()
|
|
761
|
+
role, raw_pid = decode_handshake_frame(raw)
|
|
762
|
+
except BaseException as exc:
|
|
763
|
+
try:
|
|
764
|
+
conn.close()
|
|
765
|
+
except OSError:
|
|
766
|
+
pass
|
|
767
|
+
if not self._cancelled.is_set():
|
|
768
|
+
self.error = exc
|
|
769
|
+
return
|
|
770
|
+
|
|
771
|
+
if role == HANDSHAKE_ROLE_CANCEL:
|
|
772
|
+
try:
|
|
773
|
+
conn.close()
|
|
774
|
+
except OSError:
|
|
775
|
+
pass
|
|
776
|
+
return
|
|
777
|
+
|
|
778
|
+
with self._lock:
|
|
779
|
+
if self._cancelled.is_set():
|
|
780
|
+
try:
|
|
781
|
+
conn.close()
|
|
782
|
+
except OSError:
|
|
783
|
+
pass
|
|
784
|
+
return
|
|
785
|
+
if raw_pid is not None:
|
|
786
|
+
self.worker_pid = raw_pid
|
|
787
|
+
if role == HANDSHAKE_ROLE_UPLINK and self.uplink is None:
|
|
788
|
+
self.uplink = conn
|
|
789
|
+
elif role == HANDSHAKE_ROLE_DOWNLINK and self.downlink is None:
|
|
790
|
+
self.downlink = conn
|
|
791
|
+
else:
|
|
792
|
+
try:
|
|
793
|
+
conn.close()
|
|
794
|
+
except OSError:
|
|
795
|
+
pass
|
|
796
|
+
self.error = RuntimeError(f"Unexpected worker handshake role: {role!r}")
|
|
797
|
+
return
|
|
798
|
+
if self.uplink is not None and self.downlink is not None:
|
|
799
|
+
return
|
|
800
|
+
finally:
|
|
801
|
+
self._done.set()
|
|
802
|
+
|
|
803
|
+
def cancel_and_join(self, timeout: float = 2.0) -> None:
|
|
804
|
+
"""置取消标志并在必要时经短生命周期守护线程自连发送 cancel 帧,绝不阻塞主线程。"""
|
|
805
|
+
self._cancelled.set()
|
|
806
|
+
if self._done.is_set():
|
|
807
|
+
self._thread.join(timeout=timeout)
|
|
808
|
+
return
|
|
809
|
+
if not self._thread.is_alive():
|
|
810
|
+
return
|
|
811
|
+
|
|
812
|
+
def _send_cancel_wake() -> None:
|
|
813
|
+
try:
|
|
814
|
+
wake_conn = Client(
|
|
815
|
+
address=self._address,
|
|
816
|
+
family=self._family,
|
|
817
|
+
authkey=self._authkey,
|
|
818
|
+
)
|
|
819
|
+
try:
|
|
820
|
+
wake_conn.send_bytes(encode_handshake_frame(HANDSHAKE_ROLE_CANCEL))
|
|
821
|
+
finally:
|
|
822
|
+
wake_conn.close()
|
|
823
|
+
except BaseException:
|
|
824
|
+
pass
|
|
825
|
+
|
|
826
|
+
waker = threading.Thread(
|
|
827
|
+
target=_send_cancel_wake,
|
|
828
|
+
name="tasklite-ipc-handshake-cancel",
|
|
829
|
+
daemon=True,
|
|
830
|
+
)
|
|
831
|
+
waker.start()
|
|
832
|
+
self._thread.join(timeout=timeout)
|
|
833
|
+
|
|
834
|
+
def join(self, timeout: float = 2.0) -> None:
|
|
835
|
+
if self._thread.is_alive():
|
|
836
|
+
self._thread.join(timeout=timeout)
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
# ── 两阶段引导元数据构造 ─────────────────────────────────────────────────────
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
def _should_skip_main_name(spec_name: str) -> bool:
|
|
843
|
+
if spec_name == "__main__" or spec_name.endswith(".__main__"):
|
|
844
|
+
return True
|
|
845
|
+
root = spec_name.split(".", 1)[0]
|
|
846
|
+
return root in {"pytest", "_pytest"}
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _should_skip_main_path(main_path: str) -> bool:
|
|
850
|
+
if not main_path or "<" in main_path or ">" in main_path:
|
|
851
|
+
return True
|
|
852
|
+
path = Path(main_path)
|
|
853
|
+
if path.suffix.lower() not in {".py", ".pyw"}:
|
|
854
|
+
return True
|
|
855
|
+
stem = path.stem.lower()
|
|
856
|
+
if stem in {"__main__", "pytest", "py.test", "pytest-script"} or stem.startswith("pytest"):
|
|
857
|
+
return True
|
|
858
|
+
return not path.is_file()
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
def _build_prep_data(spec: WorkerLaunchSpec) -> dict[str, Any]:
|
|
862
|
+
"""构造子进程纯 JSON 引导数据(路径、工作目录、__mp_main__ 与执行身份)。"""
|
|
863
|
+
expanded_sys_path = [
|
|
864
|
+
_ORIGINAL_DIR if entry == "" else entry for entry in sys.path
|
|
865
|
+
]
|
|
866
|
+
init_main_from_name: str | None = None
|
|
867
|
+
init_main_from_path: str | None = None
|
|
868
|
+
|
|
869
|
+
main_mod = sys.modules.get("__main__")
|
|
870
|
+
if main_mod is not None:
|
|
871
|
+
main_spec = getattr(main_mod, "__spec__", None)
|
|
872
|
+
spec_name = getattr(main_spec, "name", None) if main_spec is not None else None
|
|
873
|
+
if isinstance(spec_name, str) and spec_name:
|
|
874
|
+
if not _should_skip_main_name(spec_name):
|
|
875
|
+
init_main_from_name = spec_name
|
|
876
|
+
else:
|
|
877
|
+
main_file = getattr(main_mod, "__file__", None)
|
|
878
|
+
if isinstance(main_file, str) and main_file:
|
|
879
|
+
resolved = (
|
|
880
|
+
main_file
|
|
881
|
+
if os.path.isabs(main_file)
|
|
882
|
+
else os.path.normpath(os.path.join(_ORIGINAL_DIR, main_file))
|
|
883
|
+
)
|
|
884
|
+
if not _should_skip_main_path(resolved):
|
|
885
|
+
init_main_from_path = resolved
|
|
886
|
+
|
|
887
|
+
return {
|
|
888
|
+
"job_uid": spec.job.uid,
|
|
889
|
+
"execution_id": spec.execution_id,
|
|
890
|
+
"run_id": spec.run_id or None,
|
|
891
|
+
"attempt_no": spec.attempt_no,
|
|
892
|
+
"cwd": os.getcwd(),
|
|
893
|
+
"orig_dir": _ORIGINAL_DIR,
|
|
894
|
+
"sys_path": expanded_sys_path,
|
|
895
|
+
"init_main_from_name": init_main_from_name,
|
|
896
|
+
"init_main_from_path": init_main_from_path,
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _build_bootstrap_env(address: str, family: str, authkey: bytes) -> dict[str, str]:
|
|
901
|
+
"""组装子进程环境变量:注入私有 IPC 凭据并确保 tasklite 包可被 -m 定位。"""
|
|
902
|
+
env = os.environ.copy()
|
|
903
|
+
env[ENV_IPC_ADDR] = address
|
|
904
|
+
env[ENV_IPC_FAMILY] = family
|
|
905
|
+
env[ENV_IPC_AUTHKEY] = authkey.hex()
|
|
906
|
+
|
|
907
|
+
existing_pythonpath = env.get("PYTHONPATH", "")
|
|
908
|
+
path_entries = [p for p in existing_pythonpath.split(os.pathsep) if p] if existing_pythonpath else []
|
|
909
|
+
norm_existing = {os.path.normcase(os.path.abspath(p)) for p in path_entries}
|
|
910
|
+
if os.path.normcase(os.path.abspath(_PACKAGE_ROOT)) not in norm_existing:
|
|
911
|
+
path_entries.append(_PACKAGE_ROOT)
|
|
912
|
+
env["PYTHONPATH"] = os.pathsep.join(path_entries)
|
|
913
|
+
return env
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
# ── 真实子进程启动器(单 Listener 双物理连接 + 两阶段握手屏障)────────────────
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
class SubprocessWorkerLauncher(WorkerLauncher):
|
|
920
|
+
"""基于 Popen + Listener/Client 握手屏障与进程树适配器的真实子进程启动器。"""
|
|
921
|
+
|
|
922
|
+
def __init__(
|
|
923
|
+
self,
|
|
924
|
+
*,
|
|
925
|
+
tree_adapter: ProcessTreeAdapter | None = None,
|
|
926
|
+
handshake_timeout: float = _DEFAULT_HANDSHAKE_TIMEOUT,
|
|
927
|
+
) -> None:
|
|
928
|
+
self._tree_adapter = tree_adapter or default_process_tree_adapter()
|
|
929
|
+
self._handshake_timeout = handshake_timeout
|
|
930
|
+
|
|
931
|
+
@staticmethod
|
|
932
|
+
def _cleanup_socket_file(address: str, family: str) -> None:
|
|
933
|
+
if family == "AF_UNIX" and address:
|
|
934
|
+
try:
|
|
935
|
+
os.unlink(address)
|
|
936
|
+
except OSError:
|
|
937
|
+
pass
|
|
938
|
+
|
|
939
|
+
def prepare(
|
|
940
|
+
self,
|
|
941
|
+
spec: WorkerLaunchSpec,
|
|
942
|
+
*,
|
|
943
|
+
on_message: Callable[[], None] | None,
|
|
944
|
+
cancel_event: threading.Event,
|
|
945
|
+
) -> PreparedWorker:
|
|
946
|
+
if cancel_event.is_set():
|
|
947
|
+
raise _WorkerLaunchCancelled(
|
|
948
|
+
f"stop requested before worker launch for {spec.job.uid}"
|
|
949
|
+
)
|
|
950
|
+
|
|
951
|
+
prep_bytes = encode_json_frame(_build_prep_data(spec))
|
|
952
|
+
payload_bytes = pickle.dumps(
|
|
953
|
+
{
|
|
954
|
+
"handler": spec.handler,
|
|
955
|
+
"job": spec.job,
|
|
956
|
+
"job_context": spec.job_context,
|
|
957
|
+
"execution_id": spec.execution_id,
|
|
958
|
+
"timeout_seconds": spec.timeout_seconds,
|
|
959
|
+
"run_id": spec.run_id,
|
|
960
|
+
"attempt_no": spec.attempt_no,
|
|
961
|
+
}
|
|
962
|
+
)
|
|
963
|
+
|
|
964
|
+
address, family = _make_ipc_address()
|
|
965
|
+
authkey = os.urandom(32)
|
|
966
|
+
listener: Listener | None = None
|
|
967
|
+
acceptor: _HandshakeAcceptor | None = None
|
|
968
|
+
proc: subprocess.Popen[Any] | None = None
|
|
969
|
+
managed_handle: ManagedWorkerHandle | None = None
|
|
970
|
+
uplink: Any = None
|
|
971
|
+
downlink: Any = None
|
|
972
|
+
receiver: WorkerPipeReceiver | None = None
|
|
973
|
+
reply_sender: WorkerReplySender | None = None
|
|
974
|
+
|
|
975
|
+
try:
|
|
976
|
+
listener = Listener(address=address, family=family, authkey=authkey)
|
|
977
|
+
acceptor = _HandshakeAcceptor(
|
|
978
|
+
listener,
|
|
979
|
+
address=str(listener.address),
|
|
980
|
+
family=family,
|
|
981
|
+
authkey=authkey,
|
|
982
|
+
)
|
|
983
|
+
acceptor.start()
|
|
984
|
+
|
|
985
|
+
if cancel_event.is_set():
|
|
986
|
+
raise _WorkerLaunchCancelled(
|
|
987
|
+
f"stop requested before worker process creation for {spec.job.uid}"
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
env = _build_bootstrap_env(str(listener.address), family, authkey)
|
|
991
|
+
popen_kwargs = self._tree_adapter.popen_kwargs()
|
|
992
|
+
proc = subprocess.Popen(
|
|
993
|
+
[sys.executable, "-m", "tasklite.engine._worker_bootstrap"],
|
|
994
|
+
env=env,
|
|
995
|
+
**popen_kwargs,
|
|
996
|
+
)
|
|
997
|
+
|
|
998
|
+
deadline = time.monotonic() + self._handshake_timeout
|
|
999
|
+
while not acceptor.wait_done(_HANDSHAKE_POLL_SLICE):
|
|
1000
|
+
if cancel_event.is_set():
|
|
1001
|
+
raise _WorkerLaunchCancelled(
|
|
1002
|
+
f"stop requested during worker handshake for {spec.job.uid}"
|
|
1003
|
+
)
|
|
1004
|
+
if proc.poll() is not None:
|
|
1005
|
+
raise RuntimeError(
|
|
1006
|
+
f"Worker bootstrap process exited prematurely with code {proc.returncode}"
|
|
1007
|
+
)
|
|
1008
|
+
if time.monotonic() >= deadline:
|
|
1009
|
+
raise TimeoutError(
|
|
1010
|
+
f"Worker bootstrap handshake timed out after {self._handshake_timeout}s"
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
acceptor.join(timeout=2.0)
|
|
1014
|
+
uplink = acceptor.uplink
|
|
1015
|
+
downlink = acceptor.downlink
|
|
1016
|
+
|
|
1017
|
+
if acceptor.error is not None:
|
|
1018
|
+
raise RuntimeError(f"Worker handshake failed: {acceptor.error}") from acceptor.error
|
|
1019
|
+
if cancel_event.is_set():
|
|
1020
|
+
raise _WorkerLaunchCancelled(
|
|
1021
|
+
f"stop requested after worker handshake for {spec.job.uid}"
|
|
1022
|
+
)
|
|
1023
|
+
if uplink is None or downlink is None:
|
|
1024
|
+
raise RuntimeError("Worker handshake ended without both uplink and downlink connections")
|
|
1025
|
+
|
|
1026
|
+
# 收齐双连接后立即关闭 Listener,不再接受任何连接
|
|
1027
|
+
listener.close()
|
|
1028
|
+
listener = None
|
|
1029
|
+
self._cleanup_socket_file(address, family)
|
|
1030
|
+
|
|
1031
|
+
# 零竞态进程树绑定:此时 worker 阻塞在 downlink.recv_bytes(),尚未执行用户代码
|
|
1032
|
+
managed_handle = self._tree_adapter.attach_and_wrap(
|
|
1033
|
+
proc,
|
|
1034
|
+
worker_pid=acceptor.worker_pid,
|
|
1035
|
+
)
|
|
1036
|
+
|
|
1037
|
+
if cancel_event.is_set():
|
|
1038
|
+
raise _WorkerLaunchCancelled(
|
|
1039
|
+
f"stop requested after process tree attach for {spec.job.uid}"
|
|
1040
|
+
)
|
|
1041
|
+
|
|
1042
|
+
# 第一阶段下发预序列化数据(worker 仅缓存字节,仍阻塞于 b"GO" 放行门)
|
|
1043
|
+
downlink.send_bytes(prep_bytes)
|
|
1044
|
+
downlink.send_bytes(payload_bytes)
|
|
1045
|
+
|
|
1046
|
+
receiver = WorkerPipeReceiver(
|
|
1047
|
+
uplink,
|
|
1048
|
+
job_uid=spec.job.uid,
|
|
1049
|
+
execution_id=spec.execution_id,
|
|
1050
|
+
run_id=spec.run_id or None,
|
|
1051
|
+
attempt_no=spec.attempt_no,
|
|
1052
|
+
on_message=on_message,
|
|
1053
|
+
)
|
|
1054
|
+
uplink = None # 所有权转移给 receiver
|
|
1055
|
+
|
|
1056
|
+
reply_sender = WorkerReplySender(
|
|
1057
|
+
downlink,
|
|
1058
|
+
job_uid=spec.job.uid,
|
|
1059
|
+
execution_id=spec.execution_id,
|
|
1060
|
+
run_id=spec.run_id or None,
|
|
1061
|
+
attempt_no=spec.attempt_no,
|
|
1062
|
+
)
|
|
1063
|
+
|
|
1064
|
+
bound_handle = managed_handle
|
|
1065
|
+
bound_receiver = receiver
|
|
1066
|
+
bound_reply_sender = reply_sender
|
|
1067
|
+
bound_downlink = downlink
|
|
1068
|
+
bound_cancel = cancel_event
|
|
1069
|
+
|
|
1070
|
+
def _release_subprocess() -> None:
|
|
1071
|
+
if bound_cancel.is_set():
|
|
1072
|
+
raise _WorkerLaunchCancelled(
|
|
1073
|
+
f"stop requested before release for {spec.job.uid}"
|
|
1074
|
+
)
|
|
1075
|
+
bound_downlink.send_bytes(LAUNCH_GO_TOKEN)
|
|
1076
|
+
|
|
1077
|
+
return PreparedWorker(
|
|
1078
|
+
handle=bound_handle,
|
|
1079
|
+
receiver=bound_receiver,
|
|
1080
|
+
reply_sender=bound_reply_sender,
|
|
1081
|
+
on_release=_release_subprocess,
|
|
1082
|
+
)
|
|
1083
|
+
except BaseException:
|
|
1084
|
+
# 清理顺序红线:先非阻塞唤醒 acceptor 并尽快关 listener(打断任何未接通的
|
|
1085
|
+
# 唤醒自连),再终止进程树,随后复核 acceptor 退出并关闭 receiver/reply_sender 与句柄。
|
|
1086
|
+
if acceptor is not None:
|
|
1087
|
+
acceptor.cancel_and_join(timeout=0.5)
|
|
1088
|
+
if listener is not None:
|
|
1089
|
+
try:
|
|
1090
|
+
listener.close()
|
|
1091
|
+
except Exception:
|
|
1092
|
+
pass
|
|
1093
|
+
listener = None
|
|
1094
|
+
if acceptor is not None:
|
|
1095
|
+
if uplink is None and acceptor.uplink is not None and receiver is None:
|
|
1096
|
+
uplink = acceptor.uplink
|
|
1097
|
+
if downlink is None and acceptor.downlink is not None and reply_sender is None:
|
|
1098
|
+
downlink = acceptor.downlink
|
|
1099
|
+
if managed_handle is not None:
|
|
1100
|
+
try:
|
|
1101
|
+
managed_handle.kill_tree()
|
|
1102
|
+
except Exception:
|
|
1103
|
+
pass
|
|
1104
|
+
try:
|
|
1105
|
+
managed_handle.join(timeout=5.0)
|
|
1106
|
+
except Exception:
|
|
1107
|
+
pass
|
|
1108
|
+
elif proc is not None:
|
|
1109
|
+
# 尚未绑定进程树前先关裸连接,令阻塞于握手屏障的 worker 立即因 EOF 退出
|
|
1110
|
+
for raw_conn in (downlink, uplink):
|
|
1111
|
+
if raw_conn is not None:
|
|
1112
|
+
try:
|
|
1113
|
+
raw_conn.close()
|
|
1114
|
+
except OSError:
|
|
1115
|
+
pass
|
|
1116
|
+
downlink = None
|
|
1117
|
+
uplink = None
|
|
1118
|
+
self._tree_adapter.kill_unattached(proc)
|
|
1119
|
+
|
|
1120
|
+
if acceptor is not None:
|
|
1121
|
+
acceptor.join(timeout=1.5)
|
|
1122
|
+
if uplink is None and acceptor.uplink is not None and receiver is None:
|
|
1123
|
+
uplink = acceptor.uplink
|
|
1124
|
+
if downlink is None and acceptor.downlink is not None and reply_sender is None:
|
|
1125
|
+
downlink = acceptor.downlink
|
|
1126
|
+
|
|
1127
|
+
if reply_sender is not None:
|
|
1128
|
+
try:
|
|
1129
|
+
reply_sender.close()
|
|
1130
|
+
except Exception:
|
|
1131
|
+
pass
|
|
1132
|
+
elif downlink is not None:
|
|
1133
|
+
try:
|
|
1134
|
+
downlink.close()
|
|
1135
|
+
except OSError:
|
|
1136
|
+
pass
|
|
1137
|
+
|
|
1138
|
+
if receiver is not None:
|
|
1139
|
+
try:
|
|
1140
|
+
receiver.close(timeout=2.0)
|
|
1141
|
+
except Exception:
|
|
1142
|
+
pass
|
|
1143
|
+
elif uplink is not None:
|
|
1144
|
+
try:
|
|
1145
|
+
uplink.close()
|
|
1146
|
+
except OSError:
|
|
1147
|
+
pass
|
|
1148
|
+
|
|
1149
|
+
if managed_handle is not None:
|
|
1150
|
+
try:
|
|
1151
|
+
managed_handle.close()
|
|
1152
|
+
except Exception:
|
|
1153
|
+
pass
|
|
1154
|
+
|
|
1155
|
+
self._cleanup_socket_file(address, family)
|
|
1156
|
+
raise
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
__all__ = [
|
|
1160
|
+
"LaunchedWorker",
|
|
1161
|
+
"ManagedWorkerHandle",
|
|
1162
|
+
"PosixManagedWorkerHandle",
|
|
1163
|
+
"PosixProcessTreeAdapter",
|
|
1164
|
+
"PreparedWorker",
|
|
1165
|
+
"ProcessTreeAdapter",
|
|
1166
|
+
"SubprocessWorkerLauncher",
|
|
1167
|
+
"WindowsManagedWorkerHandle",
|
|
1168
|
+
"WindowsProcessTreeAdapter",
|
|
1169
|
+
"WorkerLauncher",
|
|
1170
|
+
"_BasePopenWorkerHandle",
|
|
1171
|
+
"_WorkerLaunchCancelled",
|
|
1172
|
+
"default_process_tree_adapter",
|
|
1173
|
+
]
|