tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
tasklite/engine/__init__.py
CHANGED
|
@@ -7,15 +7,14 @@
|
|
|
7
7
|
admission(trigger 准入与 RequeuePolicy 重入队策略);
|
|
8
8
|
- 调度与资源:scheduler(只读扫描选择点,OrderingPolicy seam)、
|
|
9
9
|
resource(资源体系与两阶段租约)、wait(等待决策);
|
|
10
|
-
- 机器群:dispatch
|
|
11
|
-
in_flight(在飞追踪)、completion(完成机器)、
|
|
12
|
-
store(状态仓库)、governor
|
|
13
|
-
- 运行期:session(run
|
|
14
|
-
ops(Run 外管理接缝)。
|
|
10
|
+
- 机器群:authorization(文件声明与三类授权等待协调)、dispatch(预检与派发)、
|
|
11
|
+
channel(子进程执行通道)、in_flight(在飞追踪)、completion(完成机器)、
|
|
12
|
+
recovery(恢复编排)、store(状态仓库)、governor(死锁治理与排除停摆);
|
|
13
|
+
- 运行期:session(run 生命周期与排除真相源)、runtime(EngineRuntime 事件泵)。
|
|
15
14
|
|
|
16
15
|
调度逻辑仅留 seam:选择点收敛在 scheduler 的 scan_next_runnable(访问
|
|
17
16
|
序经 OrderingPolicy),重试节奏收敛在 RequeuePolicy(默认立即重入队)。
|
|
18
17
|
|
|
19
|
-
分层红线:本包属核心层,严禁 import ``tasklite/wrappers
|
|
20
|
-
|
|
18
|
+
分层红线:本包属核心层,严禁 import ``tasklite/wrappers/``;
|
|
19
|
+
可依赖 ``tasklite/{models,utils,backend}``。
|
|
21
20
|
"""
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""隔离子进程两阶段引导入口与 Fork-Bomb 重入守卫。
|
|
2
|
+
|
|
3
|
+
由 ``SubprocessWorkerLauncher`` 以 ``python -m tasklite.engine._worker_bootstrap`` 启动,
|
|
4
|
+
仅导入纯 worker 侧模块,严格执行两阶段握手协议:
|
|
5
|
+
1. 立即清除私有 IPC 环境变量并屏蔽停止信号;
|
|
6
|
+
2. 依次向父进程 ``Listener`` 建立 ``uplink`` 与 ``downlink`` 双物理连接;
|
|
7
|
+
3. 阻塞接收预序列化的 ``prep_bytes`` 与 ``payload_bytes``(仅缓存字节,不导入用户代码);
|
|
8
|
+
4. 阻塞等待父进程 ``b"GO"`` 放行门(此时进程树绑定已零竞态完成);
|
|
9
|
+
5. 放行后置位 ``_IMPORTING_MP_MAIN`` 重入守卫,还原 ``sys.path`` / ``cwd`` / ``__mp_main__``,
|
|
10
|
+
反序列化执行载荷并进入 ``run_worker_entry``;引导期任何故障经 ``uplink`` 回传
|
|
11
|
+
结构化 ``failure`` 帧,绝不静默退出。
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import pickle
|
|
18
|
+
import runpy
|
|
19
|
+
import sys
|
|
20
|
+
import types
|
|
21
|
+
from multiprocessing.connection import Client
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from ..utils.ipc import (
|
|
25
|
+
HANDSHAKE_ROLE_DOWNLINK,
|
|
26
|
+
HANDSHAKE_ROLE_UPLINK,
|
|
27
|
+
WorkerPipeSender,
|
|
28
|
+
WorkerReplyReceiver,
|
|
29
|
+
decode_json_object_frame,
|
|
30
|
+
encode_handshake_frame,
|
|
31
|
+
)
|
|
32
|
+
from ._worker_runtime import (
|
|
33
|
+
WorkerLaunchSpec,
|
|
34
|
+
build_exception_dict,
|
|
35
|
+
build_failure_result_frame,
|
|
36
|
+
ignore_worker_stop_signals,
|
|
37
|
+
run_worker_entry,
|
|
38
|
+
)
|
|
39
|
+
from .types import (
|
|
40
|
+
ENV_IPC_ADDR,
|
|
41
|
+
ENV_IPC_AUTHKEY,
|
|
42
|
+
ENV_IPC_FAMILY,
|
|
43
|
+
LAUNCH_GO_TOKEN,
|
|
44
|
+
_set_importing_mp_main,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _import_mp_main_from_name(mod_name: str) -> None:
|
|
49
|
+
"""按模块名在 ``__mp_main__`` 命名空间中导入主模块并绑定到 ``__main__``。"""
|
|
50
|
+
module_globals = runpy.run_module(mod_name, run_name="__mp_main__", alter_sys=False)
|
|
51
|
+
mp_main = types.ModuleType("__mp_main__")
|
|
52
|
+
mp_main.__dict__.update(module_globals)
|
|
53
|
+
sys.modules["__mp_main__"] = mp_main
|
|
54
|
+
sys.modules["__main__"] = mp_main
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _import_mp_main_from_path(main_path: str) -> None:
|
|
58
|
+
"""按脚本路径在 ``__mp_main__`` 命名空间中导入主模块并绑定到 ``__main__``。"""
|
|
59
|
+
module_globals = runpy.run_path(main_path, run_name="__mp_main__")
|
|
60
|
+
mp_main = types.ModuleType("__mp_main__")
|
|
61
|
+
mp_main.__dict__.update(module_globals)
|
|
62
|
+
sys.modules["__mp_main__"] = mp_main
|
|
63
|
+
sys.modules["__main__"] = mp_main
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _restore_environment(prep_data: dict[str, Any]) -> None:
|
|
67
|
+
"""还原父进程 ``sys.path``、工作目录与 ``__mp_main__`` 模块环境。"""
|
|
68
|
+
sys_path = prep_data.get("sys_path")
|
|
69
|
+
if isinstance(sys_path, list):
|
|
70
|
+
sys.path[:] = [str(entry) for entry in sys_path]
|
|
71
|
+
|
|
72
|
+
cwd = prep_data.get("cwd")
|
|
73
|
+
if isinstance(cwd, str) and cwd:
|
|
74
|
+
os.chdir(cwd)
|
|
75
|
+
|
|
76
|
+
init_from_name = prep_data.get("init_main_from_name")
|
|
77
|
+
init_from_path = prep_data.get("init_main_from_path")
|
|
78
|
+
if not init_from_name and not init_from_path:
|
|
79
|
+
return
|
|
80
|
+
|
|
81
|
+
_set_importing_mp_main(True)
|
|
82
|
+
try:
|
|
83
|
+
if isinstance(init_from_name, str) and init_from_name:
|
|
84
|
+
_import_mp_main_from_name(init_from_name)
|
|
85
|
+
elif isinstance(init_from_path, str) and init_from_path:
|
|
86
|
+
_import_mp_main_from_path(init_from_path)
|
|
87
|
+
finally:
|
|
88
|
+
_set_importing_mp_main(False)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _send_bootstrap_failure(
|
|
92
|
+
pipe_sender: WorkerPipeSender,
|
|
93
|
+
exc: BaseException,
|
|
94
|
+
) -> None:
|
|
95
|
+
"""将引导或反序列化失败封装为结构化 failure 帧回传主进程。"""
|
|
96
|
+
hint = "Task handler 须定义在模块顶层,不得置于 `if __name__ == '__main__':` 内"
|
|
97
|
+
message = f"Worker bootstrap failed ({type(exc).__name__}: {exc}). {hint}"
|
|
98
|
+
payload = build_failure_result_frame(
|
|
99
|
+
message,
|
|
100
|
+
exception=build_exception_dict(exc),
|
|
101
|
+
)
|
|
102
|
+
try:
|
|
103
|
+
pipe_sender.send_result(payload)
|
|
104
|
+
except OSError:
|
|
105
|
+
pass
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _bootstrap_main() -> None:
|
|
109
|
+
"""子进程引导主流程。"""
|
|
110
|
+
addr = os.environ.pop(ENV_IPC_ADDR, None)
|
|
111
|
+
family = os.environ.pop(ENV_IPC_FAMILY, None)
|
|
112
|
+
authkey_hex = os.environ.pop(ENV_IPC_AUTHKEY, None)
|
|
113
|
+
if not addr or not family or not authkey_hex:
|
|
114
|
+
sys.stderr.write("tasklite worker bootstrap missing required IPC environment variables\n")
|
|
115
|
+
sys.exit(2)
|
|
116
|
+
|
|
117
|
+
ignore_worker_stop_signals()
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
authkey = bytes.fromhex(authkey_hex)
|
|
121
|
+
except ValueError:
|
|
122
|
+
sys.stderr.write("tasklite worker bootstrap received invalid IPC authkey\n")
|
|
123
|
+
sys.exit(2)
|
|
124
|
+
|
|
125
|
+
uplink = None
|
|
126
|
+
downlink = None
|
|
127
|
+
try:
|
|
128
|
+
uplink = Client(address=addr, family=family, authkey=authkey)
|
|
129
|
+
uplink.send_bytes(
|
|
130
|
+
encode_handshake_frame(HANDSHAKE_ROLE_UPLINK, pid=os.getpid())
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
downlink = Client(address=addr, family=family, authkey=authkey)
|
|
134
|
+
downlink.send_bytes(
|
|
135
|
+
encode_handshake_frame(HANDSHAKE_ROLE_DOWNLINK, pid=os.getpid())
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# 第一阶段:仅缓存预序列化字节,等待父进程完成进程树绑定与放行门校验
|
|
139
|
+
prep_bytes = downlink.recv_bytes()
|
|
140
|
+
payload_bytes = downlink.recv_bytes()
|
|
141
|
+
gate = downlink.recv_bytes()
|
|
142
|
+
except (EOFError, OSError):
|
|
143
|
+
for conn in (downlink, uplink):
|
|
144
|
+
if conn is not None:
|
|
145
|
+
try:
|
|
146
|
+
conn.close()
|
|
147
|
+
except OSError:
|
|
148
|
+
pass
|
|
149
|
+
sys.exit(0)
|
|
150
|
+
|
|
151
|
+
if gate != LAUNCH_GO_TOKEN:
|
|
152
|
+
for conn in (downlink, uplink):
|
|
153
|
+
try:
|
|
154
|
+
conn.close()
|
|
155
|
+
except OSError:
|
|
156
|
+
pass
|
|
157
|
+
sys.exit(0)
|
|
158
|
+
|
|
159
|
+
try:
|
|
160
|
+
prep_data = decode_json_object_frame(prep_bytes, context="bootstrap prep_data")
|
|
161
|
+
except Exception as exc:
|
|
162
|
+
sys.stderr.write(f"tasklite worker bootstrap failed to decode prep_data: {exc}\n")
|
|
163
|
+
for conn in (downlink, uplink):
|
|
164
|
+
try:
|
|
165
|
+
conn.close()
|
|
166
|
+
except OSError:
|
|
167
|
+
pass
|
|
168
|
+
sys.exit(2)
|
|
169
|
+
|
|
170
|
+
job_uid = str(prep_data.get("job_uid", ""))
|
|
171
|
+
execution_id = str(prep_data.get("execution_id", ""))
|
|
172
|
+
run_id = prep_data.get("run_id")
|
|
173
|
+
if run_id is not None:
|
|
174
|
+
run_id = str(run_id)
|
|
175
|
+
attempt_no = prep_data.get("attempt_no")
|
|
176
|
+
if not isinstance(attempt_no, int) or isinstance(attempt_no, bool):
|
|
177
|
+
attempt_no = None
|
|
178
|
+
|
|
179
|
+
pipe_sender = WorkerPipeSender(
|
|
180
|
+
uplink,
|
|
181
|
+
job_uid=job_uid,
|
|
182
|
+
execution_id=execution_id,
|
|
183
|
+
run_id=run_id,
|
|
184
|
+
attempt_no=attempt_no,
|
|
185
|
+
)
|
|
186
|
+
reply_receiver = WorkerReplyReceiver(
|
|
187
|
+
downlink,
|
|
188
|
+
job_uid=job_uid,
|
|
189
|
+
execution_id=execution_id,
|
|
190
|
+
run_id=run_id,
|
|
191
|
+
attempt_no=attempt_no,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
try:
|
|
195
|
+
_restore_environment(prep_data)
|
|
196
|
+
spec_payload = pickle.loads(payload_bytes)
|
|
197
|
+
launch_spec = WorkerLaunchSpec(
|
|
198
|
+
handler=spec_payload["handler"],
|
|
199
|
+
job=spec_payload["job"],
|
|
200
|
+
job_context=spec_payload["job_context"],
|
|
201
|
+
execution_id=spec_payload["execution_id"],
|
|
202
|
+
timeout_seconds=spec_payload["timeout_seconds"],
|
|
203
|
+
pipe_sender=pipe_sender,
|
|
204
|
+
reply_receiver=reply_receiver,
|
|
205
|
+
run_id=spec_payload["run_id"],
|
|
206
|
+
attempt_no=spec_payload["attempt_no"],
|
|
207
|
+
)
|
|
208
|
+
except BaseException as exc:
|
|
209
|
+
_send_bootstrap_failure(pipe_sender, exc)
|
|
210
|
+
pipe_sender.close()
|
|
211
|
+
reply_receiver.close()
|
|
212
|
+
return
|
|
213
|
+
|
|
214
|
+
run_worker_entry(launch_spec)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
if __name__ == "__main__":
|
|
218
|
+
# 通过正式包路径运行主流程,保证 _IMPORTING_MP_MAIN 与引擎侧引用同一模块实例。
|
|
219
|
+
from tasklite.engine._worker_bootstrap import _bootstrap_main as _run_bootstrap
|
|
220
|
+
|
|
221
|
+
_run_bootstrap()
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""Worker 侧纯执行模块与跨平台停止信号单一事实源。
|
|
2
|
+
|
|
3
|
+
本模块仅依赖模型层、IPC 底层工具与异常定义,严禁导入 ``channel``、``launcher``、
|
|
4
|
+
``dispatch``、``runtime`` 等父进程调度模块,确保隔离子进程冷启动路径轻量化且彻底
|
|
5
|
+
消除 ``channel <-> launcher`` 环形依赖。
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import signal
|
|
13
|
+
import threading
|
|
14
|
+
import traceback
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ..exceptions import _ArtifactIOError
|
|
20
|
+
from ..models.context import JobContext
|
|
21
|
+
from ..models.job import Job
|
|
22
|
+
from ..models.result import TaskResult
|
|
23
|
+
from ..utils.ipc import WorkerPipeSender, WorkerReplyReceiver
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("tasklite")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def stop_signals(signal_module: Any = signal) -> tuple[Any, ...]:
|
|
29
|
+
"""返回当前平台统一的停止信号集合(主进程注册与 worker 屏蔽共用唯一事实源)。"""
|
|
30
|
+
sigs: list[Any] = [signal_module.SIGTERM, signal_module.SIGINT]
|
|
31
|
+
sigbreak = getattr(signal_module, "SIGBREAK", None)
|
|
32
|
+
if os.name == "nt" and sigbreak is not None:
|
|
33
|
+
sigs.append(sigbreak)
|
|
34
|
+
return tuple(sigs)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def ignore_worker_stop_signals() -> None:
|
|
38
|
+
"""在 worker 主线程忽略平台停止信号,避免主进程停机信号穿透 worker。
|
|
39
|
+
|
|
40
|
+
进程内线程替身在非主线程运行,不修改宿主主线程信号表。
|
|
41
|
+
"""
|
|
42
|
+
if threading.current_thread() is not threading.main_thread():
|
|
43
|
+
return
|
|
44
|
+
for sig in stop_signals():
|
|
45
|
+
signal.signal(sig, signal.SIG_IGN)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class WorkerLaunchSpec:
|
|
50
|
+
"""进程 seam 具名契约——下发 worker 执行体的不可变启动载荷。
|
|
51
|
+
|
|
52
|
+
``execution_id``(``{run_id}.{dispatch_no}`` 执行身份)与管道端点归本 spec,
|
|
53
|
+
不借道 ``JobContext`` 公开属性穿透进程边界,亦不携带父进程控制回调。
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
handler: Callable[[Job, JobContext], Any]
|
|
57
|
+
job: Job
|
|
58
|
+
job_context: JobContext
|
|
59
|
+
execution_id: str
|
|
60
|
+
timeout_seconds: float
|
|
61
|
+
pipe_sender: WorkerPipeSender | None = None
|
|
62
|
+
reply_receiver: WorkerReplyReceiver | None = None
|
|
63
|
+
run_id: str = ""
|
|
64
|
+
attempt_no: int = 1
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def build_exception_dict(
|
|
68
|
+
exc: BaseException,
|
|
69
|
+
*,
|
|
70
|
+
tb: str | None = None,
|
|
71
|
+
) -> dict[str, str]:
|
|
72
|
+
"""构造跨进程传递的结构化异常字典(type / message / traceback)。"""
|
|
73
|
+
return {
|
|
74
|
+
"type": type(exc).__name__,
|
|
75
|
+
"message": str(exc),
|
|
76
|
+
"traceback": traceback.format_exc() if tb is None else tb,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def build_success_result_frame(
|
|
81
|
+
*,
|
|
82
|
+
message: str | None,
|
|
83
|
+
metadata: dict[str, Any],
|
|
84
|
+
spawned_jobs: list[dict[str, Any]],
|
|
85
|
+
cursor_updates: dict[str, str | None],
|
|
86
|
+
) -> dict[str, Any]:
|
|
87
|
+
"""构造 worker 成功结果帧。"""
|
|
88
|
+
return {
|
|
89
|
+
"status": "success",
|
|
90
|
+
"message": message,
|
|
91
|
+
"metadata": metadata,
|
|
92
|
+
"spawned_jobs": spawned_jobs,
|
|
93
|
+
"cursor_updates": cursor_updates,
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def build_failure_result_frame(
|
|
98
|
+
message: str | None,
|
|
99
|
+
*,
|
|
100
|
+
metadata: dict[str, Any] | None = None,
|
|
101
|
+
exception: dict[str, str] | None = None,
|
|
102
|
+
) -> dict[str, Any]:
|
|
103
|
+
"""构造 worker 失败结果帧(供执行失败与引导失败共用)。"""
|
|
104
|
+
return {
|
|
105
|
+
"status": "failure",
|
|
106
|
+
"message": message,
|
|
107
|
+
"metadata": {} if metadata is None else metadata,
|
|
108
|
+
"exception": exception,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_retry_result_frame(
|
|
113
|
+
*,
|
|
114
|
+
message: str | None,
|
|
115
|
+
metadata: dict[str, Any],
|
|
116
|
+
consume_budget: bool,
|
|
117
|
+
) -> dict[str, Any]:
|
|
118
|
+
"""构造 worker 重试结果帧。"""
|
|
119
|
+
return {
|
|
120
|
+
"status": "retry",
|
|
121
|
+
"message": message,
|
|
122
|
+
"metadata": metadata,
|
|
123
|
+
"consume_budget": consume_budget,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def build_artifact_error_frame(error_message: str) -> dict[str, Any]:
|
|
128
|
+
"""构造产物 IO 失败结果帧。"""
|
|
129
|
+
return {
|
|
130
|
+
"status": "artifact_error",
|
|
131
|
+
"error_message": error_message,
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def run_worker_entry(spec: WorkerLaunchSpec) -> None:
|
|
136
|
+
"""Worker 执行体入口:隔离信号、执行 handler 并经瞬时管道单向回传结果。
|
|
137
|
+
|
|
138
|
+
不变式:执行体任何出口(含未捕获异常)均在 ``finally`` 中关闭管道端点。
|
|
139
|
+
"""
|
|
140
|
+
ignore_worker_stop_signals()
|
|
141
|
+
handler_func = spec.handler
|
|
142
|
+
job = spec.job
|
|
143
|
+
if getattr(spec, "attempt_no", None) is not None:
|
|
144
|
+
job._attempt_no = spec.attempt_no
|
|
145
|
+
ctx = spec.job_context
|
|
146
|
+
job_uid = job.uid
|
|
147
|
+
execution_id = spec.execution_id
|
|
148
|
+
if not execution_id:
|
|
149
|
+
raise RuntimeError(
|
|
150
|
+
f"worker started without execution_id for {job_uid}: "
|
|
151
|
+
f"fencing requires WorkerLaunchSpec.execution_id"
|
|
152
|
+
)
|
|
153
|
+
pipe_sender = spec.pipe_sender
|
|
154
|
+
reply_receiver = spec.reply_receiver
|
|
155
|
+
if pipe_sender is not None or reply_receiver is not None:
|
|
156
|
+
ctx._bind_pipe_sender(pipe_sender, reply_receiver)
|
|
157
|
+
|
|
158
|
+
def _write_result(payload: dict[str, Any]) -> None:
|
|
159
|
+
if pipe_sender is None:
|
|
160
|
+
return
|
|
161
|
+
try:
|
|
162
|
+
pipe_sender.send_result(payload)
|
|
163
|
+
except OSError as exc:
|
|
164
|
+
logger.error(
|
|
165
|
+
f"worker pipe result send failed for {job_uid} ({execution_id}): {exc}"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
try:
|
|
170
|
+
raw_result = handler_func(job, ctx)
|
|
171
|
+
if isinstance(raw_result, TaskResult):
|
|
172
|
+
if raw_result.status == "success":
|
|
173
|
+
_write_result(
|
|
174
|
+
build_success_result_frame(
|
|
175
|
+
message=raw_result.message,
|
|
176
|
+
metadata=raw_result.metadata,
|
|
177
|
+
spawned_jobs=[j.to_dict() for j in ctx._spawned_jobs],
|
|
178
|
+
cursor_updates=dict(ctx._cursor_updates),
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
elif raw_result.status == "failure":
|
|
182
|
+
_write_result(
|
|
183
|
+
build_failure_result_frame(
|
|
184
|
+
raw_result.message,
|
|
185
|
+
metadata=raw_result.metadata,
|
|
186
|
+
)
|
|
187
|
+
)
|
|
188
|
+
else:
|
|
189
|
+
_write_result(
|
|
190
|
+
build_retry_result_frame(
|
|
191
|
+
message=raw_result.message,
|
|
192
|
+
metadata=raw_result.metadata,
|
|
193
|
+
consume_budget=raw_result.consume_budget,
|
|
194
|
+
)
|
|
195
|
+
)
|
|
196
|
+
else:
|
|
197
|
+
_write_result(
|
|
198
|
+
build_failure_result_frame(
|
|
199
|
+
f"invalid handler return type: expected TaskResult, "
|
|
200
|
+
f"got {type(raw_result).__name__}",
|
|
201
|
+
)
|
|
202
|
+
)
|
|
203
|
+
except _ArtifactIOError as e:
|
|
204
|
+
_write_result(build_artifact_error_frame(str(e)))
|
|
205
|
+
except BaseException as e:
|
|
206
|
+
_write_result(
|
|
207
|
+
build_failure_result_frame(
|
|
208
|
+
type(e).__name__ if not str(e) else f"{type(e).__name__}: {e}",
|
|
209
|
+
exception=build_exception_dict(e),
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
finally:
|
|
213
|
+
if pipe_sender is not None:
|
|
214
|
+
pipe_sender.close()
|
|
215
|
+
if reply_receiver is not None:
|
|
216
|
+
reply_receiver.close()
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
__all__ = [
|
|
220
|
+
"WorkerLaunchSpec",
|
|
221
|
+
"build_artifact_error_frame",
|
|
222
|
+
"build_exception_dict",
|
|
223
|
+
"build_failure_result_frame",
|
|
224
|
+
"build_retry_result_frame",
|
|
225
|
+
"build_success_result_frame",
|
|
226
|
+
"ignore_worker_stop_signals",
|
|
227
|
+
"run_worker_entry",
|
|
228
|
+
"stop_signals",
|
|
229
|
+
]
|
tasklite/engine/admission.py
CHANGED
|
@@ -1,20 +1,19 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""触发器求值与重入队接缝:核心仅负责注册与派发接线。"""
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import copy
|
|
5
5
|
from abc import ABC, abstractmethod
|
|
6
6
|
from dataclasses import dataclass
|
|
7
|
-
import os
|
|
8
|
-
from pathlib import Path
|
|
9
7
|
from types import MappingProxyType
|
|
10
|
-
from typing import Any, Callable, Mapping,
|
|
8
|
+
from typing import Any, Callable, Mapping, TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from ..models.state import PipelineState
|
|
11
12
|
|
|
12
13
|
from ..exceptions import TaskLiteError
|
|
13
|
-
from ..models.artifact import resolve_job_files_config
|
|
14
14
|
from ..models.job import Job
|
|
15
|
-
from ..models.task import
|
|
16
|
-
from ..models.trigger import TriggerContext
|
|
17
|
-
from ..triggers import BUILTIN_TRIGGER_FUNCTIONS
|
|
15
|
+
from ..models.task import validate_trigger
|
|
16
|
+
from ..models.trigger import BUILTIN_TRIGGER_FUNCTIONS, TriggerContext
|
|
18
17
|
from ..utils.clock import parse_utc_iso_to_us
|
|
19
18
|
|
|
20
19
|
|
|
@@ -23,7 +22,7 @@ def extract_trusted_metadata_timestamp_us(
|
|
|
23
22
|
) -> int | None:
|
|
24
23
|
"""从 successes 档案 metadata 中提取并校验框架写入的可信完成时间戳(整数微秒)。
|
|
25
24
|
|
|
26
|
-
|
|
25
|
+
过滤人工档案的空字典 ``{}``、缺失或非正整数的 ``success_count``、
|
|
27
26
|
非法 ``last_success_run_id`` 以及缺失或无法解析的 ``last_succeeded_at``。
|
|
28
27
|
"""
|
|
29
28
|
if not isinstance(metadata, Mapping) or not metadata:
|
|
@@ -48,68 +47,26 @@ def extract_trusted_metadata_timestamp_us(
|
|
|
48
47
|
return None
|
|
49
48
|
|
|
50
49
|
|
|
51
|
-
def is_trusted_success_metadata(metadata: Mapping[str, Any] | None) -> bool:
|
|
52
|
-
"""判断 successes 档案 metadata 是否具备完整的框架成功运行元数据。"""
|
|
53
|
-
return extract_trusted_metadata_timestamp_us(metadata) is not None
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
EFFECTIVE_JOB_CONFIG_KEYS = (
|
|
57
|
-
"task_type",
|
|
58
|
-
"job_id",
|
|
59
|
-
"payload",
|
|
60
|
-
"depends_on",
|
|
61
|
-
"files",
|
|
62
|
-
"resources",
|
|
63
|
-
"max_retries_per_run",
|
|
64
|
-
"timeout_seconds",
|
|
65
|
-
"trigger",
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def extract_effective_job_config(job_snapshot: Mapping[str, Any]) -> dict[str, Any]:
|
|
70
|
-
"""提取已解析作业快照的有效配置投影(供同 UID 幂等/冲突比较)。"""
|
|
71
|
-
return {
|
|
72
|
-
"task_type": job_snapshot.get("task_type"),
|
|
73
|
-
"job_id": job_snapshot.get("job_id"),
|
|
74
|
-
"payload": job_snapshot.get("payload") or {},
|
|
75
|
-
"depends_on": list(job_snapshot.get("depends_on") or []),
|
|
76
|
-
"files": job_snapshot.get("files") or {},
|
|
77
|
-
"resources": dict(job_snapshot.get("resources") or {}),
|
|
78
|
-
"max_retries_per_run": job_snapshot.get("max_retries_per_run"),
|
|
79
|
-
"timeout_seconds": job_snapshot.get("timeout_seconds"),
|
|
80
|
-
"trigger": job_snapshot.get("trigger"),
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
class TerminalHistory(Protocol):
|
|
85
|
-
"""准入判定消费的终态历史契约(PipelineState 满足)。"""
|
|
86
|
-
|
|
87
|
-
@property
|
|
88
|
-
def successes(self) -> Mapping[str, dict]: ...
|
|
89
|
-
|
|
90
|
-
@property
|
|
91
|
-
def failed(self) -> Mapping[str, dict]: ...
|
|
92
|
-
|
|
93
|
-
|
|
94
50
|
class TriggerRegistry:
|
|
95
|
-
"""
|
|
51
|
+
"""触发器名称注册与主进程 ``TriggerContext`` 求值的唯一出口。"""
|
|
96
52
|
|
|
97
53
|
def __init__(
|
|
98
54
|
self,
|
|
99
55
|
*,
|
|
100
|
-
tasks: TaskRegistry | None = None,
|
|
101
56
|
functions: Mapping[str, Callable[[TriggerContext], bool]] | None = None,
|
|
102
|
-
base_dir: str | Path | None = None,
|
|
103
57
|
) -> None:
|
|
104
|
-
self.tasks = tasks
|
|
105
58
|
self.functions = dict(
|
|
106
59
|
BUILTIN_TRIGGER_FUNCTIONS if functions is None else functions
|
|
107
60
|
)
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
61
|
+
|
|
62
|
+
def __contains__(self, name: object) -> bool:
|
|
63
|
+
return isinstance(name, str) and name in self.functions
|
|
64
|
+
|
|
65
|
+
def require_registered(self, name: Any) -> str:
|
|
66
|
+
"""校验触发器名称已注册;未注册时 fail-loud 抛 TaskLiteError。"""
|
|
67
|
+
if not isinstance(name, str) or name not in self.functions:
|
|
68
|
+
raise TaskLiteError(f"unregistered trigger {name!r}")
|
|
69
|
+
return name
|
|
113
70
|
|
|
114
71
|
def register(self, name: str, fn: Callable[[TriggerContext], bool]) -> None:
|
|
115
72
|
validate_trigger(name)
|
|
@@ -119,56 +76,10 @@ class TriggerRegistry:
|
|
|
119
76
|
raise TaskLiteError("trigger function must be callable")
|
|
120
77
|
self.functions[name] = fn
|
|
121
78
|
|
|
122
|
-
def normalize_job_snapshot(self, job_snapshot: dict, task_type: str) -> bool:
|
|
123
|
-
"""摄入时落实 Task 四项规格与 FileSpec 快照解析(普通路径按固定基准绝对化、校验上游引用直接依赖)。"""
|
|
124
|
-
task = self.tasks.get(task_type) if self.tasks is not None else None
|
|
125
|
-
merged_resources: dict[str, float] = (
|
|
126
|
-
dict(task.resources) if task and task.resources else {}
|
|
127
|
-
)
|
|
128
|
-
defaults: dict[str, Any] = {
|
|
129
|
-
"resources": merged_resources,
|
|
130
|
-
"max_retries_per_run": task.max_retries_per_run if task else 3,
|
|
131
|
-
"timeout_seconds": task.timeout_seconds if task else 3600,
|
|
132
|
-
"trigger": task.trigger if task else "once",
|
|
133
|
-
}
|
|
134
|
-
changed = False
|
|
135
|
-
for key, value in defaults.items():
|
|
136
|
-
if key == "resources":
|
|
137
|
-
merged_resources.update(job_snapshot.get(key) or {})
|
|
138
|
-
if merged_resources != job_snapshot.get(key):
|
|
139
|
-
changed = True
|
|
140
|
-
job_snapshot[key] = merged_resources
|
|
141
|
-
elif job_snapshot.get(key) is None:
|
|
142
|
-
job_snapshot[key] = value
|
|
143
|
-
changed = True
|
|
144
|
-
trigger = job_snapshot.get("trigger")
|
|
145
|
-
if trigger not in self.functions:
|
|
146
|
-
raise TaskLiteError(f"unregistered trigger {trigger!r}")
|
|
147
|
-
if job_snapshot.get("payload") is None:
|
|
148
|
-
job_snapshot["payload"] = {}
|
|
149
|
-
changed = True
|
|
150
|
-
else:
|
|
151
|
-
job_snapshot["payload"] = copy.deepcopy(job_snapshot["payload"])
|
|
152
|
-
if job_snapshot.get("depends_on") is None:
|
|
153
|
-
job_snapshot["depends_on"] = []
|
|
154
|
-
changed = True
|
|
155
|
-
else:
|
|
156
|
-
job_snapshot["depends_on"] = list(job_snapshot["depends_on"])
|
|
157
|
-
raw_files = job_snapshot.get("files")
|
|
158
|
-
normalized_files = resolve_job_files_config(
|
|
159
|
-
raw_files,
|
|
160
|
-
base_dir=self.base_dir,
|
|
161
|
-
depends_on=job_snapshot["depends_on"],
|
|
162
|
-
)
|
|
163
|
-
if raw_files is None or normalized_files != raw_files:
|
|
164
|
-
changed = True
|
|
165
|
-
job_snapshot["files"] = normalized_files
|
|
166
|
-
return changed
|
|
167
|
-
|
|
168
79
|
def admit(
|
|
169
80
|
self,
|
|
170
81
|
job_snapshot: dict,
|
|
171
|
-
history:
|
|
82
|
+
history: "PipelineState",
|
|
172
83
|
*,
|
|
173
84
|
baseline_lookup: Callable[[str], int | None] | None = None,
|
|
174
85
|
) -> bool:
|
|
@@ -205,8 +116,7 @@ class RequeuePlan:
|
|
|
205
116
|
|
|
206
117
|
``front`` 为 True 表示插队首(同轮内优先重扫,重试不被新入队作业
|
|
207
118
|
排挤到饥饿尾部);``delay_seconds`` 是延迟类策略的表达位——默认
|
|
208
|
-
实现恒为 0
|
|
209
|
-
RequeuePolicy 接缝填充此字段即可生效,无需改动调用方。
|
|
119
|
+
实现恒为 0(立即)。
|
|
210
120
|
"""
|
|
211
121
|
|
|
212
122
|
front: bool
|
|
@@ -229,13 +139,9 @@ class ImmediateRequeuePolicy(RequeuePolicy):
|
|
|
229
139
|
|
|
230
140
|
|
|
231
141
|
__all__ = [
|
|
232
|
-
"EFFECTIVE_JOB_CONFIG_KEYS",
|
|
233
142
|
"ImmediateRequeuePolicy",
|
|
234
143
|
"RequeuePlan",
|
|
235
144
|
"RequeuePolicy",
|
|
236
145
|
"TriggerRegistry",
|
|
237
|
-
"TerminalHistory",
|
|
238
|
-
"extract_effective_job_config",
|
|
239
146
|
"extract_trusted_metadata_timestamp_us",
|
|
240
|
-
"is_trusted_success_metadata",
|
|
241
147
|
]
|