tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
tasklite/models/state.py
CHANGED
|
@@ -17,6 +17,7 @@ from typing import Any, Callable
|
|
|
17
17
|
|
|
18
18
|
from ..exceptions import TaskLiteError
|
|
19
19
|
from ..utils.encoding import make_job_uid
|
|
20
|
+
from .graph import build_dependents_index, collect_downstream, find_cycle_members
|
|
20
21
|
from .job import Job
|
|
21
22
|
|
|
22
23
|
|
|
@@ -76,7 +77,6 @@ class PipelineState:
|
|
|
76
77
|
self._failed_uids: set[str] = set(self._failed)
|
|
77
78
|
self._queue_uids: set[str] = {job_uid_from_snapshot(j) for j in self._queue}
|
|
78
79
|
self._in_flight_uids: set[str] = set()
|
|
79
|
-
self._trigger_skipped_uids: set[str] = set()
|
|
80
80
|
|
|
81
81
|
@property
|
|
82
82
|
def successes(self) -> dict[str, dict[str, Any]]:
|
|
@@ -115,7 +115,6 @@ class PipelineState:
|
|
|
115
115
|
self._queue[0:0] = list(job_snapshots)
|
|
116
116
|
else:
|
|
117
117
|
self._queue.extend(job_snapshots)
|
|
118
|
-
self._trigger_skipped_uids.difference_update(job_uid_from_snapshot(j) for j in job_snapshots)
|
|
119
118
|
self._queue_uids.update(job_uid_from_snapshot(j) for j in job_snapshots)
|
|
120
119
|
if __debug__:
|
|
121
120
|
self._assert_uids_consistent()
|
|
@@ -160,39 +159,30 @@ class PipelineState:
|
|
|
160
159
|
"""整体替换队列(死锁批量移除肇事者场景),重建 uid 索引。"""
|
|
161
160
|
self._queue = list(job_snapshots)
|
|
162
161
|
self._queue_uids = {job_uid_from_snapshot(j) for j in self._queue}
|
|
163
|
-
self._trigger_skipped_uids.difference_update(self._queue_uids)
|
|
164
162
|
if __debug__:
|
|
165
163
|
self._assert_uids_consistent()
|
|
166
164
|
|
|
167
|
-
#
|
|
165
|
+
# 依赖图与级联失败(复用无状态纯图原语)-------------------------
|
|
168
166
|
|
|
169
|
-
def
|
|
170
|
-
|
|
171
|
-
|
|
167
|
+
def _queue_dependency_edges(
|
|
168
|
+
self, *, node_uids: AbstractSet[str] | None = None
|
|
169
|
+
) -> dict[str, list[str]]:
|
|
170
|
+
"""提取当前 _queue 的正向依赖映射 ``job_uid -> depends_on``。"""
|
|
171
|
+
edges: dict[str, list[str]] = {}
|
|
172
172
|
for job_snapshot in self._queue:
|
|
173
173
|
try:
|
|
174
174
|
job = Job.from_dict(job_snapshot)
|
|
175
175
|
except (KeyError, TypeError, ValueError, TaskLiteError):
|
|
176
176
|
continue
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
return
|
|
177
|
+
if node_uids is not None and job.uid not in node_uids:
|
|
178
|
+
continue
|
|
179
|
+
edges[job.uid] = list(job.depends_on)
|
|
180
|
+
return edges
|
|
181
181
|
|
|
182
182
|
def find_all_downstream(self, job_uid: str) -> list[str]:
|
|
183
183
|
"""沿反向依赖递归找出所有直接与间接下游作业 UID(不分状态)。"""
|
|
184
|
-
dependents = self.
|
|
185
|
-
|
|
186
|
-
stack = [job_uid]
|
|
187
|
-
seen = {job_uid}
|
|
188
|
-
while stack:
|
|
189
|
-
cur = stack.pop()
|
|
190
|
-
for dep in dependents.get(cur, ()):
|
|
191
|
-
if dep not in seen:
|
|
192
|
-
seen.add(dep)
|
|
193
|
-
downstream.append(dep)
|
|
194
|
-
stack.append(dep)
|
|
195
|
-
return downstream
|
|
184
|
+
dependents = build_dependents_index(self._queue_dependency_edges())
|
|
185
|
+
return collect_downstream(job_uid, dependents)
|
|
196
186
|
|
|
197
187
|
def cascade_fail(
|
|
198
188
|
self,
|
|
@@ -205,84 +195,21 @@ class PipelineState:
|
|
|
205
195
|
``excluded_uids`` 是传播终止点(执行选择排除的作业):既不加入级联
|
|
206
196
|
集合,也不向其下游继续传播——这些作业原样留队,由后续 run 消费。
|
|
207
197
|
"""
|
|
208
|
-
dependents = self.
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
seen = {failed_uid}
|
|
212
|
-
while stack:
|
|
213
|
-
cur = stack.pop()
|
|
214
|
-
for downstream in dependents.get(cur, ()):
|
|
215
|
-
if downstream in excluded_uids:
|
|
216
|
-
continue
|
|
217
|
-
if downstream not in seen and downstream not in self._successes \
|
|
218
|
-
and downstream not in self._failed:
|
|
219
|
-
seen.add(downstream)
|
|
220
|
-
cascade.append(downstream)
|
|
221
|
-
stack.append(downstream)
|
|
222
|
-
return cascade
|
|
198
|
+
dependents = build_dependents_index(self._queue_dependency_edges())
|
|
199
|
+
stop_at = excluded_uids | self._success_uids | self._failed_uids
|
|
200
|
+
return collect_downstream(failed_uid, dependents, stop_at=stop_at)
|
|
223
201
|
|
|
224
202
|
def find_dependency_cycles(
|
|
225
203
|
self, *, node_uids: AbstractSet[str] | None = None
|
|
226
204
|
) -> list[str]:
|
|
227
|
-
"""在当前 _queue
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
不变式:返回全部实际环成员;成员可能因多条回边重复出现,调用方按集合
|
|
232
|
-
归因,不依赖遍历顺序或闭点重复次数。
|
|
233
|
-
实现为显式栈仿真调用栈:栈帧持有节点的依赖迭代器,弹帧即回溯置黑,
|
|
234
|
-
遍历序与递归形式逐一致且深度不受解释器递归上限约束——深链队列
|
|
235
|
-
(深度超递归限制)不得因环检测本身崩溃。
|
|
236
|
-
"""
|
|
237
|
-
edges: dict[str, set[str]] = {}
|
|
238
|
-
for job_snapshot in self._queue:
|
|
239
|
-
try:
|
|
240
|
-
job = Job.from_dict(job_snapshot)
|
|
241
|
-
except (KeyError, TypeError, ValueError, TaskLiteError):
|
|
242
|
-
continue
|
|
243
|
-
if node_uids is not None and job.uid not in node_uids:
|
|
244
|
-
continue
|
|
245
|
-
edges[job.uid] = set(job.depends_on)
|
|
246
|
-
|
|
247
|
-
color: dict[str, int] = {}
|
|
248
|
-
cycle_members: list[str] = []
|
|
249
|
-
path_stack: list[str] = []
|
|
250
|
-
frame_deps: list[Any] = []
|
|
251
|
-
|
|
252
|
-
for root in list(edges):
|
|
253
|
-
if color.get(root, 0) != 0:
|
|
254
|
-
continue
|
|
255
|
-
color[root] = 1
|
|
256
|
-
path_stack.append(root)
|
|
257
|
-
frame_deps.append(iter(edges.get(root, ())))
|
|
258
|
-
while frame_deps:
|
|
259
|
-
try:
|
|
260
|
-
dep = next(frame_deps[-1])
|
|
261
|
-
except StopIteration:
|
|
262
|
-
frame_deps.pop()
|
|
263
|
-
color[path_stack.pop()] = 2
|
|
264
|
-
continue
|
|
265
|
-
if dep not in edges:
|
|
266
|
-
continue
|
|
267
|
-
dep_color = color.get(dep, 0)
|
|
268
|
-
if dep_color == 1:
|
|
269
|
-
try:
|
|
270
|
-
start = path_stack.index(dep)
|
|
271
|
-
except ValueError:
|
|
272
|
-
start = 0
|
|
273
|
-
cycle_members.extend(path_stack[start:])
|
|
274
|
-
cycle_members.append(dep)
|
|
275
|
-
elif dep_color == 0:
|
|
276
|
-
color[dep] = 1
|
|
277
|
-
path_stack.append(dep)
|
|
278
|
-
frame_deps.append(iter(edges.get(dep, ())))
|
|
279
|
-
return cycle_members
|
|
205
|
+
"""在当前 _queue 的依赖图中找出所有依赖环成员(迭代式 Kosaraju,零递归深度限制)。"""
|
|
206
|
+
edges = self._queue_dependency_edges(node_uids=node_uids)
|
|
207
|
+
return find_cycle_members(edges.keys(), edges)
|
|
280
208
|
|
|
281
209
|
# 完成/失败记录 ------------------------------------------------
|
|
282
210
|
|
|
283
211
|
def mark_success(self, job_uid: str, metadata: dict[str, Any]) -> None:
|
|
284
212
|
"""终态转移:uid → successes(成功)。单一入口维护 successes/failed 索引。"""
|
|
285
|
-
self._trigger_skipped_uids.discard(job_uid)
|
|
286
213
|
self._failed.pop(job_uid, None)
|
|
287
214
|
self._failed_uids.discard(job_uid)
|
|
288
215
|
self._successes[job_uid] = copy.deepcopy(metadata)
|
|
@@ -292,7 +219,6 @@ class PipelineState:
|
|
|
292
219
|
|
|
293
220
|
def mark_failed(self, job_uid: str, metadata: dict[str, Any]) -> None:
|
|
294
221
|
"""终态转移:uid → failed(失败档案)。单一入口维护 successes/failed 索引。"""
|
|
295
|
-
self._trigger_skipped_uids.discard(job_uid)
|
|
296
222
|
self._successes.pop(job_uid, None)
|
|
297
223
|
self._success_uids.discard(job_uid)
|
|
298
224
|
self._failed[job_uid] = copy.deepcopy(metadata)
|
|
@@ -308,29 +234,6 @@ class PipelineState:
|
|
|
308
234
|
else:
|
|
309
235
|
self._cursors[k] = v
|
|
310
236
|
|
|
311
|
-
# 运维管理受控变更 ------------------------------------------------
|
|
312
|
-
|
|
313
|
-
def discard_success(self, job_uid: str) -> None:
|
|
314
|
-
"""从 successes 集合移除指定 uid(运维接缝,幂等)。"""
|
|
315
|
-
self._successes.pop(job_uid, None)
|
|
316
|
-
self._success_uids.discard(job_uid)
|
|
317
|
-
if __debug__:
|
|
318
|
-
self._assert_terminal_uids_consistent()
|
|
319
|
-
|
|
320
|
-
def discard_failed(self, job_uid: str) -> None:
|
|
321
|
-
"""从 failed 集合移除指定 uid(运维接缝,幂等)。"""
|
|
322
|
-
self._failed.pop(job_uid, None)
|
|
323
|
-
self._failed_uids.discard(job_uid)
|
|
324
|
-
if __debug__:
|
|
325
|
-
self._assert_terminal_uids_consistent()
|
|
326
|
-
|
|
327
|
-
def add_success(self, job_uid: str, metadata: dict[str, Any]) -> None:
|
|
328
|
-
"""向 successes 集合添加 uid(运维接缝,seed_successes 场景)。"""
|
|
329
|
-
self._successes[job_uid] = metadata
|
|
330
|
-
self._success_uids.add(job_uid)
|
|
331
|
-
if __debug__:
|
|
332
|
-
self._assert_terminal_uids_consistent()
|
|
333
|
-
|
|
334
237
|
def get_cursor(self, key: str) -> str | None:
|
|
335
238
|
"""获取单个游标键值。"""
|
|
336
239
|
return self._cursors.get(key)
|
|
@@ -349,15 +252,6 @@ class PipelineState:
|
|
|
349
252
|
"""队列 uid 的活索引。"""
|
|
350
253
|
return self._queue_uids
|
|
351
254
|
|
|
352
|
-
@property
|
|
353
|
-
def trigger_skipped_uids(self) -> frozenset[str]:
|
|
354
|
-
"""本次 run 中触发条件判定无需执行的依赖就绪标记。"""
|
|
355
|
-
return frozenset(self._trigger_skipped_uids)
|
|
356
|
-
|
|
357
|
-
def mark_trigger_skipped(self, job_uid: str) -> None:
|
|
358
|
-
"""仅在跳过提交成功后记录;不伪造历史成功。"""
|
|
359
|
-
self._trigger_skipped_uids.add(job_uid)
|
|
360
|
-
|
|
361
255
|
def check_dependency_status(
|
|
362
256
|
self,
|
|
363
257
|
dep_uid: str,
|
tasklite/models/trigger.py
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""主进程触发条件的只读判定上下文与内置触发策略。
|
|
2
|
+
|
|
3
|
+
分层约束:本模块位于 ``tasklite/models/``,严禁导入 ``engine`` / ``backend`` / ``wrappers``。
|
|
4
|
+
"""
|
|
2
5
|
from __future__ import annotations
|
|
3
6
|
|
|
7
|
+
import os
|
|
4
8
|
from dataclasses import dataclass
|
|
5
|
-
from typing import Any, Mapping
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Callable, Mapping
|
|
10
|
+
|
|
11
|
+
from ..exceptions import TaskLiteError
|
|
12
|
+
from ..utils.clock import mtime_ns_to_us
|
|
6
13
|
|
|
7
|
-
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from .job import Job
|
|
8
16
|
|
|
9
17
|
|
|
10
18
|
@dataclass(frozen=True)
|
|
@@ -16,3 +24,55 @@ class TriggerContext:
|
|
|
16
24
|
has_failed: bool
|
|
17
25
|
success_metadata: Mapping[str, Any] | None
|
|
18
26
|
last_succeeded_at_us: int | None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def once(ctx: TriggerContext) -> bool:
|
|
30
|
+
"""只在尚无历史终态时执行。"""
|
|
31
|
+
return not ctx.has_succeeded and not ctx.has_failed
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def until_success(ctx: TriggerContext) -> bool:
|
|
35
|
+
"""尚未成功的作业可执行。"""
|
|
36
|
+
return not ctx.has_succeeded
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def always(ctx: TriggerContext) -> bool:
|
|
40
|
+
"""每次入队均执行。"""
|
|
41
|
+
return True
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def on_input_change(ctx: TriggerContext) -> bool:
|
|
45
|
+
"""仅比较输入路径自身的微秒 mtime,缺基线时保守执行。"""
|
|
46
|
+
if not ctx.has_succeeded or ctx.last_succeeded_at_us is None:
|
|
47
|
+
return True
|
|
48
|
+
inputs = ctx.success_metadata.get("inputs") if ctx.success_metadata else None
|
|
49
|
+
if not isinstance(inputs, list) or not inputs:
|
|
50
|
+
raise TaskLiteError("on_input_change requires a non-empty inputs list")
|
|
51
|
+
changed = False
|
|
52
|
+
for path in inputs:
|
|
53
|
+
if not isinstance(path, str) or not path:
|
|
54
|
+
raise TaskLiteError("on_input_change inputs must contain non-empty paths")
|
|
55
|
+
# 所有输入都须存在,不能让较早命中的变化掩盖后续缺失路径。
|
|
56
|
+
changed |= mtime_ns_to_us(os.stat(path).st_mtime_ns) > ctx.last_succeeded_at_us
|
|
57
|
+
return changed
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
BUILTIN_TRIGGER_FUNCTIONS: dict[str, Callable[[TriggerContext], bool]] = {
|
|
61
|
+
"once": once,
|
|
62
|
+
"until_success": until_success,
|
|
63
|
+
"always": always,
|
|
64
|
+
"on_input_change": on_input_change,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
# 内置触发器名称单点派生自 BUILTIN_TRIGGER_FUNCTIONS,消除重复硬编码。
|
|
68
|
+
BUILTIN_TRIGGERS: tuple[str, ...] = tuple(BUILTIN_TRIGGER_FUNCTIONS)
|
|
69
|
+
|
|
70
|
+
__all__ = [
|
|
71
|
+
"BUILTIN_TRIGGERS",
|
|
72
|
+
"BUILTIN_TRIGGER_FUNCTIONS",
|
|
73
|
+
"TriggerContext",
|
|
74
|
+
"always",
|
|
75
|
+
"on_input_change",
|
|
76
|
+
"once",
|
|
77
|
+
"until_success",
|
|
78
|
+
]
|
tasklite/pipeline.py
CHANGED
|
@@ -7,28 +7,31 @@
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import logging
|
|
10
|
-
import multiprocessing as mp
|
|
11
10
|
import os
|
|
12
11
|
from pathlib import Path
|
|
13
|
-
from typing import Any, Callable,
|
|
12
|
+
from typing import Any, Callable, Literal
|
|
14
13
|
|
|
15
14
|
from .backend.base import AbstractStateBackend
|
|
16
15
|
from .backend.memory import InMemoryStateBackend
|
|
17
16
|
from .backend.sqlite_backend import SQLiteStateBackend
|
|
17
|
+
from .edit import EditTransaction
|
|
18
18
|
from .engine.admission import TriggerRegistry
|
|
19
|
-
from .models.trigger import TriggerContext
|
|
19
|
+
from .models.trigger import BUILTIN_TRIGGER_FUNCTIONS, TriggerContext
|
|
20
20
|
from .engine.channel import ExecutionChannel
|
|
21
21
|
from .engine.config import RunConfig
|
|
22
|
-
from .engine.execution_access import exclusive_state_access
|
|
23
22
|
from .engine.governor import DeadlockGovernor
|
|
24
|
-
from .engine.
|
|
25
|
-
from .
|
|
23
|
+
from .engine.resource import CapacityResource, ResourceManager, adapt_resource
|
|
24
|
+
from .resource import (
|
|
25
|
+
CapacityResource as PublicCapacityResource,
|
|
26
|
+
IntervalResource as PublicIntervalResource,
|
|
27
|
+
)
|
|
26
28
|
from .engine.runtime import EngineRuntime
|
|
27
|
-
from .engine.types import ExecutionOptions
|
|
29
|
+
from .engine.types import ExecutionOptions, is_importing_mp_main
|
|
28
30
|
from .exceptions import TaskLiteError
|
|
29
31
|
from .models.job import Job, WORKER_RESOURCE
|
|
30
32
|
from .models.task import Task, TaskRegistry
|
|
31
33
|
from .selection import Exclude
|
|
34
|
+
from .utils.artifact import PathPolicy
|
|
32
35
|
|
|
33
36
|
logger = logging.getLogger("tasklite")
|
|
34
37
|
|
|
@@ -50,18 +53,14 @@ class TaskLite:
|
|
|
50
53
|
backend: "sqlite"(ACID 保证)或 "memory"(内存状态)。
|
|
51
54
|
state_path: SQLite 数据库文件路径(backend="sqlite" 时必填,memory 默认 None)。
|
|
52
55
|
max_workers: 最大并发子进程数(默认 4)。
|
|
53
|
-
output_roots: 产物根目录(单根或多根序列);设置后沙盒校验在根内,构造不创建目录。
|
|
54
56
|
"""
|
|
55
57
|
|
|
56
|
-
output_roots: Path | list[Path] | None = None
|
|
57
|
-
|
|
58
58
|
def __init__(
|
|
59
59
|
self,
|
|
60
60
|
*,
|
|
61
61
|
backend: Literal["sqlite", "memory"],
|
|
62
62
|
state_path: str | Path | None = None,
|
|
63
63
|
max_workers: int = _DEFAULT_MAX_WORKERS,
|
|
64
|
-
output_roots: str | Path | Sequence[str | Path] | None = None,
|
|
65
64
|
) -> None:
|
|
66
65
|
if backend not in (_BACKEND_SQLITE, _BACKEND_MEMORY):
|
|
67
66
|
if isinstance(backend, str):
|
|
@@ -75,9 +74,9 @@ class TaskLite:
|
|
|
75
74
|
if backend == _BACKEND_SQLITE:
|
|
76
75
|
if state_path is None or (isinstance(state_path, str) and not state_path.strip()):
|
|
77
76
|
raise TaskLiteError("state_path is required when backend is 'sqlite'")
|
|
78
|
-
self.
|
|
77
|
+
self._state_path: Path | None = Path(state_path).resolve()
|
|
79
78
|
else:
|
|
80
|
-
self.
|
|
79
|
+
self._state_path = None
|
|
81
80
|
|
|
82
81
|
if (
|
|
83
82
|
not isinstance(max_workers, int)
|
|
@@ -89,30 +88,23 @@ class TaskLite:
|
|
|
89
88
|
)
|
|
90
89
|
|
|
91
90
|
self._closed: bool = False
|
|
92
|
-
self.
|
|
93
|
-
self.
|
|
91
|
+
self._editing: bool = False
|
|
92
|
+
self._base_dir: Path = Path(os.path.abspath(os.getcwd()))
|
|
94
93
|
self._backend_type: str = backend
|
|
95
94
|
self._backend_handle: AbstractStateBackend | None = None
|
|
95
|
+
self._path_policy: PathPolicy = PathPolicy(
|
|
96
|
+
base_dir=self._base_dir,
|
|
97
|
+
state_path=self._state_path,
|
|
98
|
+
)
|
|
96
99
|
|
|
97
100
|
try:
|
|
98
101
|
self._backend_handle = self._build_backend(backend)
|
|
99
102
|
self._tasks = TaskRegistry()
|
|
100
|
-
|
|
101
|
-
self._triggers = TriggerRegistry(
|
|
102
|
-
tasks=self._tasks,
|
|
103
|
-
functions=BUILTIN_TRIGGER_FUNCTIONS,
|
|
104
|
-
base_dir=self.base_dir,
|
|
105
|
-
)
|
|
103
|
+
self._triggers = TriggerRegistry(functions=BUILTIN_TRIGGER_FUNCTIONS)
|
|
106
104
|
self._resources = self._build_resources(self._tasks, max_workers)
|
|
107
|
-
self.
|
|
108
|
-
self._channel = ExecutionChannel(
|
|
109
|
-
process_context=self._process_context,
|
|
110
|
-
output_roots=self.output_roots,
|
|
111
|
-
protected_roots=((self.state_path,) if self.state_path is not None else ()),
|
|
112
|
-
)
|
|
105
|
+
self._channel = ExecutionChannel()
|
|
113
106
|
self._runtime_config = self._build_run_config(self._backend_handle)
|
|
114
107
|
self._runtime = EngineRuntime(config=self._runtime_config)
|
|
115
|
-
self._console = OpsConsole(self._runtime.store)
|
|
116
108
|
except BaseException:
|
|
117
109
|
self._cleanup_failed_init()
|
|
118
110
|
raise
|
|
@@ -145,9 +137,14 @@ class TaskLite:
|
|
|
145
137
|
self.close()
|
|
146
138
|
|
|
147
139
|
def close(self) -> None:
|
|
148
|
-
"""
|
|
140
|
+
"""释放资源且运行期或图修整事务期间拒绝调用,重复关闭无操作。"""
|
|
149
141
|
if self.is_running:
|
|
150
142
|
raise TaskLiteError("Cannot close TaskLite while run() is in progress.")
|
|
143
|
+
if self._editing:
|
|
144
|
+
raise TaskLiteError(
|
|
145
|
+
"Cannot close TaskLite while an edit() transaction is active "
|
|
146
|
+
"(concurrent operations on the same TaskLite instance are forbidden)."
|
|
147
|
+
)
|
|
151
148
|
if self._closed:
|
|
152
149
|
return
|
|
153
150
|
self._closed = True
|
|
@@ -169,27 +166,11 @@ class TaskLite:
|
|
|
169
166
|
|
|
170
167
|
# ── 构造期装配分区 ────────────────────────────────────────────
|
|
171
168
|
|
|
172
|
-
@staticmethod
|
|
173
|
-
def _resolve_output_roots(
|
|
174
|
-
output_roots: str | Path | Sequence[str | Path] | None,
|
|
175
|
-
) -> Path | list[Path] | None:
|
|
176
|
-
"""产物根装配:单根/多根(多根时声明路径属任一根即过沙盒),不创建目录。"""
|
|
177
|
-
if output_roots is None:
|
|
178
|
-
return None
|
|
179
|
-
if isinstance(output_roots, (list, tuple)):
|
|
180
|
-
return [Path(r).resolve() for r in output_roots]
|
|
181
|
-
if isinstance(output_roots, (str, Path)):
|
|
182
|
-
return Path(output_roots).resolve()
|
|
183
|
-
raise TaskLiteError(
|
|
184
|
-
f"output_roots must be str/Path or a sequence thereof, "
|
|
185
|
-
f"got {type(output_roots).__name__}"
|
|
186
|
-
)
|
|
187
|
-
|
|
188
169
|
def _build_backend(self, backend: str) -> AbstractStateBackend:
|
|
189
170
|
"""持久化后端装配。"""
|
|
190
171
|
if backend == _BACKEND_SQLITE:
|
|
191
|
-
assert self.
|
|
192
|
-
return SQLiteStateBackend(self.
|
|
172
|
+
assert self._state_path is not None
|
|
173
|
+
return SQLiteStateBackend(self._state_path)
|
|
193
174
|
return InMemoryStateBackend()
|
|
194
175
|
|
|
195
176
|
@staticmethod
|
|
@@ -207,15 +188,13 @@ class TaskLite:
|
|
|
207
188
|
) -> RunConfig:
|
|
208
189
|
"""RunConfig 静态装配快照。"""
|
|
209
190
|
return RunConfig.resolve(
|
|
210
|
-
|
|
211
|
-
protected_roots=((self.state_path,) if self.state_path is not None else ()),
|
|
191
|
+
path_policy=self._path_policy,
|
|
212
192
|
backend=backend,
|
|
213
193
|
resources=self._resources,
|
|
214
194
|
tasks=self._tasks,
|
|
215
195
|
channel=self._channel,
|
|
216
196
|
governor=DeadlockGovernor(),
|
|
217
197
|
trigger_registry=self._triggers,
|
|
218
|
-
output_roots=self.output_roots,
|
|
219
198
|
)
|
|
220
199
|
|
|
221
200
|
# ── 运行期只读接缝 ─────────────────────────────────────────────
|
|
@@ -225,15 +204,25 @@ class TaskLite:
|
|
|
225
204
|
"""当前管线是否正在执行中。"""
|
|
226
205
|
return self._runtime.is_running
|
|
227
206
|
|
|
228
|
-
def
|
|
229
|
-
"""
|
|
207
|
+
def _open_snapshot_source(self) -> tuple[Literal["sqlite", "memory"], Path | dict[str, Any]]:
|
|
208
|
+
"""受控包内快照源接缝:在门面内完成关闭校验与 SQLite 路径 / 内存一致快照分发。"""
|
|
209
|
+
if self._closed:
|
|
210
|
+
raise TaskLiteError(
|
|
211
|
+
"Cannot open StateSnapshot from a closed TaskLite instance."
|
|
212
|
+
)
|
|
230
213
|
backend = self._backend_handle
|
|
231
214
|
if backend is None:
|
|
232
215
|
raise TaskLiteError("TaskLite backend is unavailable")
|
|
233
|
-
|
|
216
|
+
if self._state_path is not None:
|
|
217
|
+
return ("sqlite", self._state_path)
|
|
218
|
+
return ("memory", backend.load_consistent_snapshot())
|
|
219
|
+
|
|
220
|
+
def _set_editing(self, active: bool) -> None:
|
|
221
|
+
"""受控事务态切换回调,仅由 EditTransaction.__enter__ / __exit__ 成对维护。"""
|
|
222
|
+
self._editing = active
|
|
234
223
|
|
|
235
224
|
def _ensure_active(self, api_name: str) -> None:
|
|
236
|
-
"""管理/入队/注册 API
|
|
225
|
+
"""管理/入队/注册 API 的状态守卫(关闭、运行或图修整事务期间拒绝)。"""
|
|
237
226
|
if self._closed:
|
|
238
227
|
raise TaskLiteError(
|
|
239
228
|
f"Cannot call {api_name}() on a closed TaskLite instance."
|
|
@@ -243,29 +232,33 @@ class TaskLite:
|
|
|
243
232
|
f"{api_name}() is only allowed outside run(); "
|
|
244
233
|
f"current run is in progress."
|
|
245
234
|
)
|
|
235
|
+
if self._editing:
|
|
236
|
+
raise TaskLiteError(
|
|
237
|
+
f"Cannot call {api_name}() while an edit() transaction is active "
|
|
238
|
+
f"(concurrent operations on the same TaskLite instance are forbidden)."
|
|
239
|
+
)
|
|
246
240
|
|
|
247
241
|
# ── 注册面(六步之二/三)───────────────────────────────────────
|
|
248
242
|
|
|
249
|
-
def register_resource(
|
|
250
|
-
|
|
243
|
+
def register_resource(
|
|
244
|
+
self,
|
|
245
|
+
resource: PublicCapacityResource | PublicIntervalResource,
|
|
246
|
+
) -> None:
|
|
247
|
+
"""注册一个官方内置资源配置(CapacityResource 或 IntervalResource)。
|
|
251
248
|
|
|
252
249
|
同名资源重复注册明确拒绝,内部 worker 容量不能通过资源注册覆盖,
|
|
253
250
|
最大并发只通过 max_workers 配置。
|
|
254
251
|
"""
|
|
255
252
|
self._ensure_active("register_resource")
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
f"register_resource expects a Resource instance, "
|
|
259
|
-
f"got {type(resource).__name__}"
|
|
260
|
-
)
|
|
261
|
-
if resource.name == WORKER_RESOURCE:
|
|
253
|
+
adapted = adapt_resource(resource)
|
|
254
|
+
if adapted.name == WORKER_RESOURCE:
|
|
262
255
|
raise TaskLiteError(
|
|
263
256
|
f"Cannot register internal worker resource {WORKER_RESOURCE!r}; "
|
|
264
257
|
f"worker concurrency must be configured via max_workers."
|
|
265
258
|
)
|
|
266
|
-
if
|
|
267
|
-
raise TaskLiteError(f"Resource {
|
|
268
|
-
self._resources[
|
|
259
|
+
if adapted.name in self._resources:
|
|
260
|
+
raise TaskLiteError(f"Resource {adapted.name!r} already registered")
|
|
261
|
+
self._resources[adapted.name] = adapted
|
|
269
262
|
|
|
270
263
|
def register_task(
|
|
271
264
|
self,
|
|
@@ -347,7 +340,7 @@ class TaskLite:
|
|
|
347
340
|
raise TaskLiteError(
|
|
348
341
|
f"enqueue() expects a Job, got {type(job).__name__}"
|
|
349
342
|
)
|
|
350
|
-
with
|
|
343
|
+
with self._runtime.store.exclusive_access():
|
|
351
344
|
inserted = self._runtime.store.enqueue_job(job)
|
|
352
345
|
if inserted:
|
|
353
346
|
logger.info("Enqueued 1 job(s).")
|
|
@@ -357,31 +350,25 @@ class TaskLite:
|
|
|
357
350
|
|
|
358
351
|
# ── 管理面(run() 外运维委托)──────────────────────────────────
|
|
359
352
|
|
|
360
|
-
def
|
|
353
|
+
def edit(
|
|
361
354
|
self,
|
|
362
|
-
job_or_uid: Job | str,
|
|
363
|
-
status: Literal["pending", "succeeded", "failed"],
|
|
364
355
|
*,
|
|
365
|
-
|
|
356
|
+
dry_run: bool = True,
|
|
366
357
|
note: str | None = None,
|
|
367
|
-
) ->
|
|
368
|
-
"""
|
|
369
|
-
self._ensure_active("
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
)
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
"""从指定档案和/或 Attempt 历史中删除单个 job 的记录,限 Run 外。"""
|
|
382
|
-
self._ensure_active("clear_history")
|
|
383
|
-
with exclusive_state_access(self._runtime.store.backend):
|
|
384
|
-
return self._console.clear_history(uid, records=records)
|
|
358
|
+
) -> EditTransaction:
|
|
359
|
+
"""进入 Run 外原子图修整事务上下文(默认 dry_run=True 预览,显式 dry_run=False 落盘)。"""
|
|
360
|
+
self._ensure_active("edit")
|
|
361
|
+
if type(dry_run) is not bool:
|
|
362
|
+
raise TaskLiteError(f"dry_run must be a bool, got {type(dry_run).__name__}")
|
|
363
|
+
if note is not None and not isinstance(note, str):
|
|
364
|
+
raise TaskLiteError(f"note must be a str or None, got {type(note).__name__}")
|
|
365
|
+
return EditTransaction(
|
|
366
|
+
self._runtime.store,
|
|
367
|
+
dry_run=dry_run,
|
|
368
|
+
note=note,
|
|
369
|
+
ensure_active=self._ensure_active,
|
|
370
|
+
set_editing=self._set_editing,
|
|
371
|
+
)
|
|
385
372
|
|
|
386
373
|
def get_cursor(self, key: str) -> str | None:
|
|
387
374
|
"""读取指定游标键的最新已提交值,限 Run 外。"""
|
|
@@ -391,19 +378,19 @@ class TaskLite:
|
|
|
391
378
|
def set_cursor(self, key: str, value: str | None) -> None:
|
|
392
379
|
"""设置或删除游标键值,写入直接持久化提交,限 Run 外。"""
|
|
393
380
|
self._ensure_active("set_cursor")
|
|
394
|
-
with
|
|
381
|
+
with self._runtime.store.exclusive_access():
|
|
395
382
|
self._runtime.store.set_cursor(key, value)
|
|
396
383
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
self.
|
|
406
|
-
|
|
384
|
+
def grant_authorization(
|
|
385
|
+
self,
|
|
386
|
+
path: str | Path,
|
|
387
|
+
*,
|
|
388
|
+
target_type: Literal["file", "directory"],
|
|
389
|
+
) -> dict[str, Any]:
|
|
390
|
+
"""在 Run 外显式预授权单个文件或目录路径,重复或父子类型冲突抛 TaskLiteError。"""
|
|
391
|
+
self._ensure_active("grant_authorization")
|
|
392
|
+
with self._runtime.store.exclusive_access():
|
|
393
|
+
return self._runtime.store.grant_authorization(path, target_type=target_type)
|
|
407
394
|
|
|
408
395
|
def list_authorizations(self) -> list[dict[str, Any]]:
|
|
409
396
|
"""只读查询全部已持久化的授权记录,限 Run 外。"""
|
|
@@ -413,7 +400,7 @@ class TaskLite:
|
|
|
413
400
|
def revoke_authorization(self, path: str | Path) -> bool:
|
|
414
401
|
"""按规范绝对路径精确撤销授权,限 Run 外。"""
|
|
415
402
|
self._ensure_active("revoke_authorization")
|
|
416
|
-
with
|
|
403
|
+
with self._runtime.store.exclusive_access():
|
|
417
404
|
return self._runtime.store.revoke_authorization(path)
|
|
418
405
|
|
|
419
406
|
# ── 运行与停机(六步之五/六)───────────────────────────────────
|
|
@@ -435,6 +422,11 @@ class TaskLite:
|
|
|
435
422
|
install_signals: bool = True,
|
|
436
423
|
) -> None:
|
|
437
424
|
"""运行管线直到队列排空(或收到排空请求)。"""
|
|
425
|
+
if is_importing_mp_main():
|
|
426
|
+
raise RuntimeError(
|
|
427
|
+
"TaskLite.run() cannot be called while bootstrapping a worker process. "
|
|
428
|
+
"Wrap top-level execution in `if __name__ == '__main__':`."
|
|
429
|
+
)
|
|
438
430
|
self._ensure_active("run")
|
|
439
431
|
if exclude is not None and not isinstance(exclude, Exclude):
|
|
440
432
|
raise TaskLiteError(
|