tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
tasklite/engine/types.py
CHANGED
|
@@ -7,11 +7,9 @@ Handler 与默认执行配置归属 Task 规格层(``models/task.py``)。
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import enum
|
|
10
|
-
import time
|
|
11
10
|
from dataclasses import dataclass
|
|
12
11
|
from typing import TYPE_CHECKING, Any
|
|
13
12
|
|
|
14
|
-
from ..models.artifact import FileSpec
|
|
15
13
|
from ..models.job import Job
|
|
16
14
|
|
|
17
15
|
if TYPE_CHECKING:
|
|
@@ -26,7 +24,7 @@ class StopMode(enum.Enum):
|
|
|
26
24
|
|
|
27
25
|
|
|
28
26
|
class AuthorizationStatus(str, enum.Enum):
|
|
29
|
-
"""
|
|
27
|
+
"""主进程文件授权的统一状态。"""
|
|
30
28
|
|
|
31
29
|
PENDING = "pending"
|
|
32
30
|
APPROVED = "approved"
|
|
@@ -35,18 +33,6 @@ class AuthorizationStatus(str, enum.Enum):
|
|
|
35
33
|
PREPARATION_FAILED = "preparation_failed"
|
|
36
34
|
|
|
37
35
|
|
|
38
|
-
@dataclass
|
|
39
|
-
class PendingDeclaration:
|
|
40
|
-
"""一次等待主进程决定的动态文件声明。"""
|
|
41
|
-
|
|
42
|
-
request_id: str
|
|
43
|
-
name: str
|
|
44
|
-
source: FileSpec
|
|
45
|
-
bound: FileSpec
|
|
46
|
-
resolved_path: str
|
|
47
|
-
status: AuthorizationStatus = AuthorizationStatus.PENDING
|
|
48
|
-
|
|
49
|
-
|
|
50
36
|
@dataclass(frozen=True)
|
|
51
37
|
class AuthorizationOutcome:
|
|
52
38
|
"""受阻执行的终态授权结果及可持久化原因。"""
|
|
@@ -68,11 +54,11 @@ class ExitReason(str, enum.Enum):
|
|
|
68
54
|
|
|
69
55
|
@dataclass
|
|
70
56
|
class JobHandle:
|
|
71
|
-
"""
|
|
57
|
+
"""一个在途子进程的纯句柄值对象(在飞追踪与收割看门狗共用)。
|
|
72
58
|
|
|
73
59
|
``timeout_at_monotonic``(monotonic 时刻)与 ``timeout_seconds`` 驱动看门狗超时判定;
|
|
74
60
|
``execution_id``(``run_id.dispatch_no``)是本次执行体的身份串,
|
|
75
|
-
|
|
61
|
+
结果认证与归属都锚定它。进程对象本身以 ``Any`` 承载——进程 seam
|
|
76
62
|
的具体形态由执行通道装配,本值对象不绑定。
|
|
77
63
|
"""
|
|
78
64
|
|
|
@@ -84,30 +70,8 @@ class JobHandle:
|
|
|
84
70
|
execution_id: str | None = None
|
|
85
71
|
run_id: str | None = None
|
|
86
72
|
attempt_no: int | None = None
|
|
87
|
-
paused_remaining: float | None = None
|
|
88
|
-
pending_declaration: PendingDeclaration | None = None
|
|
89
|
-
authorization_outcome: AuthorizationOutcome | None = None
|
|
90
73
|
started_at: str | None = None
|
|
91
74
|
|
|
92
|
-
@property
|
|
93
|
-
def waiting_auth(self) -> bool:
|
|
94
|
-
"""当前执行是否正在等待动态文件声明授权。"""
|
|
95
|
-
return (
|
|
96
|
-
self.pending_declaration is not None
|
|
97
|
-
and self.pending_declaration.status is AuthorizationStatus.PENDING
|
|
98
|
-
)
|
|
99
|
-
|
|
100
|
-
def pause_timeout(self) -> None:
|
|
101
|
-
"""等待外部授权或操作时暂停超时计时。"""
|
|
102
|
-
if self.paused_remaining is None:
|
|
103
|
-
self.paused_remaining = max(0.0, self.timeout_at_monotonic - time.monotonic())
|
|
104
|
-
|
|
105
|
-
def resume_timeout(self) -> None:
|
|
106
|
-
"""恢复超时计时。"""
|
|
107
|
-
if self.paused_remaining is not None:
|
|
108
|
-
self.timeout_at_monotonic = time.monotonic() + self.paused_remaining
|
|
109
|
-
self.paused_remaining = None
|
|
110
|
-
|
|
111
75
|
|
|
112
76
|
@dataclass(frozen=True)
|
|
113
77
|
class ExecutionOptions:
|
|
@@ -132,13 +96,38 @@ class StepOutcome:
|
|
|
132
96
|
exit_reason: str | None = None
|
|
133
97
|
|
|
134
98
|
|
|
99
|
+
# 子进程引导私有 IPC 环境变量键名与放行令牌(集中定义,避免导入链提前加载 _worker_bootstrap)
|
|
100
|
+
ENV_IPC_ADDR = "_TASKLITE_IPC_ADDR"
|
|
101
|
+
ENV_IPC_FAMILY = "_TASKLITE_IPC_FAMILY"
|
|
102
|
+
ENV_IPC_AUTHKEY = "_TASKLITE_IPC_AUTHKEY"
|
|
103
|
+
LAUNCH_GO_TOKEN = b"GO"
|
|
104
|
+
|
|
105
|
+
# 引导期 __mp_main__ 重入守卫:拦截缺失 if __name__ == "__main__" 时的递归启动
|
|
106
|
+
_IMPORTING_MP_MAIN: bool = False
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def is_importing_mp_main() -> bool:
|
|
110
|
+
"""当前进程是否正处于 worker 引导导入 ``__mp_main__`` 阶段。"""
|
|
111
|
+
return _IMPORTING_MP_MAIN
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _set_importing_mp_main(value: bool) -> None:
|
|
115
|
+
"""内部接口:仅由 worker 引导模块在导入 __mp_main__ 前后切换重入标志。"""
|
|
116
|
+
global _IMPORTING_MP_MAIN
|
|
117
|
+
_IMPORTING_MP_MAIN = value
|
|
118
|
+
|
|
119
|
+
|
|
135
120
|
__all__ = [
|
|
136
121
|
"AuthorizationOutcome",
|
|
137
122
|
"AuthorizationStatus",
|
|
123
|
+
"ENV_IPC_ADDR",
|
|
124
|
+
"ENV_IPC_AUTHKEY",
|
|
125
|
+
"ENV_IPC_FAMILY",
|
|
138
126
|
"StopMode",
|
|
139
127
|
"ExitReason",
|
|
140
128
|
"ExecutionOptions",
|
|
141
129
|
"JobHandle",
|
|
142
|
-
"
|
|
130
|
+
"LAUNCH_GO_TOKEN",
|
|
143
131
|
"StepOutcome",
|
|
132
|
+
"is_importing_mp_main",
|
|
144
133
|
]
|
tasklite/engine/wait.py
CHANGED
|
@@ -79,3 +79,11 @@ def decide_wait(facts: LoopFacts) -> WaitDecision:
|
|
|
79
79
|
if facts.dispatched == 0:
|
|
80
80
|
return WaitDecision(should_wait=True, wait_time=float("inf"))
|
|
81
81
|
return WaitDecision(should_wait=False, wait_time=0.0)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
__all__ = [
|
|
85
|
+
"AUTHORIZATION_RECHECK_SECONDS",
|
|
86
|
+
"LoopFacts",
|
|
87
|
+
"WaitDecision",
|
|
88
|
+
"decide_wait",
|
|
89
|
+
]
|
tasklite/exceptions.py
CHANGED
tasklite/models/artifact.py
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
"""磁盘文件声明值与配置解析:FileSpec 统一声明值对象及提交期快照规范化。"""
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
-
from collections.abc import Mapping, Sequence
|
|
4
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
5
5
|
from dataclasses import dataclass
|
|
6
|
-
import os
|
|
7
6
|
from pathlib import Path
|
|
8
7
|
from typing import Any, Literal, cast
|
|
9
8
|
|
|
10
9
|
from ..exceptions import TaskLiteError
|
|
10
|
+
from ..utils.artifact import (
|
|
11
|
+
InspectMode,
|
|
12
|
+
PathPolicy,
|
|
13
|
+
is_platform_abs_path,
|
|
14
|
+
resolve_lexical_path,
|
|
15
|
+
validate_and_normalize_lexical,
|
|
16
|
+
)
|
|
11
17
|
|
|
12
18
|
FILE_ROLES: tuple[str, ...] = ("input", "output", "cache")
|
|
13
19
|
FileRole = Literal["input", "output", "cache"]
|
|
@@ -30,41 +36,37 @@ _OUTPUT_REF_SPEC_KEYS = frozenset({
|
|
|
30
36
|
|
|
31
37
|
|
|
32
38
|
def _normalize_lexical_path(path: str | Path) -> str:
|
|
33
|
-
"""
|
|
39
|
+
"""校验并词法规范化路径字符串(复用于 PathPolicy 唯一词法规则,不触碰磁盘)。"""
|
|
34
40
|
if not isinstance(path, (str, Path)):
|
|
35
41
|
raise TaskLiteError(
|
|
36
42
|
f"FileSpec path must be a str or Path, got {type(path).__name__}"
|
|
37
43
|
)
|
|
38
44
|
if isinstance(path, str) and not path:
|
|
39
45
|
raise TaskLiteError("FileSpec path must be non-empty")
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if os.name == "posix" and norm.startswith("//"):
|
|
45
|
-
norm = "/" + norm.lstrip("/")
|
|
46
|
-
return norm
|
|
46
|
+
try:
|
|
47
|
+
return validate_and_normalize_lexical(path)
|
|
48
|
+
except ValueError as exc:
|
|
49
|
+
raise TaskLiteError(str(exc)) from exc
|
|
47
50
|
|
|
48
51
|
|
|
49
52
|
def resolve_path_against_base(path: str | Path, base_dir: str | Path) -> str:
|
|
50
53
|
"""按固定基准目录将路径解析为词法规范的绝对路径(不创建目录或访问磁盘)。"""
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return resolved
|
|
54
|
+
if not isinstance(path, (str, Path)):
|
|
55
|
+
raise TaskLiteError(
|
|
56
|
+
f"FileSpec path must be a str or Path, got {type(path).__name__}"
|
|
57
|
+
)
|
|
58
|
+
if isinstance(path, str) and not path:
|
|
59
|
+
raise TaskLiteError("FileSpec path must be non-empty")
|
|
60
|
+
if not isinstance(base_dir, (str, Path)):
|
|
61
|
+
raise TaskLiteError(
|
|
62
|
+
f"FileSpec base_dir must be a str or Path, got {type(base_dir).__name__}"
|
|
63
|
+
)
|
|
64
|
+
if isinstance(base_dir, str) and not base_dir:
|
|
65
|
+
raise TaskLiteError("FileSpec base_dir must be non-empty")
|
|
66
|
+
try:
|
|
67
|
+
return resolve_lexical_path(path, base_dir)
|
|
68
|
+
except ValueError as exc:
|
|
69
|
+
raise TaskLiteError(str(exc)) from exc
|
|
68
70
|
|
|
69
71
|
|
|
70
72
|
def _validate_role_and_cleanup(
|
|
@@ -164,7 +166,7 @@ class FileSpec:
|
|
|
164
166
|
"""是否为上游具名产物引用(from_output)。"""
|
|
165
167
|
return self.producer_uid is not None
|
|
166
168
|
|
|
167
|
-
def
|
|
169
|
+
def _resolve_against(self, base_dir: str | Path) -> FileSpec:
|
|
168
170
|
"""按固定基准目录解析普通路径为绝对路径;上游引用保持原样。"""
|
|
169
171
|
_normalize_lexical_path(base_dir)
|
|
170
172
|
if self.is_output_ref:
|
|
@@ -181,7 +183,7 @@ class FileSpec:
|
|
|
181
183
|
delete_on_success=self.delete_on_success,
|
|
182
184
|
)
|
|
183
185
|
|
|
184
|
-
def
|
|
186
|
+
def _bind_output_path(
|
|
185
187
|
self,
|
|
186
188
|
resolved_path: str | Path,
|
|
187
189
|
*,
|
|
@@ -192,7 +194,7 @@ class FileSpec:
|
|
|
192
194
|
abs_path = resolve_path_against_base(resolved_path, base_dir)
|
|
193
195
|
else:
|
|
194
196
|
abs_path = _normalize_lexical_path(resolved_path)
|
|
195
|
-
if not
|
|
197
|
+
if not is_platform_abs_path(abs_path):
|
|
196
198
|
raise TaskLiteError(
|
|
197
199
|
f"resolved_path must be an absolute path when base_dir is None, got {resolved_path!r}"
|
|
198
200
|
)
|
|
@@ -326,7 +328,7 @@ def resolve_job_files_config(
|
|
|
326
328
|
)
|
|
327
329
|
resolved_spec = spec
|
|
328
330
|
else:
|
|
329
|
-
resolved_spec = spec.
|
|
331
|
+
resolved_spec = spec._resolve_against(base_dir)
|
|
330
332
|
resolved[name] = resolved_spec.to_dict()
|
|
331
333
|
return resolved
|
|
332
334
|
|
|
@@ -357,12 +359,12 @@ def merge_named_file_spec(
|
|
|
357
359
|
f"FileSpec {validated_name!r} references producer {spec.producer_uid!r} "
|
|
358
360
|
f"which is not in explicit direct depends_on {list(depends_on)!r}"
|
|
359
361
|
)
|
|
360
|
-
canonical = spec.
|
|
362
|
+
canonical = spec._resolve_against(base_dir) if base_dir is not None else spec
|
|
361
363
|
merged: dict[str, FileSpec] = {}
|
|
362
364
|
for raw_key, raw_val in existing.items():
|
|
363
365
|
k = validate_file_name(raw_key)
|
|
364
366
|
v = coerce_file_spec(raw_val, allow_dict=True)
|
|
365
|
-
merged[k] = v.
|
|
367
|
+
merged[k] = v._resolve_against(base_dir) if base_dir is not None else v
|
|
366
368
|
if validated_name in merged:
|
|
367
369
|
previous = merged[validated_name]
|
|
368
370
|
if previous == canonical:
|
|
@@ -375,34 +377,54 @@ def merge_named_file_spec(
|
|
|
375
377
|
return merged, True
|
|
376
378
|
|
|
377
379
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
if not
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
380
|
+
def resolve_file_spec(
|
|
381
|
+
spec: FileSpec,
|
|
382
|
+
*,
|
|
383
|
+
path_policy: PathPolicy,
|
|
384
|
+
depends_on: Sequence[str] = (),
|
|
385
|
+
artifact_lookup: Callable[[str, str], Mapping[str, Any] | None] | None = None,
|
|
386
|
+
enforce_safety: bool | None = None,
|
|
387
|
+
) -> FileSpec:
|
|
388
|
+
"""基于 PathPolicy 单次原子解析 FileSpec(上游引用、安全边界、存在性判定与路径绑定)。
|
|
389
|
+
|
|
390
|
+
全程经由 ``path_policy.inspect``(100% 通过 ``FilesystemAdapter``),不直调
|
|
391
|
+
``os.path.exists``,且单次完成规范化、清理安全校验与绑定,消除重复往返。
|
|
392
|
+
"""
|
|
393
|
+
if spec.is_output_ref:
|
|
394
|
+
assert spec.producer_uid is not None and spec.output_name is not None
|
|
395
|
+
if spec.producer_uid not in depends_on:
|
|
396
|
+
raise TaskLiteError(
|
|
397
|
+
f"Producer {spec.producer_uid!r} not in explicit direct depends_on {list(depends_on)!r}"
|
|
398
|
+
)
|
|
399
|
+
desc = (
|
|
400
|
+
artifact_lookup(spec.producer_uid, spec.output_name)
|
|
401
|
+
if artifact_lookup is not None
|
|
402
|
+
else None
|
|
403
|
+
)
|
|
404
|
+
if not desc or "path" not in desc:
|
|
405
|
+
raise TaskLiteError(
|
|
406
|
+
f"Producer {spec.producer_uid!r} did not publish output {spec.output_name!r}"
|
|
407
|
+
)
|
|
408
|
+
raw_path: str = desc["path"]
|
|
409
|
+
else:
|
|
410
|
+
if spec.path is None:
|
|
411
|
+
raise TaskLiteError("FileSpec resolved without a path")
|
|
412
|
+
raw_path = spec.path
|
|
413
|
+
|
|
414
|
+
need_safety = (
|
|
415
|
+
spec.delete_existing or spec.delete_on_success or spec.delete_on_failure
|
|
416
|
+
if enforce_safety is None
|
|
417
|
+
else enforce_safety
|
|
418
|
+
)
|
|
419
|
+
mode: InspectMode = (
|
|
420
|
+
("pre_delete" if spec.delete_existing else "strict")
|
|
421
|
+
if need_safety
|
|
422
|
+
else "lookup"
|
|
423
|
+
)
|
|
424
|
+
inspected = path_policy.inspect(raw_path, mode=mode)
|
|
425
|
+
if spec.role == "input" and (not inspected.exists or inspected.kind == "broken_symlink"):
|
|
426
|
+
raise FileNotFoundError(f"Missing input file: {inspected.canonical_path}")
|
|
427
|
+
return spec._bind_output_path(inspected.canonical_path)
|
|
406
428
|
|
|
407
429
|
|
|
408
430
|
__all__ = [
|
|
@@ -412,8 +434,8 @@ __all__ = [
|
|
|
412
434
|
"coerce_file_spec",
|
|
413
435
|
"merge_named_file_spec",
|
|
414
436
|
"parse_job_files_config",
|
|
437
|
+
"resolve_file_spec",
|
|
415
438
|
"resolve_job_files_config",
|
|
416
439
|
"resolve_path_against_base",
|
|
417
440
|
"validate_file_name",
|
|
418
|
-
"ArtifactDeclaration",
|
|
419
441
|
]
|
tasklite/models/context.py
CHANGED
|
@@ -15,6 +15,7 @@ from pathlib import Path
|
|
|
15
15
|
from typing import Any
|
|
16
16
|
|
|
17
17
|
from ..exceptions import TaskLiteError, _ArtifactIOError
|
|
18
|
+
from ..utils.artifact import PathPolicy
|
|
18
19
|
from ..utils.ipc import (
|
|
19
20
|
WorkerPipeSender,
|
|
20
21
|
WorkerReplyReceiver,
|
|
@@ -22,6 +23,7 @@ from ..utils.ipc import (
|
|
|
22
23
|
from .artifact import (
|
|
23
24
|
FileSpec,
|
|
24
25
|
coerce_file_spec,
|
|
26
|
+
resolve_file_spec,
|
|
25
27
|
resolve_job_files_config,
|
|
26
28
|
validate_file_name,
|
|
27
29
|
)
|
|
@@ -37,7 +39,6 @@ class JobContext:
|
|
|
37
39
|
success_job_uids: set | frozenset,
|
|
38
40
|
failed_job_uids: set | frozenset,
|
|
39
41
|
cursors: dict[str, str],
|
|
40
|
-
output_roots: str | Path | Sequence[str | Path] | None = None,
|
|
41
42
|
artifact_descriptions: dict[str, dict[str, dict[str, str]]] | None = None,
|
|
42
43
|
protected_roots: tuple[str | Path, ...] = (),
|
|
43
44
|
base_dir: str | Path | None = None,
|
|
@@ -46,7 +47,7 @@ class JobContext:
|
|
|
46
47
|
):
|
|
47
48
|
self._artifact_descriptions = copy.deepcopy(artifact_descriptions or {})
|
|
48
49
|
self._protected_roots = tuple(protected_roots)
|
|
49
|
-
self.
|
|
50
|
+
self._base_dir: Path = (
|
|
50
51
|
Path(os.path.abspath(os.fspath(base_dir)))
|
|
51
52
|
if base_dir is not None
|
|
52
53
|
else Path(os.path.abspath(os.getcwd()))
|
|
@@ -66,6 +67,7 @@ class JobContext:
|
|
|
66
67
|
source = self._canonical_source(
|
|
67
68
|
coerce_file_spec(raw_source, allow_dict=True)
|
|
68
69
|
)
|
|
70
|
+
self._validate_output_ref_dependency(name, source)
|
|
69
71
|
resolved = (
|
|
70
72
|
coerce_file_spec(resolved_files[name], allow_dict=True)
|
|
71
73
|
if name in resolved_files
|
|
@@ -82,9 +84,6 @@ class JobContext:
|
|
|
82
84
|
)
|
|
83
85
|
self._cursors = cursors
|
|
84
86
|
self._cursor_updates: dict[str, str | None] = {}
|
|
85
|
-
# output_roots 支持多根(跨盘输出场景)——解析时
|
|
86
|
-
# 路径属于**任一**根即通过沙盒校验。
|
|
87
|
-
self.output_roots = output_roots
|
|
88
87
|
|
|
89
88
|
def _bind_pipe_sender(
|
|
90
89
|
self,
|
|
@@ -97,26 +96,40 @@ class JobContext:
|
|
|
97
96
|
|
|
98
97
|
def _canonical_source(self, spec: FileSpec) -> FileSpec:
|
|
99
98
|
"""将普通路径声明按固定基准绝对化;上游引用保留配置身份。"""
|
|
100
|
-
return spec if spec.is_output_ref else spec.
|
|
99
|
+
return spec if spec.is_output_ref else spec._resolve_against(self._base_dir)
|
|
101
100
|
|
|
102
|
-
def
|
|
103
|
-
"""
|
|
101
|
+
def _validate_output_ref_dependency(self, name: str, spec: FileSpec) -> None:
|
|
102
|
+
"""校验上游产物引用必须显式列于当前 Job 的直接依赖中。"""
|
|
104
103
|
if spec.is_output_ref:
|
|
105
|
-
assert spec.producer_uid is not None
|
|
106
|
-
|
|
107
|
-
if not desc or "path" not in desc:
|
|
104
|
+
assert spec.producer_uid is not None
|
|
105
|
+
if spec.producer_uid not in self.job.depends_on:
|
|
108
106
|
raise TaskLiteError(
|
|
109
|
-
f"producer {spec.producer_uid!r}
|
|
110
|
-
f"
|
|
107
|
+
f"FileSpec {name!r} references producer {spec.producer_uid!r} "
|
|
108
|
+
f"which is not in explicit direct depends_on {list(self.job.depends_on)!r}"
|
|
111
109
|
)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
110
|
+
|
|
111
|
+
def _resolve_local_file(self, spec: FileSpec) -> FileSpec:
|
|
112
|
+
"""无回复通道时复用统一 PathPolicy 解析声明供隔离测试使用;不执行任何文件删除。"""
|
|
113
|
+
policy = PathPolicy(
|
|
114
|
+
base_dir=self._base_dir,
|
|
115
|
+
protected_roots=self._protected_roots,
|
|
116
|
+
)
|
|
117
|
+
enforce_safety = bool(self._protected_roots) and (
|
|
118
|
+
spec.role != "input"
|
|
119
|
+
or spec.delete_existing
|
|
120
|
+
or spec.delete_on_success
|
|
121
|
+
or spec.delete_on_failure
|
|
122
|
+
)
|
|
123
|
+
try:
|
|
124
|
+
return resolve_file_spec(
|
|
125
|
+
spec,
|
|
126
|
+
path_policy=policy,
|
|
127
|
+
depends_on=self.job.depends_on,
|
|
128
|
+
artifact_lookup=lambda puid, oname: self._artifact_descriptions.get(puid, {}).get(oname),
|
|
129
|
+
enforce_safety=enforce_safety,
|
|
130
|
+
)
|
|
131
|
+
except ValueError as exc:
|
|
132
|
+
raise TaskLiteError(str(exc)) from exc
|
|
120
133
|
|
|
121
134
|
def file_path(self, name: str) -> Path:
|
|
122
135
|
"""返回当前 Job 已声明文件(预声明或已完成动态声明)的规范绝对 Path。"""
|
|
@@ -134,13 +147,7 @@ class JobContext:
|
|
|
134
147
|
if not isinstance(spec, FileSpec):
|
|
135
148
|
raise TaskLiteError(f"declare_file expects a FileSpec, got {type(spec).__name__}")
|
|
136
149
|
|
|
137
|
-
|
|
138
|
-
assert spec.producer_uid is not None
|
|
139
|
-
if spec.producer_uid not in self.job.depends_on:
|
|
140
|
-
raise TaskLiteError(
|
|
141
|
-
f"FileSpec {validated_name!r} references producer {spec.producer_uid!r} "
|
|
142
|
-
f"which is not in explicit direct depends_on {list(self.job.depends_on)!r}"
|
|
143
|
-
)
|
|
150
|
+
self._validate_output_ref_dependency(validated_name, spec)
|
|
144
151
|
|
|
145
152
|
canonical_source = self._canonical_source(spec)
|
|
146
153
|
if validated_name in self._file_sources:
|
|
@@ -164,7 +171,7 @@ class JobContext:
|
|
|
164
171
|
status = reply.get("status")
|
|
165
172
|
if status == "ok":
|
|
166
173
|
resolved_path = str(reply["resolved_path"])
|
|
167
|
-
bound_spec = canonical_source.
|
|
174
|
+
bound_spec = canonical_source._bind_output_path(resolved_path)
|
|
168
175
|
self._file_sources[validated_name] = canonical_source
|
|
169
176
|
self._files[validated_name] = bound_spec
|
|
170
177
|
return Path(resolved_path)
|
|
@@ -198,7 +205,7 @@ class JobContext:
|
|
|
198
205
|
job_dict = job.to_dict()
|
|
199
206
|
job_dict["files"] = resolve_job_files_config(
|
|
200
207
|
job_dict.get("files"),
|
|
201
|
-
base_dir=self.
|
|
208
|
+
base_dir=self._base_dir,
|
|
202
209
|
depends_on=job_dict.get("depends_on") or (),
|
|
203
210
|
)
|
|
204
211
|
self._spawned_jobs.append(Job.from_dict(job_dict))
|
tasklite/models/graph.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""无状态纯依赖图原语:反向依赖索引、下游闭包遍历与迭代式 Kosaraju 环检测。
|
|
2
|
+
|
|
3
|
+
分层约束:本模块仅依赖 Python 标准库,供 ``PipelineState`` 与 ``EditTransaction``
|
|
4
|
+
同源复用,零递归深度限制。
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections import deque
|
|
9
|
+
from collections.abc import Collection, Iterator, Mapping, Set as AbstractSet
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_dependents_index(edges: Mapping[str, Collection[str]]) -> dict[str, set[str]]:
|
|
13
|
+
"""由正向依赖边 ``node -> depends_on`` 构建反向依赖索引 ``dep -> {node...}``。"""
|
|
14
|
+
dependents: dict[str, set[str]] = {}
|
|
15
|
+
for node, deps in edges.items():
|
|
16
|
+
for dep in deps:
|
|
17
|
+
if isinstance(dep, str):
|
|
18
|
+
dependents.setdefault(dep, set()).add(node)
|
|
19
|
+
return dependents
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def collect_downstream(
|
|
23
|
+
root: str,
|
|
24
|
+
dependents: Mapping[str, Collection[str]],
|
|
25
|
+
*,
|
|
26
|
+
stop_at: AbstractSet[str] = frozenset(),
|
|
27
|
+
) -> list[str]:
|
|
28
|
+
"""沿反向依赖索引迭代收集 ``root`` 的全部直接与间接下游(确定性字典序 BFS)。
|
|
29
|
+
|
|
30
|
+
``stop_at`` 中的节点为传播终止点:既不收入结果,也不向其下游继续扩散。
|
|
31
|
+
"""
|
|
32
|
+
downstream: list[str] = []
|
|
33
|
+
visited = {root}
|
|
34
|
+
frontier: deque[str] = deque([root])
|
|
35
|
+
while frontier:
|
|
36
|
+
cur = frontier.popleft()
|
|
37
|
+
for child in sorted(dependents.get(cur, ())):
|
|
38
|
+
if child in visited or child in stop_at:
|
|
39
|
+
continue
|
|
40
|
+
visited.add(child)
|
|
41
|
+
downstream.append(child)
|
|
42
|
+
frontier.append(child)
|
|
43
|
+
return downstream
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def find_cycle_components(
|
|
47
|
+
nodes: Collection[str],
|
|
48
|
+
edges: Mapping[str, Collection[str]],
|
|
49
|
+
) -> list[list[str]]:
|
|
50
|
+
"""迭代式 Kosaraju 强连通分量检测,返回所有含环节点分量(零递归深度限制)。"""
|
|
51
|
+
node_set = set(nodes)
|
|
52
|
+
adj: dict[str, list[str]] = {
|
|
53
|
+
u: [v for v in edges.get(u, ()) if v in node_set]
|
|
54
|
+
for u in node_set
|
|
55
|
+
}
|
|
56
|
+
rev: dict[str, list[str]] = {u: [] for u in node_set}
|
|
57
|
+
for u, nbrs in adj.items():
|
|
58
|
+
for v in nbrs:
|
|
59
|
+
rev[v].append(u)
|
|
60
|
+
|
|
61
|
+
visited: set[str] = set()
|
|
62
|
+
order: list[str] = []
|
|
63
|
+
for root in sorted(node_set):
|
|
64
|
+
if root in visited:
|
|
65
|
+
continue
|
|
66
|
+
visited.add(root)
|
|
67
|
+
stack: list[tuple[str, Iterator[str]]] = [(root, iter(adj[root]))]
|
|
68
|
+
while stack:
|
|
69
|
+
cur, it = stack[-1]
|
|
70
|
+
try:
|
|
71
|
+
nxt = next(it)
|
|
72
|
+
except StopIteration:
|
|
73
|
+
stack.pop()
|
|
74
|
+
order.append(cur)
|
|
75
|
+
continue
|
|
76
|
+
if nxt not in visited:
|
|
77
|
+
visited.add(nxt)
|
|
78
|
+
stack.append((nxt, iter(adj[nxt])))
|
|
79
|
+
|
|
80
|
+
assigned: set[str] = set()
|
|
81
|
+
cycles: list[list[str]] = []
|
|
82
|
+
for root in reversed(order):
|
|
83
|
+
if root in assigned:
|
|
84
|
+
continue
|
|
85
|
+
comp: list[str] = []
|
|
86
|
+
dfs_stack = [root]
|
|
87
|
+
assigned.add(root)
|
|
88
|
+
while dfs_stack:
|
|
89
|
+
cur = dfs_stack.pop()
|
|
90
|
+
comp.append(cur)
|
|
91
|
+
for nxt in rev[cur]:
|
|
92
|
+
if nxt not in assigned:
|
|
93
|
+
assigned.add(nxt)
|
|
94
|
+
dfs_stack.append(nxt)
|
|
95
|
+
if len(comp) > 1 or (len(comp) == 1 and comp[0] in adj[comp[0]]):
|
|
96
|
+
cycles.append(sorted(comp))
|
|
97
|
+
cycles.sort()
|
|
98
|
+
return cycles
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def find_cycle_members(
|
|
102
|
+
nodes: Collection[str],
|
|
103
|
+
edges: Mapping[str, Collection[str]],
|
|
104
|
+
) -> list[str]:
|
|
105
|
+
"""返回参与任意依赖环的全部节点列表(按分量有序去重)。"""
|
|
106
|
+
return [uid for comp in find_cycle_components(nodes, edges) for uid in comp]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
__all__ = [
|
|
110
|
+
"build_dependents_index",
|
|
111
|
+
"collect_downstream",
|
|
112
|
+
"find_cycle_components",
|
|
113
|
+
"find_cycle_members",
|
|
114
|
+
]
|
tasklite/models/job.py
CHANGED
|
@@ -20,12 +20,10 @@ from .task import (
|
|
|
20
20
|
validate_timeout_seconds,
|
|
21
21
|
validate_trigger,
|
|
22
22
|
)
|
|
23
|
+
from .trigger import BUILTIN_TRIGGERS
|
|
23
24
|
|
|
24
25
|
WORKER_RESOURCE = "__workers__"
|
|
25
26
|
|
|
26
|
-
# 内置触发器名称;自定义触发器同样仅以名称引用。
|
|
27
|
-
BUILTIN_TRIGGERS = ("once", "until_success", "always", "on_input_change")
|
|
28
|
-
|
|
29
27
|
|
|
30
28
|
def inject_worker_resource(job_snapshot: dict) -> None:
|
|
31
29
|
"""给 job_snapshot 的 resources 注入默认 worker 槽位。"""
|
|
@@ -104,12 +102,12 @@ def _normalize_depends_on(depends_on: Any) -> list[str]:
|
|
|
104
102
|
def _validate_attempt_no(attempt_no: Any) -> None:
|
|
105
103
|
"""attempt_no 入口校验:非 bool 的 int 且 >= 1(1 起始含首次执行)。"""
|
|
106
104
|
if not isinstance(attempt_no, int) or isinstance(attempt_no, bool):
|
|
107
|
-
raise
|
|
105
|
+
raise TaskLiteError(
|
|
108
106
|
f"attempt_no must be an int, got {type(attempt_no).__name__} "
|
|
109
107
|
f"({attempt_no!r})"
|
|
110
108
|
)
|
|
111
109
|
if attempt_no < 1:
|
|
112
|
-
raise
|
|
110
|
+
raise TaskLiteError(f"attempt_no must be >= 1, got {attempt_no}")
|
|
113
111
|
|
|
114
112
|
|
|
115
113
|
class Job:
|