tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
- tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
tasklite/engine/scheduler.py
CHANGED
|
@@ -16,8 +16,8 @@ from dataclasses import dataclass, field
|
|
|
16
16
|
from ..exceptions import TaskLiteError
|
|
17
17
|
from ..models.job import Job
|
|
18
18
|
from ..models.state import DependencyStatus, PipelineState, job_uid_from_snapshot
|
|
19
|
-
from ..selection import Exclude
|
|
20
19
|
from .resource import Resource, ResourceManager
|
|
20
|
+
from .session import RunSession
|
|
21
21
|
|
|
22
22
|
logger = logging.getLogger("tasklite")
|
|
23
23
|
|
|
@@ -109,7 +109,6 @@ class ScheduleResult:
|
|
|
109
109
|
# 死锁归因值对象(UID 集合)
|
|
110
110
|
attribution: DeadlockAttribution = field(default_factory=DeadlockAttribution)
|
|
111
111
|
candidate_uid: str | None = None
|
|
112
|
-
impossible_reason: str | None = None
|
|
113
112
|
has_resource_waiters: bool = False
|
|
114
113
|
|
|
115
114
|
@property
|
|
@@ -175,6 +174,7 @@ class JobScheduler:
|
|
|
175
174
|
tasks: Mapping[str, object] | None = None,
|
|
176
175
|
*,
|
|
177
176
|
ordering_policy: OrderingPolicy | None = None,
|
|
177
|
+
session: RunSession | None = None,
|
|
178
178
|
):
|
|
179
179
|
if isinstance(resources, ResourceManager):
|
|
180
180
|
self.resource_mgr = resources
|
|
@@ -183,43 +183,14 @@ class JobScheduler:
|
|
|
183
183
|
self.resources = resources
|
|
184
184
|
self.resource_mgr = ResourceManager(resources, tasks=tasks)
|
|
185
185
|
self.ordering_policy: OrderingPolicy = ordering_policy if ordering_policy is not None else FifoOrderingPolicy()
|
|
186
|
-
self.
|
|
187
|
-
self._temporary_excluded_uids: set[str] = set()
|
|
186
|
+
self._session = session if session is not None else RunSession()
|
|
188
187
|
self._job_cache: dict[tuple[str, str], JobFacts] = {}
|
|
189
188
|
self._JOB_CACHE_MAX = 100_000
|
|
190
189
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
def clear_temporary_exclusions(self) -> None:
|
|
196
|
-
"""清空临时排除 UID 集合(每次 Run 启动时重置)。"""
|
|
197
|
-
self._temporary_excluded_uids.clear()
|
|
198
|
-
|
|
199
|
-
def is_job_excluded(self, job_uid: str, task_type: str) -> bool:
|
|
200
|
-
"""统一判断作业是否被排除(用户 exclude 规则或运行期临时排除)。"""
|
|
201
|
-
if job_uid in self._temporary_excluded_uids:
|
|
202
|
-
return True
|
|
203
|
-
if self.exclude is not None and self.exclude.matches_task_type(task_type):
|
|
204
|
-
return True
|
|
205
|
-
return False
|
|
206
|
-
|
|
207
|
-
def is_job_excluded_by_uid(self, job_uid: str) -> bool:
|
|
208
|
-
"""按 UID 查询是否被排除(临时排除或根据 task_type 前缀比对 exclude 规则)。"""
|
|
209
|
-
if job_uid in self._temporary_excluded_uids:
|
|
210
|
-
return True
|
|
211
|
-
if self.exclude is None:
|
|
212
|
-
return False
|
|
213
|
-
parts = job_uid.split("::", 1)
|
|
214
|
-
if len(parts) == 2:
|
|
215
|
-
return self.exclude.matches_task_type(parts[0])
|
|
216
|
-
return False
|
|
217
|
-
|
|
218
|
-
def reset_run(self, exclude: Exclude | None = None) -> None:
|
|
219
|
-
"""Run 生命周期重置:固化排除规则并清空临时排除与调度缓存。"""
|
|
220
|
-
self.exclude = exclude
|
|
221
|
-
self._temporary_excluded_uids.clear()
|
|
222
|
-
self.begin_round()
|
|
190
|
+
@property
|
|
191
|
+
def session(self) -> RunSession:
|
|
192
|
+
"""当前关联的运行期会话(运行期排除状态唯一接口)。"""
|
|
193
|
+
return self._session
|
|
223
194
|
|
|
224
195
|
def begin_round(self) -> None:
|
|
225
196
|
"""清空调度缓存(**run 生命周期**,由运行主循环加载期调用)。
|
|
@@ -285,14 +256,6 @@ class JobScheduler:
|
|
|
285
256
|
cached_res = dict(job.resources or {}) if not isinstance(job.resources, dict) else job.resources
|
|
286
257
|
return list(job.depends_on) == job_dep and cached_res == job_res
|
|
287
258
|
|
|
288
|
-
def _effective_resources(self, job) -> dict[str, float]:
|
|
289
|
-
"""job 实际会 acquire 的资源集(委托给 ResourceManager 单点真相源)。"""
|
|
290
|
-
if isinstance(job, JobFacts):
|
|
291
|
-
job_resources = job.resource_map()
|
|
292
|
-
else:
|
|
293
|
-
job_resources = dict(job.resources)
|
|
294
|
-
return self.resource_mgr.effective_resources(job.task_type, job_resources)
|
|
295
|
-
|
|
296
259
|
def scan_next_runnable(
|
|
297
260
|
self,
|
|
298
261
|
state: PipelineState,
|
|
@@ -332,7 +295,7 @@ class JobScheduler:
|
|
|
332
295
|
continue
|
|
333
296
|
|
|
334
297
|
# 排除规则与临时排除:命中排除的作业在依赖/资源评估之前整体跳过。
|
|
335
|
-
if self.
|
|
298
|
+
if self._session.is_excluded(job.uid, job.task_type):
|
|
336
299
|
continue
|
|
337
300
|
|
|
338
301
|
# 依赖屏障与就绪判定:统一调用 state.check_dependency_status
|
|
@@ -347,7 +310,7 @@ class JobScheduler:
|
|
|
347
310
|
status = state.check_dependency_status(
|
|
348
311
|
dep_uid,
|
|
349
312
|
in_flight_uids=in_flight_uids,
|
|
350
|
-
is_dep_excluded=self.
|
|
313
|
+
is_dep_excluded=self._session.is_excluded_by_uid,
|
|
351
314
|
)
|
|
352
315
|
if status == DependencyStatus.ACTIVE or status == DependencyStatus.EXCLUDED:
|
|
353
316
|
waiting_for_dep = True
|
tasklite/engine/session.py
CHANGED
|
@@ -2,21 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
不变式:
|
|
4
4
|
- stop_mode 单调转移(NONE→DRAINING,唯一入口 request_stop,幂等且不升级强停);
|
|
5
|
-
- begin() 是新 run 的复位入口(run_id/dispatch_no/stop_mode
|
|
5
|
+
- begin() 是新 run 的复位入口(run_id/dispatch_no/stop_mode/候选来源/重试计数/排除状态归零)。
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
|
-
from
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
12
|
|
|
13
|
+
from ..exceptions import TaskLiteError
|
|
14
|
+
from ..selection import Exclude
|
|
15
|
+
from ..utils.encoding import parse_job_uid
|
|
13
16
|
from .types import ExitReason, StopMode
|
|
14
17
|
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from ..models.state import PipelineState
|
|
20
|
+
|
|
15
21
|
logger = logging.getLogger("tasklite")
|
|
16
22
|
|
|
17
23
|
|
|
18
24
|
class RunSession:
|
|
19
|
-
"""一次 run()
|
|
25
|
+
"""一次 run() 的可变生命周期状态与运行期排除单一真相源。"""
|
|
20
26
|
|
|
21
27
|
def __init__(self) -> None:
|
|
22
28
|
self.run_id: str | None = None
|
|
@@ -24,24 +30,63 @@ class RunSession:
|
|
|
24
30
|
self.stop_mode: StopMode = StopMode.NONE
|
|
25
31
|
self.retry_counts: dict[str, int] = {}
|
|
26
32
|
self.retry_candidates: set[str] = set()
|
|
27
|
-
self.
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
self
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
self._exclude_chain_handler(job_uid)
|
|
37
|
-
|
|
38
|
-
def begin(self, run_id: str | None = None) -> None:
|
|
33
|
+
self.exclude: Exclude | None = None
|
|
34
|
+
self.temporary_excluded_uids: set[str] = set()
|
|
35
|
+
|
|
36
|
+
def begin(
|
|
37
|
+
self,
|
|
38
|
+
run_id: str | None = None,
|
|
39
|
+
*,
|
|
40
|
+
exclude: Exclude | None = None,
|
|
41
|
+
) -> None:
|
|
39
42
|
"""新 run 的复位入口:全部生命周期状态归零,run_id 待分配时置 None。"""
|
|
40
43
|
self.run_id = run_id
|
|
41
44
|
self.dispatch_no = 0
|
|
42
45
|
self.stop_mode = StopMode.NONE
|
|
43
46
|
self.retry_counts = {}
|
|
44
47
|
self.retry_candidates = set()
|
|
48
|
+
self.exclude = exclude
|
|
49
|
+
self.temporary_excluded_uids = set()
|
|
50
|
+
|
|
51
|
+
def temporarily_exclude(self, job_uid: str) -> None:
|
|
52
|
+
"""登记单个运行期临时排除 UID(供授权拒绝等复用)。"""
|
|
53
|
+
self.temporary_excluded_uids.add(job_uid)
|
|
54
|
+
|
|
55
|
+
def temporarily_exclude_chain(self, job_uid: str, state: PipelineState) -> None:
|
|
56
|
+
"""将指定作业及其所有直接和间接下游作业加入本 Run 临时排除。"""
|
|
57
|
+
self.temporarily_exclude(job_uid)
|
|
58
|
+
for down_uid in state.find_all_downstream(job_uid):
|
|
59
|
+
self.temporarily_exclude(down_uid)
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def has_exclusions(self) -> bool:
|
|
63
|
+
"""当前 Run 是否启用了用户排除规则或运行期临时排除。"""
|
|
64
|
+
return self.exclude is not None or bool(self.temporary_excluded_uids)
|
|
65
|
+
|
|
66
|
+
def clear_exclusions(self, exclude: Exclude | None = None) -> None:
|
|
67
|
+
"""重置本 Run 的排除规则并清空临时排除集合。"""
|
|
68
|
+
self.exclude = exclude
|
|
69
|
+
self.temporary_excluded_uids.clear()
|
|
70
|
+
|
|
71
|
+
def is_excluded(self, job_uid: str, task_type: str) -> bool:
|
|
72
|
+
"""判断作业是否被排除(用户 exclude 规则或运行期临时排除)。"""
|
|
73
|
+
if job_uid in self.temporary_excluded_uids:
|
|
74
|
+
return True
|
|
75
|
+
if self.exclude is not None and self.exclude.matches_task_type(task_type):
|
|
76
|
+
return True
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
def is_excluded_by_uid(self, job_uid: str) -> bool:
|
|
80
|
+
"""按规范 UID 解码 task_type 后判断是否被排除。"""
|
|
81
|
+
if job_uid in self.temporary_excluded_uids:
|
|
82
|
+
return True
|
|
83
|
+
if self.exclude is None:
|
|
84
|
+
return False
|
|
85
|
+
try:
|
|
86
|
+
task_type, _ = parse_job_uid(job_uid)
|
|
87
|
+
except TaskLiteError:
|
|
88
|
+
return False
|
|
89
|
+
return self.exclude.matches_task_type(task_type)
|
|
45
90
|
|
|
46
91
|
def get_retry_count(self, job_uid: str) -> int:
|
|
47
92
|
"""查询当前 Run 内指定 uid 已消耗的重试预算次数。"""
|