tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. tasklite/__init__.py +42 -18
  2. tasklite/__main__.py +205 -0
  3. tasklite/_report.py +562 -0
  4. tasklite/backend/__init__.py +19 -2
  5. tasklite/backend/base.py +327 -156
  6. tasklite/backend/memory.py +261 -339
  7. tasklite/backend/sqlite_backend.py +416 -488
  8. tasklite/edit.py +1276 -0
  9. tasklite/engine/__init__.py +6 -7
  10. tasklite/engine/_worker_bootstrap.py +221 -0
  11. tasklite/engine/_worker_runtime.py +229 -0
  12. tasklite/engine/admission.py +20 -114
  13. tasklite/engine/authorization.py +989 -0
  14. tasklite/engine/channel.py +531 -468
  15. tasklite/engine/completion.py +193 -275
  16. tasklite/engine/config.py +5 -11
  17. tasklite/engine/dispatch.py +299 -384
  18. tasklite/engine/governor.py +134 -69
  19. tasklite/engine/in_flight.py +20 -88
  20. tasklite/engine/launcher.py +1173 -0
  21. tasklite/engine/recovery.py +147 -100
  22. tasklite/engine/resource.py +50 -103
  23. tasklite/engine/runtime.py +158 -525
  24. tasklite/engine/scheduler.py +9 -46
  25. tasklite/engine/session.py +60 -15
  26. tasklite/engine/store.py +235 -462
  27. tasklite/engine/types.py +29 -40
  28. tasklite/engine/wait.py +8 -0
  29. tasklite/exceptions.py +0 -1
  30. tasklite/models/artifact.py +84 -62
  31. tasklite/models/context.py +37 -30
  32. tasklite/models/graph.py +114 -0
  33. tasklite/models/job.py +3 -5
  34. tasklite/models/records.py +482 -0
  35. tasklite/models/state.py +19 -125
  36. tasklite/models/trigger.py +63 -3
  37. tasklite/pipeline.py +92 -100
  38. tasklite/resource.py +79 -2
  39. tasklite/snapshot.py +608 -405
  40. tasklite/testing.py +424 -11
  41. tasklite/utils/artifact.py +1091 -179
  42. tasklite/utils/encoding.py +21 -40
  43. tasklite/utils/ipc.py +49 -1
  44. tasklite/utils/lockfile.py +50 -19
  45. tasklite/wrappers/__init__.py +1 -1
  46. tasklite/wrappers/discovery.py +13 -16
  47. tasklite/wrappers/http.py +77 -43
  48. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
  49. tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
  50. tasklite/contrib/__init__.py +0 -7
  51. tasklite/engine/execution_access.py +0 -39
  52. tasklite/engine/files.py +0 -52
  53. tasklite/engine/ops.py +0 -25
  54. tasklite/engine/selection.py +0 -186
  55. tasklite/hooks.py +0 -49
  56. tasklite/triggers.py +0 -47
  57. tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
  58. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
  59. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
  60. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
@@ -16,8 +16,8 @@ from dataclasses import dataclass, field
16
16
  from ..exceptions import TaskLiteError
17
17
  from ..models.job import Job
18
18
  from ..models.state import DependencyStatus, PipelineState, job_uid_from_snapshot
19
- from ..selection import Exclude
20
19
  from .resource import Resource, ResourceManager
20
+ from .session import RunSession
21
21
 
22
22
  logger = logging.getLogger("tasklite")
23
23
 
@@ -109,7 +109,6 @@ class ScheduleResult:
109
109
  # 死锁归因值对象(UID 集合)
110
110
  attribution: DeadlockAttribution = field(default_factory=DeadlockAttribution)
111
111
  candidate_uid: str | None = None
112
- impossible_reason: str | None = None
113
112
  has_resource_waiters: bool = False
114
113
 
115
114
  @property
@@ -175,6 +174,7 @@ class JobScheduler:
175
174
  tasks: Mapping[str, object] | None = None,
176
175
  *,
177
176
  ordering_policy: OrderingPolicy | None = None,
177
+ session: RunSession | None = None,
178
178
  ):
179
179
  if isinstance(resources, ResourceManager):
180
180
  self.resource_mgr = resources
@@ -183,43 +183,14 @@ class JobScheduler:
183
183
  self.resources = resources
184
184
  self.resource_mgr = ResourceManager(resources, tasks=tasks)
185
185
  self.ordering_policy: OrderingPolicy = ordering_policy if ordering_policy is not None else FifoOrderingPolicy()
186
- self.exclude: Exclude | None = None
187
- self._temporary_excluded_uids: set[str] = set()
186
+ self._session = session if session is not None else RunSession()
188
187
  self._job_cache: dict[tuple[str, str], JobFacts] = {}
189
188
  self._JOB_CACHE_MAX = 100_000
190
189
 
191
- def temporarily_exclude(self, job_uid: str) -> None:
192
- """内部排除入口:登记单个临时排除 UID(供授权拒绝等复用)。"""
193
- self._temporary_excluded_uids.add(job_uid)
194
-
195
- def clear_temporary_exclusions(self) -> None:
196
- """清空临时排除 UID 集合(每次 Run 启动时重置)。"""
197
- self._temporary_excluded_uids.clear()
198
-
199
- def is_job_excluded(self, job_uid: str, task_type: str) -> bool:
200
- """统一判断作业是否被排除(用户 exclude 规则或运行期临时排除)。"""
201
- if job_uid in self._temporary_excluded_uids:
202
- return True
203
- if self.exclude is not None and self.exclude.matches_task_type(task_type):
204
- return True
205
- return False
206
-
207
- def is_job_excluded_by_uid(self, job_uid: str) -> bool:
208
- """按 UID 查询是否被排除(临时排除或根据 task_type 前缀比对 exclude 规则)。"""
209
- if job_uid in self._temporary_excluded_uids:
210
- return True
211
- if self.exclude is None:
212
- return False
213
- parts = job_uid.split("::", 1)
214
- if len(parts) == 2:
215
- return self.exclude.matches_task_type(parts[0])
216
- return False
217
-
218
- def reset_run(self, exclude: Exclude | None = None) -> None:
219
- """Run 生命周期重置:固化排除规则并清空临时排除与调度缓存。"""
220
- self.exclude = exclude
221
- self._temporary_excluded_uids.clear()
222
- self.begin_round()
190
+ @property
191
+ def session(self) -> RunSession:
192
+ """当前关联的运行期会话(运行期排除状态唯一接口)。"""
193
+ return self._session
223
194
 
224
195
  def begin_round(self) -> None:
225
196
  """清空调度缓存(**run 生命周期**,由运行主循环加载期调用)。
@@ -285,14 +256,6 @@ class JobScheduler:
285
256
  cached_res = dict(job.resources or {}) if not isinstance(job.resources, dict) else job.resources
286
257
  return list(job.depends_on) == job_dep and cached_res == job_res
287
258
 
288
- def _effective_resources(self, job) -> dict[str, float]:
289
- """job 实际会 acquire 的资源集(委托给 ResourceManager 单点真相源)。"""
290
- if isinstance(job, JobFacts):
291
- job_resources = job.resource_map()
292
- else:
293
- job_resources = dict(job.resources)
294
- return self.resource_mgr.effective_resources(job.task_type, job_resources)
295
-
296
259
  def scan_next_runnable(
297
260
  self,
298
261
  state: PipelineState,
@@ -332,7 +295,7 @@ class JobScheduler:
332
295
  continue
333
296
 
334
297
  # 排除规则与临时排除:命中排除的作业在依赖/资源评估之前整体跳过。
335
- if self.is_job_excluded(job.uid, job.task_type):
298
+ if self._session.is_excluded(job.uid, job.task_type):
336
299
  continue
337
300
 
338
301
  # 依赖屏障与就绪判定:统一调用 state.check_dependency_status
@@ -347,7 +310,7 @@ class JobScheduler:
347
310
  status = state.check_dependency_status(
348
311
  dep_uid,
349
312
  in_flight_uids=in_flight_uids,
350
- is_dep_excluded=self.is_job_excluded_by_uid,
313
+ is_dep_excluded=self._session.is_excluded_by_uid,
351
314
  )
352
315
  if status == DependencyStatus.ACTIVE or status == DependencyStatus.EXCLUDED:
353
316
  waiting_for_dep = True
@@ -2,21 +2,27 @@
2
2
 
3
3
  不变式:
4
4
  - stop_mode 单调转移(NONE→DRAINING,唯一入口 request_stop,幂等且不升级强停);
5
- - begin() 是新 run 的复位入口(run_id/dispatch_no/stop_mode/候选来源与重试计数归零)。
5
+ - begin() 是新 run 的复位入口(run_id/dispatch_no/stop_mode/候选来源/重试计数/排除状态归零)。
6
6
  """
7
7
 
8
8
  from __future__ import annotations
9
9
 
10
10
  import logging
11
- from collections.abc import Callable
11
+ from typing import TYPE_CHECKING
12
12
 
13
+ from ..exceptions import TaskLiteError
14
+ from ..selection import Exclude
15
+ from ..utils.encoding import parse_job_uid
13
16
  from .types import ExitReason, StopMode
14
17
 
18
+ if TYPE_CHECKING:
19
+ from ..models.state import PipelineState
20
+
15
21
  logger = logging.getLogger("tasklite")
16
22
 
17
23
 
18
24
  class RunSession:
19
- """一次 run() 的可变生命周期状态。"""
25
+ """一次 run() 的可变生命周期状态与运行期排除单一真相源。"""
20
26
 
21
27
  def __init__(self) -> None:
22
28
  self.run_id: str | None = None
@@ -24,24 +30,63 @@ class RunSession:
24
30
  self.stop_mode: StopMode = StopMode.NONE
25
31
  self.retry_counts: dict[str, int] = {}
26
32
  self.retry_candidates: set[str] = set()
27
- self._exclude_chain_handler: Callable[[str], None] | None = None
28
-
29
- def bind_exclude_chain_handler(self, handler: Callable[[str], None] | None) -> None:
30
- """绑定临时级联排除处理器。"""
31
- self._exclude_chain_handler = handler
32
-
33
- def temporarily_exclude_chain(self, job_uid: str) -> None:
34
- """委托绑定的处理器将指定作业及其全部下游加入本 Run 临时排除。"""
35
- if self._exclude_chain_handler is not None and callable(self._exclude_chain_handler):
36
- self._exclude_chain_handler(job_uid)
37
-
38
- def begin(self, run_id: str | None = None) -> None:
33
+ self.exclude: Exclude | None = None
34
+ self.temporary_excluded_uids: set[str] = set()
35
+
36
+ def begin(
37
+ self,
38
+ run_id: str | None = None,
39
+ *,
40
+ exclude: Exclude | None = None,
41
+ ) -> None:
39
42
  """新 run 的复位入口:全部生命周期状态归零,run_id 待分配时置 None。"""
40
43
  self.run_id = run_id
41
44
  self.dispatch_no = 0
42
45
  self.stop_mode = StopMode.NONE
43
46
  self.retry_counts = {}
44
47
  self.retry_candidates = set()
48
+ self.exclude = exclude
49
+ self.temporary_excluded_uids = set()
50
+
51
+ def temporarily_exclude(self, job_uid: str) -> None:
52
+ """登记单个运行期临时排除 UID(供授权拒绝等复用)。"""
53
+ self.temporary_excluded_uids.add(job_uid)
54
+
55
+ def temporarily_exclude_chain(self, job_uid: str, state: PipelineState) -> None:
56
+ """将指定作业及其所有直接和间接下游作业加入本 Run 临时排除。"""
57
+ self.temporarily_exclude(job_uid)
58
+ for down_uid in state.find_all_downstream(job_uid):
59
+ self.temporarily_exclude(down_uid)
60
+
61
+ @property
62
+ def has_exclusions(self) -> bool:
63
+ """当前 Run 是否启用了用户排除规则或运行期临时排除。"""
64
+ return self.exclude is not None or bool(self.temporary_excluded_uids)
65
+
66
+ def clear_exclusions(self, exclude: Exclude | None = None) -> None:
67
+ """重置本 Run 的排除规则并清空临时排除集合。"""
68
+ self.exclude = exclude
69
+ self.temporary_excluded_uids.clear()
70
+
71
+ def is_excluded(self, job_uid: str, task_type: str) -> bool:
72
+ """判断作业是否被排除(用户 exclude 规则或运行期临时排除)。"""
73
+ if job_uid in self.temporary_excluded_uids:
74
+ return True
75
+ if self.exclude is not None and self.exclude.matches_task_type(task_type):
76
+ return True
77
+ return False
78
+
79
+ def is_excluded_by_uid(self, job_uid: str) -> bool:
80
+ """按规范 UID 解码 task_type 后判断是否被排除。"""
81
+ if job_uid in self.temporary_excluded_uids:
82
+ return True
83
+ if self.exclude is None:
84
+ return False
85
+ try:
86
+ task_type, _ = parse_job_uid(job_uid)
87
+ except TaskLiteError:
88
+ return False
89
+ return self.exclude.matches_task_type(task_type)
45
90
 
46
91
  def get_retry_count(self, job_uid: str) -> int:
47
92
  """查询当前 Run 内指定 uid 已消耗的重试预算次数。"""