tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. tasklite/__init__.py +42 -18
  2. tasklite/__main__.py +205 -0
  3. tasklite/_report.py +562 -0
  4. tasklite/backend/__init__.py +19 -2
  5. tasklite/backend/base.py +327 -156
  6. tasklite/backend/memory.py +261 -339
  7. tasklite/backend/sqlite_backend.py +416 -488
  8. tasklite/edit.py +1276 -0
  9. tasklite/engine/__init__.py +6 -7
  10. tasklite/engine/_worker_bootstrap.py +221 -0
  11. tasklite/engine/_worker_runtime.py +229 -0
  12. tasklite/engine/admission.py +20 -114
  13. tasklite/engine/authorization.py +989 -0
  14. tasklite/engine/channel.py +531 -468
  15. tasklite/engine/completion.py +193 -275
  16. tasklite/engine/config.py +5 -11
  17. tasklite/engine/dispatch.py +299 -384
  18. tasklite/engine/governor.py +134 -69
  19. tasklite/engine/in_flight.py +20 -88
  20. tasklite/engine/launcher.py +1173 -0
  21. tasklite/engine/recovery.py +147 -100
  22. tasklite/engine/resource.py +50 -103
  23. tasklite/engine/runtime.py +158 -525
  24. tasklite/engine/scheduler.py +9 -46
  25. tasklite/engine/session.py +60 -15
  26. tasklite/engine/store.py +235 -462
  27. tasklite/engine/types.py +29 -40
  28. tasklite/engine/wait.py +8 -0
  29. tasklite/exceptions.py +0 -1
  30. tasklite/models/artifact.py +84 -62
  31. tasklite/models/context.py +37 -30
  32. tasklite/models/graph.py +114 -0
  33. tasklite/models/job.py +3 -5
  34. tasklite/models/records.py +482 -0
  35. tasklite/models/state.py +19 -125
  36. tasklite/models/trigger.py +63 -3
  37. tasklite/pipeline.py +92 -100
  38. tasklite/resource.py +79 -2
  39. tasklite/snapshot.py +608 -405
  40. tasklite/testing.py +424 -11
  41. tasklite/utils/artifact.py +1091 -179
  42. tasklite/utils/encoding.py +21 -40
  43. tasklite/utils/ipc.py +49 -1
  44. tasklite/utils/lockfile.py +50 -19
  45. tasklite/wrappers/__init__.py +1 -1
  46. tasklite/wrappers/discovery.py +13 -16
  47. tasklite/wrappers/http.py +77 -43
  48. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
  49. tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
  50. tasklite/contrib/__init__.py +0 -7
  51. tasklite/engine/execution_access.py +0 -39
  52. tasklite/engine/files.py +0 -52
  53. tasklite/engine/ops.py +0 -25
  54. tasklite/engine/selection.py +0 -186
  55. tasklite/hooks.py +0 -49
  56. tasklite/triggers.py +0 -47
  57. tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
  58. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
  59. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
  60. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
tasklite/testing.py CHANGED
@@ -1,4 +1,4 @@
1
- """v2 官方测试工具(单测 JobContext 构造器与运行状态模拟)。
1
+ """v2 官方测试工具(单测 JobContext 构造器、进程内 WorkerLauncher 与运行状态模拟)。
2
2
 
3
3
  注意:本模块为测试辅助工具,本版本暂不承诺向后兼容与冻结扩展协议。
4
4
  handler 单测构造 JobContext,提供对文件声明、游标、依赖引用与派生的隔离测试支持。
@@ -17,16 +17,434 @@ Examples:
17
17
  """
18
18
  from __future__ import annotations
19
19
 
20
+ import dataclasses
20
21
  import tempfile
21
- from collections.abc import Mapping
22
+ import threading
23
+ import time
24
+ from collections.abc import Callable, Mapping
22
25
  from contextlib import contextmanager
23
26
  from pathlib import Path
24
27
  from typing import Any, Iterable, Iterator
25
28
 
29
+ from .engine._worker_runtime import WorkerLaunchSpec, run_worker_entry
30
+ from .engine.launcher import (
31
+ PreparedWorker,
32
+ WorkerLauncher,
33
+ _WorkerLaunchCancelled,
34
+ )
26
35
  from .models.context import JobContext
27
36
  from .models.job import Job
37
+ from .utils.ipc import (
38
+ WorkerPipeSender,
39
+ WorkerReplyReceiver,
40
+ create_worker_pipe,
41
+ create_worker_reply_pipe,
42
+ )
28
43
 
29
- __all__ = ["fake_ctx", "running"]
44
+ __all__ = [
45
+ "FakeClock",
46
+ "InProcessWorkerLauncher",
47
+ "authorizer",
48
+ "cleanup_hook",
49
+ "cursor_updates",
50
+ "fake_ctx",
51
+ "in_process_workers",
52
+ "monotonic_clock",
53
+ "observe_wait",
54
+ "running",
55
+ "spawned_jobs",
56
+ ]
57
+
58
+
59
+ class _ThreadWorkerHandle:
60
+ """进程内线程 Worker 句柄:直接实现 ``ManagedWorkerHandle`` 并管理 worker 侧管道端点。
61
+
62
+ 不变式:与真实 OS 子进程退出/强杀时内核立即关闭写端 FD 保持一致,线程退出、
63
+ ``kill_tree()`` 或 ``close()`` 均会幂等关闭 worker 侧 ``pipe_sender`` 与
64
+ ``reply_receiver``,令父进程侧 ``drain_to_eof`` 立即观测到 EOF,无需
65
+ ``ExecutionChannel`` 对进程内替身保留特殊清理分支。
66
+ """
67
+
68
+ def __init__(
69
+ self,
70
+ *,
71
+ entry: Callable[[], None],
72
+ pipe_sender: WorkerPipeSender,
73
+ reply_receiver: WorkerReplyReceiver,
74
+ ) -> None:
75
+ self._entry = entry
76
+ self._pipe_sender: WorkerPipeSender | None = pipe_sender
77
+ self._reply_receiver: WorkerReplyReceiver | None = reply_receiver
78
+ self._lock = threading.RLock()
79
+ self._thread: threading.Thread | None = None
80
+ self._exitcode: int | None = None
81
+ self._was_force_killed = False
82
+ self._closed = False
83
+
84
+ def _close_worker_pipes(self) -> None:
85
+ with self._lock:
86
+ sender = self._pipe_sender
87
+ reply_receiver = self._reply_receiver
88
+ self._pipe_sender = None
89
+ self._reply_receiver = None
90
+ if sender is not None:
91
+ try:
92
+ sender.close()
93
+ except OSError:
94
+ pass
95
+ if reply_receiver is not None:
96
+ try:
97
+ reply_receiver.close()
98
+ except OSError:
99
+ pass
100
+
101
+ def _run(self) -> None:
102
+ try:
103
+ self._entry()
104
+ with self._lock:
105
+ if self._exitcode is None:
106
+ self._exitcode = 0
107
+ except BaseException:
108
+ with self._lock:
109
+ if self._exitcode is None:
110
+ self._exitcode = 1
111
+ finally:
112
+ self._close_worker_pipes()
113
+
114
+ def start(self) -> None:
115
+ """启动后台线程运行 worker 执行体。"""
116
+ with self._lock:
117
+ if self._closed:
118
+ return
119
+ thread = threading.Thread(target=self._run, daemon=True)
120
+ self._thread = thread
121
+ thread.start()
122
+
123
+ @property
124
+ def pid(self) -> int | None:
125
+ return None
126
+
127
+ @property
128
+ def exitcode(self) -> int | None:
129
+ with self._lock:
130
+ return self._exitcode
131
+
132
+ @property
133
+ def was_force_killed(self) -> bool:
134
+ with self._lock:
135
+ return self._was_force_killed
136
+
137
+ def is_alive(self) -> bool:
138
+ with self._lock:
139
+ if self._closed:
140
+ return False
141
+ thread = self._thread
142
+ return thread is not None and thread.is_alive()
143
+
144
+ def join(self, timeout: float | None = None) -> None:
145
+ with self._lock:
146
+ if self._closed:
147
+ return
148
+ thread = self._thread
149
+ if thread is not None:
150
+ thread.join(timeout=timeout)
151
+ if not self.is_alive():
152
+ self._close_worker_pipes()
153
+
154
+ def kill_tree(self) -> None:
155
+ with self._lock:
156
+ self._was_force_killed = True
157
+ if self._exitcode is None:
158
+ self._exitcode = -9
159
+ self._close_worker_pipes()
160
+
161
+ def kill(self) -> None:
162
+ self.kill_tree()
163
+
164
+ def close(self) -> None:
165
+ with self._lock:
166
+ if self._closed:
167
+ return
168
+ self._closed = True
169
+ self._thread = None
170
+ self._close_worker_pipes()
171
+
172
+
173
+ class InProcessWorkerLauncher(WorkerLauncher):
174
+ """基于真实双工 Pipe 与后台线程的进程内标准 ``WorkerLauncher`` 实现。"""
175
+
176
+ def __init__(
177
+ self,
178
+ *,
179
+ entry: Callable[[WorkerLaunchSpec], None] = run_worker_entry,
180
+ ) -> None:
181
+ self._entry = entry
182
+
183
+ def prepare(
184
+ self,
185
+ spec: WorkerLaunchSpec,
186
+ *,
187
+ on_message: Callable[[], None] | None,
188
+ cancel_event: threading.Event,
189
+ ) -> PreparedWorker:
190
+ if cancel_event.is_set():
191
+ raise _WorkerLaunchCancelled(
192
+ f"stop requested before worker launch for {spec.job.uid}"
193
+ )
194
+
195
+ receiver, sender = create_worker_pipe(
196
+ job_uid=spec.job.uid,
197
+ execution_id=spec.execution_id,
198
+ run_id=spec.run_id or None,
199
+ attempt_no=spec.attempt_no,
200
+ on_message=on_message,
201
+ )
202
+ reply_receiver, reply_sender = create_worker_reply_pipe(
203
+ job_uid=spec.job.uid,
204
+ execution_id=spec.execution_id,
205
+ run_id=spec.run_id or None,
206
+ attempt_no=spec.attempt_no,
207
+ )
208
+ launch_spec = dataclasses.replace(
209
+ spec,
210
+ pipe_sender=sender,
211
+ reply_receiver=reply_receiver,
212
+ )
213
+ handle = _ThreadWorkerHandle(
214
+ entry=lambda: self._entry(launch_spec),
215
+ pipe_sender=sender,
216
+ reply_receiver=reply_receiver,
217
+ )
218
+
219
+ def _release() -> None:
220
+ if cancel_event.is_set():
221
+ raise _WorkerLaunchCancelled(
222
+ f"stop requested before worker start for {spec.job.uid}"
223
+ )
224
+ handle.start()
225
+
226
+ return PreparedWorker(
227
+ handle=handle,
228
+ receiver=receiver,
229
+ reply_sender=reply_sender,
230
+ on_release=_release,
231
+ )
232
+
233
+
234
+ def _resolve_launcher_channels(target: Any) -> list[Any]:
235
+ channels: list[Any] = []
236
+ if callable(getattr(target, "set_launcher", None)):
237
+ channels.append(target)
238
+ channel = getattr(target, "_channel", getattr(target, "channel", None))
239
+ if (
240
+ channel is not None
241
+ and callable(getattr(channel, "set_launcher", None))
242
+ and channel not in channels
243
+ ):
244
+ channels.append(channel)
245
+ if not channels:
246
+ raise TypeError(
247
+ f"in_process_workers target has no ExecutionChannel with set_launcher: "
248
+ f"{type(target).__name__}"
249
+ )
250
+ return channels
251
+
252
+
253
+ def _install_launcher(
254
+ target: Any,
255
+ launcher: WorkerLauncher | None = None,
256
+ ) -> list[tuple[Any, WorkerLauncher]]:
257
+ if launcher is not None and not isinstance(launcher, WorkerLauncher):
258
+ raise TypeError(
259
+ f"in_process_workers expects a WorkerLauncher instance or None, "
260
+ f"got {type(launcher).__name__}"
261
+ )
262
+ effective_launcher: WorkerLauncher = (
263
+ launcher if launcher is not None else InProcessWorkerLauncher()
264
+ )
265
+ channels = _resolve_launcher_channels(target)
266
+ previous = [(channel, channel.set_launcher(effective_launcher)) for channel in channels]
267
+ return previous
268
+
269
+
270
+ @contextmanager
271
+ def authorizer(
272
+ target: Any,
273
+ fn: Callable[[str, str], bool | None] | None,
274
+ ) -> Iterator[Any]:
275
+ """临时为 TaskLite / EngineRuntime / StateStore 注入授权回调。
276
+
277
+ 进入上下文时保存原 ``authorizer`` 并替换为 ``fn``,退出时自动恢复原值,
278
+ 避免测试直接修改 ``_runtime.store.authorizer`` 导致跨用例污染。
279
+
280
+ Yields:
281
+ 被注入授权回调的底层 ``StateStore`` 实例。
282
+ """
283
+ rt = getattr(target, "_runtime", target)
284
+ store = getattr(rt, "store", rt)
285
+ if not hasattr(store, "authorizer"):
286
+ raise TypeError(
287
+ f"authorizer target has no store.authorizer attribute: {type(target).__name__}"
288
+ )
289
+ previous = store.authorizer
290
+ store.authorizer = fn
291
+ try:
292
+ yield store
293
+ finally:
294
+ store.authorizer = previous
295
+
296
+
297
+ @contextmanager
298
+ def in_process_workers(
299
+ target: Any,
300
+ launcher: WorkerLauncher | None = None,
301
+ ) -> Iterator[Any]:
302
+ """在测试上下文中启用进程内/自定义 ``WorkerLauncher`` 替身。
303
+
304
+ 保留真实单向 ``Pipe`` 通信边界,仅将子进程启动器替换为标准 ``WorkerLauncher``
305
+ 实例(默认使用线程替身 ``InProcessWorkerLauncher()``),退出上下文时自动恢复
306
+ 原有的 ``_launcher``。
307
+
308
+ Yields:
309
+ 传入的 ``target`` 对象本身。
310
+ """
311
+ previous = _install_launcher(target, launcher)
312
+ try:
313
+ yield target
314
+ finally:
315
+ for channel, orig in previous:
316
+ channel.set_launcher(orig)
317
+
318
+
319
+ def _resolve_filesystem_adapter(target: Any) -> Any:
320
+ """从 TaskLite / EngineRuntime / StateStore / PathPolicy / FilesystemAdapter 解析文件适配器。"""
321
+ current = getattr(target, "_runtime", target)
322
+ current = getattr(current, "store", current)
323
+ current = getattr(current, "path_policy", current)
324
+ adapter = getattr(current, "adapter", current)
325
+ if not callable(getattr(adapter, "delete_target", None)):
326
+ raise TypeError(
327
+ f"cleanup_hook target has no filesystem adapter delete_target: {type(target).__name__}"
328
+ )
329
+ return adapter
330
+
331
+
332
+ @contextmanager
333
+ def cleanup_hook(
334
+ target: Any,
335
+ handler: Callable[..., Any] | BaseException,
336
+ ) -> Iterator[Any]:
337
+ """受控注入产物清理故障或拦截钩子,退出时自动恢复原文件系统适配器。
338
+
339
+ Args:
340
+ target: TaskLite、EngineRuntime、StateStore、PathPolicy 或 FilesystemAdapter。
341
+ handler:
342
+ - 若传入 ``BaseException`` 实例,每次调用 ``delete_target`` 时直接抛出该异常;
343
+ - 若传入可调用对象,以 ``handler(original_delete, *args, **kwargs)`` 调用,
344
+ 支持先执行观察/故障/篡改逻辑再按需委托原始 ``original_delete``。
345
+ """
346
+ adapter = _resolve_filesystem_adapter(target)
347
+ had_instance_attr = "delete_target" in getattr(adapter, "__dict__", {})
348
+ original_delete = adapter.delete_target
349
+
350
+ if isinstance(handler, BaseException):
351
+ exc = handler
352
+
353
+ def _patched_delete(*args: Any, **kwargs: Any) -> Any:
354
+ raise exc
355
+ else:
356
+ def _patched_delete(*args: Any, **kwargs: Any) -> Any:
357
+ return handler(original_delete, *args, **kwargs)
358
+
359
+ adapter.delete_target = _patched_delete
360
+ try:
361
+ yield original_delete
362
+ finally:
363
+ if had_instance_attr:
364
+ adapter.delete_target = original_delete
365
+ else:
366
+ delattr(adapter, "delete_target")
367
+
368
+
369
+ @contextmanager
370
+ def observe_wait(
371
+ target: Any,
372
+ on_wait: Callable[[float | None], None] | None = None,
373
+ ) -> Iterator[list[float | None]]:
374
+ """受控观察主循环调度等待,记录每次等待超时并在进入等待时触发回调。
375
+
376
+ 退出上下文时自动恢复 ``EngineRuntime._wait``,避免测试直接反射篡改运行期方法。
377
+ """
378
+ rt = getattr(target, "_runtime", target)
379
+ had_instance_attr = "_wait" in getattr(rt, "__dict__", {})
380
+ original_wait = getattr(rt, "_wait", None)
381
+ if not callable(original_wait):
382
+ raise TypeError(
383
+ f"observe_wait target has no EngineRuntime._wait: {type(target).__name__}"
384
+ )
385
+ observed: list[float | None] = []
386
+
387
+ def _tracked_wait(timeout: float | None, last_seq: int | None = None) -> None:
388
+ observed.append(timeout)
389
+ if on_wait is not None:
390
+ on_wait(timeout)
391
+ original_wait(timeout, last_seq=last_seq)
392
+
393
+ rt._wait = _tracked_wait
394
+ try:
395
+ yield observed
396
+ finally:
397
+ if had_instance_attr:
398
+ rt._wait = original_wait
399
+ else:
400
+ delattr(rt, "_wait")
401
+
402
+
403
+ class FakeClock:
404
+ """可控单调时钟(供资源节流、超时等确定性测试使用)。"""
405
+
406
+ def __init__(self, start: float = 0.0) -> None:
407
+ self._now = float(start)
408
+
409
+ def __call__(self) -> float:
410
+ return self._now
411
+
412
+ def advance(self, seconds: float) -> float:
413
+ """向前推进单调时钟并返回新时间。"""
414
+ if seconds < 0:
415
+ raise ValueError(f"FakeClock.advance requires non-negative seconds, got {seconds}")
416
+ self._now += float(seconds)
417
+ return self._now
418
+
419
+ def set(self, value: float) -> None:
420
+ """直接设置当前单调时钟读数。"""
421
+ self._now = float(value)
422
+
423
+
424
+ @contextmanager
425
+ def monotonic_clock(start: float = 0.0) -> Iterator[FakeClock]:
426
+ """在测试上下文内替换 ``time.monotonic`` 为可控 ``FakeClock``,退出自动恢复。"""
427
+ clock = FakeClock(start=start)
428
+ original_monotonic = time.monotonic
429
+ time.monotonic = clock
430
+ try:
431
+ yield clock
432
+ finally:
433
+ time.monotonic = original_monotonic
434
+
435
+
436
+ def spawned_jobs(ctx: JobContext) -> list[Job]:
437
+ """返回 ``JobContext`` 在本次执行期间通过 ``ctx.spawn()`` 登记的子作业浅拷贝。"""
438
+ if not isinstance(ctx, JobContext):
439
+ raise TypeError(f"spawned_jobs expects a JobContext, got {type(ctx).__name__}")
440
+ return list(ctx._spawned_jobs)
441
+
442
+
443
+ def cursor_updates(ctx: JobContext) -> dict[str, str | None]:
444
+ """返回 ``JobContext`` 在本次执行期间通过 ``ctx.set_cursor()`` 暂存的游标更新浅拷贝。"""
445
+ if not isinstance(ctx, JobContext):
446
+ raise TypeError(f"cursor_updates expects a JobContext, got {type(ctx).__name__}")
447
+ return dict(ctx._cursor_updates)
30
448
 
31
449
 
32
450
  @contextmanager
@@ -58,7 +476,6 @@ def fake_ctx(
58
476
  failed: Iterable[str] | None = None,
59
477
  cursors: dict[str, str] | None = None,
60
478
  tmp_root: Any | None = None,
61
- output_roots: Any | None = None,
62
479
  base_dir: str | Path | None = None,
63
480
  known_job_uids: Iterable[str] | None = None,
64
481
  files: Mapping[str, Any] | None = None,
@@ -72,10 +489,9 @@ def fake_ctx(
72
489
  protected_roots: 不可声明为产物的框架状态目录。
73
490
  failed: 已进失败档案的 uid 集合。
74
491
  cursors: 游标初值映射(``ctx.get_cursor`` 命中源)。
75
- tmp_root: 提供时在其下创建一次性临时目录作为 ``output_roots``。
492
+ tmp_root: 提供时在其下创建一次性临时目录作为相对路径解析基准(``base_dir``)。
76
493
  目录清理由调用方的 tmp_root 机制(如 pytest tmp_path 的会话级
77
494
  回收)统一承担。
78
- output_roots: 直接指定产物根目录(与 tmp_root 互斥使用)。
79
495
  base_dir: 相对路径解析基准目录(默认当前工作目录)。
80
496
  known_job_uids: 允许被引用的已知 Job UID 集合(派生校验用)。
81
497
  files: 预声明文件规范字典(默认继承 job.files)。
@@ -83,19 +499,16 @@ def fake_ctx(
83
499
  Returns:
84
500
  JobContext: 内部容器结构由本函数兜底装配的真实上下文实例。
85
501
  """
86
- if tmp_root is not None:
87
- base = Path(tempfile.mkdtemp(dir=str(tmp_root)))
88
- output_roots = base
502
+ if tmp_root is not None and base_dir is None:
503
+ base_dir = Path(tempfile.mkdtemp(dir=str(tmp_root)))
89
504
  return JobContext(
90
505
  job,
91
506
  set(successes) if successes is not None else set(),
92
507
  set(failed) if failed is not None else set(),
93
508
  dict(cursors) if cursors is not None else {},
94
- output_roots=Path(output_roots) if isinstance(output_roots, str) else output_roots,
95
509
  artifact_descriptions=artifacts,
96
510
  protected_roots=protected_roots,
97
511
  base_dir=base_dir,
98
512
  known_job_uids=frozenset(known_job_uids) if known_job_uids is not None else None,
99
513
  files=files,
100
514
  )
101
-