tasklite-engine 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/PKG-INFO +14 -5
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/README.md +13 -4
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/__init__.py +18 -5
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/dispatch.py +3 -16
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/errorclass.py +16 -1
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/governor.py +147 -8
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/runtime.py +89 -9
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/scheduler.py +16 -0
- tasklite_engine-2.1.0/tasklite/engine/selection.py +244 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/store.py +126 -4
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/types.py +1 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/models/state.py +33 -6
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/pipeline.py +20 -0
- tasklite_engine-2.1.0/tasklite/selection.py +116 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite_engine.egg-info/PKG-INFO +14 -5
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite_engine.egg-info/SOURCES.txt +2 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/LICENSE +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/pyproject.toml +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/setup.cfg +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/backend/__init__.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/backend/base.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/backend/memory.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/backend/sqlite_backend.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/contrib/__init__.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/__init__.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/admission.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/channel.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/completion.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/config.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/in_flight.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/ops.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/recovery.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/resource.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/session.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/engine/wait.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/exceptions.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/hooks.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/models/__init__.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/models/attempt.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/models/context.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/models/job.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/models/task.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/py.typed +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/testing.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/utils/__init__.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/utils/clock.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/utils/encoding.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/utils/ipc.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/utils/jsonutil.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/utils/lockfile.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/wrappers/__init__.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/wrappers/discovery.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite/wrappers/http.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite_engine.egg-info/dependency_links.txt +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite_engine.egg-info/requires.txt +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tasklite_engine.egg-info/top_level.txt +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_encoding.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_exceptions.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_hooks.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_jsonutil.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_package_surface.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_packaging.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_pipeline.py +0 -0
- {tasklite_engine-2.0.0 → tasklite_engine-2.1.0}/tests/test_testing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasklite-engine
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
@@ -144,6 +144,13 @@ flowchart TD
|
|
|
144
144
|
- `SQLiteSnapshotStore` 提供单射键原始响应快照持久化,支持全离线幂等重放;
|
|
145
145
|
- 内置标准 Netscape `cookies.txt` 解析工具。
|
|
146
146
|
|
|
147
|
+
### 8. 🎛️ 执行选择 (Selection):Exclude 延迟留队 + Skip 终态跳过
|
|
148
|
+
- **第三类调度扩展 seam(候选准入/选择,与 OrderingPolicy / RequeuePolicy 并列,决策见 [ADR-0006](docs/adr/0006-selection-scheduling-seam.md))**:`pipeline.selection(Exclude(type="pdf_to_md"), Skip(type="ocr"))` 以方法调用配置 pipeline 级规则,两类语义严格区分:
|
|
149
|
+
- `Exclude`(延迟):命中作业不派发、不消耗资源与重试预算,队列行原样保留;下游允许作业本次报 `ERR_DEPENDENCY_EXCLUDED` 并计入 `stats["excluded_blocked"]`,下一次 run 自动恢复执行;
|
|
150
|
+
- `Skip`(终态):命中作业移出队列(不进 wall / 失败档案),下游整链级联跳过(attempt 记 `DEPENDENCY_SKIPPED`),下次运行不再重试;同一 task_type 同时命中时 Skip 优先。
|
|
151
|
+
- 停摆仲裁用 **Kahn 入度消减**(波前 = wall ∪ excluded)严格区分「真死锁」(依赖环/缺口 → 既有归因熔断)与「仅被 Exclude 阻塞」(留队 + COMPLETED 收敛);不存在固定等待兜底与忙旋,混合真实死锁不会被选择规则掩盖;
|
|
152
|
+
- 默认空规则与扩展前完全等价;`Rule` 协议要求 `matches_type(task_type)`(缺少则注册期 fail-loud),顶层导出 `Exclude` / `Selection` / `Skip`。
|
|
153
|
+
|
|
147
154
|
---
|
|
148
155
|
|
|
149
156
|
|
|
@@ -366,6 +373,7 @@ pipeline.clear_history("download::")
|
|
|
366
373
|
```
|
|
367
374
|
tasklite/
|
|
368
375
|
├── pipeline.py # TaskLite 核心门面与配置
|
|
376
|
+
├── selection.py # 执行选择规则值对象 Exclude/Skip(候选准入 seam,ADR-0006)
|
|
369
377
|
├── hooks.py # 用户工具函数(job_ref / progress_hook / slice_list)
|
|
370
378
|
├── engine/ # 核心执行机器群
|
|
371
379
|
│ ├── runtime.py # 核心运行期深模块、主循环事件泵 (EngineRuntime)
|
|
@@ -373,15 +381,16 @@ tasklite/
|
|
|
373
381
|
│ ├── session.py # RunSession 单次 run 生命周期状态与钩子单一出口
|
|
374
382
|
│ ├── wait.py # 事件泵等待/空闲决策纯函数
|
|
375
383
|
│ ├── types.py # 引擎公共值对象单一真相源
|
|
376
|
-
│ ├── store.py # 状态事务、3-strike
|
|
384
|
+
│ ├── store.py # 状态事务、3-strike 崩溃与失败写入隔离 (StateStore)
|
|
377
385
|
│ ├── ops.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
|
|
378
386
|
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
379
387
|
│ ├── completion.py # 任务完成与提交 (CompletionMachine)
|
|
380
388
|
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
|
|
381
389
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
382
|
-
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler / OrderingPolicy seam)
|
|
383
|
-
│ ├── admission.py # rerun 准入 (RerunPolicy) 与重入队节奏 (RequeuePolicy)
|
|
384
|
-
│ ├── governor.py #
|
|
390
|
+
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler / OrderingPolicy 排序 seam / Selection 候选准入 seam)
|
|
391
|
+
│ ├── admission.py # rerun 准入 (RerunPolicy) 与重入队节奏 (RequeuePolicy seam)
|
|
392
|
+
│ ├── governor.py # 死锁归因仲裁与依赖宽限,消费选择停摆事实 (DeadlockGovernor)
|
|
393
|
+
│ ├── selection.py # Skip 闭包与选择停摆图事实 (SelectionMachine)
|
|
385
394
|
│ ├── in_flight.py # 在途作业租约追踪 (InFlightTracker)
|
|
386
395
|
│ ├── errorclass.py # 错误分类与校验分类深模块 (ErrorClassifier)
|
|
387
396
|
│ └── resource.py # 令牌桶限速与并发容量资源
|
|
@@ -115,6 +115,13 @@ flowchart TD
|
|
|
115
115
|
- `SQLiteSnapshotStore` 提供单射键原始响应快照持久化,支持全离线幂等重放;
|
|
116
116
|
- 内置标准 Netscape `cookies.txt` 解析工具。
|
|
117
117
|
|
|
118
|
+
### 8. 🎛️ 执行选择 (Selection):Exclude 延迟留队 + Skip 终态跳过
|
|
119
|
+
- **第三类调度扩展 seam(候选准入/选择,与 OrderingPolicy / RequeuePolicy 并列,决策见 [ADR-0006](docs/adr/0006-selection-scheduling-seam.md))**:`pipeline.selection(Exclude(type="pdf_to_md"), Skip(type="ocr"))` 以方法调用配置 pipeline 级规则,两类语义严格区分:
|
|
120
|
+
- `Exclude`(延迟):命中作业不派发、不消耗资源与重试预算,队列行原样保留;下游允许作业本次报 `ERR_DEPENDENCY_EXCLUDED` 并计入 `stats["excluded_blocked"]`,下一次 run 自动恢复执行;
|
|
121
|
+
- `Skip`(终态):命中作业移出队列(不进 wall / 失败档案),下游整链级联跳过(attempt 记 `DEPENDENCY_SKIPPED`),下次运行不再重试;同一 task_type 同时命中时 Skip 优先。
|
|
122
|
+
- 停摆仲裁用 **Kahn 入度消减**(波前 = wall ∪ excluded)严格区分「真死锁」(依赖环/缺口 → 既有归因熔断)与「仅被 Exclude 阻塞」(留队 + COMPLETED 收敛);不存在固定等待兜底与忙旋,混合真实死锁不会被选择规则掩盖;
|
|
123
|
+
- 默认空规则与扩展前完全等价;`Rule` 协议要求 `matches_type(task_type)`(缺少则注册期 fail-loud),顶层导出 `Exclude` / `Selection` / `Skip`。
|
|
124
|
+
|
|
118
125
|
---
|
|
119
126
|
|
|
120
127
|
|
|
@@ -337,6 +344,7 @@ pipeline.clear_history("download::")
|
|
|
337
344
|
```
|
|
338
345
|
tasklite/
|
|
339
346
|
├── pipeline.py # TaskLite 核心门面与配置
|
|
347
|
+
├── selection.py # 执行选择规则值对象 Exclude/Skip(候选准入 seam,ADR-0006)
|
|
340
348
|
├── hooks.py # 用户工具函数(job_ref / progress_hook / slice_list)
|
|
341
349
|
├── engine/ # 核心执行机器群
|
|
342
350
|
│ ├── runtime.py # 核心运行期深模块、主循环事件泵 (EngineRuntime)
|
|
@@ -344,15 +352,16 @@ tasklite/
|
|
|
344
352
|
│ ├── session.py # RunSession 单次 run 生命周期状态与钩子单一出口
|
|
345
353
|
│ ├── wait.py # 事件泵等待/空闲决策纯函数
|
|
346
354
|
│ ├── types.py # 引擎公共值对象单一真相源
|
|
347
|
-
│ ├── store.py # 状态事务、3-strike
|
|
355
|
+
│ ├── store.py # 状态事务、3-strike 崩溃与失败写入隔离 (StateStore)
|
|
348
356
|
│ ├── ops.py # OpsConsole:run() 外纯运维接缝(管理 API 委托目标)
|
|
349
357
|
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
350
358
|
│ ├── completion.py # 任务完成与提交 (CompletionMachine)
|
|
351
359
|
│ ├── recovery.py # 崩溃检测与恢复 (RecoveryOrchestrator)
|
|
352
360
|
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel / WorkerLaunchSpec)
|
|
353
|
-
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler / OrderingPolicy seam)
|
|
354
|
-
│ ├── admission.py # rerun 准入 (RerunPolicy) 与重入队节奏 (RequeuePolicy)
|
|
355
|
-
│ ├── governor.py #
|
|
361
|
+
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler / OrderingPolicy 排序 seam / Selection 候选准入 seam)
|
|
362
|
+
│ ├── admission.py # rerun 准入 (RerunPolicy) 与重入队节奏 (RequeuePolicy seam)
|
|
363
|
+
│ ├── governor.py # 死锁归因仲裁与依赖宽限,消费选择停摆事实 (DeadlockGovernor)
|
|
364
|
+
│ ├── selection.py # Skip 闭包与选择停摆图事实 (SelectionMachine)
|
|
356
365
|
│ ├── in_flight.py # 在途作业租约追踪 (InFlightTracker)
|
|
357
366
|
│ ├── errorclass.py # 错误分类与校验分类深模块 (ErrorClassifier)
|
|
358
367
|
│ └── resource.py # 令牌桶限速与并发容量资源
|
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
|
|
8
8
|
公开面:TaskLite 门面(六步调用契约)、模型层、异常族、错误分类
|
|
9
9
|
(ErrorClassifier 与 ERR_* 错误码)、单射编码族、资源体系、失败档案
|
|
10
|
-
条目、用户钩子三件套、准入与重入队策略 seam、排序策略 seam
|
|
11
|
-
|
|
12
|
-
...`` 是唯一承诺稳定的导入面)。``__version__``
|
|
13
|
-
|
|
10
|
+
条目、用户钩子三件套、准入与重入队策略 seam、排序策略 seam、执行选择
|
|
11
|
+
规则值对象、持久化后端与运行摘要值对象,均经本文件集中导出
|
|
12
|
+
(``from tasklite import ...`` 是唯一承诺稳定的导入面)。``__version__``
|
|
13
|
+
为版本单一事实源(pyproject 动态读取点)。
|
|
14
14
|
|
|
15
15
|
分层单向依赖红线:``tasklite/models/`` 严禁 import
|
|
16
16
|
``tasklite/engine/``(IPC 声明读写一律下沉 ``tasklite/utils/ipc.py``),
|
|
@@ -22,11 +22,14 @@
|
|
|
22
22
|
"""
|
|
23
23
|
from __future__ import annotations
|
|
24
24
|
|
|
25
|
-
__version__ = "2.
|
|
25
|
+
__version__ = "2.1.0"
|
|
26
26
|
|
|
27
27
|
# ── 门面(六步调用契约的唯一入口)────────────────────────────────────
|
|
28
28
|
from .pipeline import TaskLite
|
|
29
29
|
|
|
30
|
+
# ── 执行选择(pipeline 级规则值对象与规则集)─────────────────────────
|
|
31
|
+
from .selection import Exclude, Selection, Skip
|
|
32
|
+
|
|
30
33
|
# ── 模型层:规格 / 实例 / 轨迹 / 上下文 ──────────────────────────────
|
|
31
34
|
from .models.attempt import AttemptRecord
|
|
32
35
|
from .models.context import JobContext
|
|
@@ -49,6 +52,8 @@ from .engine.errorclass import (
|
|
|
49
52
|
ERR_COMMIT_FAILURE,
|
|
50
53
|
ERR_DEADLOCK_GAP,
|
|
51
54
|
ERR_DEPENDENCY_DEADLOCK,
|
|
55
|
+
ERR_DEPENDENCY_EXCLUDED,
|
|
56
|
+
ERR_DEPENDENCY_SKIPPED,
|
|
52
57
|
ERR_DISPATCH_FAILURE,
|
|
53
58
|
ERR_IPC_WRITE_DEGRADED_PREFIX,
|
|
54
59
|
ERR_JOB_DEPENDENCY,
|
|
@@ -60,6 +65,7 @@ from .engine.errorclass import (
|
|
|
60
65
|
ERR_PROCESS_CRASH_PREFIX,
|
|
61
66
|
ERR_PROCESS_SIGNAL_DEATH,
|
|
62
67
|
ERR_RESOURCE_DEADLOCK,
|
|
68
|
+
ERR_SKIPPED,
|
|
63
69
|
ERR_TIMEOUT_PREFIX,
|
|
64
70
|
FATAL_EXCEPTIONS,
|
|
65
71
|
TRANSIENT_EXCEPTIONS,
|
|
@@ -123,6 +129,10 @@ from .utils import jsonutil
|
|
|
123
129
|
__all__ = [
|
|
124
130
|
# 门面
|
|
125
131
|
"TaskLite",
|
|
132
|
+
# 执行选择
|
|
133
|
+
"Exclude",
|
|
134
|
+
"Selection",
|
|
135
|
+
"Skip",
|
|
126
136
|
# 模型层
|
|
127
137
|
"AttemptRecord",
|
|
128
138
|
"Job",
|
|
@@ -160,6 +170,9 @@ __all__ = [
|
|
|
160
170
|
"ERR_COMMIT_FAILURE",
|
|
161
171
|
"ERR_DISPATCH_FAILURE",
|
|
162
172
|
"ERR_DEADLOCK_GAP",
|
|
173
|
+
"ERR_DEPENDENCY_EXCLUDED",
|
|
174
|
+
"ERR_DEPENDENCY_SKIPPED",
|
|
175
|
+
"ERR_SKIPPED",
|
|
163
176
|
"ERR_TIMEOUT_PREFIX",
|
|
164
177
|
"ERR_PROCESS_CRASH_PREFIX",
|
|
165
178
|
"ERR_NO_IPC_RESULT",
|
|
@@ -59,7 +59,7 @@ if TYPE_CHECKING:
|
|
|
59
59
|
def __contains__(self, task_type: object) -> bool: ...
|
|
60
60
|
|
|
61
61
|
from ..exceptions import _CommitCrashSignal, _JobTerminated
|
|
62
|
-
from ..models.attempt import ATTEMPT_REQUEUED
|
|
62
|
+
from ..models.attempt import ATTEMPT_REQUEUED
|
|
63
63
|
from ..models.context import JobContext
|
|
64
64
|
from ..models.job import Job, JobRuntimeState
|
|
65
65
|
from .channel import ArtifactCleanupMode, WorkerLaunchSpec
|
|
@@ -75,7 +75,6 @@ from .resource import RateLimitUnavailable, apply_suspend_signals
|
|
|
75
75
|
from .scheduler import ScheduleResult, StandstillFacts
|
|
76
76
|
from .types import AttemptFinish
|
|
77
77
|
from .wait import IN_FLIGHT_POLL_SECONDS
|
|
78
|
-
from ..utils.clock import utc_now_iso
|
|
79
78
|
|
|
80
79
|
logger = logging.getLogger("tasklite")
|
|
81
80
|
|
|
@@ -400,23 +399,11 @@ class DispatchMachine:
|
|
|
400
399
|
attempts 是旁路观测面,观测缺口不得击落可正常执行的派发。
|
|
401
400
|
"""
|
|
402
401
|
incarnation = f"{self._session.run_id}.{self._session.next_dispatch_seq()}"
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
activation_no=job.activation_no,
|
|
406
|
-
attempt_no=job.attempt_no,
|
|
402
|
+
attempt_id = self._store.record_attempt_dispatch(
|
|
403
|
+
job,
|
|
407
404
|
incarnation=incarnation,
|
|
408
405
|
run_id=self._session.run_id,
|
|
409
|
-
started_at=utc_now_iso(),
|
|
410
|
-
outcome=ATTEMPT_RUNNING,
|
|
411
406
|
)
|
|
412
|
-
try:
|
|
413
|
-
attempt_id = self._store.backend.append_attempt(record)
|
|
414
|
-
except Exception as e:
|
|
415
|
-
logger.warning(
|
|
416
|
-
f"Attempt trace append degraded for {job.uid}: {e}; "
|
|
417
|
-
f"dispatching without trace row"
|
|
418
|
-
)
|
|
419
|
-
return None, incarnation
|
|
420
407
|
return attempt_id, incarnation
|
|
421
408
|
|
|
422
409
|
def _spawn_entry(
|
|
@@ -20,6 +20,13 @@ from ..utils.clock import utc_now_iso
|
|
|
20
20
|
|
|
21
21
|
_UnionType = getattr(types, "UnionType", None)
|
|
22
22
|
|
|
23
|
+
# SIGKILL 仅 POSIX 提供,Windows 无该常量。oom_hint 是「进程被 OOM killer
|
|
24
|
+
# 击杀」的启发式标记:无 SIGKILL 的平台恒为 False(负 exitcode 仍按信号
|
|
25
|
+
# 死亡归因)。平台差异而非降级——强杀路径在 Windows 由 proc.kill() 经
|
|
26
|
+
# TerminateProcess 等价完成(看门狗/中止路径共用),本条只影响 OOM 归因位,
|
|
27
|
+
# 绝不因平台缺常量而抛 AttributeError。
|
|
28
|
+
_SIGKILL: int | None = getattr(signal, "SIGKILL", None)
|
|
29
|
+
|
|
23
30
|
# ── 框架错误码常量(持久化协议:失败档案与重试账目按这些串归因,
|
|
24
31
|
# 串值属跨进程数据契约,不做词汇改写) ──────────────────────────────
|
|
25
32
|
ERR_DEPENDENCY_DEADLOCK = "DEPENDENCY_DEADLOCK"
|
|
@@ -32,6 +39,11 @@ ERR_MALFORMED_JOB = "MALFORMED_JOB"
|
|
|
32
39
|
ERR_COMMIT_FAILURE = "COMMIT_FAILURE"
|
|
33
40
|
ERR_DISPATCH_FAILURE = "DISPATCH_FAILURE"
|
|
34
41
|
ERR_DEADLOCK_GAP = "DEADLOCK_CLASSIFICATION_GAP"
|
|
42
|
+
# 执行选择(Selection)归因:旁路观测/运行日志口径,不进入失败档案
|
|
43
|
+
# (Skip 为终态跳过,Exclude 下游为延迟留队,均非失败)。
|
|
44
|
+
ERR_SKIPPED = "SKIPPED"
|
|
45
|
+
ERR_DEPENDENCY_SKIPPED = "DEPENDENCY_SKIPPED"
|
|
46
|
+
ERR_DEPENDENCY_EXCLUDED = "DEPENDENCY_EXCLUDED"
|
|
35
47
|
|
|
36
48
|
# ── 子进程死亡归因错误串前缀(classify_process_death 决策表产出,
|
|
37
49
|
# 与 _classify_string 映射同源;失败档案落盘串按前缀匹配分类) ──────
|
|
@@ -429,7 +441,7 @@ class ErrorClassifier:
|
|
|
429
441
|
result_meta={
|
|
430
442
|
"error": f"{ERR_PROCESS_CRASH_PREFIX}{exitcode}",
|
|
431
443
|
"signal": sig_name,
|
|
432
|
-
"oom_hint": -exitcode ==
|
|
444
|
+
"oom_hint": _SIGKILL is not None and -exitcode == _SIGKILL,
|
|
433
445
|
},
|
|
434
446
|
retry_error=f"{ERR_PROCESS_SIGNAL_DEATH}: killed by {sig_name} ({exitcode})",
|
|
435
447
|
failed_error_type=ErrorCategory.TRANSIENT_EXHAUSTED.value,
|
|
@@ -931,6 +943,9 @@ __all__ = [
|
|
|
931
943
|
"ERR_COMMIT_FAILURE",
|
|
932
944
|
"ERR_DISPATCH_FAILURE",
|
|
933
945
|
"ERR_DEADLOCK_GAP",
|
|
946
|
+
"ERR_SKIPPED",
|
|
947
|
+
"ERR_DEPENDENCY_SKIPPED",
|
|
948
|
+
"ERR_DEPENDENCY_EXCLUDED",
|
|
934
949
|
# 死亡归因错误串前缀
|
|
935
950
|
"ERR_TIMEOUT_PREFIX",
|
|
936
951
|
"ERR_PROCESS_CRASH_PREFIX",
|
|
@@ -26,6 +26,7 @@ from .errorclass import (
|
|
|
26
26
|
ERR_RESOURCE_DEADLOCK,
|
|
27
27
|
)
|
|
28
28
|
from .scheduler import JobFacts, StandstillFacts
|
|
29
|
+
from .selection import SelectionStandstill
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger("tasklite")
|
|
31
32
|
|
|
@@ -66,16 +67,26 @@ class ArbitrationStore(Protocol):
|
|
|
66
67
|
) -> BulkFailureOutcome: ...
|
|
67
68
|
|
|
68
69
|
|
|
70
|
+
class SelectionStandstillSource(Protocol):
|
|
71
|
+
"""死锁仲裁消费的执行选择停摆事实源契约(SelectionMachine 自动满足)。"""
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def is_active(self) -> bool: ...
|
|
75
|
+
|
|
76
|
+
def partition_standstill(self) -> SelectionStandstill | None: ...
|
|
77
|
+
|
|
78
|
+
|
|
69
79
|
@dataclass(frozen=True)
|
|
70
80
|
class DeadlockDecision:
|
|
71
81
|
"""死锁治理裁决结果(纯值对象,无阻塞副作用)。
|
|
72
82
|
|
|
73
|
-
- action: "resolved" (已归因并移入失败档案) | "
|
|
74
|
-
| "gap_retrying" (分类缺口重试中) | "none" (未检测到死锁)
|
|
75
|
-
- should_terminate:
|
|
83
|
+
- action: "resolved" (已归因并移入失败档案) | "selection_blocked" (队列仅因执行选择排除而延迟)
|
|
84
|
+
| "grace_waiting" (正在依赖宽限期中) | "gap_retrying" (分类缺口重试中) | "none" (未检测到死锁)
|
|
85
|
+
- should_terminate: 是否应终止主循环(如全队列熔断完毕且队列清空,或选择排除干净收敛)
|
|
76
86
|
- wait_time: 建议外层事件泵等待的时延秒数(宽限或重试时为 0.5s,否则 0.0s)
|
|
77
87
|
- failed_uids: 本轮判定失败的 UID 列表
|
|
78
88
|
- cascaded_uids: 本轮级联失败的 UID 列表
|
|
89
|
+
- standstill: 选择停摆事实快照(仅在 action="selection_blocked" 时附带)
|
|
79
90
|
"""
|
|
80
91
|
|
|
81
92
|
action: str
|
|
@@ -83,6 +94,7 @@ class DeadlockDecision:
|
|
|
83
94
|
wait_time: float = 0.0
|
|
84
95
|
failed_uids: list[str] = field(default_factory=list)
|
|
85
96
|
cascaded_uids: list[str] = field(default_factory=list)
|
|
97
|
+
standstill: SelectionStandstill | None = None
|
|
86
98
|
|
|
87
99
|
|
|
88
100
|
@dataclass
|
|
@@ -281,10 +293,14 @@ class DeadlockGovernor:
|
|
|
281
293
|
self,
|
|
282
294
|
facts: StandstillFacts | None,
|
|
283
295
|
store: ArbitrationStore,
|
|
296
|
+
*,
|
|
297
|
+
selection_machine: SelectionStandstillSource | None = None,
|
|
284
298
|
) -> DeadlockDecision:
|
|
285
299
|
"""自闭环死锁仲裁单一入口。
|
|
286
300
|
|
|
287
301
|
评估停摆事实:
|
|
302
|
+
0. 若传入 selection_machine 且激活,且无已知死锁成因,先做 Kahn 拓扑消减;
|
|
303
|
+
若判定为「仅因 Exclude 阻塞」,闭环返回 action="selection_blocked";
|
|
288
304
|
1. 若 min_wait == inf,直接触发 resolve_deadlock 进行细粒度归因与批量熔断;
|
|
289
305
|
2. 若 min_wait 为有限值,但处于 waiting_for_dependency 且存在拓扑成环
|
|
290
306
|
(find_dependency_cycles),自动识别隐式死锁并触发 resolve_deadlock;
|
|
@@ -294,17 +310,61 @@ class DeadlockGovernor:
|
|
|
294
310
|
self.deadlock_gap_rounds = 0
|
|
295
311
|
return DeadlockDecision(action="none", should_terminate=False, wait_time=0.0)
|
|
296
312
|
|
|
313
|
+
selection_active = selection_machine is not None and selection_machine.is_active
|
|
314
|
+
selection_standstill = (
|
|
315
|
+
selection_machine.partition_standstill()
|
|
316
|
+
if selection_active and selection_machine is not None
|
|
317
|
+
else None
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
# 0. 执行选择先验判定:无明确死锁成因时先检查是否仅因 Exclude 阻塞
|
|
321
|
+
if (
|
|
322
|
+
selection_active
|
|
323
|
+
and selection_machine is not None
|
|
324
|
+
and not facts.attribution.has_deadlock_causes
|
|
325
|
+
):
|
|
326
|
+
if (
|
|
327
|
+
selection_standstill is not None
|
|
328
|
+
and selection_standstill.only_blocked_by_exclusion
|
|
329
|
+
):
|
|
330
|
+
self.deadlock_gap_rounds = 0
|
|
331
|
+
return DeadlockDecision(
|
|
332
|
+
action="selection_blocked",
|
|
333
|
+
should_terminate=True,
|
|
334
|
+
wait_time=0.0,
|
|
335
|
+
standstill=selection_standstill,
|
|
336
|
+
)
|
|
337
|
+
|
|
297
338
|
if facts.min_wait == float("inf"):
|
|
298
|
-
return self.
|
|
339
|
+
return self._resolve_deadlock(
|
|
340
|
+
facts,
|
|
341
|
+
store=store,
|
|
342
|
+
selection_machine=selection_machine,
|
|
343
|
+
selection_standstill=selection_standstill,
|
|
344
|
+
scheduler=None,
|
|
345
|
+
)
|
|
299
346
|
|
|
300
347
|
if facts.waiting_for_dependency and store.state is not None:
|
|
301
|
-
|
|
348
|
+
if selection_active:
|
|
349
|
+
cycle_uids = (
|
|
350
|
+
selection_standstill.cycle_uids
|
|
351
|
+
if selection_standstill is not None
|
|
352
|
+
else ()
|
|
353
|
+
)
|
|
354
|
+
else:
|
|
355
|
+
cycle_uids = store.state.find_dependency_cycles()
|
|
302
356
|
if cycle_uids:
|
|
303
357
|
logger.error(
|
|
304
358
|
f"Deadlock detected during wait: dependency cycle "
|
|
305
359
|
f"{sorted(set(cycle_uids))} masked by finite min_wait."
|
|
306
360
|
)
|
|
307
|
-
return self.
|
|
361
|
+
return self._resolve_deadlock(
|
|
362
|
+
facts,
|
|
363
|
+
store=store,
|
|
364
|
+
selection_machine=selection_machine,
|
|
365
|
+
selection_standstill=selection_standstill,
|
|
366
|
+
scheduler=None,
|
|
367
|
+
)
|
|
308
368
|
|
|
309
369
|
# 正常等待/背压结论(有限等待且无环)证明上一缺口 episode 已消解——
|
|
310
370
|
# 缺口升级只统计连续轮次,非连续缺口从零计数
|
|
@@ -317,11 +377,36 @@ class DeadlockGovernor:
|
|
|
317
377
|
store: ArbitrationStore,
|
|
318
378
|
*,
|
|
319
379
|
scheduler: ScanCache | None = None,
|
|
380
|
+
selection_machine: SelectionStandstillSource | None = None,
|
|
381
|
+
) -> DeadlockDecision:
|
|
382
|
+
"""直接执行死锁归因,并在入口读取当前选择停摆事实。"""
|
|
383
|
+
selection_standstill = (
|
|
384
|
+
selection_machine.partition_standstill()
|
|
385
|
+
if selection_machine is not None and selection_machine.is_active
|
|
386
|
+
else None
|
|
387
|
+
)
|
|
388
|
+
return self._resolve_deadlock(
|
|
389
|
+
facts,
|
|
390
|
+
store=store,
|
|
391
|
+
scheduler=scheduler,
|
|
392
|
+
selection_machine=selection_machine,
|
|
393
|
+
selection_standstill=selection_standstill,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def _resolve_deadlock(
|
|
397
|
+
self,
|
|
398
|
+
facts: StandstillFacts,
|
|
399
|
+
store: ArbitrationStore,
|
|
400
|
+
*,
|
|
401
|
+
scheduler: ScanCache | None,
|
|
402
|
+
selection_machine: SelectionStandstillSource | None,
|
|
403
|
+
selection_standstill: SelectionStandstill | None,
|
|
320
404
|
) -> DeadlockDecision:
|
|
321
|
-
"""
|
|
405
|
+
"""以同一停摆快照完成归因与状态提交。"""
|
|
322
406
|
effective_state = store.state
|
|
323
407
|
effective_grace = self.dep_grace_seconds
|
|
324
408
|
effective_gap_max = self.deadlock_gap_max_rounds
|
|
409
|
+
selection_active = selection_machine is not None and selection_machine.is_active
|
|
325
410
|
|
|
326
411
|
attr = facts.attribution
|
|
327
412
|
malformed_uids = set(attr.malformed_uids)
|
|
@@ -365,7 +450,14 @@ class DeadlockGovernor:
|
|
|
365
450
|
list(effective_state.queue), impossible_uids, ERR_RESOURCE_DEADLOCK
|
|
366
451
|
)
|
|
367
452
|
elif facts.waiting_for_dependency:
|
|
368
|
-
|
|
453
|
+
if selection_active:
|
|
454
|
+
cycle_uids = set(
|
|
455
|
+
selection_standstill.cycle_uids
|
|
456
|
+
if selection_standstill is not None
|
|
457
|
+
else ()
|
|
458
|
+
)
|
|
459
|
+
else:
|
|
460
|
+
cycle_uids = set(effective_state.find_dependency_cycles())
|
|
369
461
|
if not cycle_uids:
|
|
370
462
|
escalated = self.check_gap_or_escalate(
|
|
371
463
|
"Deadlock classification gap (waiting_for_dependency without cycle)",
|
|
@@ -406,6 +498,52 @@ class DeadlockGovernor:
|
|
|
406
498
|
# 提交批量死锁失败(失败档案单一出口)
|
|
407
499
|
outcome = store.apply_bulk_failure(uids_metas)
|
|
408
500
|
self.deadlock_gap_rounds = 0
|
|
501
|
+
|
|
502
|
+
# resolved 但零归档时的选择收敛闭环(如归档全部被选择过滤)
|
|
503
|
+
if (
|
|
504
|
+
not outcome.failed_uids
|
|
505
|
+
and selection_machine is not None
|
|
506
|
+
and selection_machine.is_active
|
|
507
|
+
):
|
|
508
|
+
# 批量写入可能改变队列;归档为零时再基于当前队列收敛。
|
|
509
|
+
standstill = selection_machine.partition_standstill()
|
|
510
|
+
if standstill is not None and standstill.only_blocked_by_exclusion:
|
|
511
|
+
return DeadlockDecision(
|
|
512
|
+
action="selection_blocked",
|
|
513
|
+
should_terminate=True,
|
|
514
|
+
wait_time=0.0,
|
|
515
|
+
standstill=standstill,
|
|
516
|
+
)
|
|
517
|
+
if standstill is not None and standstill.stuck_uids:
|
|
518
|
+
logger.error(
|
|
519
|
+
"Deadlock arbitration resolved without archiving while allowed "
|
|
520
|
+
f"jobs remain stuck ({list(standstill.stuck_uids)}); attributing "
|
|
521
|
+
f"them to {ERR_DEADLOCK_GAP} to guarantee queue reduction."
|
|
522
|
+
)
|
|
523
|
+
fallback_outcome = store.apply_bulk_failure(
|
|
524
|
+
[
|
|
525
|
+
(uid, {"error": ERR_DEADLOCK_GAP, "root_cause": True})
|
|
526
|
+
for uid in standstill.stuck_uids
|
|
527
|
+
]
|
|
528
|
+
)
|
|
529
|
+
if fallback_outcome.failed_uids:
|
|
530
|
+
return DeadlockDecision(
|
|
531
|
+
action="resolved",
|
|
532
|
+
should_terminate=len(effective_state.queue) == 0,
|
|
533
|
+
wait_time=0.0,
|
|
534
|
+
failed_uids=fallback_outcome.failed_uids,
|
|
535
|
+
cascaded_uids=fallback_outcome.cascaded_uids,
|
|
536
|
+
)
|
|
537
|
+
logger.critical(
|
|
538
|
+
"Deadlock arbitration reported 'resolved' without failing any job; "
|
|
539
|
+
"terminating the run instead of spinning (queue preserved on disk)."
|
|
540
|
+
)
|
|
541
|
+
return DeadlockDecision(
|
|
542
|
+
action="resolved",
|
|
543
|
+
should_terminate=True,
|
|
544
|
+
wait_time=0.0,
|
|
545
|
+
)
|
|
546
|
+
|
|
409
547
|
should_terminate = len(outcome.failed_uids) > 0 and len(effective_state.queue) == 0
|
|
410
548
|
return DeadlockDecision(
|
|
411
549
|
action="resolved",
|
|
@@ -424,4 +562,5 @@ __all__ = [
|
|
|
424
562
|
"DeadlockDecision",
|
|
425
563
|
"DeadlockGovernor",
|
|
426
564
|
"ScanCache",
|
|
565
|
+
"SelectionStandstill",
|
|
427
566
|
]
|
|
@@ -21,10 +21,14 @@ from .config import RunConfig
|
|
|
21
21
|
from .dispatch import DispatchKind, DispatchMachine, DispatchOutcome
|
|
22
22
|
from .channel import ExecutionChannel
|
|
23
23
|
from .completion import CompletionMachine
|
|
24
|
-
from .errorclass import
|
|
24
|
+
from .errorclass import (
|
|
25
|
+
ERR_DEPENDENCY_EXCLUDED,
|
|
26
|
+
ErrorClassifier,
|
|
27
|
+
)
|
|
28
|
+
from .selection import SelectionMachine, SelectionStandstill
|
|
25
29
|
from .in_flight import InFlightTracker
|
|
26
30
|
from .recovery import RecoveryOrchestrator
|
|
27
|
-
from .scheduler import JobScheduler
|
|
31
|
+
from .scheduler import DeadlockAttribution, JobScheduler
|
|
28
32
|
from .session import RunSession
|
|
29
33
|
from .store import StateStore
|
|
30
34
|
from .types import (
|
|
@@ -38,7 +42,9 @@ from .types import (
|
|
|
38
42
|
from .wait import LoopFacts, decide_wait
|
|
39
43
|
from ..backend.base import AbstractStateBackend
|
|
40
44
|
from ..exceptions import _CommitCrashSignal, _JobTerminated
|
|
41
|
-
from ..models.
|
|
45
|
+
from ..models.job import Job
|
|
46
|
+
from ..models.state import PipelineState, uid_from_job_dict
|
|
47
|
+
from ..selection import Selection
|
|
42
48
|
from ..utils.lockfile import release_lock, try_acquire_lock
|
|
43
49
|
|
|
44
50
|
logger = logging.getLogger("tasklite")
|
|
@@ -132,6 +138,27 @@ class EngineRuntime:
|
|
|
132
138
|
|
|
133
139
|
self._run_lock_fd: int | None = None
|
|
134
140
|
self._is_running: bool = False
|
|
141
|
+
# 执行选择(pipeline 属性):None = 默认全执行;门面经 set_selection
|
|
142
|
+
# 注入后同步到调度器与 StateStore(终态写入出口),execute() 启动期
|
|
143
|
+
# 再固化一次即 per-run 快照。
|
|
144
|
+
self._selection: Selection | None = None
|
|
145
|
+
self.selection_machine: SelectionMachine | None = None
|
|
146
|
+
# Skip 终态跳过的 run 级状态:脏标记驱动跳过阶段(run 启动 + 派生作业落队后)
|
|
147
|
+
self._skip_dirty: bool = False
|
|
148
|
+
|
|
149
|
+
def set_selection(self, selection: Selection | None) -> None:
|
|
150
|
+
"""注入执行选择(仅 run() 外;门面已守卫,此处提供运行期双保险)。"""
|
|
151
|
+
if self._is_running:
|
|
152
|
+
raise RuntimeError(
|
|
153
|
+
"selection() is only allowed outside run(); current run is in progress."
|
|
154
|
+
)
|
|
155
|
+
self._selection = selection
|
|
156
|
+
if selection is not None and selection.is_active:
|
|
157
|
+
self.selection_machine = SelectionMachine(selection, self.store)
|
|
158
|
+
else:
|
|
159
|
+
self.selection_machine = None
|
|
160
|
+
self.scheduler.selection = selection
|
|
161
|
+
self.store._set_selection(selection)
|
|
135
162
|
|
|
136
163
|
@property
|
|
137
164
|
def session(self) -> RunSession:
|
|
@@ -184,6 +211,11 @@ class EngineRuntime:
|
|
|
184
211
|
if self._is_running:
|
|
185
212
|
raise RuntimeError("Pipeline run() already in progress on this instance.")
|
|
186
213
|
self._is_running = True
|
|
214
|
+
if self.selection_machine is not None:
|
|
215
|
+
self.selection_machine.reset_run()
|
|
216
|
+
self.scheduler.selection = self._selection
|
|
217
|
+
self.store._set_selection(self._selection)
|
|
218
|
+
self._skip_dirty = False
|
|
187
219
|
|
|
188
220
|
old_sigterm = None
|
|
189
221
|
old_sigint = None
|
|
@@ -319,6 +351,9 @@ class EngineRuntime:
|
|
|
319
351
|
dispatched += 1
|
|
320
352
|
continue
|
|
321
353
|
if outcome.kind is DispatchKind.HANDLED_NO_SUBPROCESS:
|
|
354
|
+
# 残留结果认领可派生作业,必须在后续派发与仲裁前收敛 Skip。
|
|
355
|
+
if self._session.stop_mode is StopMode.NONE:
|
|
356
|
+
self._apply_selection_skips()
|
|
322
357
|
continue
|
|
323
358
|
break
|
|
324
359
|
return last_outcome, worker_wait, dispatched
|
|
@@ -326,7 +361,7 @@ class EngineRuntime:
|
|
|
326
361
|
def _arbitrate_deadlock(
|
|
327
362
|
self, last_outcome: DispatchOutcome | None, store: StateStore
|
|
328
363
|
) -> DeadlockArbitration:
|
|
329
|
-
"""
|
|
364
|
+
"""相二·无可运行候选时的死锁归因(委托 DeadlockGovernor 自闭环仲裁)。"""
|
|
330
365
|
deadlock_detected = False
|
|
331
366
|
should_terminate = False
|
|
332
367
|
wait_seconds = 0.0
|
|
@@ -334,11 +369,23 @@ class EngineRuntime:
|
|
|
334
369
|
if store.state.is_empty and not self._in_flight:
|
|
335
370
|
should_terminate = True
|
|
336
371
|
elif not self._in_flight:
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
372
|
+
if self.selection_machine is not None:
|
|
373
|
+
decision = self.governor.arbitrate(
|
|
374
|
+
last_outcome.standstill,
|
|
375
|
+
store=self.store,
|
|
376
|
+
selection_machine=self.selection_machine,
|
|
377
|
+
)
|
|
378
|
+
else:
|
|
379
|
+
decision = self.governor.arbitrate(
|
|
380
|
+
last_outcome.standstill,
|
|
381
|
+
store=self.store,
|
|
382
|
+
)
|
|
383
|
+
if decision.action in ("resolved", "selection_blocked"):
|
|
384
|
+
if decision.action == "selection_blocked":
|
|
385
|
+
if decision.standstill is not None:
|
|
386
|
+
self._report_selection_blocked(decision.standstill)
|
|
387
|
+
else:
|
|
388
|
+
deadlock_detected = True
|
|
342
389
|
if decision.should_terminate:
|
|
343
390
|
should_terminate = True
|
|
344
391
|
elif decision.action in ("grace_waiting", "gap_retrying"):
|
|
@@ -350,6 +397,30 @@ class EngineRuntime:
|
|
|
350
397
|
wait_seconds=wait_seconds,
|
|
351
398
|
)
|
|
352
399
|
|
|
400
|
+
def _report_selection_blocked(self, standstill: SelectionStandstill) -> None:
|
|
401
|
+
"""记录「允许执行但仅因 Exclude 而本次无法完成」的下游(留队待下次)。"""
|
|
402
|
+
upstream = dict(standstill.blocked_by)
|
|
403
|
+
for uid in standstill.blocked_uids:
|
|
404
|
+
logger.warning(
|
|
405
|
+
f"SELECTION: allowed job {uid} blocked by excluded upstream "
|
|
406
|
+
f"{upstream.get(uid, '?')} ({ERR_DEPENDENCY_EXCLUDED}); kept in "
|
|
407
|
+
f"queue for a later run."
|
|
408
|
+
)
|
|
409
|
+
if standstill.blocked_uids:
|
|
410
|
+
self.stats["excluded_blocked"] += len(standstill.blocked_uids)
|
|
411
|
+
logger.info(
|
|
412
|
+
"Remaining queued jobs are deferred by selection; finishing run as "
|
|
413
|
+
"completed with the queue preserved."
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
def _apply_selection_skips(self) -> None:
|
|
417
|
+
"""终态跳过:委托 SelectionMachine 收敛 Skip 闭包与审计轨迹。"""
|
|
418
|
+
if self.selection_machine is not None and self.selection_machine.has_skip_rules:
|
|
419
|
+
self.selection_machine.apply_skips(
|
|
420
|
+
run_id=self._session.run_id,
|
|
421
|
+
seq_supplier=self._session.next_dispatch_seq,
|
|
422
|
+
)
|
|
423
|
+
|
|
353
424
|
def _drain_and_settle(self) -> int:
|
|
354
425
|
"""相三·回收在途结果并统一结算,返回本轮完成计数(无在途为 0)。"""
|
|
355
426
|
if not self._in_flight:
|
|
@@ -383,6 +454,11 @@ class EngineRuntime:
|
|
|
383
454
|
logger.info("Pipeline drained. Saving queue and exiting.")
|
|
384
455
|
self._recovery.save_queue_crash_safe()
|
|
385
456
|
return self._terminal_outcome()
|
|
457
|
+
|
|
458
|
+
# 0.5 执行选择·Skip:run 启动与派生作业落队后把终态跳过收敛掉
|
|
459
|
+
if self._skip_dirty:
|
|
460
|
+
self._skip_dirty = False
|
|
461
|
+
self._apply_selection_skips()
|
|
386
462
|
if store.state.is_empty and not self._in_flight:
|
|
387
463
|
return self._terminal_outcome()
|
|
388
464
|
|
|
@@ -400,6 +476,9 @@ class EngineRuntime:
|
|
|
400
476
|
|
|
401
477
|
# 3. Drain 回收在途结果并统一结算
|
|
402
478
|
completed_count = self._drain_and_settle()
|
|
479
|
+
if completed_count:
|
|
480
|
+
# 派生作业可能带 Skip 命中或依赖已跳过上游,下一拍先收敛跳过闭包
|
|
481
|
+
self._skip_dirty = True
|
|
403
482
|
|
|
404
483
|
# 4. 等待/空闲决策(唯一实现见 wait.decide_wait)
|
|
405
484
|
is_idle = store.state.is_empty and not self._in_flight
|
|
@@ -514,6 +593,7 @@ class EngineRuntime:
|
|
|
514
593
|
def _run_body(self) -> None:
|
|
515
594
|
"""主执行体:启动屏障(装载 + fencing)→ 带承重网的主循环。"""
|
|
516
595
|
self.prepare_run_state()
|
|
596
|
+
self._skip_dirty = True
|
|
517
597
|
self._run_with_crash_net()
|
|
518
598
|
|
|
519
599
|
def _handle_signal(self, signum: int, frame: Any) -> None:
|