tasklite-engine 2.2.1__tar.gz → 3.0.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite_engine-3.0.0.dev1/PKG-INFO +313 -0
- tasklite_engine-3.0.0.dev1/README.md +282 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/pyproject.toml +2 -2
- tasklite_engine-3.0.0.dev1/tasklite/__init__.py +77 -0
- tasklite_engine-3.0.0.dev1/tasklite/__main__.py +205 -0
- tasklite_engine-3.0.0.dev1/tasklite/_report.py +562 -0
- tasklite_engine-3.0.0.dev1/tasklite/backend/__init__.py +37 -0
- tasklite_engine-3.0.0.dev1/tasklite/backend/base.py +534 -0
- tasklite_engine-3.0.0.dev1/tasklite/backend/memory.py +611 -0
- tasklite_engine-3.0.0.dev1/tasklite/backend/sqlite_backend.py +1082 -0
- tasklite_engine-3.0.0.dev1/tasklite/edit.py +1276 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/engine/__init__.py +7 -8
- tasklite_engine-3.0.0.dev1/tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/_worker_runtime.py +229 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/admission.py +147 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/authorization.py +989 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/channel.py +1084 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/completion.py +678 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/config.py +71 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/dispatch.py +616 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/governor.py +473 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/in_flight.py +142 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/launcher.py +1173 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/recovery.py +194 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/resource.py +470 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/runtime.py +550 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/engine/scheduler.py +70 -98
- tasklite_engine-3.0.0.dev1/tasklite/engine/session.py +141 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/store.py +748 -0
- tasklite_engine-3.0.0.dev1/tasklite/engine/types.py +133 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/engine/wait.py +38 -16
- tasklite_engine-3.0.0.dev1/tasklite/exceptions.py +20 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/__init__.py +26 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/artifact.py +441 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/context.py +237 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/graph.py +114 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/job.py +264 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/records.py +482 -0
- tasklite_engine-3.0.0.dev1/tasklite/models/result.py +246 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/models/state.py +99 -125
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/models/task.py +52 -105
- tasklite_engine-3.0.0.dev1/tasklite/models/trigger.py +78 -0
- tasklite_engine-3.0.0.dev1/tasklite/pipeline.py +446 -0
- tasklite_engine-3.0.0.dev1/tasklite/resource.py +89 -0
- tasklite_engine-3.0.0.dev1/tasklite/selection.py +28 -0
- tasklite_engine-3.0.0.dev1/tasklite/snapshot.py +842 -0
- tasklite_engine-3.0.0.dev1/tasklite/testing.py +514 -0
- tasklite_engine-3.0.0.dev1/tasklite/utils/artifact.py +1183 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/utils/clock.py +3 -3
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/utils/encoding.py +107 -31
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/utils/ipc.py +289 -243
- tasklite_engine-3.0.0.dev1/tasklite/utils/lockfile.py +127 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/wrappers/__init__.py +2 -2
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/wrappers/discovery.py +161 -167
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/wrappers/http.py +142 -521
- tasklite_engine-3.0.0.dev1/tasklite_engine.egg-info/PKG-INFO +313 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite_engine.egg-info/SOURCES.txt +13 -17
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite_engine.egg-info/requires.txt +3 -0
- tasklite_engine-2.2.1/PKG-INFO +0 -370
- tasklite_engine-2.2.1/README.md +0 -340
- tasklite_engine-2.2.1/tasklite/__init__.py +0 -219
- tasklite_engine-2.2.1/tasklite/backend/__init__.py +0 -23
- tasklite_engine-2.2.1/tasklite/backend/base.py +0 -391
- tasklite_engine-2.2.1/tasklite/backend/memory.py +0 -452
- tasklite_engine-2.2.1/tasklite/backend/sqlite_backend.py +0 -882
- tasklite_engine-2.2.1/tasklite/contrib/__init__.py +0 -7
- tasklite_engine-2.2.1/tasklite/engine/admission.py +0 -203
- tasklite_engine-2.2.1/tasklite/engine/channel.py +0 -913
- tasklite_engine-2.2.1/tasklite/engine/completion.py +0 -473
- tasklite_engine-2.2.1/tasklite/engine/config.py +0 -201
- tasklite_engine-2.2.1/tasklite/engine/dispatch.py +0 -612
- tasklite_engine-2.2.1/tasklite/engine/errorclass.py +0 -971
- tasklite_engine-2.2.1/tasklite/engine/governor.py +0 -571
- tasklite_engine-2.2.1/tasklite/engine/in_flight.py +0 -208
- tasklite_engine-2.2.1/tasklite/engine/ops.py +0 -363
- tasklite_engine-2.2.1/tasklite/engine/recovery.py +0 -347
- tasklite_engine-2.2.1/tasklite/engine/resource.py +0 -656
- tasklite_engine-2.2.1/tasklite/engine/runtime.py +0 -626
- tasklite_engine-2.2.1/tasklite/engine/selection.py +0 -244
- tasklite_engine-2.2.1/tasklite/engine/session.py +0 -124
- tasklite_engine-2.2.1/tasklite/engine/store.py +0 -944
- tasklite_engine-2.2.1/tasklite/engine/types.py +0 -149
- tasklite_engine-2.2.1/tasklite/exceptions.py +0 -74
- tasklite_engine-2.2.1/tasklite/hooks.py +0 -69
- tasklite_engine-2.2.1/tasklite/models/__init__.py +0 -11
- tasklite_engine-2.2.1/tasklite/models/artifact.py +0 -80
- tasklite_engine-2.2.1/tasklite/models/attempt.py +0 -141
- tasklite_engine-2.2.1/tasklite/models/context.py +0 -261
- tasklite_engine-2.2.1/tasklite/models/job.py +0 -417
- tasklite_engine-2.2.1/tasklite/models/trigger.py +0 -18
- tasklite_engine-2.2.1/tasklite/pipeline.py +0 -709
- tasklite_engine-2.2.1/tasklite/selection.py +0 -116
- tasklite_engine-2.2.1/tasklite/testing.py +0 -103
- tasklite_engine-2.2.1/tasklite/triggers.py +0 -46
- tasklite_engine-2.2.1/tasklite/utils/artifact.py +0 -203
- tasklite_engine-2.2.1/tasklite/utils/lockfile.py +0 -121
- tasklite_engine-2.2.1/tasklite_engine.egg-info/PKG-INFO +0 -370
- tasklite_engine-2.2.1/tests/test_encoding.py +0 -533
- tasklite_engine-2.2.1/tests/test_exceptions.py +0 -99
- tasklite_engine-2.2.1/tests/test_hooks.py +0 -118
- tasklite_engine-2.2.1/tests/test_jsonutil.py +0 -97
- tasklite_engine-2.2.1/tests/test_package_surface.py +0 -254
- tasklite_engine-2.2.1/tests/test_packaging.py +0 -42
- tasklite_engine-2.2.1/tests/test_pipeline.py +0 -956
- tasklite_engine-2.2.1/tests/test_testing.py +0 -189
- tasklite_engine-2.2.1/tests/test_trigger_contract.py +0 -298
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/LICENSE +0 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/setup.cfg +0 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/py.typed +0 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/utils/__init__.py +0 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite/utils/jsonutil.py +0 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite_engine.egg-info/dependency_links.txt +0 -0
- {tasklite_engine-2.2.1 → tasklite_engine-3.0.0.dev1}/tasklite_engine.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tasklite-engine
|
|
3
|
+
Version: 3.0.0.dev1
|
|
4
|
+
Summary: A lightweight task orchestration engine with zero external dependencies, process isolation, and ACID persistence
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/CaptainLi321/tasklite
|
|
7
|
+
Project-URL: Repository, https://github.com/CaptainLi321/tasklite
|
|
8
|
+
Keywords: task-queue,task-orchestration,background-jobs,subprocess,process-isolation,acid,workflow
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest; extra == "dev"
|
|
23
|
+
Requires-Dist: hypothesis>=6.0; extra == "dev"
|
|
24
|
+
Requires-Dist: mutmut>=3.0; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-timeout>=2.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
27
|
+
Requires-Dist: requests>=2.28; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
29
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11" and extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# ⚡ TaskLite — 零依赖、进程隔离的轻量任务编排引擎
|
|
33
|
+
|
|
34
|
+
> **A Lightweight Task Orchestration Engine for Agent & Batch Pipelines.**
|
|
35
|
+
> 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
|
|
36
|
+
> 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用主控守护与物理子进程隔离架构,内置 SQLite WAL 事务持久化:已提交终态跨进程保留;异常退出遗留的 `running` 下次启动按失败收敛,未派发的 `pending` 留队,不自动接续旧执行。
|
|
37
|
+
|
|
38
|
+
[](https://www.python.org/)
|
|
39
|
+
[](LICENSE)
|
|
40
|
+
[](https://github.com/CaptainLi321/tasklite/actions/workflows/ci.yml)
|
|
41
|
+
|
|
42
|
+
当前开发版本为 **3.0.0.dev1**,公开接口与兼容性说明见 [API 参考手册](docs/API.md)。
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## 💡 为什么选择 TaskLite?
|
|
47
|
+
|
|
48
|
+
TaskLite 面向 AI Agent 批量调用、大模型推理批处理、多媒体文件处理与离线数据流转,提供**零依赖的单机任务编排能力**,通过 `pip install tasklite-engine` 即可安装。
|
|
49
|
+
|
|
50
|
+
1. **多 Worker 并发执行**:以独立子进程并行处理任务,支持配置并发数量与资源容量;
|
|
51
|
+
2. **持久状态防重跑**:以 SQLite WAL 事务表记录状态,已提交成功由后续 Trigger 判断是否跳过;handler 显式请求重试时按本次 Run 的预算重入队,预算耗尽后进入失败档案,减少个别失败导致的全量重跑;
|
|
52
|
+
3. **进程隔离与超时守护**:业务逻辑在独立子进程中运行,超时后终止执行并回收资源;已提交状态由 SQLite 持久保存,异常退出后的遗留执行按恢复规则收敛。
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 🏗️ 核心架构与设计哲学
|
|
57
|
+
|
|
58
|
+
```mermaid
|
|
59
|
+
flowchart TD
|
|
60
|
+
subgraph Host ["主调度进程 (Master Scheduler) - 永不运行业务 Handler"]
|
|
61
|
+
State[("SQLite WAL 状态机\n(Schema 6: successes / queue / failed / attempts)")]
|
|
62
|
+
Scheduler["JobScheduler (资源调度 / 拓扑依赖)"]
|
|
63
|
+
Watchdog["进程看门狗 (超时 SIGKILL / 单库级锁防护)"]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
subgraph Workers ["独立 Worker 子进程 (Isolated Subprocesses)"]
|
|
67
|
+
W1["Worker 1 (Handler)"]
|
|
68
|
+
W2["Worker 2 (Handler)"]
|
|
69
|
+
WN["Worker N (Handler)"]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
Scheduler -->|"1. 检查资源 & 拓扑依赖"| State
|
|
73
|
+
Scheduler -->|"2. Spawn 独立子进程 (隔离)"| Workers
|
|
74
|
+
Workers -.->|"3. 独立单向 Pipe 传递声明与 TaskResult"| State
|
|
75
|
+
Watchdog -->|"4. 轮询超时 SIGKILL 强杀"| Workers
|
|
76
|
+
State -->|"5. BEGIN IMMEDIATE 原子收尾"| State
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### 三层可靠性边界
|
|
80
|
+
1. **执行代标识与通道绑定 (Generation Fencing)**:每 worker 绑定独立单向 `Pipe` 与执行代标识,校验 `job_uid` 与 `execution_id`,旧执行与伪造消息无法污染状态机;
|
|
81
|
+
2. **已结算状态事务持久化**:IPC 仅负责本次运行的瞬时通信;已提交成功、失败与 `attempts` 由 SQLite 强事务持久化。异常退出遗留的 `running` 下次按失败收敛,`pending` 留队,不自动接续旧执行;
|
|
82
|
+
3. **单库级文件锁防护 (Library Lock Fencing)**:通过 `<state_path>.lock`(如 `state.db.lock`)互斥保护,同一状态库同时只允许一个活动调度实例或管理写操作,避免并发写入破坏。
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## 🌟 核心特性
|
|
87
|
+
|
|
88
|
+
### 1. 🛡️ 进程隔离与守护 (Process Isolation)
|
|
89
|
+
- 主进程作为总控调度器,不执行具体的业务代码;
|
|
90
|
+
- 每一个任务均在独立子进程中隔离执行(POSIX 独立进程组,Windows 无窗口独立控制台 + Job Object 绑定);
|
|
91
|
+
- 当子进程发生死循环或超时,看门狗将原子回收整棵子进程树(含多层孙进程)并自动释放资源;
|
|
92
|
+
- 支持 POSIX `SIGTERM`/`SIGINT` 与 Windows `SIGTERM`/`SIGINT`/`SIGBREAK` 优雅排空停机,子进程异常退出不影响主调度进程稳定运行。
|
|
93
|
+
|
|
94
|
+
### 2. ⚡ SQLite WAL 状态持久化 (ACID)
|
|
95
|
+
- 基于 SQLite WAL + `synchronous=FULL` 模式;
|
|
96
|
+
- 状态机(`successes` 成功档案 / `queue` 待办 / `failed` 失败档案 / `attempts` 轨迹)在单次事务中原子转移;
|
|
97
|
+
- 已提交终态不会因之后异常退出被改写;遗留 `running` 以“上次执行未完成结算,实际结果未知”登记失败,不根据残留产物或迟到结果补造成功。
|
|
98
|
+
|
|
99
|
+
### 3. 🔍 函数式增量探索 (Functional Discovery)
|
|
100
|
+
- 针对数据同步、采集与拉取场景的无游标 (Cursor-less) 增量扫描;
|
|
101
|
+
- 直接复用成功档案 (`successes`) 即时判定已见内容,整页命中自动刹车,天然免疫上游记录删除、插入导致的游标漂移。
|
|
102
|
+
|
|
103
|
+
### 4. ⏸️ 内置容量与限速控制 (Resource Control)
|
|
104
|
+
- 内置 `IntervalResource`(调用间隔限速)与 `CapacityResource`(并发量限制);
|
|
105
|
+
- 核心零调度计算,调度器以事件驱动等待与主动通知推进,避免轮询忙旋。
|
|
106
|
+
|
|
107
|
+
### 5. 🔗 DAG 拓扑依赖与级联熔断
|
|
108
|
+
- 任务通过 `depends_on` 声明依赖列表;
|
|
109
|
+
- 上游失败进入失败档案时,受影响下游按依赖规则级联失败;未执行的下游不生成 Attempt,避免无效计算。
|
|
110
|
+
|
|
111
|
+
### 6. 📦 统一文件声明与主进程清理授权
|
|
112
|
+
- `FileSpec` 明确文件角色(`output` / `input` / `cache`)与清理权限(`delete_existing` / `delete_on_failure` / `delete_on_success`);
|
|
113
|
+
- 主进程持有排他文件授权(含 Run 外 `grant_authorization` / `list_authorizations` / `revoke_authorization`)与跨平台(Linux / Windows)安全清理职责,收尾时先执行授权清理后提交事务。
|
|
114
|
+
|
|
115
|
+
### 7. 🎛️ 原子图修整、只读快照与运维 CLI
|
|
116
|
+
- Run 外通过 `pipeline.edit(dry_run=True)` 原子执行作业增、删、替换与状态流转,提交前全图闭环校验并支持 `tx.diff()` 预览;`attempts` 物理轨迹永不可删;
|
|
117
|
+
- 只读 `StateSnapshot.open(..., mode="lazy"|"copy")` 与 `StateSnapshot.report()` 提供零句柄运行前后对比与耗时/ETA 报告;并内置极简运维 CLI(`python -m tasklite status | set-status | remove`);
|
|
118
|
+
- `pipeline.run(exclude=Exclude(task_type="..."))` 支持按任务类型将作业安全留队至下次运行。
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## 🚀 快速上手 (Quickstart)
|
|
123
|
+
|
|
124
|
+
### 安装
|
|
125
|
+
|
|
126
|
+
要求 **Python ≥ 3.10**,零外部依赖:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install tasklite-engine
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
> **三方命名对照**:仓库名 `tasklite`,PyPI 包名 `tasklite-engine`(PyPI 的 `tasklite` 名已被无关项目占用),Python 导入名 `tasklite`——安装 `tasklite-engine`、`import tasklite`。
|
|
133
|
+
|
|
134
|
+
### 最小示例
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from tasklite import Job, StateSnapshot, TaskLite, TaskResult
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def greet(job, ctx):
|
|
141
|
+
return TaskResult.success(metadata={"message": f"你好,{job.payload['name']}"})
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
if __name__ == "__main__":
|
|
145
|
+
with TaskLite(backend="memory") as pipeline:
|
|
146
|
+
pipeline.register_task("greet", greet)
|
|
147
|
+
pipeline.enqueue(Job("greet", "one", payload={"name": "TaskLite"}))
|
|
148
|
+
pipeline.run()
|
|
149
|
+
pipeline.stop()
|
|
150
|
+
with StateSnapshot.open(pipeline) as state:
|
|
151
|
+
print(state.get_job("greet::one"))
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
> 引擎采用 **Task(静态规格) / Job(逻辑实例) / Attempt(物理轨迹)** 三层模型,完整 API 签名、只读快照(`StateSnapshot`)、文件授权、状态管理与运维接口请查阅 **[`docs/GUIDE.md`](docs/GUIDE.md)**(使用手册)。
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
## 🍳 常见场景速览 (Recipes)
|
|
159
|
+
|
|
160
|
+
### Recipe 1: 增量分页扫描器 (Discovery)
|
|
161
|
+
|
|
162
|
+
无需游标,增量同步时只要发现整页内容已全部处理过,自动刹车停止扫描:
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
from tasklite import TaskResult
|
|
166
|
+
from tasklite.wrappers.discovery import register_discovery
|
|
167
|
+
|
|
168
|
+
def fetch_page(job, ctx, page: int):
|
|
169
|
+
return api.get_author_posts(author_id=job.payload["author_id"], page=page)
|
|
170
|
+
|
|
171
|
+
def item_id(post) -> str:
|
|
172
|
+
return str(post["id"])
|
|
173
|
+
|
|
174
|
+
def process_item(job, ctx, post, content_id: str):
|
|
175
|
+
ctx.spawn(Job("download", content_id, payload={"url": post["url"]}))
|
|
176
|
+
|
|
177
|
+
register_discovery(
|
|
178
|
+
pipeline,
|
|
179
|
+
task_type="discover_author",
|
|
180
|
+
fetch_func=fetch_page,
|
|
181
|
+
id_func=item_id,
|
|
182
|
+
process_item_func=process_item,
|
|
183
|
+
process_task_type="download",
|
|
184
|
+
resources={"api": 1.0},
|
|
185
|
+
)
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### Recipe 2: DAG 依赖级联与产物传递
|
|
189
|
+
|
|
190
|
+
通过 `depends_on` 声明拓扑依赖,上游成功后再执行下游,上游失败则下游自动级联阻断(完整可运行示例见 [`examples/trigger_chain.py`](examples/trigger_chain.py)):
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
download_job = Job("download", "video_101", payload={"url": "https://example.com/v.mp4"})
|
|
194
|
+
transcode_job = Job(
|
|
195
|
+
"transcode",
|
|
196
|
+
"video_101",
|
|
197
|
+
payload={"preset": "1080p"},
|
|
198
|
+
depends_on=[download_job.uid], # 声明上游依赖
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
pipeline.enqueue(download_job)
|
|
202
|
+
pipeline.enqueue(transcode_job)
|
|
203
|
+
pipeline.run()
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
### Recipe 3: 周期限速调用与重试
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from tasklite import TaskResult
|
|
210
|
+
|
|
211
|
+
def fetch_handler(job, ctx):
|
|
212
|
+
resp = requests.get(job.payload["url"])
|
|
213
|
+
if resp.status_code == 429:
|
|
214
|
+
# 请求重试并指明重试原因
|
|
215
|
+
return TaskResult.retry(message="Rate limited (429), retry later")
|
|
216
|
+
if resp.status_code != 200:
|
|
217
|
+
return TaskResult.failure(message=f"HTTP status {resp.status_code}")
|
|
218
|
+
return TaskResult.success(metadata=resp.json())
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
### Recipe 4: AI Agent / 大模型批量任务持久化
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
import json
|
|
225
|
+
from pathlib import Path
|
|
226
|
+
from tasklite import TaskLite, Task, Job, TaskResult, FileSpec
|
|
227
|
+
from tasklite.resource import IntervalResource
|
|
228
|
+
|
|
229
|
+
def agent_worker(job: Job, ctx) -> TaskResult:
|
|
230
|
+
result = call_llm(job.payload["prompt"])
|
|
231
|
+
out_file = ctx.declare_file(
|
|
232
|
+
"res",
|
|
233
|
+
FileSpec(f"results/{job.job_id}.json", role="output", delete_existing=True),
|
|
234
|
+
)
|
|
235
|
+
out_file.write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
|
|
236
|
+
return TaskResult.success(metadata={"tokens": result.get("usage", 0)})
|
|
237
|
+
|
|
238
|
+
pipeline = TaskLite(backend="sqlite", state_path="./agent_state.db", max_workers=8)
|
|
239
|
+
# 含清理标志的文件声明可通过交互授权或 TASKLITE_DANGEROUS_AUTO_APPROVE=1 确认
|
|
240
|
+
pipeline.register_resource(IntervalResource("llm_rpm", interval_seconds=0.1))
|
|
241
|
+
pipeline.register_task(Task("analyze", agent_worker, resources={"llm_rpm": 1.0}))
|
|
242
|
+
|
|
243
|
+
for i in range(100):
|
|
244
|
+
pipeline.enqueue(Job("analyze", f"doc_{i}", payload={"prompt": f"分析文本 {i}"}))
|
|
245
|
+
|
|
246
|
+
pipeline.run()
|
|
247
|
+
pipeline.stop()
|
|
248
|
+
pipeline.close()
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
---
|
|
252
|
+
|
|
253
|
+
## 📚 文档导航与职责划分
|
|
254
|
+
|
|
255
|
+
仓库内各文档按受众与职责严格正交划分,避免重复维护:
|
|
256
|
+
|
|
257
|
+
| 文档 | 定位与职责 |
|
|
258
|
+
| :--- | :--- |
|
|
259
|
+
| **[`README.md`](README.md)** | **项目门户与快速上手**:解决痛点、架构概览、安装与最小示例、常见场景 Recipes 速览、目录结构 |
|
|
260
|
+
| **[`docs/GUIDE.md`](docs/GUIDE.md)** | **使用手册**:公开导入面、三层模型与 Schema 6、`JobContext` 与统一文件声明授权、失败档案与运维 API、调度扩展接缝 |
|
|
261
|
+
| **[`docs/API.md`](docs/API.md)** | **公开 API 权威参考手册**:全公开类、方法、属性签名、严格 JSON 规则、返回结构与异常契约速查 |
|
|
262
|
+
| **[`CONTEXT.md`](CONTEXT.md)** | **领域通用语言词典**:核心领域模型与引擎内部组件的标准术语定义及禁用词表 |
|
|
263
|
+
| **[`docs/adr/`](docs/adr/)** | **架构决策记录(ADR)**:关键架构演进、设计取舍与边界裁决依据 |
|
|
264
|
+
| **[`docs/ROADMAP.md`](docs/ROADMAP.md)** | **未来演进路线图**:已分析但尚未立项实施的储备方向(实施前须立项 ADR) |
|
|
265
|
+
| **[`AGENTS.md`](AGENTS.md)** | **研发红线与工作流规范**:测试门槛、原子提交纪律、分层依赖红线与注释规范 |
|
|
266
|
+
| **[`CHANGELOG.md`](CHANGELOG.md)** | **版本变更日志**:各发布版本的显著特性与破坏性变更记录 |
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
## 📁 目录结构
|
|
271
|
+
|
|
272
|
+
```
|
|
273
|
+
tasklite/
|
|
274
|
+
├── pipeline.py # TaskLite 核心门面与装配
|
|
275
|
+
├── edit.py # EditTransaction:Run 外原子图修整事务与闭环校验
|
|
276
|
+
├── snapshot.py # StateSnapshot:只读一致状态快照、copy 模式与 JSON 查询
|
|
277
|
+
├── _report.py # StateSnapshot.report 状态/耗时/ETA 与前后对比渲染
|
|
278
|
+
├── __main__.py # 极简运维 CLI(python -m tasklite status / set-status / remove)
|
|
279
|
+
├── selection.py # 执行选择规则值对象 Exclude(候选准入 seam)
|
|
280
|
+
├── resource.py # 内置资源公共导出门面 (CapacityResource, IntervalResource)
|
|
281
|
+
├── exceptions.py # 异常族 (TaskLiteError)
|
|
282
|
+
├── testing.py # 测试辅助工具(fake_ctx / running / authorizer / in_process_workers 等)
|
|
283
|
+
├── engine/ # 核心调度与执行引擎群
|
|
284
|
+
│ ├── runtime.py # 核心运行期深模块、主循环事件泵 (EngineRuntime)
|
|
285
|
+
│ ├── config.py # RunConfig 静态装配快照、默认值唯一解析点
|
|
286
|
+
│ ├── session.py # RunSession 单次 run 生命周期状态
|
|
287
|
+
│ ├── wait.py # 事件泵等待/空闲决策纯函数
|
|
288
|
+
│ ├── types.py # 引擎公共值对象单一真相源
|
|
289
|
+
│ ├── store.py # 状态事务与失败写入隔离 (StateStore)
|
|
290
|
+
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
291
|
+
│ ├── completion.py # 任务完成与提交单一收口 (CompletionMachine)
|
|
292
|
+
│ ├── authorization.py # 主进程文件声明与三类授权等待协调器 (AuthorizationCoordinator)
|
|
293
|
+
│ ├── recovery.py # 遗留 running 收敛与在途中止编排 (RecoveryOrchestrator)
|
|
294
|
+
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel)
|
|
295
|
+
│ ├── launcher.py # 跨平台 Worker 启动器与进程树原子回收 (WorkerLauncher / ProcessTreeAdapter)
|
|
296
|
+
│ ├── _worker_runtime.py # 纯 Worker 侧执行入口、启动规格与跨平台停止信号单一事实源
|
|
297
|
+
│ ├── _worker_bootstrap.py # Worker 子进程独立引导与握手屏障
|
|
298
|
+
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler / OrderingPolicy 排序 seam)
|
|
299
|
+
│ ├── admission.py # 重入队节奏 (RequeuePolicy seam)
|
|
300
|
+
│ ├── governor.py # 死锁归因仲裁与依赖宽限 (DeadlockGovernor)
|
|
301
|
+
│ ├── in_flight.py # 在途作业租约追踪 (InFlightTracker)
|
|
302
|
+
│ └── resource.py # 内置资源实现
|
|
303
|
+
├── backend/ # SQLite WAL 强一致事务持久化后端与内存后端
|
|
304
|
+
├── models/ # Task / Job / Attempt / JobContext / TaskResult / FileSpec 数据模型
|
|
305
|
+
├── wrappers/ # 可选工具:discovery.py(增量扫描)/ http.py(HTTP 请求、重试与快照)
|
|
306
|
+
└── utils/ # 单射编码 (%XX 转义) / 瞬时 Pipe 通道 / 跨平台路径策略 / 文件锁 / 时钟 / JSON 校验
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
---
|
|
310
|
+
|
|
311
|
+
## 📄 开源许可证
|
|
312
|
+
|
|
313
|
+
本项目基于 [MIT License](LICENSE) 开源。
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# ⚡ TaskLite — 零依赖、进程隔离的轻量任务编排引擎
|
|
2
|
+
|
|
3
|
+
> **A Lightweight Task Orchestration Engine for Agent & Batch Pipelines.**
|
|
4
|
+
> 专为 **AI Agent 批量处理**、数据同步与采集、音视频转码、离线 ETL 与后台批处理任务打造。
|
|
5
|
+
> 核心解决「单线程处理耗时长」、「中途失败需全量重跑」、「子任务卡死拖垮主流程」等工程痛点。采用主控守护与物理子进程隔离架构,内置 SQLite WAL 事务持久化:已提交终态跨进程保留;异常退出遗留的 `running` 下次启动按失败收敛,未派发的 `pending` 留队,不自动接续旧执行。
|
|
6
|
+
|
|
7
|
+
[](https://www.python.org/)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
[](https://github.com/CaptainLi321/tasklite/actions/workflows/ci.yml)
|
|
10
|
+
|
|
11
|
+
当前开发版本为 **3.0.0.dev1**,公开接口与兼容性说明见 [API 参考手册](docs/API.md)。
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## 💡 为什么选择 TaskLite?
|
|
16
|
+
|
|
17
|
+
TaskLite 面向 AI Agent 批量调用、大模型推理批处理、多媒体文件处理与离线数据流转,提供**零依赖的单机任务编排能力**,通过 `pip install tasklite-engine` 即可安装。
|
|
18
|
+
|
|
19
|
+
1. **多 Worker 并发执行**:以独立子进程并行处理任务,支持配置并发数量与资源容量;
|
|
20
|
+
2. **持久状态防重跑**:以 SQLite WAL 事务表记录状态,已提交成功由后续 Trigger 判断是否跳过;handler 显式请求重试时按本次 Run 的预算重入队,预算耗尽后进入失败档案,减少个别失败导致的全量重跑;
|
|
21
|
+
3. **进程隔离与超时守护**:业务逻辑在独立子进程中运行,超时后终止执行并回收资源;已提交状态由 SQLite 持久保存,异常退出后的遗留执行按恢复规则收敛。
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## 🏗️ 核心架构与设计哲学
|
|
26
|
+
|
|
27
|
+
```mermaid
|
|
28
|
+
flowchart TD
|
|
29
|
+
subgraph Host ["主调度进程 (Master Scheduler) - 永不运行业务 Handler"]
|
|
30
|
+
State[("SQLite WAL 状态机\n(Schema 6: successes / queue / failed / attempts)")]
|
|
31
|
+
Scheduler["JobScheduler (资源调度 / 拓扑依赖)"]
|
|
32
|
+
Watchdog["进程看门狗 (超时 SIGKILL / 单库级锁防护)"]
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
subgraph Workers ["独立 Worker 子进程 (Isolated Subprocesses)"]
|
|
36
|
+
W1["Worker 1 (Handler)"]
|
|
37
|
+
W2["Worker 2 (Handler)"]
|
|
38
|
+
WN["Worker N (Handler)"]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
Scheduler -->|"1. 检查资源 & 拓扑依赖"| State
|
|
42
|
+
Scheduler -->|"2. Spawn 独立子进程 (隔离)"| Workers
|
|
43
|
+
Workers -.->|"3. 独立单向 Pipe 传递声明与 TaskResult"| State
|
|
44
|
+
Watchdog -->|"4. 轮询超时 SIGKILL 强杀"| Workers
|
|
45
|
+
State -->|"5. BEGIN IMMEDIATE 原子收尾"| State
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### 三层可靠性边界
|
|
49
|
+
1. **执行代标识与通道绑定 (Generation Fencing)**:每 worker 绑定独立单向 `Pipe` 与执行代标识,校验 `job_uid` 与 `execution_id`,旧执行与伪造消息无法污染状态机;
|
|
50
|
+
2. **已结算状态事务持久化**:IPC 仅负责本次运行的瞬时通信;已提交成功、失败与 `attempts` 由 SQLite 强事务持久化。异常退出遗留的 `running` 下次按失败收敛,`pending` 留队,不自动接续旧执行;
|
|
51
|
+
3. **单库级文件锁防护 (Library Lock Fencing)**:通过 `<state_path>.lock`(如 `state.db.lock`)互斥保护,同一状态库同时只允许一个活动调度实例或管理写操作,避免并发写入破坏。
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 🌟 核心特性
|
|
56
|
+
|
|
57
|
+
### 1. 🛡️ 进程隔离与守护 (Process Isolation)
|
|
58
|
+
- 主进程作为总控调度器,不执行具体的业务代码;
|
|
59
|
+
- 每一个任务均在独立子进程中隔离执行(POSIX 独立进程组,Windows 无窗口独立控制台 + Job Object 绑定);
|
|
60
|
+
- 当子进程发生死循环或超时,看门狗将原子回收整棵子进程树(含多层孙进程)并自动释放资源;
|
|
61
|
+
- 支持 POSIX `SIGTERM`/`SIGINT` 与 Windows `SIGTERM`/`SIGINT`/`SIGBREAK` 优雅排空停机,子进程异常退出不影响主调度进程稳定运行。
|
|
62
|
+
|
|
63
|
+
### 2. ⚡ SQLite WAL 状态持久化 (ACID)
|
|
64
|
+
- 基于 SQLite WAL + `synchronous=FULL` 模式;
|
|
65
|
+
- 状态机(`successes` 成功档案 / `queue` 待办 / `failed` 失败档案 / `attempts` 轨迹)在单次事务中原子转移;
|
|
66
|
+
- 已提交终态不会因之后异常退出被改写;遗留 `running` 以“上次执行未完成结算,实际结果未知”登记失败,不根据残留产物或迟到结果补造成功。
|
|
67
|
+
|
|
68
|
+
### 3. 🔍 函数式增量探索 (Functional Discovery)
|
|
69
|
+
- 针对数据同步、采集与拉取场景的无游标 (Cursor-less) 增量扫描;
|
|
70
|
+
- 直接复用成功档案 (`successes`) 即时判定已见内容,整页命中自动刹车,天然免疫上游记录删除、插入导致的游标漂移。
|
|
71
|
+
|
|
72
|
+
### 4. ⏸️ 内置容量与限速控制 (Resource Control)
|
|
73
|
+
- 内置 `IntervalResource`(调用间隔限速)与 `CapacityResource`(并发量限制);
|
|
74
|
+
- 核心零调度计算,调度器以事件驱动等待与主动通知推进,避免轮询忙旋。
|
|
75
|
+
|
|
76
|
+
### 5. 🔗 DAG 拓扑依赖与级联熔断
|
|
77
|
+
- 任务通过 `depends_on` 声明依赖列表;
|
|
78
|
+
- 上游失败进入失败档案时,受影响下游按依赖规则级联失败;未执行的下游不生成 Attempt,避免无效计算。
|
|
79
|
+
|
|
80
|
+
### 6. 📦 统一文件声明与主进程清理授权
|
|
81
|
+
- `FileSpec` 明确文件角色(`output` / `input` / `cache`)与清理权限(`delete_existing` / `delete_on_failure` / `delete_on_success`);
|
|
82
|
+
- 主进程持有排他文件授权(含 Run 外 `grant_authorization` / `list_authorizations` / `revoke_authorization`)与跨平台(Linux / Windows)安全清理职责,收尾时先执行授权清理后提交事务。
|
|
83
|
+
|
|
84
|
+
### 7. 🎛️ 原子图修整、只读快照与运维 CLI
|
|
85
|
+
- Run 外通过 `pipeline.edit(dry_run=True)` 原子执行作业增、删、替换与状态流转,提交前全图闭环校验并支持 `tx.diff()` 预览;`attempts` 物理轨迹永不可删;
|
|
86
|
+
- 只读 `StateSnapshot.open(..., mode="lazy"|"copy")` 与 `StateSnapshot.report()` 提供零句柄运行前后对比与耗时/ETA 报告;并内置极简运维 CLI(`python -m tasklite status | set-status | remove`);
|
|
87
|
+
- `pipeline.run(exclude=Exclude(task_type="..."))` 支持按任务类型将作业安全留队至下次运行。
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## 🚀 快速上手 (Quickstart)
|
|
92
|
+
|
|
93
|
+
### 安装
|
|
94
|
+
|
|
95
|
+
要求 **Python ≥ 3.10**,零外部依赖:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
pip install tasklite-engine
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
> **三方命名对照**:仓库名 `tasklite`,PyPI 包名 `tasklite-engine`(PyPI 的 `tasklite` 名已被无关项目占用),Python 导入名 `tasklite`——安装 `tasklite-engine`、`import tasklite`。
|
|
102
|
+
|
|
103
|
+
### 最小示例
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from tasklite import Job, StateSnapshot, TaskLite, TaskResult
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def greet(job, ctx):
|
|
110
|
+
return TaskResult.success(metadata={"message": f"你好,{job.payload['name']}"})
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
with TaskLite(backend="memory") as pipeline:
|
|
115
|
+
pipeline.register_task("greet", greet)
|
|
116
|
+
pipeline.enqueue(Job("greet", "one", payload={"name": "TaskLite"}))
|
|
117
|
+
pipeline.run()
|
|
118
|
+
pipeline.stop()
|
|
119
|
+
with StateSnapshot.open(pipeline) as state:
|
|
120
|
+
print(state.get_job("greet::one"))
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
> 引擎采用 **Task(静态规格) / Job(逻辑实例) / Attempt(物理轨迹)** 三层模型,完整 API 签名、只读快照(`StateSnapshot`)、文件授权、状态管理与运维接口请查阅 **[`docs/GUIDE.md`](docs/GUIDE.md)**(使用手册)。
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## 🍳 常见场景速览 (Recipes)
|
|
128
|
+
|
|
129
|
+
### Recipe 1: 增量分页扫描器 (Discovery)
|
|
130
|
+
|
|
131
|
+
无需游标,增量同步时只要发现整页内容已全部处理过,自动刹车停止扫描:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from tasklite import TaskResult
|
|
135
|
+
from tasklite.wrappers.discovery import register_discovery
|
|
136
|
+
|
|
137
|
+
def fetch_page(job, ctx, page: int):
|
|
138
|
+
return api.get_author_posts(author_id=job.payload["author_id"], page=page)
|
|
139
|
+
|
|
140
|
+
def item_id(post) -> str:
|
|
141
|
+
return str(post["id"])
|
|
142
|
+
|
|
143
|
+
def process_item(job, ctx, post, content_id: str):
|
|
144
|
+
ctx.spawn(Job("download", content_id, payload={"url": post["url"]}))
|
|
145
|
+
|
|
146
|
+
register_discovery(
|
|
147
|
+
pipeline,
|
|
148
|
+
task_type="discover_author",
|
|
149
|
+
fetch_func=fetch_page,
|
|
150
|
+
id_func=item_id,
|
|
151
|
+
process_item_func=process_item,
|
|
152
|
+
process_task_type="download",
|
|
153
|
+
resources={"api": 1.0},
|
|
154
|
+
)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Recipe 2: DAG 依赖级联与产物传递
|
|
158
|
+
|
|
159
|
+
通过 `depends_on` 声明拓扑依赖,上游成功后再执行下游,上游失败则下游自动级联阻断(完整可运行示例见 [`examples/trigger_chain.py`](examples/trigger_chain.py)):
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
download_job = Job("download", "video_101", payload={"url": "https://example.com/v.mp4"})
|
|
163
|
+
transcode_job = Job(
|
|
164
|
+
"transcode",
|
|
165
|
+
"video_101",
|
|
166
|
+
payload={"preset": "1080p"},
|
|
167
|
+
depends_on=[download_job.uid], # 声明上游依赖
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
pipeline.enqueue(download_job)
|
|
171
|
+
pipeline.enqueue(transcode_job)
|
|
172
|
+
pipeline.run()
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### Recipe 3: 周期限速调用与重试
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from tasklite import TaskResult
|
|
179
|
+
|
|
180
|
+
def fetch_handler(job, ctx):
|
|
181
|
+
resp = requests.get(job.payload["url"])
|
|
182
|
+
if resp.status_code == 429:
|
|
183
|
+
# 请求重试并指明重试原因
|
|
184
|
+
return TaskResult.retry(message="Rate limited (429), retry later")
|
|
185
|
+
if resp.status_code != 200:
|
|
186
|
+
return TaskResult.failure(message=f"HTTP status {resp.status_code}")
|
|
187
|
+
return TaskResult.success(metadata=resp.json())
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Recipe 4: AI Agent / 大模型批量任务持久化
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
import json
|
|
194
|
+
from pathlib import Path
|
|
195
|
+
from tasklite import TaskLite, Task, Job, TaskResult, FileSpec
|
|
196
|
+
from tasklite.resource import IntervalResource
|
|
197
|
+
|
|
198
|
+
def agent_worker(job: Job, ctx) -> TaskResult:
|
|
199
|
+
result = call_llm(job.payload["prompt"])
|
|
200
|
+
out_file = ctx.declare_file(
|
|
201
|
+
"res",
|
|
202
|
+
FileSpec(f"results/{job.job_id}.json", role="output", delete_existing=True),
|
|
203
|
+
)
|
|
204
|
+
out_file.write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8")
|
|
205
|
+
return TaskResult.success(metadata={"tokens": result.get("usage", 0)})
|
|
206
|
+
|
|
207
|
+
pipeline = TaskLite(backend="sqlite", state_path="./agent_state.db", max_workers=8)
|
|
208
|
+
# 含清理标志的文件声明可通过交互授权或 TASKLITE_DANGEROUS_AUTO_APPROVE=1 确认
|
|
209
|
+
pipeline.register_resource(IntervalResource("llm_rpm", interval_seconds=0.1))
|
|
210
|
+
pipeline.register_task(Task("analyze", agent_worker, resources={"llm_rpm": 1.0}))
|
|
211
|
+
|
|
212
|
+
for i in range(100):
|
|
213
|
+
pipeline.enqueue(Job("analyze", f"doc_{i}", payload={"prompt": f"分析文本 {i}"}))
|
|
214
|
+
|
|
215
|
+
pipeline.run()
|
|
216
|
+
pipeline.stop()
|
|
217
|
+
pipeline.close()
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## 📚 文档导航与职责划分
|
|
223
|
+
|
|
224
|
+
仓库内各文档按受众与职责严格正交划分,避免重复维护:
|
|
225
|
+
|
|
226
|
+
| 文档 | 定位与职责 |
|
|
227
|
+
| :--- | :--- |
|
|
228
|
+
| **[`README.md`](README.md)** | **项目门户与快速上手**:解决痛点、架构概览、安装与最小示例、常见场景 Recipes 速览、目录结构 |
|
|
229
|
+
| **[`docs/GUIDE.md`](docs/GUIDE.md)** | **使用手册**:公开导入面、三层模型与 Schema 6、`JobContext` 与统一文件声明授权、失败档案与运维 API、调度扩展接缝 |
|
|
230
|
+
| **[`docs/API.md`](docs/API.md)** | **公开 API 权威参考手册**:全公开类、方法、属性签名、严格 JSON 规则、返回结构与异常契约速查 |
|
|
231
|
+
| **[`CONTEXT.md`](CONTEXT.md)** | **领域通用语言词典**:核心领域模型与引擎内部组件的标准术语定义及禁用词表 |
|
|
232
|
+
| **[`docs/adr/`](docs/adr/)** | **架构决策记录(ADR)**:关键架构演进、设计取舍与边界裁决依据 |
|
|
233
|
+
| **[`docs/ROADMAP.md`](docs/ROADMAP.md)** | **未来演进路线图**:已分析但尚未立项实施的储备方向(实施前须立项 ADR) |
|
|
234
|
+
| **[`AGENTS.md`](AGENTS.md)** | **研发红线与工作流规范**:测试门槛、原子提交纪律、分层依赖红线与注释规范 |
|
|
235
|
+
| **[`CHANGELOG.md`](CHANGELOG.md)** | **版本变更日志**:各发布版本的显著特性与破坏性变更记录 |
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## 📁 目录结构
|
|
240
|
+
|
|
241
|
+
```
|
|
242
|
+
tasklite/
|
|
243
|
+
├── pipeline.py # TaskLite 核心门面与装配
|
|
244
|
+
├── edit.py # EditTransaction:Run 外原子图修整事务与闭环校验
|
|
245
|
+
├── snapshot.py # StateSnapshot:只读一致状态快照、copy 模式与 JSON 查询
|
|
246
|
+
├── _report.py # StateSnapshot.report 状态/耗时/ETA 与前后对比渲染
|
|
247
|
+
├── __main__.py # 极简运维 CLI(python -m tasklite status / set-status / remove)
|
|
248
|
+
├── selection.py # 执行选择规则值对象 Exclude(候选准入 seam)
|
|
249
|
+
├── resource.py # 内置资源公共导出门面 (CapacityResource, IntervalResource)
|
|
250
|
+
├── exceptions.py # 异常族 (TaskLiteError)
|
|
251
|
+
├── testing.py # 测试辅助工具(fake_ctx / running / authorizer / in_process_workers 等)
|
|
252
|
+
├── engine/ # 核心调度与执行引擎群
|
|
253
|
+
│ ├── runtime.py # 核心运行期深模块、主循环事件泵 (EngineRuntime)
|
|
254
|
+
│ ├── config.py # RunConfig 静态装配快照、默认值唯一解析点
|
|
255
|
+
│ ├── session.py # RunSession 单次 run 生命周期状态
|
|
256
|
+
│ ├── wait.py # 事件泵等待/空闲决策纯函数
|
|
257
|
+
│ ├── types.py # 引擎公共值对象单一真相源
|
|
258
|
+
│ ├── store.py # 状态事务与失败写入隔离 (StateStore)
|
|
259
|
+
│ ├── dispatch.py # 任务派发状态机 (DispatchMachine)
|
|
260
|
+
│ ├── completion.py # 任务完成与提交单一收口 (CompletionMachine)
|
|
261
|
+
│ ├── authorization.py # 主进程文件声明与三类授权等待协调器 (AuthorizationCoordinator)
|
|
262
|
+
│ ├── recovery.py # 遗留 running 收敛与在途中止编排 (RecoveryOrchestrator)
|
|
263
|
+
│ ├── channel.py # IPC 与子进程执行通道 (ExecutionChannel)
|
|
264
|
+
│ ├── launcher.py # 跨平台 Worker 启动器与进程树原子回收 (WorkerLauncher / ProcessTreeAdapter)
|
|
265
|
+
│ ├── _worker_runtime.py # 纯 Worker 侧执行入口、启动规格与跨平台停止信号单一事实源
|
|
266
|
+
│ ├── _worker_bootstrap.py # Worker 子进程独立引导与握手屏障
|
|
267
|
+
│ ├── scheduler.py # 资源调度与 DAG 依赖 (JobScheduler / OrderingPolicy 排序 seam)
|
|
268
|
+
│ ├── admission.py # 重入队节奏 (RequeuePolicy seam)
|
|
269
|
+
│ ├── governor.py # 死锁归因仲裁与依赖宽限 (DeadlockGovernor)
|
|
270
|
+
│ ├── in_flight.py # 在途作业租约追踪 (InFlightTracker)
|
|
271
|
+
│ └── resource.py # 内置资源实现
|
|
272
|
+
├── backend/ # SQLite WAL 强一致事务持久化后端与内存后端
|
|
273
|
+
├── models/ # Task / Job / Attempt / JobContext / TaskResult / FileSpec 数据模型
|
|
274
|
+
├── wrappers/ # 可选工具:discovery.py(增量扫描)/ http.py(HTTP 请求、重试与快照)
|
|
275
|
+
└── utils/ # 单射编码 (%XX 转义) / 瞬时 Pipe 通道 / 跨平台路径策略 / 文件锁 / 时钟 / JSON 校验
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## 📄 开源许可证
|
|
281
|
+
|
|
282
|
+
本项目基于 [MIT License](LICENSE) 开源。
|
|
@@ -30,7 +30,7 @@ Homepage = "https://github.com/CaptainLi321/tasklite"
|
|
|
30
30
|
Repository = "https://github.com/CaptainLi321/tasklite"
|
|
31
31
|
|
|
32
32
|
[project.optional-dependencies]
|
|
33
|
-
dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0", "requests>=2.28", "mypy>=1.0"]
|
|
33
|
+
dev = ["pytest", "hypothesis>=6.0", "mutmut>=3.0", "pytest-timeout>=2.0", "pytest-cov>=4.0", "requests>=2.28", "mypy>=1.0", "tomli>=2.0; python_version < '3.11'"]
|
|
34
34
|
|
|
35
35
|
[tool.setuptools.packages.find]
|
|
36
36
|
include = ["tasklite", "tasklite.*"]
|
|
@@ -74,7 +74,7 @@ show_missing = true
|
|
|
74
74
|
skip_covered = false
|
|
75
75
|
|
|
76
76
|
[tool.mutmut]
|
|
77
|
-
#
|
|
77
|
+
# 变异测试目标:核心层与门面
|
|
78
78
|
source_paths = [
|
|
79
79
|
"tasklite/backend/",
|
|
80
80
|
"tasklite/engine/",
|