celestialflow 3.0.5__tar.gz → 3.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celestialflow-3.0.5 → celestialflow-3.0.7}/PKG-INFO +33 -21
- {celestialflow-3.0.5 → celestialflow-3.0.7}/README.md +32 -20
- {celestialflow-3.0.5 → celestialflow-3.0.7}/pyproject.toml +1 -1
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/__init__.py +7 -0
- celestialflow-3.0.7/src/celestialflow/adapters/__init__.py +0 -0
- celestialflow-3.0.7/src/celestialflow/adapters/celestialtree/__init__.py +2 -0
- celestialflow-3.0.7/src/celestialflow/adapters/celestialtree/client.py +198 -0
- celestialflow-3.0.7/src/celestialflow/adapters/celestialtree/tools.py +41 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_graph.py +60 -19
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_logging.py +88 -15
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_manage.py +240 -249
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_nodes.py +76 -17
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_queue.py +6 -3
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_report.py +25 -36
- celestialflow-3.0.7/src/celestialflow/task_stage.py +186 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_tools.py +17 -17
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_types.py +25 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_web.py +42 -10
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow.egg-info/PKG-INFO +33 -21
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow.egg-info/SOURCES.txt +5 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/tests/test_graph.py +28 -18
- celestialflow-3.0.7/tests/test_manage.py +119 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/tests/test_nodes.py +58 -24
- {celestialflow-3.0.5 → celestialflow-3.0.7}/tests/test_structure.py +57 -55
- celestialflow-3.0.5/tests/test_manage.py +0 -64
- {celestialflow-3.0.5 → celestialflow-3.0.7}/setup.cfg +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/css/base.css +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/css/dashboard.css +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/css/errors.css +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/css/inject.css +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/favicon.ico +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/main.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/task_errors.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/task_injection.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/task_statuses.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/task_structure.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/task_topology.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/static/js/utils.js +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_progress.py +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/task_structure.py +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow/templates/index.html +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow.egg-info/dependency_links.txt +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow.egg-info/entry_points.txt +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow.egg-info/requires.txt +0 -0
- {celestialflow-3.0.5 → celestialflow-3.0.7}/src/celestialflow.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celestialflow
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.7
|
|
4
4
|
Summary: A flexible GRAPH-based task orchestration framework.
|
|
5
5
|
Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -36,10 +36,17 @@ Requires-Dist: jinja2
|
|
|
36
36
|
<a href="https://pypi.org/project/celestialflow/"><img src="https://img.shields.io/pypi/pyversions/celestialflow.svg"></a>
|
|
37
37
|
</p>
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://img.shields.io/badge/Task%20Graph-DAG-blueviolet">
|
|
41
|
+
<img src="https://img.shields.io/badge/Workflow-Orchestrator-7c3aed">
|
|
42
|
+
<img src="https://img.shields.io/badge/IPC-Redis%20Ready-red">
|
|
43
|
+
<img src="https://img.shields.io/badge/Distributed-Worker%20Friendly-orange">
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
**CelestialFlow** 是一个轻量级但功能完全的任务流框架,适合需要 **复杂依赖关系**、**灵活执行模型**、**跨设备运行**与**实时可视化监控** 的中/大型 Python 任务系统。
|
|
40
47
|
|
|
41
48
|
- 相比 Airflow/Dagster 更轻、更快开始
|
|
42
|
-
- 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph
|
|
49
|
+
- 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph 等复杂依赖模式
|
|
43
50
|
|
|
44
51
|
框架的基本单元为 **TaskStage**(由 `TaskManager` 派生),每个 stage 内部绑定一个独立的执行函数,并支持四种运行模式:
|
|
45
52
|
|
|
@@ -48,16 +55,16 @@ Requires-Dist: jinja2
|
|
|
48
55
|
* **多进程(process)**
|
|
49
56
|
* **协程(async)**
|
|
50
57
|
|
|
51
|
-
每个 stage 均可独立运行,也可作为节点互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage
|
|
58
|
+
每个 stage 均可独立运行,也可作为节点互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
|
|
52
59
|
|
|
53
|
-
|
|
60
|
+
在图级别上,每个 Stage 支持两种上下文模式:
|
|
54
61
|
|
|
55
|
-
* **线性执行(serial layout
|
|
56
|
-
* **并行执行(process layout
|
|
62
|
+
* **线性执行(serial layout)**:当前节点执行完毕再启动下一节点(下游节点可提前接收任务但不会立即执行)。
|
|
63
|
+
* **并行执行(process layout)**:当前节点启动后立刻前去启动下一节点。
|
|
57
64
|
|
|
58
|
-
TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达
|
|
65
|
+
TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达 **树形(Tree)**、**环形(loop)** 乃至于 **完全图(Complete Graph)** 形式的任务依赖。
|
|
59
66
|
|
|
60
|
-
|
|
67
|
+
在此基础上,CelestialFlow 支持 Web 可视化监控,并可通过 Redis 实现跨进程、跨设备协作;同时引入基于 Go 的外部 worker(通过 Redis 通信),用于承载 CPU 密集型任务,弥补 Python 在该场景下的性能瓶颈。
|
|
61
68
|
|
|
62
69
|
## 项目结构(Project Structure)
|
|
63
70
|
|
|
@@ -313,21 +320,26 @@ flowchart TD
|
|
|
313
320
|
|
|
314
321
|
## 更新日志(Change Log)
|
|
315
322
|
|
|
316
|
-
-
|
|
317
|
-
-
|
|
318
|
-
-
|
|
319
|
-
-
|
|
320
|
-
-
|
|
321
|
-
-
|
|
322
|
-
-
|
|
323
|
-
-
|
|
324
|
-
-
|
|
325
|
-
-
|
|
326
|
-
-
|
|
323
|
+
- 2021: 建立一个支持多线程与单线程处理函数的类
|
|
324
|
+
- 2023: 在GPT4帮助下添加多进程与携程运行模式
|
|
325
|
+
- 5/9/2024: 将原有的处理类抽象为节点, 添加TaskChain类, 可以线性连接多个节点, 并设定节点在Chain中的运行模式, 支持serial和process两种, 后者Chain所有节点同时运行
|
|
326
|
+
- 12/12/2024-12/16/2024: 在原有链式结构基础上允许节点有复数下级节点, 实现Tree结构; 将原有TaskChain改名为TaskTree
|
|
327
|
+
- 3/16/2025: 支持Web端任务完成情况可视化
|
|
328
|
+
- 6/9/2025: 支持节点拥有复数上级节点, 脱离纯Tree结构, 为之后循环图做准备
|
|
329
|
+
- 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁出
|
|
330
|
+
- 6/12/2025: 支持循环图, 下级节点可指向上级节点
|
|
331
|
+
- 6/13/2025: 支持loop结构, 即节点可指向自己
|
|
332
|
+
- 6/14/2025: 支持forest结构, 即可有多个根节点
|
|
333
|
+
- 6/16/2025: 多轮评测后, 当前框架已支持完整有向图结构, 将TaskTree改名为TaskGraph
|
|
334
|
+
- 3.0.1: 上线Pypi, 可喜可贺
|
|
335
|
+
- 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
|
|
336
|
+
- 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
|
|
337
|
+
- 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
|
|
338
|
+
- 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
|
|
327
339
|
|
|
328
340
|
## Star 历史趋势(Star History)
|
|
329
341
|
|
|
330
|
-
|
|
342
|
+
如果对项目感兴趣的话,欢迎star。如果有问题或者建议的话, 欢迎提交[Issues](https://github.com/Mr-xiaotian/CelestialFlow/issues)或者在[Discussion](https://github.com/Mr-xiaotian/CelestialFlow/discussions)中告诉我。
|
|
331
343
|
|
|
332
344
|
[](https://star-history.com/#Mr-xiaotian/CelestialFlow&Date)
|
|
333
345
|
|
|
@@ -11,10 +11,17 @@
|
|
|
11
11
|
<a href="https://pypi.org/project/celestialflow/"><img src="https://img.shields.io/pypi/pyversions/celestialflow.svg"></a>
|
|
12
12
|
</p>
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
<p align="center">
|
|
15
|
+
<img src="https://img.shields.io/badge/Task%20Graph-DAG-blueviolet">
|
|
16
|
+
<img src="https://img.shields.io/badge/Workflow-Orchestrator-7c3aed">
|
|
17
|
+
<img src="https://img.shields.io/badge/IPC-Redis%20Ready-red">
|
|
18
|
+
<img src="https://img.shields.io/badge/Distributed-Worker%20Friendly-orange">
|
|
19
|
+
</p>
|
|
20
|
+
|
|
21
|
+
**CelestialFlow** 是一个轻量级但功能完全的任务流框架,适合需要 **复杂依赖关系**、**灵活执行模型**、**跨设备运行**与**实时可视化监控** 的中/大型 Python 任务系统。
|
|
15
22
|
|
|
16
23
|
- 相比 Airflow/Dagster 更轻、更快开始
|
|
17
|
-
- 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph
|
|
24
|
+
- 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph 等复杂依赖模式
|
|
18
25
|
|
|
19
26
|
框架的基本单元为 **TaskStage**(由 `TaskManager` 派生),每个 stage 内部绑定一个独立的执行函数,并支持四种运行模式:
|
|
20
27
|
|
|
@@ -23,16 +30,16 @@
|
|
|
23
30
|
* **多进程(process)**
|
|
24
31
|
* **协程(async)**
|
|
25
32
|
|
|
26
|
-
每个 stage 均可独立运行,也可作为节点互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage
|
|
33
|
+
每个 stage 均可独立运行,也可作为节点互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
|
|
27
34
|
|
|
28
|
-
|
|
35
|
+
在图级别上,每个 Stage 支持两种上下文模式:
|
|
29
36
|
|
|
30
|
-
* **线性执行(serial layout
|
|
31
|
-
* **并行执行(process layout
|
|
37
|
+
* **线性执行(serial layout)**:当前节点执行完毕再启动下一节点(下游节点可提前接收任务但不会立即执行)。
|
|
38
|
+
* **并行执行(process layout)**:当前节点启动后立刻前去启动下一节点。
|
|
32
39
|
|
|
33
|
-
TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达
|
|
40
|
+
TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达 **树形(Tree)**、**环形(loop)** 乃至于 **完全图(Complete Graph)** 形式的任务依赖。
|
|
34
41
|
|
|
35
|
-
|
|
42
|
+
在此基础上,CelestialFlow 支持 Web 可视化监控,并可通过 Redis 实现跨进程、跨设备协作;同时引入基于 Go 的外部 worker(通过 Redis 通信),用于承载 CPU 密集型任务,弥补 Python 在该场景下的性能瓶颈。
|
|
36
43
|
|
|
37
44
|
## 项目结构(Project Structure)
|
|
38
45
|
|
|
@@ -288,21 +295,26 @@ flowchart TD
|
|
|
288
295
|
|
|
289
296
|
## 更新日志(Change Log)
|
|
290
297
|
|
|
291
|
-
-
|
|
292
|
-
-
|
|
293
|
-
-
|
|
294
|
-
-
|
|
295
|
-
-
|
|
296
|
-
-
|
|
297
|
-
-
|
|
298
|
-
-
|
|
299
|
-
-
|
|
300
|
-
-
|
|
301
|
-
-
|
|
298
|
+
- 2021: 建立一个支持多线程与单线程处理函数的类
|
|
299
|
+
- 2023: 在GPT4帮助下添加多进程与携程运行模式
|
|
300
|
+
- 5/9/2024: 将原有的处理类抽象为节点, 添加TaskChain类, 可以线性连接多个节点, 并设定节点在Chain中的运行模式, 支持serial和process两种, 后者Chain所有节点同时运行
|
|
301
|
+
- 12/12/2024-12/16/2024: 在原有链式结构基础上允许节点有复数下级节点, 实现Tree结构; 将原有TaskChain改名为TaskTree
|
|
302
|
+
- 3/16/2025: 支持Web端任务完成情况可视化
|
|
303
|
+
- 6/9/2025: 支持节点拥有复数上级节点, 脱离纯Tree结构, 为之后循环图做准备
|
|
304
|
+
- 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁出
|
|
305
|
+
- 6/12/2025: 支持循环图, 下级节点可指向上级节点
|
|
306
|
+
- 6/13/2025: 支持loop结构, 即节点可指向自己
|
|
307
|
+
- 6/14/2025: 支持forest结构, 即可有多个根节点
|
|
308
|
+
- 6/16/2025: 多轮评测后, 当前框架已支持完整有向图结构, 将TaskTree改名为TaskGraph
|
|
309
|
+
- 3.0.1: 上线Pypi, 可喜可贺
|
|
310
|
+
- 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
|
|
311
|
+
- 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
|
|
312
|
+
- 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
|
|
313
|
+
- 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
|
|
302
314
|
|
|
303
315
|
## Star 历史趋势(Star History)
|
|
304
316
|
|
|
305
|
-
|
|
317
|
+
如果对项目感兴趣的话,欢迎star。如果有问题或者建议的话, 欢迎提交[Issues](https://github.com/Mr-xiaotian/CelestialFlow/issues)或者在[Discussion](https://github.com/Mr-xiaotian/CelestialFlow/discussions)中告诉我。
|
|
306
318
|
|
|
307
319
|
[](https://star-history.com/#Mr-xiaotian/CelestialFlow&Date)
|
|
308
320
|
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
from .task_graph import TaskGraph
|
|
2
2
|
from .task_manage import TaskManager
|
|
3
|
+
from .task_stage import TaskStage
|
|
3
4
|
from .task_nodes import (
|
|
4
5
|
TaskSplitter,
|
|
5
6
|
TaskRedisSink,
|
|
6
7
|
TaskRedisSource,
|
|
7
8
|
TaskRedisAck,
|
|
9
|
+
TaskRouter,
|
|
8
10
|
)
|
|
9
11
|
from .task_structure import (
|
|
10
12
|
TaskChain,
|
|
@@ -22,6 +24,7 @@ from .task_tools import (
|
|
|
22
24
|
format_table,
|
|
23
25
|
)
|
|
24
26
|
from .task_web import TaskWebServer
|
|
27
|
+
from .adapters.celestialtree import Client as CelestialTreeClient, format_tree_root
|
|
25
28
|
|
|
26
29
|
__all__ = [
|
|
27
30
|
"TaskGraph",
|
|
@@ -32,12 +35,16 @@ __all__ = [
|
|
|
32
35
|
"TaskWheel",
|
|
33
36
|
"TaskGrid",
|
|
34
37
|
"TaskManager",
|
|
38
|
+
"TaskStage",
|
|
35
39
|
"TaskSplitter",
|
|
36
40
|
"TaskRedisSink",
|
|
37
41
|
"TaskRedisSource",
|
|
38
42
|
"TaskRedisAck",
|
|
43
|
+
"TaskRouter",
|
|
39
44
|
"TerminationSignal",
|
|
40
45
|
"TaskWebServer",
|
|
46
|
+
"CelestialTreeClient",
|
|
47
|
+
"format_tree_root",
|
|
41
48
|
"load_task_by_stage",
|
|
42
49
|
"load_task_by_error",
|
|
43
50
|
"make_hashable",
|
|
File without changes
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import threading
|
|
3
|
+
import requests
|
|
4
|
+
from typing import List, Optional, Dict, Any, Callable
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Client:
|
|
8
|
+
"""
|
|
9
|
+
Python client for CelestialTree HTTP API.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, host: str = "127.0.0.1", port: int = 7777, timeout: float = 5.0):
|
|
13
|
+
self.base_url = f"http://{host}:{port}"
|
|
14
|
+
self.timeout = timeout
|
|
15
|
+
|
|
16
|
+
def init_session(self):
|
|
17
|
+
if hasattr(self, "session"):
|
|
18
|
+
return
|
|
19
|
+
|
|
20
|
+
self.session = requests.Session()
|
|
21
|
+
self.session.headers.update(
|
|
22
|
+
{
|
|
23
|
+
"Content-Type": "application/json",
|
|
24
|
+
"Accept": "application/json",
|
|
25
|
+
}
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
# ---------- Core APIs ----------
|
|
29
|
+
|
|
30
|
+
def emit(
|
|
31
|
+
self,
|
|
32
|
+
type_: str,
|
|
33
|
+
parents: Optional[List[int]] = None,
|
|
34
|
+
message: Optional[str] = None,
|
|
35
|
+
payload: Optional[bytes | dict] = None,
|
|
36
|
+
) -> int:
|
|
37
|
+
"""
|
|
38
|
+
Emit a new event into CelestialTree.
|
|
39
|
+
"""
|
|
40
|
+
self.init_session()
|
|
41
|
+
|
|
42
|
+
body = {
|
|
43
|
+
"type": type_,
|
|
44
|
+
"parents": parents or [],
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if message is not None:
|
|
48
|
+
body["message"] = message
|
|
49
|
+
|
|
50
|
+
if payload is not None:
|
|
51
|
+
if isinstance(payload, (dict, list)):
|
|
52
|
+
body["payload"] = json.dumps(payload).encode("utf-8")
|
|
53
|
+
elif isinstance(payload, (bytes, bytearray)):
|
|
54
|
+
body["payload"] = payload
|
|
55
|
+
else:
|
|
56
|
+
raise TypeError("payload must be bytes or dict")
|
|
57
|
+
|
|
58
|
+
r = self.session.post(
|
|
59
|
+
f"{self.base_url}/emit",
|
|
60
|
+
data=json.dumps(body),
|
|
61
|
+
timeout=self.timeout,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
if not (200 <= r.status_code < 300):
|
|
65
|
+
raise RuntimeError(r.json()["error"])
|
|
66
|
+
return r.json()["id"]
|
|
67
|
+
|
|
68
|
+
def get_event(self, event_id: int) -> Dict[str, Any]:
|
|
69
|
+
self.init_session()
|
|
70
|
+
|
|
71
|
+
r = self.session.get(
|
|
72
|
+
f"{self.base_url}/event/{event_id}",
|
|
73
|
+
timeout=self.timeout,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
if not (200 <= r.status_code < 300):
|
|
77
|
+
raise RuntimeError(r.json()["error"])
|
|
78
|
+
return r.json()
|
|
79
|
+
|
|
80
|
+
def children(self, event_id: int) -> List[int]:
|
|
81
|
+
self.init_session()
|
|
82
|
+
|
|
83
|
+
r = self.session.get(
|
|
84
|
+
f"{self.base_url}/children/{event_id}",
|
|
85
|
+
timeout=self.timeout,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
if not (200 <= r.status_code < 300):
|
|
89
|
+
raise RuntimeError(r.json()["error"])
|
|
90
|
+
return r.json()["children"]
|
|
91
|
+
|
|
92
|
+
def descendants(self, event_id: int) -> Dict[str, Any]:
|
|
93
|
+
self.init_session()
|
|
94
|
+
|
|
95
|
+
r = self.session.get(
|
|
96
|
+
f"{self.base_url}/descendants/{event_id}",
|
|
97
|
+
timeout=self.timeout,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if not (200 <= r.status_code < 300):
|
|
101
|
+
raise RuntimeError(r.json()["error"])
|
|
102
|
+
return r.json()
|
|
103
|
+
|
|
104
|
+
def heads(self) -> List[int]:
|
|
105
|
+
self.init_session()
|
|
106
|
+
|
|
107
|
+
r = self.session.get(
|
|
108
|
+
f"{self.base_url}/heads",
|
|
109
|
+
timeout=self.timeout,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if not (200 <= r.status_code < 300):
|
|
113
|
+
raise RuntimeError(r.json()["error"])
|
|
114
|
+
return r.json()["heads"]
|
|
115
|
+
|
|
116
|
+
def health(self) -> bool:
|
|
117
|
+
self.init_session()
|
|
118
|
+
try:
|
|
119
|
+
r = self.session.get(
|
|
120
|
+
f"{self.base_url}/healthz",
|
|
121
|
+
timeout=self.timeout,
|
|
122
|
+
)
|
|
123
|
+
return r.status_code == 200
|
|
124
|
+
except Exception:
|
|
125
|
+
return False
|
|
126
|
+
|
|
127
|
+
def version(self) -> Dict[str, Any]:
|
|
128
|
+
self.init_session()
|
|
129
|
+
|
|
130
|
+
r = self.session.get(
|
|
131
|
+
f"{self.base_url}/version",
|
|
132
|
+
timeout=self.timeout,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
if not (200 <= r.status_code < 300):
|
|
136
|
+
raise RuntimeError(r.json()["error"])
|
|
137
|
+
return r.json()
|
|
138
|
+
|
|
139
|
+
# ---------- SSE Subscribe ----------
|
|
140
|
+
|
|
141
|
+
def subscribe(
|
|
142
|
+
self,
|
|
143
|
+
on_event: Callable[[Dict[str, Any]], None],
|
|
144
|
+
daemon: bool = True,
|
|
145
|
+
) -> threading.Thread:
|
|
146
|
+
"""
|
|
147
|
+
Subscribe to SSE stream.
|
|
148
|
+
on_event will be called for each emitted Event.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
def _run():
|
|
152
|
+
with self.session.get(
|
|
153
|
+
f"{self.base_url}/subscribe",
|
|
154
|
+
stream=True,
|
|
155
|
+
timeout=None,
|
|
156
|
+
) as r:
|
|
157
|
+
r.raise_for_status()
|
|
158
|
+
buf = ""
|
|
159
|
+
for line in r.iter_lines(decode_unicode=True):
|
|
160
|
+
if not line:
|
|
161
|
+
continue
|
|
162
|
+
|
|
163
|
+
if line.startswith("data:"):
|
|
164
|
+
data = line[len("data:") :].strip()
|
|
165
|
+
try:
|
|
166
|
+
ev = json.loads(data)
|
|
167
|
+
on_event(ev)
|
|
168
|
+
except Exception:
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
self.init_session()
|
|
172
|
+
|
|
173
|
+
t = threading.Thread(target=_run, daemon=daemon)
|
|
174
|
+
t.start()
|
|
175
|
+
return t
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class NullClient:
|
|
179
|
+
event_id = 0
|
|
180
|
+
|
|
181
|
+
def emit(self, *args, **kwargs):
|
|
182
|
+
self.event_id += 1
|
|
183
|
+
return self.event_id
|
|
184
|
+
|
|
185
|
+
def get_event(self, *args, **kwargs):
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
def children(self, *args, **kwargs):
|
|
189
|
+
return []
|
|
190
|
+
|
|
191
|
+
def descendants(self, *args, **kwargs):
|
|
192
|
+
return None
|
|
193
|
+
|
|
194
|
+
def heads(self):
|
|
195
|
+
return []
|
|
196
|
+
|
|
197
|
+
def subscribe(self, *args, **kwargs):
|
|
198
|
+
return None
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
def format_tree(node: dict, prefix: str = "", is_last: bool = True) -> str:
|
|
2
|
+
"""
|
|
3
|
+
将 {"id": x, "children": [...]} 结构格式化为树状文本。
|
|
4
|
+
|
|
5
|
+
:param node: 当前节点
|
|
6
|
+
:param prefix: 前缀(递归用)
|
|
7
|
+
:param is_last: 是否是同级最后一个节点
|
|
8
|
+
:return: 树状字符串
|
|
9
|
+
"""
|
|
10
|
+
lines = []
|
|
11
|
+
|
|
12
|
+
connector = "└── " if is_last else "├── "
|
|
13
|
+
|
|
14
|
+
label = str(node["id"])
|
|
15
|
+
if node.get("is_ref"):
|
|
16
|
+
label += " (ref)"
|
|
17
|
+
|
|
18
|
+
lines.append(f"{prefix}{connector}{label}")
|
|
19
|
+
|
|
20
|
+
children = node.get("children", [])
|
|
21
|
+
if children:
|
|
22
|
+
next_prefix = prefix + (" " if is_last else "│ ")
|
|
23
|
+
for i, child in enumerate(children):
|
|
24
|
+
last = i == len(children) - 1
|
|
25
|
+
lines.append(format_tree(child, next_prefix, last))
|
|
26
|
+
|
|
27
|
+
return "\n".join(lines)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def format_tree_root(tree: dict) -> str:
|
|
31
|
+
"""
|
|
32
|
+
格式化整棵树(根节点无连接符)
|
|
33
|
+
"""
|
|
34
|
+
lines = [str(tree["id"])]
|
|
35
|
+
|
|
36
|
+
children = tree.get("children", [])
|
|
37
|
+
for i, child in enumerate(children):
|
|
38
|
+
last = i == len(children) - 1
|
|
39
|
+
lines.append(format_tree(child, "", last))
|
|
40
|
+
|
|
41
|
+
return "\n".join(lines)
|
|
@@ -5,16 +5,15 @@ from datetime import datetime
|
|
|
5
5
|
from multiprocessing import Queue as MPQueue
|
|
6
6
|
from typing import Any, Dict, List
|
|
7
7
|
|
|
8
|
-
from .
|
|
9
|
-
from .task_report import TaskReporter
|
|
8
|
+
from .task_stage import TaskStage
|
|
9
|
+
from .task_report import TaskReporter, NullTaskReporter
|
|
10
10
|
from .task_logging import LogListener, TaskLogger
|
|
11
11
|
from .task_queue import TaskQueue
|
|
12
|
-
from .task_types import StageStatus, TerminationSignal, TERMINATION_SIGNAL
|
|
12
|
+
from .task_types import TaskEnvelope, StageStatus, TerminationSignal, TERMINATION_SIGNAL
|
|
13
13
|
from .task_tools import (
|
|
14
14
|
format_duration,
|
|
15
15
|
format_timestamp,
|
|
16
16
|
cleanup_mpqueue,
|
|
17
|
-
make_hashable,
|
|
18
17
|
build_structure_graph,
|
|
19
18
|
format_structure_list_from_graph,
|
|
20
19
|
append_jsonl_log,
|
|
@@ -26,19 +25,23 @@ from .task_tools import (
|
|
|
26
25
|
load_task_by_error,
|
|
27
26
|
format_repr,
|
|
28
27
|
)
|
|
28
|
+
from .adapters.celestialtree import (
|
|
29
|
+
Client as CelestialTreeClient,
|
|
30
|
+
NullClient as NullCelestialTreeClient,
|
|
31
|
+
)
|
|
29
32
|
|
|
30
33
|
|
|
31
34
|
class TaskGraph:
|
|
32
|
-
def __init__(self, root_stages: List[
|
|
35
|
+
def __init__(self, root_stages: List[TaskStage], layout_mode: str = "process"):
|
|
33
36
|
"""
|
|
34
37
|
初始化 TaskGraph 实例。
|
|
35
38
|
|
|
36
|
-
TaskGraph 表示一组
|
|
39
|
+
TaskGraph 表示一组 TaskStage 节点所构成的任务图,可用于构建并行、串行、
|
|
37
40
|
分层等多种形式的任务执行流程。通过分析图结构和调度布局策略,实现灵活的
|
|
38
41
|
DAG 任务调度控制。
|
|
39
42
|
|
|
40
|
-
:param root_stages : List[
|
|
41
|
-
根节点
|
|
43
|
+
:param root_stages : List[TaskStage]
|
|
44
|
+
根节点 TaskStage 列表,用于构建任务图的入口节点。
|
|
42
45
|
支持多根节点(森林结构),系统将自动构建整个任务依赖图。
|
|
43
46
|
|
|
44
47
|
:param layout_mode : str, optional, default = 'process'
|
|
@@ -61,6 +64,7 @@ class TaskGraph:
|
|
|
61
64
|
self.analyze_graph()
|
|
62
65
|
self.set_layout_mode(layout_mode)
|
|
63
66
|
self.set_reporter()
|
|
67
|
+
self.set_ctree()
|
|
64
68
|
|
|
65
69
|
def init_env(self):
|
|
66
70
|
"""
|
|
@@ -126,7 +130,7 @@ class TaskGraph:
|
|
|
126
130
|
queue.extend(stage.next_stages)
|
|
127
131
|
|
|
128
132
|
for stage_tag in self.stages_status_dict:
|
|
129
|
-
stage:
|
|
133
|
+
stage: TaskStage = self.stages_status_dict[stage_tag]["stage"]
|
|
130
134
|
in_queue: TaskQueue = self.stages_status_dict[stage_tag]["in_queue"]
|
|
131
135
|
|
|
132
136
|
# 遍历每个前驱,创建边队列
|
|
@@ -158,7 +162,7 @@ class TaskGraph:
|
|
|
158
162
|
"""
|
|
159
163
|
self.structure_json = build_structure_graph(self.root_stages)
|
|
160
164
|
|
|
161
|
-
def set_root_stages(self, root_stages: List[
|
|
165
|
+
def set_root_stages(self, root_stages: List[TaskStage]):
|
|
162
166
|
"""
|
|
163
167
|
设置根节点
|
|
164
168
|
|
|
@@ -188,8 +192,32 @@ class TaskGraph:
|
|
|
188
192
|
:param host: 报告器主机地址
|
|
189
193
|
:param port: 报告器端口
|
|
190
194
|
"""
|
|
191
|
-
|
|
192
|
-
|
|
195
|
+
if is_report:
|
|
196
|
+
self.reporter = TaskReporter(
|
|
197
|
+
self, self.log_listener.get_queue(), host, port
|
|
198
|
+
)
|
|
199
|
+
else:
|
|
200
|
+
self.reporter = NullTaskReporter()
|
|
201
|
+
|
|
202
|
+
def set_ctree(self, use_ctree=False, host="127.0.0.1", port=7777):
|
|
203
|
+
"""
|
|
204
|
+
设定事件树客户端
|
|
205
|
+
|
|
206
|
+
:param use_ctree: 是否使用事件树
|
|
207
|
+
:param host: 事件树主机地址
|
|
208
|
+
:param port: 事件树端口
|
|
209
|
+
"""
|
|
210
|
+
self._use_ctree = use_ctree
|
|
211
|
+
self._ctree_host = host
|
|
212
|
+
self._ctree_port = port
|
|
213
|
+
|
|
214
|
+
if use_ctree:
|
|
215
|
+
self.ctree_client = CelestialTreeClient(host=host, port=port)
|
|
216
|
+
if not self.ctree_client.health():
|
|
217
|
+
self._use_ctree = False
|
|
218
|
+
self.ctree_client = NullCelestialTreeClient()
|
|
219
|
+
else:
|
|
220
|
+
self.ctree_client = NullCelestialTreeClient()
|
|
193
221
|
|
|
194
222
|
def set_graph_mode(self, stage_mode: str, execution_mode: str):
|
|
195
223
|
"""
|
|
@@ -199,7 +227,7 @@ class TaskGraph:
|
|
|
199
227
|
:param execution_mode: 节点内部执行模式, 可选值为 'serial' 或 'thread''
|
|
200
228
|
"""
|
|
201
229
|
|
|
202
|
-
def set_subsequent_stage_mode(stage:
|
|
230
|
+
def set_subsequent_stage_mode(stage: TaskStage):
|
|
203
231
|
stage.set_stage_mode(stage_mode)
|
|
204
232
|
stage.set_execution_mode(execution_mode)
|
|
205
233
|
visited_stages.add(stage)
|
|
@@ -222,7 +250,7 @@ class TaskGraph:
|
|
|
222
250
|
:param put_termination_signal: 是否放入终止信号
|
|
223
251
|
"""
|
|
224
252
|
for tag, tasks in tasks_dict.items():
|
|
225
|
-
stage:
|
|
253
|
+
stage: TaskStage = self.stages_status_dict[tag]["stage"]
|
|
226
254
|
in_queue: TaskQueue = self.stages_status_dict[tag]["in_queue"]
|
|
227
255
|
|
|
228
256
|
for task in tasks:
|
|
@@ -230,8 +258,18 @@ class TaskGraph:
|
|
|
230
258
|
in_queue.put(TERMINATION_SIGNAL)
|
|
231
259
|
continue
|
|
232
260
|
|
|
233
|
-
|
|
261
|
+
task_id = self.ctree_client.emit(
|
|
262
|
+
"task.input", message=f"In '{stage.get_stage_tag()}'"
|
|
263
|
+
)
|
|
264
|
+
envelope = TaskEnvelope.wrap(task, task_id)
|
|
265
|
+
in_queue.put_first(envelope)
|
|
234
266
|
stage.task_counter.add_init_value(1)
|
|
267
|
+
self.task_logger.task_inject(
|
|
268
|
+
stage.get_func_name(),
|
|
269
|
+
stage.get_task_info(task),
|
|
270
|
+
stage.get_stage_tag(),
|
|
271
|
+
f"[{task_id}]",
|
|
272
|
+
)
|
|
235
273
|
|
|
236
274
|
if put_termination_signal:
|
|
237
275
|
for root_stage in self.root_stages:
|
|
@@ -253,7 +291,7 @@ class TaskGraph:
|
|
|
253
291
|
self.start_time = time.time()
|
|
254
292
|
self.task_logger.start_graph(self.get_structure_list())
|
|
255
293
|
self._persist_structure_metadata()
|
|
256
|
-
self.reporter.start()
|
|
294
|
+
self.reporter.start()
|
|
257
295
|
|
|
258
296
|
self.put_stage_queue(init_tasks_dict, put_termination_signal)
|
|
259
297
|
self._excute_stages()
|
|
@@ -288,7 +326,7 @@ class TaskGraph:
|
|
|
288
326
|
|
|
289
327
|
processes = []
|
|
290
328
|
for stage_tag in layer:
|
|
291
|
-
stage:
|
|
329
|
+
stage: TaskStage = self.stages_status_dict[stage_tag]["stage"]
|
|
292
330
|
self._execute_stage(stage)
|
|
293
331
|
if stage.stage_mode == "process":
|
|
294
332
|
processes.append(self.processes[-1]) # 最新的进程
|
|
@@ -301,7 +339,7 @@ class TaskGraph:
|
|
|
301
339
|
|
|
302
340
|
self.task_logger.end_layer(layer, time.time() - start_time)
|
|
303
341
|
|
|
304
|
-
def _execute_stage(self, stage:
|
|
342
|
+
def _execute_stage(self, stage: TaskStage):
|
|
305
343
|
"""
|
|
306
344
|
执行单个节点
|
|
307
345
|
|
|
@@ -318,6 +356,9 @@ class TaskGraph:
|
|
|
318
356
|
self.stages_status_dict[stage_tag]["status"] = StageStatus.RUNNING
|
|
319
357
|
self.stages_status_dict[stage_tag]["start_time"] = time.time()
|
|
320
358
|
|
|
359
|
+
if self._use_ctree:
|
|
360
|
+
stage.set_ctree(self._ctree_host, self._ctree_port)
|
|
361
|
+
|
|
321
362
|
if stage.stage_mode == "process":
|
|
322
363
|
p = multiprocessing.Process(
|
|
323
364
|
target=stage.start_stage,
|
|
@@ -458,7 +499,7 @@ class TaskGraph:
|
|
|
458
499
|
interval = self.reporter.interval
|
|
459
500
|
|
|
460
501
|
for tag, stage_status_dict in self.stages_status_dict.items():
|
|
461
|
-
stage:
|
|
502
|
+
stage: TaskStage = stage_status_dict["stage"]
|
|
462
503
|
last_stage_status_dict: dict = self.last_status_dict.get(tag, {})
|
|
463
504
|
|
|
464
505
|
status = stage_status_dict.get("status", StageStatus.NOT_STARTED)
|