celestialflow 3.0.9__tar.gz → 3.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {celestialflow-3.0.9 → celestialflow-3.1.1}/PKG-INFO +45 -108
  2. {celestialflow-3.0.9 → celestialflow-3.1.1}/README.md +44 -107
  3. {celestialflow-3.0.9 → celestialflow-3.1.1}/pyproject.toml +66 -60
  4. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/__init__.py +2 -2
  5. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/base.css +34 -0
  6. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/dashboard.css +34 -70
  7. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/main.js +17 -9
  8. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_errors.js +1 -1
  9. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_statuses.js +3 -24
  10. celestialflow-3.1.1/src/celestialflow/static/js/task_summary.js +36 -0
  11. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/utils.js +34 -1
  12. celestialflow-3.1.1/src/celestialflow/task_errors.py +78 -0
  13. celestialflow-3.0.9/src/celestialflow/task_manage.py → celestialflow-3.1.1/src/celestialflow/task_executor.py +168 -144
  14. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_graph.py +197 -177
  15. celestialflow-3.0.9/src/celestialflow/task_logging.py → celestialflow-3.1.1/src/celestialflow/task_logger.py +48 -28
  16. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_nodes.py +9 -13
  17. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_progress.py +16 -16
  18. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_queue.py +114 -104
  19. celestialflow-3.1.1/src/celestialflow/task_report.py +235 -0
  20. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_stage.py +69 -24
  21. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_structure.py +33 -12
  22. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_tools.py +244 -34
  23. celestialflow-3.1.1/src/celestialflow/task_types.py +131 -0
  24. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_web.py +68 -12
  25. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/templates/index.html +17 -8
  26. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/PKG-INFO +45 -108
  27. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/SOURCES.txt +5 -3
  28. celestialflow-3.0.9/tests/test_manage.py → celestialflow-3.1.1/tests/test_executor.py +24 -22
  29. {celestialflow-3.0.9 → celestialflow-3.1.1}/tests/test_graph.py +17 -6
  30. {celestialflow-3.0.9 → celestialflow-3.1.1}/tests/test_nodes.py +14 -6
  31. {celestialflow-3.0.9 → celestialflow-3.1.1}/tests/test_structure.py +43 -35
  32. celestialflow-3.0.9/src/celestialflow/task_report.py +0 -142
  33. celestialflow-3.0.9/src/celestialflow/task_types.py +0 -100
  34. {celestialflow-3.0.9 → celestialflow-3.1.1}/setup.cfg +0 -0
  35. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/errors.css +0 -0
  36. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/inject.css +0 -0
  37. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/favicon.ico +0 -0
  38. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_injection.js +0 -0
  39. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_structure.js +0 -0
  40. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_topology.js +0 -0
  41. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/dependency_links.txt +0 -0
  42. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/entry_points.txt +0 -0
  43. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/requires.txt +0 -0
  44. {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: celestialflow
3
- Version: 3.0.9
3
+ Version: 3.1.1
4
4
  Summary: A flexible GRAPH-based task orchestration framework.
5
5
  Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
6
6
  License: MIT
@@ -49,16 +49,16 @@ Requires-Dist: celestialtree
49
49
  - 相比 Airflow/Dagster 更轻、更快开始
50
50
  - 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph 等复杂依赖模式
51
51
 
52
- 框架的基本单元为 **TaskManager**,可独立运行,并支持四种执行模式:
52
+ 框架的基本单元为 **TaskExecutor**,可独立运行,并支持四种执行模式:
53
53
 
54
54
  * **线性(serial)**
55
55
  * **多线程(thread)**
56
56
  * **多进程(process)**
57
57
  * **协程(async)**
58
58
 
59
- TaskManager 实现了对任务的结果缓存,任务去重,进度条显示,多执行模式比较等功能,单独使用也很好用。
59
+ TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示,多执行模式比较等功能,单独使用也很好用。
60
60
 
61
- 但除去直接使用 TaskManager,更重要的是使用其子类**TaskStage**。TaskStage 可以互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
61
+ 但除去直接使用 TaskExecutor,更重要的是使用其子类**TaskStage**。TaskStage 可以互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
62
62
 
63
63
  TaskStage 的任务执行模式只有两种:
64
64
 
@@ -130,13 +130,17 @@ flowchart LR
130
130
  安装 CelestialFlow:
131
131
 
132
132
  ```bash
133
- pip install celestialflow
133
+ # 推荐使用 `uv` 管理依赖与环境
134
+ uv pip install celestialflow
135
+
136
+ # 不过也可以直接使用 `pip`
137
+ uv pip install celestialflow
134
138
  ```
135
139
 
136
140
  一个简单的可运行代码:
137
141
 
138
142
  ```python
139
- from celestialflow import TaskManager, TaskGraph
143
+ from celestialflow import TaskStage, TaskGraph
140
144
 
141
145
  def add(x, y):
142
146
  return x + y
@@ -146,8 +150,8 @@ def square(x):
146
150
 
147
151
  if __name__ == "__main__":
148
152
  # 定义两个任务节点
149
- stage1 = TaskManager(add, execution_mode="thread", unpack_task_args=True)
150
- stage2 = TaskManager(square, execution_mode="thread")
153
+ stage1 = TaskStage(add, execution_mode="thread", unpack_task_args=True)
154
+ stage2 = TaskStage(square, execution_mode="thread")
151
155
 
152
156
  # 构建任务图结构
153
157
  stage1.set_graph_context([stage2], stage_mode="process", stage_name="Adder")
@@ -172,8 +176,9 @@ if __name__ == "__main__":
172
176
 
173
177
  若你想了解框架的整体结构与核心组件,下面的参考文档会对你有帮助:
174
178
 
175
- - [🔧TaskManage/TaskStage概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_manage.md)
176
- - [🌐TaskGrapg概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_graph.md)
179
+ - [🔧TaskExecutor概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_executor.md)
180
+ - [🔧TaskStage概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_stage.md)
181
+ - [🌐TaskGraph概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_graph.md)
177
182
  - [📚Go Worker概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/go_worker.md)
178
183
 
179
184
  推荐阅读顺序:
@@ -182,7 +187,7 @@ if __name__ == "__main__":
182
187
  flowchart TD
183
188
  classDef whiteNode fill:#ffffff,stroke:#000000,color:#000000;
184
189
 
185
- TM[TaskManage.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
190
+ TM[TaskExecutor.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
186
191
  TM --> TP[TaskProgress.md]
187
192
 
188
193
  TG --> TQ[TaskQueue.md]
@@ -201,6 +206,7 @@ flowchart TD
201
206
  - [TaskTools](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_tools.md)
202
207
  - [TaskTypes](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_types.md)
203
208
  - [TaskLogging](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_logging.md)
209
+ - [TaskErrors](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_errors.md)
204
210
 
205
211
  如果你更喜欢通过完整案例理解框架的运行方式,可以参考这篇从零开始构建 TaskGraph 的教程:
206
212
 
@@ -230,100 +236,13 @@ flowchart TD
230
236
 
231
237
  ## 文件结构(File Structure)
232
238
 
233
- ```
234
- 📁 CelestialFlow (224MB 407KB 500B)
235
- 📁 docs (34KB 944B)
236
- 📁 reference (29KB 995B)
237
- 📝 ctree_client.md (6KB 997B)
238
- 📝 go_worker.md (6KB 378B)
239
- 📝 task_graph.md (4KB 989B)
240
- 📝 task_logging.md (0B)
241
- 📝 task_manage.md (7KB 499B)
242
- 📝 task_nodes.md (4KB 68B)
243
- 📝 task_progress.md (0B)
244
- 📝 task_queue.md (0B)
245
- 📝 task_report.md (0B)
246
- 📝 task_stage.md (0B)
247
- 📝 task_structure.md (112B)
248
- 📝 task_tools.md (0B)
249
- 📝 task_types.md (0B)
250
- 📝 task_web.md (0B)
251
- 📝 quick_start.md (4KB 973B)
252
- 📝 tutorial.md (0B)
253
- 📁 experiments (15KB 111B)
254
- 🐍 benchmark_datastructures.py (6KB 52B)
255
- 🐍 benchmark_hash.py (1KB 284B)
256
- 🐍 benchmark_queue.py (5KB 504B)
257
- 🐍 benchmark_tqdm.py (1KB 160B)
258
- 🐍 experiment_tqdm.py (1KB 135B)
259
- 📁 go_worker (6MB 968KB 38B)
260
- 📁 worker (5KB 746B)
261
- 🌀 parser.go (394B)
262
- 🌀 processor.go (2KB 612B)
263
- 🌀 types.go (299B)
264
- 🌀 worker.go (2KB 465B)
265
- ❓ go.mod (292B)
266
- ❓ go.sum (1KB 68B)
267
- ❓ go_worker.exe (6MB 960KB)
268
- 🌀 main.go (980B)
269
- 📁 img (4MB 700KB 695B)
270
- 📷 logo(old).png (836KB 542B)
271
- 📷 logo.png (122KB 747B)
272
- 📷 web_display.png (129KB 545B)
273
- 🎞️ web_ui.gif (3MB 635KB 909B)
274
- 📁 src (1MB 929KB 111B)
275
- 📁 celestialflow (1MB 911KB 486B)
276
- 📁 static (1MB 417KB 367B)
277
- 📁 css (31KB 621B)
278
- 🎨 base.css (6KB 181B)
279
- 🎨 dashboard.css (7KB 414B)
280
- 🎨 errors.css (5KB 216B)
281
- 🎨 inject.css (12KB 834B)
282
- 📁 js (33KB 672B)
283
- 📜 main.js (5KB 65B)
284
- 📜 task_errors.js (4KB 595B)
285
- 📜 task_injection.js (8KB 484B)
286
- 📜 task_statuses.js (8KB 51B)
287
- 📜 task_structure.js (4KB 709B)
288
- 📜 task_topology.js (1KB 375B)
289
- 📜 utils.js (1KB 441B)
290
- ❓ favicon.ico (1MB 352KB 98B)
291
- 📁 templates (13KB 174B)
292
- 🌐 index.html (13KB 174B)
293
- 📁 [1项排除的目录] (339KB 553B)
294
- 🐍 task_graph.py (26KB 700B)
295
- 🐍 task_logging.py (10KB 282B)
296
- 🐍 task_manage.py (36KB 110B)
297
- 🐍 task_nodes.py (10KB 814B)
298
- 🐍 task_progress.py (1KB 477B)
299
- 🐍 task_queue.py (10KB 948B)
300
- 🐍 task_report.py (4KB 828B)
301
- 🐍 task_stage.py (6KB 577B)
302
- 🐍 task_structure.py (7KB 255B)
303
- 🐍 task_tools.py (17KB 19B)
304
- 🐍 task_types.py (2KB 269B)
305
- 🐍 task_web.py (6KB 235B)
306
- 🐍 __init__.py (1KB 22B)
307
- 📁 [1项排除的目录] (17KB 649B)
308
- 📁 tests (126KB 724B)
309
- 📁 [1项排除的目录] (90KB 464B)
310
- 🐍 conftest.py (49B)
311
- 📝 README.md (6KB 324B)
312
- 🐍 test_graph.py (5KB 547B)
313
- 🐍 test_manage.py (3KB 90B)
314
- 🐍 test_nodes.py (11KB 218B)
315
- 🐍 test_structure.py (10KB 56B)
316
- 📁 [6项排除的目录] (210MB 685KB 450B)
317
- ❓ .env (88B)
318
- ❓ .gitignore (272B)
319
- ❓ LICENSE (1KB 65B)
320
- ❓ Makefile (501B)
321
- ❓ pyproject.toml (1KB 594B)
322
- 📝 README.md (16KB 3B)
323
- (celestial-flow 3.0.9)
324
- ```
239
+ <p align="center">
240
+ <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
241
+ <br/>
242
+ <em>celestial-flow 3.1.1</em>
243
+ </p>
325
244
 
326
- (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。)
245
+ (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
327
246
 
328
247
  ## 更新日志(Change Log)
329
248
 
@@ -342,17 +261,35 @@ flowchart TD
342
261
  - 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
343
262
  - 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
344
263
  - 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
345
- - 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
264
+ - 3.0.7: 将TaskStage从TaskExecutor中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
346
265
  - 3.0.8: 在ctree逻辑上将"任务重试"事件后的"任务成功/失败/重试"事件视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
347
- - 3.0.9: 1/23/2026
266
+ - 3.0.9:
348
267
  - 更新前端mermaid显示中部分节点图标;
349
- - 对ctree_client的大量修改;
268
+ - 对ctree_client进行匹配CelestialTree的大量修改;
350
269
  - 将ctree_client移出为单独的project;
351
270
  - 在前端中添加error_id的显示, 为之后显示provenance_tree做准备;
352
271
  - 增加大量warning与error, 用于提醒不规范设置;
353
272
  - 优化前后端中错误数据的传输方式, 在大量错误数据时减少内存消耗;
354
273
  - 优化TaskLogger中log队列的准入机制;
355
- - 修改部分Bug
274
+ - 修改部分Bug;
275
+ - 3.1.0:
276
+ - 新增:
277
+ - 优化web端仪表盘页面的"总体状态摘要"卡片, 新增"总重复任务"与"总剩余市场", 后者由节点间拓扑关系计算而成, 后续还需要优化;
278
+ - 修复:
279
+ - 3.0.9版本下当web端与celestialflow运行端不同时, error数据无法传递的问题;
280
+ - 修复NullTaskReporter使用问题;
281
+ - 3.1.1:
282
+ - 新增:
283
+ - [Important] CelestialTree中引入grpc, 这大大减少了emit操作的耗时;
284
+ - [Important] 为TerminationSignal添加id, 并可以像task一样通过CelestialTree进行跟踪;
285
+ - 优化task_graph与task_executor的log分级, 现在默认为"SUCCESS"级;
286
+ - 将go_worker部分分离为单独project: [celestialflow-goworker]https://github.com/Mr-xiaotian/celestialflow-goworker
287
+ - 在readme中使用svg图片来展示文件夹结构;
288
+ - 优化全局剩余时间的计算;
289
+ - 优化部分代码结构;
290
+ - 修复:
291
+ - 修复节点剩余时间在小于1s时显示0的问题(这很影响判断);
292
+ - 在task_graph中使用"staged"模式时会报错的问题;
356
293
 
357
294
  ## Star 历史趋势(Star History)
358
295
 
@@ -23,16 +23,16 @@
23
23
  - 相比 Airflow/Dagster 更轻、更快开始
24
24
  - 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph 等复杂依赖模式
25
25
 
26
- 框架的基本单元为 **TaskManager**,可独立运行,并支持四种执行模式:
26
+ 框架的基本单元为 **TaskExecutor**,可独立运行,并支持四种执行模式:
27
27
 
28
28
  * **线性(serial)**
29
29
  * **多线程(thread)**
30
30
  * **多进程(process)**
31
31
  * **协程(async)**
32
32
 
33
- TaskManager 实现了对任务的结果缓存,任务去重,进度条显示,多执行模式比较等功能,单独使用也很好用。
33
+ TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示,多执行模式比较等功能,单独使用也很好用。
34
34
 
35
- 但除去直接使用 TaskManager,更重要的是使用其子类**TaskStage**。TaskStage 可以互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
35
+ 但除去直接使用 TaskExecutor,更重要的是使用其子类**TaskStage**。TaskStage 可以互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
36
36
 
37
37
  TaskStage 的任务执行模式只有两种:
38
38
 
@@ -104,13 +104,17 @@ flowchart LR
104
104
  安装 CelestialFlow:
105
105
 
106
106
  ```bash
107
- pip install celestialflow
107
+ # 推荐使用 `uv` 管理依赖与环境
108
+ uv pip install celestialflow
109
+
110
+ # 不过也可以直接使用 `pip`
111
+ uv pip install celestialflow
108
112
  ```
109
113
 
110
114
  一个简单的可运行代码:
111
115
 
112
116
  ```python
113
- from celestialflow import TaskManager, TaskGraph
117
+ from celestialflow import TaskStage, TaskGraph
114
118
 
115
119
  def add(x, y):
116
120
  return x + y
@@ -120,8 +124,8 @@ def square(x):
120
124
 
121
125
  if __name__ == "__main__":
122
126
  # 定义两个任务节点
123
- stage1 = TaskManager(add, execution_mode="thread", unpack_task_args=True)
124
- stage2 = TaskManager(square, execution_mode="thread")
127
+ stage1 = TaskStage(add, execution_mode="thread", unpack_task_args=True)
128
+ stage2 = TaskStage(square, execution_mode="thread")
125
129
 
126
130
  # 构建任务图结构
127
131
  stage1.set_graph_context([stage2], stage_mode="process", stage_name="Adder")
@@ -146,8 +150,9 @@ if __name__ == "__main__":
146
150
 
147
151
  若你想了解框架的整体结构与核心组件,下面的参考文档会对你有帮助:
148
152
 
149
- - [🔧TaskManage/TaskStage概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_manage.md)
150
- - [🌐TaskGrapg概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_graph.md)
153
+ - [🔧TaskExecutor概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_executor.md)
154
+ - [🔧TaskStage概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_stage.md)
155
+ - [🌐TaskGraph概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_graph.md)
151
156
  - [📚Go Worker概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/go_worker.md)
152
157
 
153
158
  推荐阅读顺序:
@@ -156,7 +161,7 @@ if __name__ == "__main__":
156
161
  flowchart TD
157
162
  classDef whiteNode fill:#ffffff,stroke:#000000,color:#000000;
158
163
 
159
- TM[TaskManage.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
164
+ TM[TaskExecutor.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
160
165
  TM --> TP[TaskProgress.md]
161
166
 
162
167
  TG --> TQ[TaskQueue.md]
@@ -175,6 +180,7 @@ flowchart TD
175
180
  - [TaskTools](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_tools.md)
176
181
  - [TaskTypes](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_types.md)
177
182
  - [TaskLogging](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_logging.md)
183
+ - [TaskErrors](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_errors.md)
178
184
 
179
185
  如果你更喜欢通过完整案例理解框架的运行方式,可以参考这篇从零开始构建 TaskGraph 的教程:
180
186
 
@@ -204,100 +210,13 @@ flowchart TD
204
210
 
205
211
  ## 文件结构(File Structure)
206
212
 
207
- ```
208
- 📁 CelestialFlow (224MB 407KB 500B)
209
- 📁 docs (34KB 944B)
210
- 📁 reference (29KB 995B)
211
- 📝 ctree_client.md (6KB 997B)
212
- 📝 go_worker.md (6KB 378B)
213
- 📝 task_graph.md (4KB 989B)
214
- 📝 task_logging.md (0B)
215
- 📝 task_manage.md (7KB 499B)
216
- 📝 task_nodes.md (4KB 68B)
217
- 📝 task_progress.md (0B)
218
- 📝 task_queue.md (0B)
219
- 📝 task_report.md (0B)
220
- 📝 task_stage.md (0B)
221
- 📝 task_structure.md (112B)
222
- 📝 task_tools.md (0B)
223
- 📝 task_types.md (0B)
224
- 📝 task_web.md (0B)
225
- 📝 quick_start.md (4KB 973B)
226
- 📝 tutorial.md (0B)
227
- 📁 experiments (15KB 111B)
228
- 🐍 benchmark_datastructures.py (6KB 52B)
229
- 🐍 benchmark_hash.py (1KB 284B)
230
- 🐍 benchmark_queue.py (5KB 504B)
231
- 🐍 benchmark_tqdm.py (1KB 160B)
232
- 🐍 experiment_tqdm.py (1KB 135B)
233
- 📁 go_worker (6MB 968KB 38B)
234
- 📁 worker (5KB 746B)
235
- 🌀 parser.go (394B)
236
- 🌀 processor.go (2KB 612B)
237
- 🌀 types.go (299B)
238
- 🌀 worker.go (2KB 465B)
239
- ❓ go.mod (292B)
240
- ❓ go.sum (1KB 68B)
241
- ❓ go_worker.exe (6MB 960KB)
242
- 🌀 main.go (980B)
243
- 📁 img (4MB 700KB 695B)
244
- 📷 logo(old).png (836KB 542B)
245
- 📷 logo.png (122KB 747B)
246
- 📷 web_display.png (129KB 545B)
247
- 🎞️ web_ui.gif (3MB 635KB 909B)
248
- 📁 src (1MB 929KB 111B)
249
- 📁 celestialflow (1MB 911KB 486B)
250
- 📁 static (1MB 417KB 367B)
251
- 📁 css (31KB 621B)
252
- 🎨 base.css (6KB 181B)
253
- 🎨 dashboard.css (7KB 414B)
254
- 🎨 errors.css (5KB 216B)
255
- 🎨 inject.css (12KB 834B)
256
- 📁 js (33KB 672B)
257
- 📜 main.js (5KB 65B)
258
- 📜 task_errors.js (4KB 595B)
259
- 📜 task_injection.js (8KB 484B)
260
- 📜 task_statuses.js (8KB 51B)
261
- 📜 task_structure.js (4KB 709B)
262
- 📜 task_topology.js (1KB 375B)
263
- 📜 utils.js (1KB 441B)
264
- ❓ favicon.ico (1MB 352KB 98B)
265
- 📁 templates (13KB 174B)
266
- 🌐 index.html (13KB 174B)
267
- 📁 [1项排除的目录] (339KB 553B)
268
- 🐍 task_graph.py (26KB 700B)
269
- 🐍 task_logging.py (10KB 282B)
270
- 🐍 task_manage.py (36KB 110B)
271
- 🐍 task_nodes.py (10KB 814B)
272
- 🐍 task_progress.py (1KB 477B)
273
- 🐍 task_queue.py (10KB 948B)
274
- 🐍 task_report.py (4KB 828B)
275
- 🐍 task_stage.py (6KB 577B)
276
- 🐍 task_structure.py (7KB 255B)
277
- 🐍 task_tools.py (17KB 19B)
278
- 🐍 task_types.py (2KB 269B)
279
- 🐍 task_web.py (6KB 235B)
280
- 🐍 __init__.py (1KB 22B)
281
- 📁 [1项排除的目录] (17KB 649B)
282
- 📁 tests (126KB 724B)
283
- 📁 [1项排除的目录] (90KB 464B)
284
- 🐍 conftest.py (49B)
285
- 📝 README.md (6KB 324B)
286
- 🐍 test_graph.py (5KB 547B)
287
- 🐍 test_manage.py (3KB 90B)
288
- 🐍 test_nodes.py (11KB 218B)
289
- 🐍 test_structure.py (10KB 56B)
290
- 📁 [6项排除的目录] (210MB 685KB 450B)
291
- ❓ .env (88B)
292
- ❓ .gitignore (272B)
293
- ❓ LICENSE (1KB 65B)
294
- ❓ Makefile (501B)
295
- ❓ pyproject.toml (1KB 594B)
296
- 📝 README.md (16KB 3B)
297
- (celestial-flow 3.0.9)
298
- ```
213
+ <p align="center">
214
+ <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
215
+ <br/>
216
+ <em>celestial-flow 3.1.1</em>
217
+ </p>
299
218
 
300
- (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。)
219
+ (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
301
220
 
302
221
  ## 更新日志(Change Log)
303
222
 
@@ -316,17 +235,35 @@ flowchart TD
316
235
  - 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
317
236
  - 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
318
237
  - 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
319
- - 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
238
+ - 3.0.7: 将TaskStage从TaskExecutor中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
320
239
  - 3.0.8: 在ctree逻辑上将"任务重试"事件后的"任务成功/失败/重试"事件视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
321
- - 3.0.9: 1/23/2026
240
+ - 3.0.9:
322
241
  - 更新前端mermaid显示中部分节点图标;
323
- - 对ctree_client的大量修改;
242
+ - 对ctree_client进行匹配CelestialTree的大量修改;
324
243
  - 将ctree_client移出为单独的project;
325
244
  - 在前端中添加error_id的显示, 为之后显示provenance_tree做准备;
326
245
  - 增加大量warning与error, 用于提醒不规范设置;
327
246
  - 优化前后端中错误数据的传输方式, 在大量错误数据时减少内存消耗;
328
247
  - 优化TaskLogger中log队列的准入机制;
329
- - 修改部分Bug
248
+ - 修改部分Bug;
249
+ - 3.1.0:
250
+ - 新增:
251
+ - 优化web端仪表盘页面的"总体状态摘要"卡片, 新增"总重复任务"与"总剩余市场", 后者由节点间拓扑关系计算而成, 后续还需要优化;
252
+ - 修复:
253
+ - 3.0.9版本下当web端与celestialflow运行端不同时, error数据无法传递的问题;
254
+ - 修复NullTaskReporter使用问题;
255
+ - 3.1.1:
256
+ - 新增:
257
+ - [Important] CelestialTree中引入grpc, 这大大减少了emit操作的耗时;
258
+ - [Important] 为TerminationSignal添加id, 并可以像task一样通过CelestialTree进行跟踪;
259
+ - 优化task_graph与task_executor的log分级, 现在默认为"SUCCESS"级;
260
+ - 将go_worker部分分离为单独project: [celestialflow-goworker]https://github.com/Mr-xiaotian/celestialflow-goworker
261
+ - 在readme中使用svg图片来展示文件夹结构;
262
+ - 优化全局剩余时间的计算;
263
+ - 优化部分代码结构;
264
+ - 修复:
265
+ - 修复节点剩余时间在小于1s时显示0的问题(这很影响判断);
266
+ - 在task_graph中使用"staged"模式时会报错的问题;
330
267
 
331
268
  ## Star 历史趋势(Star History)
332
269
 
@@ -1,60 +1,66 @@
1
- [build-system]
2
- requires = ["setuptools>=61.0", "wheel"]
3
- build-backend = "setuptools.build_meta"
4
-
5
- [project]
6
- name = "celestialflow"
7
- version = "3.0.9"
8
- description = "A flexible GRAPH-based task orchestration framework."
9
- readme = "README.md"
10
- license = { text = "MIT" }
11
- authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
12
- keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
13
- requires-python = ">=3.8"
14
-
15
- dependencies = [
16
- "tqdm",
17
- "loguru",
18
- "fastapi",
19
- "uvicorn",
20
- "requests",
21
- "networkx",
22
- "redis",
23
- "jinja2",
24
- "celestialtree",
25
- ]
26
-
27
- classifiers = [
28
- "Programming Language :: Python :: 3",
29
- "License :: OSI Approved :: MIT License",
30
- "Operating System :: OS Independent",
31
- "Framework :: FastAPI",
32
- "Topic :: Software Development :: Libraries",
33
- ]
34
-
35
- [project.urls]
36
- "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
37
- "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
38
-
39
- [project.scripts]
40
- celestialflow-web = "celestialflow.task_web:main_entry"
41
-
42
- [tool.setuptools]
43
- license-files = []
44
-
45
- [tool.setuptools.package-data]
46
- "celestialflow" = ["templates/*.html", "static/**/*"]
47
-
48
- [tool.pytest.ini_options]
49
- addopts = "-s"
50
- log_cli = true
51
- log_cli_level = "INFO"
52
- log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
53
- log_cli_date_format = "%Y-%m-%d %H:%M:%S"
54
-
55
- filterwarnings = [
56
- "ignore::DeprecationWarning",
57
- "ignore::pytest.PytestDeprecationWarning",
58
- ]
59
-
60
- asyncio_default_fixture_loop_scope = "function"
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "celestialflow"
7
+ version = "3.1.1"
8
+ description = "A flexible GRAPH-based task orchestration framework."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
12
+ keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
13
+ requires-python = ">=3.8"
14
+
15
+ dependencies = [
16
+ "tqdm",
17
+ "loguru",
18
+ "fastapi",
19
+ "uvicorn",
20
+ "requests",
21
+ "networkx",
22
+ "redis",
23
+ "jinja2",
24
+ "celestialtree",
25
+ ]
26
+
27
+ classifiers = [
28
+ "Programming Language :: Python :: 3",
29
+ "License :: OSI Approved :: MIT License",
30
+ "Operating System :: OS Independent",
31
+ "Framework :: FastAPI",
32
+ "Topic :: Software Development :: Libraries",
33
+ ]
34
+
35
+ [project.urls]
36
+ "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
37
+ "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
38
+
39
+ [project.scripts]
40
+ celestialflow-web = "celestialflow.task_web:main_entry"
41
+
42
+ [tool.setuptools]
43
+ license-files = []
44
+
45
+ [tool.setuptools.package-data]
46
+ "celestialflow" = ["templates/*.html", "static/**/*"]
47
+
48
+ [tool.pytest.ini_options]
49
+ addopts = "-s"
50
+ log_cli = true
51
+ log_cli_level = "INFO"
52
+ log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
53
+ log_cli_date_format = "%Y-%m-%d %H:%M:%S"
54
+
55
+ filterwarnings = [
56
+ "ignore::DeprecationWarning",
57
+ "ignore::pytest.PytestDeprecationWarning",
58
+ ]
59
+
60
+ asyncio_default_fixture_loop_scope = "function"
61
+
62
+ [dependency-groups]
63
+ dev = [
64
+ "build>=1.2.2.post1",
65
+ "twine>=6.1.0",
66
+ ]
@@ -1,5 +1,5 @@
1
1
  from .task_graph import TaskGraph
2
- from .task_manage import TaskManager
2
+ from .task_executor import TaskExecutor
3
3
  from .task_stage import TaskStage
4
4
  from .task_nodes import (
5
5
  TaskSplitter,
@@ -34,7 +34,7 @@ __all__ = [
34
34
  "TaskComplete",
35
35
  "TaskWheel",
36
36
  "TaskGrid",
37
- "TaskManager",
37
+ "TaskExecutor",
38
38
  "TaskStage",
39
39
  "TaskSplitter",
40
40
  "TaskRedisSink",
@@ -10,6 +10,40 @@
10
10
  --gray-medium: #9ca3af;
11
11
  --gray-dark: #4b5563;
12
12
  --gray-slate: #6b7280;
13
+
14
+ /* ================================
15
+ Summary palette (light mode)
16
+ ================================ */
17
+ --summary-blue-bg: #dbeafe;
18
+ --summary-orange-bg: #fed7aa;
19
+ --summary-yellow-bg: #fef3c7;
20
+ --summary-red-bg: #fee2e2;
21
+ --summary-green-bg: #d1fae5;
22
+ --summary-purple-bg: #e9d5ff;
23
+
24
+ --summary-blue-value: #2563eb;
25
+ --summary-orange-value: #ea580c;
26
+ --summary-yellow-value: #d97706;
27
+ --summary-red-value: #dc2626;
28
+ --summary-green-value: #059669;
29
+ --summary-purple-value: #7c3aed;
30
+
31
+ /* ================================
32
+ Summary palette (dark mode)
33
+ ================================ */
34
+ --summary-blue-bg-dark: #1e3a8a;
35
+ --summary-orange-bg-dark: #7c2d12;
36
+ --summary-yellow-bg-dark: #78350f;
37
+ --summary-red-bg-dark: #7f1d1d;
38
+ --summary-green-bg-dark: #14532d;
39
+ --summary-purple-bg-dark: #4c1d95;
40
+
41
+ --summary-blue-value-dark: #60a5fa;
42
+ --summary-orange-value-dark: #fdba74;
43
+ --summary-yellow-value-dark: #facc15;
44
+ --summary-red-value-dark: #f87171;
45
+ --summary-green-value-dark: #34d399;
46
+ --summary-purple-value-dark: #c4b5fd;
13
47
  }
14
48
 
15
49
  /* ================================