celestialflow 3.0.9__tar.gz → 3.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celestialflow-3.0.9 → celestialflow-3.1.1}/PKG-INFO +45 -108
- {celestialflow-3.0.9 → celestialflow-3.1.1}/README.md +44 -107
- {celestialflow-3.0.9 → celestialflow-3.1.1}/pyproject.toml +66 -60
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/__init__.py +2 -2
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/base.css +34 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/dashboard.css +34 -70
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/main.js +17 -9
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_errors.js +1 -1
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_statuses.js +3 -24
- celestialflow-3.1.1/src/celestialflow/static/js/task_summary.js +36 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/utils.js +34 -1
- celestialflow-3.1.1/src/celestialflow/task_errors.py +78 -0
- celestialflow-3.0.9/src/celestialflow/task_manage.py → celestialflow-3.1.1/src/celestialflow/task_executor.py +168 -144
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_graph.py +197 -177
- celestialflow-3.0.9/src/celestialflow/task_logging.py → celestialflow-3.1.1/src/celestialflow/task_logger.py +48 -28
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_nodes.py +9 -13
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_progress.py +16 -16
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_queue.py +114 -104
- celestialflow-3.1.1/src/celestialflow/task_report.py +235 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_stage.py +69 -24
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_structure.py +33 -12
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_tools.py +244 -34
- celestialflow-3.1.1/src/celestialflow/task_types.py +131 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/task_web.py +68 -12
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/templates/index.html +17 -8
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/PKG-INFO +45 -108
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/SOURCES.txt +5 -3
- celestialflow-3.0.9/tests/test_manage.py → celestialflow-3.1.1/tests/test_executor.py +24 -22
- {celestialflow-3.0.9 → celestialflow-3.1.1}/tests/test_graph.py +17 -6
- {celestialflow-3.0.9 → celestialflow-3.1.1}/tests/test_nodes.py +14 -6
- {celestialflow-3.0.9 → celestialflow-3.1.1}/tests/test_structure.py +43 -35
- celestialflow-3.0.9/src/celestialflow/task_report.py +0 -142
- celestialflow-3.0.9/src/celestialflow/task_types.py +0 -100
- {celestialflow-3.0.9 → celestialflow-3.1.1}/setup.cfg +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/errors.css +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/css/inject.css +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/favicon.ico +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_injection.js +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_structure.js +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow/static/js/task_topology.js +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/dependency_links.txt +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/entry_points.txt +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/requires.txt +0 -0
- {celestialflow-3.0.9 → celestialflow-3.1.1}/src/celestialflow.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celestialflow
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.1.1
|
|
4
4
|
Summary: A flexible GRAPH-based task orchestration framework.
|
|
5
5
|
Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -49,16 +49,16 @@ Requires-Dist: celestialtree
|
|
|
49
49
|
- 相比 Airflow/Dagster 更轻、更快开始
|
|
50
50
|
- 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph 等复杂依赖模式
|
|
51
51
|
|
|
52
|
-
框架的基本单元为 **
|
|
52
|
+
框架的基本单元为 **TaskExecutor**,可独立运行,并支持四种执行模式:
|
|
53
53
|
|
|
54
54
|
* **线性(serial)**
|
|
55
55
|
* **多线程(thread)**
|
|
56
56
|
* **多进程(process)**
|
|
57
57
|
* **协程(async)**
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示,多执行模式比较等功能,单独使用也很好用。
|
|
60
60
|
|
|
61
|
-
但除去直接使用
|
|
61
|
+
但除去直接使用 TaskExecutor,更重要的是使用其子类**TaskStage**。TaskStage 可以互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
|
|
62
62
|
|
|
63
63
|
TaskStage 的任务执行模式只有两种:
|
|
64
64
|
|
|
@@ -130,13 +130,17 @@ flowchart LR
|
|
|
130
130
|
安装 CelestialFlow:
|
|
131
131
|
|
|
132
132
|
```bash
|
|
133
|
-
|
|
133
|
+
# 推荐使用 `uv` 管理依赖与环境
|
|
134
|
+
uv pip install celestialflow
|
|
135
|
+
|
|
136
|
+
# 不过也可以直接使用 `pip`
|
|
137
|
+
uv pip install celestialflow
|
|
134
138
|
```
|
|
135
139
|
|
|
136
140
|
一个简单的可运行代码:
|
|
137
141
|
|
|
138
142
|
```python
|
|
139
|
-
from celestialflow import
|
|
143
|
+
from celestialflow import TaskStage, TaskGraph
|
|
140
144
|
|
|
141
145
|
def add(x, y):
|
|
142
146
|
return x + y
|
|
@@ -146,8 +150,8 @@ def square(x):
|
|
|
146
150
|
|
|
147
151
|
if __name__ == "__main__":
|
|
148
152
|
# 定义两个任务节点
|
|
149
|
-
stage1 =
|
|
150
|
-
stage2 =
|
|
153
|
+
stage1 = TaskStage(add, execution_mode="thread", unpack_task_args=True)
|
|
154
|
+
stage2 = TaskStage(square, execution_mode="thread")
|
|
151
155
|
|
|
152
156
|
# 构建任务图结构
|
|
153
157
|
stage1.set_graph_context([stage2], stage_mode="process", stage_name="Adder")
|
|
@@ -172,8 +176,9 @@ if __name__ == "__main__":
|
|
|
172
176
|
|
|
173
177
|
若你想了解框架的整体结构与核心组件,下面的参考文档会对你有帮助:
|
|
174
178
|
|
|
175
|
-
- [🔧
|
|
176
|
-
- [
|
|
179
|
+
- [🔧TaskExecutor概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_executor.md)
|
|
180
|
+
- [🔧TaskStage概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_stage.md)
|
|
181
|
+
- [🌐TaskGraph概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_graph.md)
|
|
177
182
|
- [📚Go Worker概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/go_worker.md)
|
|
178
183
|
|
|
179
184
|
推荐阅读顺序:
|
|
@@ -182,7 +187,7 @@ if __name__ == "__main__":
|
|
|
182
187
|
flowchart TD
|
|
183
188
|
classDef whiteNode fill:#ffffff,stroke:#000000,color:#000000;
|
|
184
189
|
|
|
185
|
-
TM[
|
|
190
|
+
TM[TaskExecutor.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
|
|
186
191
|
TM --> TP[TaskProgress.md]
|
|
187
192
|
|
|
188
193
|
TG --> TQ[TaskQueue.md]
|
|
@@ -201,6 +206,7 @@ flowchart TD
|
|
|
201
206
|
- [TaskTools](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_tools.md)
|
|
202
207
|
- [TaskTypes](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_types.md)
|
|
203
208
|
- [TaskLogging](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_logging.md)
|
|
209
|
+
- [TaskErrors](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_errors.md)
|
|
204
210
|
|
|
205
211
|
如果你更喜欢通过完整案例理解框架的运行方式,可以参考这篇从零开始构建 TaskGraph 的教程:
|
|
206
212
|
|
|
@@ -230,100 +236,13 @@ flowchart TD
|
|
|
230
236
|
|
|
231
237
|
## 文件结构(File Structure)
|
|
232
238
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
📝 go_worker.md (6KB 378B)
|
|
239
|
-
📝 task_graph.md (4KB 989B)
|
|
240
|
-
📝 task_logging.md (0B)
|
|
241
|
-
📝 task_manage.md (7KB 499B)
|
|
242
|
-
📝 task_nodes.md (4KB 68B)
|
|
243
|
-
📝 task_progress.md (0B)
|
|
244
|
-
📝 task_queue.md (0B)
|
|
245
|
-
📝 task_report.md (0B)
|
|
246
|
-
📝 task_stage.md (0B)
|
|
247
|
-
📝 task_structure.md (112B)
|
|
248
|
-
📝 task_tools.md (0B)
|
|
249
|
-
📝 task_types.md (0B)
|
|
250
|
-
📝 task_web.md (0B)
|
|
251
|
-
📝 quick_start.md (4KB 973B)
|
|
252
|
-
📝 tutorial.md (0B)
|
|
253
|
-
📁 experiments (15KB 111B)
|
|
254
|
-
🐍 benchmark_datastructures.py (6KB 52B)
|
|
255
|
-
🐍 benchmark_hash.py (1KB 284B)
|
|
256
|
-
🐍 benchmark_queue.py (5KB 504B)
|
|
257
|
-
🐍 benchmark_tqdm.py (1KB 160B)
|
|
258
|
-
🐍 experiment_tqdm.py (1KB 135B)
|
|
259
|
-
📁 go_worker (6MB 968KB 38B)
|
|
260
|
-
📁 worker (5KB 746B)
|
|
261
|
-
🌀 parser.go (394B)
|
|
262
|
-
🌀 processor.go (2KB 612B)
|
|
263
|
-
🌀 types.go (299B)
|
|
264
|
-
🌀 worker.go (2KB 465B)
|
|
265
|
-
❓ go.mod (292B)
|
|
266
|
-
❓ go.sum (1KB 68B)
|
|
267
|
-
❓ go_worker.exe (6MB 960KB)
|
|
268
|
-
🌀 main.go (980B)
|
|
269
|
-
📁 img (4MB 700KB 695B)
|
|
270
|
-
📷 logo(old).png (836KB 542B)
|
|
271
|
-
📷 logo.png (122KB 747B)
|
|
272
|
-
📷 web_display.png (129KB 545B)
|
|
273
|
-
🎞️ web_ui.gif (3MB 635KB 909B)
|
|
274
|
-
📁 src (1MB 929KB 111B)
|
|
275
|
-
📁 celestialflow (1MB 911KB 486B)
|
|
276
|
-
📁 static (1MB 417KB 367B)
|
|
277
|
-
📁 css (31KB 621B)
|
|
278
|
-
🎨 base.css (6KB 181B)
|
|
279
|
-
🎨 dashboard.css (7KB 414B)
|
|
280
|
-
🎨 errors.css (5KB 216B)
|
|
281
|
-
🎨 inject.css (12KB 834B)
|
|
282
|
-
📁 js (33KB 672B)
|
|
283
|
-
📜 main.js (5KB 65B)
|
|
284
|
-
📜 task_errors.js (4KB 595B)
|
|
285
|
-
📜 task_injection.js (8KB 484B)
|
|
286
|
-
📜 task_statuses.js (8KB 51B)
|
|
287
|
-
📜 task_structure.js (4KB 709B)
|
|
288
|
-
📜 task_topology.js (1KB 375B)
|
|
289
|
-
📜 utils.js (1KB 441B)
|
|
290
|
-
❓ favicon.ico (1MB 352KB 98B)
|
|
291
|
-
📁 templates (13KB 174B)
|
|
292
|
-
🌐 index.html (13KB 174B)
|
|
293
|
-
📁 [1项排除的目录] (339KB 553B)
|
|
294
|
-
🐍 task_graph.py (26KB 700B)
|
|
295
|
-
🐍 task_logging.py (10KB 282B)
|
|
296
|
-
🐍 task_manage.py (36KB 110B)
|
|
297
|
-
🐍 task_nodes.py (10KB 814B)
|
|
298
|
-
🐍 task_progress.py (1KB 477B)
|
|
299
|
-
🐍 task_queue.py (10KB 948B)
|
|
300
|
-
🐍 task_report.py (4KB 828B)
|
|
301
|
-
🐍 task_stage.py (6KB 577B)
|
|
302
|
-
🐍 task_structure.py (7KB 255B)
|
|
303
|
-
🐍 task_tools.py (17KB 19B)
|
|
304
|
-
🐍 task_types.py (2KB 269B)
|
|
305
|
-
🐍 task_web.py (6KB 235B)
|
|
306
|
-
🐍 __init__.py (1KB 22B)
|
|
307
|
-
📁 [1项排除的目录] (17KB 649B)
|
|
308
|
-
📁 tests (126KB 724B)
|
|
309
|
-
📁 [1项排除的目录] (90KB 464B)
|
|
310
|
-
🐍 conftest.py (49B)
|
|
311
|
-
📝 README.md (6KB 324B)
|
|
312
|
-
🐍 test_graph.py (5KB 547B)
|
|
313
|
-
🐍 test_manage.py (3KB 90B)
|
|
314
|
-
🐍 test_nodes.py (11KB 218B)
|
|
315
|
-
🐍 test_structure.py (10KB 56B)
|
|
316
|
-
📁 [6项排除的目录] (210MB 685KB 450B)
|
|
317
|
-
❓ .env (88B)
|
|
318
|
-
❓ .gitignore (272B)
|
|
319
|
-
❓ LICENSE (1KB 65B)
|
|
320
|
-
❓ Makefile (501B)
|
|
321
|
-
❓ pyproject.toml (1KB 594B)
|
|
322
|
-
📝 README.md (16KB 3B)
|
|
323
|
-
(celestial-flow 3.0.9)
|
|
324
|
-
```
|
|
239
|
+
<p align="center">
|
|
240
|
+
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
241
|
+
<br/>
|
|
242
|
+
<em>celestial-flow 3.1.1</em>
|
|
243
|
+
</p>
|
|
325
244
|
|
|
326
|
-
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()
|
|
245
|
+
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
327
246
|
|
|
328
247
|
## 更新日志(Change Log)
|
|
329
248
|
|
|
@@ -342,17 +261,35 @@ flowchart TD
|
|
|
342
261
|
- 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
|
|
343
262
|
- 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
|
|
344
263
|
- 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
|
|
345
|
-
- 3.0.7: 将TaskStage从
|
|
264
|
+
- 3.0.7: 将TaskStage从TaskExecutor中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
|
|
346
265
|
- 3.0.8: 在ctree逻辑上将"任务重试"事件后的"任务成功/失败/重试"事件视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
|
|
347
|
-
- 3.0.9:
|
|
266
|
+
- 3.0.9:
|
|
348
267
|
- 更新前端mermaid显示中部分节点图标;
|
|
349
|
-
- 对ctree_client的大量修改;
|
|
268
|
+
- 对ctree_client进行匹配CelestialTree的大量修改;
|
|
350
269
|
- 将ctree_client移出为单独的project;
|
|
351
270
|
- 在前端中添加error_id的显示, 为之后显示provenance_tree做准备;
|
|
352
271
|
- 增加大量warning与error, 用于提醒不规范设置;
|
|
353
272
|
- 优化前后端中错误数据的传输方式, 在大量错误数据时减少内存消耗;
|
|
354
273
|
- 优化TaskLogger中log队列的准入机制;
|
|
355
|
-
- 修改部分Bug
|
|
274
|
+
- 修改部分Bug;
|
|
275
|
+
- 3.1.0:
|
|
276
|
+
- 新增:
|
|
277
|
+
- 优化web端仪表盘页面的"总体状态摘要"卡片, 新增"总重复任务"与"总剩余市场", 后者由节点间拓扑关系计算而成, 后续还需要优化;
|
|
278
|
+
- 修复:
|
|
279
|
+
- 3.0.9版本下当web端与celestialflow运行端不同时, error数据无法传递的问题;
|
|
280
|
+
- 修复NullTaskReporter使用问题;
|
|
281
|
+
- 3.1.1:
|
|
282
|
+
- 新增:
|
|
283
|
+
- [Important] CelestialTree中引入grpc, 这大大减少了emit操作的耗时;
|
|
284
|
+
- [Important] 为TerminationSignal添加id, 并可以像task一样通过CelestialTree进行跟踪;
|
|
285
|
+
- 优化task_graph与task_executor的log分级, 现在默认为"SUCCESS"级;
|
|
286
|
+
- 将go_worker部分分离为单独project: [celestialflow-goworker]https://github.com/Mr-xiaotian/celestialflow-goworker
|
|
287
|
+
- 在readme中使用svg图片来展示文件夹结构;
|
|
288
|
+
- 优化全局剩余时间的计算;
|
|
289
|
+
- 优化部分代码结构;
|
|
290
|
+
- 修复:
|
|
291
|
+
- 修复节点剩余时间在小于1s时显示0的问题(这很影响判断);
|
|
292
|
+
- 在task_graph中使用"staged"模式时会报错的问题;
|
|
356
293
|
|
|
357
294
|
## Star 历史趋势(Star History)
|
|
358
295
|
|
|
@@ -23,16 +23,16 @@
|
|
|
23
23
|
- 相比 Airflow/Dagster 更轻、更快开始
|
|
24
24
|
- 相比 multiprocessing/threading 更结构化,可直接表达 loop / complete graph 等复杂依赖模式
|
|
25
25
|
|
|
26
|
-
框架的基本单元为 **
|
|
26
|
+
框架的基本单元为 **TaskExecutor**,可独立运行,并支持四种执行模式:
|
|
27
27
|
|
|
28
28
|
* **线性(serial)**
|
|
29
29
|
* **多线程(thread)**
|
|
30
30
|
* **多进程(process)**
|
|
31
31
|
* **协程(async)**
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示,多执行模式比较等功能,单独使用也很好用。
|
|
34
34
|
|
|
35
|
-
但除去直接使用
|
|
35
|
+
但除去直接使用 TaskExecutor,更重要的是使用其子类**TaskStage**。TaskStage 可以互相连接,形成具有上游与下游依赖关系的任务图(**TaskGraph**)。下游 stage 会自动接收上游执行完成的结果作为输入,从而形成明确的数据流。
|
|
36
36
|
|
|
37
37
|
TaskStage 的任务执行模式只有两种:
|
|
38
38
|
|
|
@@ -104,13 +104,17 @@ flowchart LR
|
|
|
104
104
|
安装 CelestialFlow:
|
|
105
105
|
|
|
106
106
|
```bash
|
|
107
|
-
|
|
107
|
+
# 推荐使用 `uv` 管理依赖与环境
|
|
108
|
+
uv pip install celestialflow
|
|
109
|
+
|
|
110
|
+
# 不过也可以直接使用 `pip`
|
|
111
|
+
uv pip install celestialflow
|
|
108
112
|
```
|
|
109
113
|
|
|
110
114
|
一个简单的可运行代码:
|
|
111
115
|
|
|
112
116
|
```python
|
|
113
|
-
from celestialflow import
|
|
117
|
+
from celestialflow import TaskStage, TaskGraph
|
|
114
118
|
|
|
115
119
|
def add(x, y):
|
|
116
120
|
return x + y
|
|
@@ -120,8 +124,8 @@ def square(x):
|
|
|
120
124
|
|
|
121
125
|
if __name__ == "__main__":
|
|
122
126
|
# 定义两个任务节点
|
|
123
|
-
stage1 =
|
|
124
|
-
stage2 =
|
|
127
|
+
stage1 = TaskStage(add, execution_mode="thread", unpack_task_args=True)
|
|
128
|
+
stage2 = TaskStage(square, execution_mode="thread")
|
|
125
129
|
|
|
126
130
|
# 构建任务图结构
|
|
127
131
|
stage1.set_graph_context([stage2], stage_mode="process", stage_name="Adder")
|
|
@@ -146,8 +150,9 @@ if __name__ == "__main__":
|
|
|
146
150
|
|
|
147
151
|
若你想了解框架的整体结构与核心组件,下面的参考文档会对你有帮助:
|
|
148
152
|
|
|
149
|
-
- [🔧
|
|
150
|
-
- [
|
|
153
|
+
- [🔧TaskExecutor概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_executor.md)
|
|
154
|
+
- [🔧TaskStage概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_stage.md)
|
|
155
|
+
- [🌐TaskGraph概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_graph.md)
|
|
151
156
|
- [📚Go Worker概念](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/go_worker.md)
|
|
152
157
|
|
|
153
158
|
推荐阅读顺序:
|
|
@@ -156,7 +161,7 @@ if __name__ == "__main__":
|
|
|
156
161
|
flowchart TD
|
|
157
162
|
classDef whiteNode fill:#ffffff,stroke:#000000,color:#000000;
|
|
158
163
|
|
|
159
|
-
TM[
|
|
164
|
+
TM[TaskExecutor.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
|
|
160
165
|
TM --> TP[TaskProgress.md]
|
|
161
166
|
|
|
162
167
|
TG --> TQ[TaskQueue.md]
|
|
@@ -175,6 +180,7 @@ flowchart TD
|
|
|
175
180
|
- [TaskTools](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_tools.md)
|
|
176
181
|
- [TaskTypes](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_types.md)
|
|
177
182
|
- [TaskLogging](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_logging.md)
|
|
183
|
+
- [TaskErrors](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/reference/task_errors.md)
|
|
178
184
|
|
|
179
185
|
如果你更喜欢通过完整案例理解框架的运行方式,可以参考这篇从零开始构建 TaskGraph 的教程:
|
|
180
186
|
|
|
@@ -204,100 +210,13 @@ flowchart TD
|
|
|
204
210
|
|
|
205
211
|
## 文件结构(File Structure)
|
|
206
212
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
📝 go_worker.md (6KB 378B)
|
|
213
|
-
📝 task_graph.md (4KB 989B)
|
|
214
|
-
📝 task_logging.md (0B)
|
|
215
|
-
📝 task_manage.md (7KB 499B)
|
|
216
|
-
📝 task_nodes.md (4KB 68B)
|
|
217
|
-
📝 task_progress.md (0B)
|
|
218
|
-
📝 task_queue.md (0B)
|
|
219
|
-
📝 task_report.md (0B)
|
|
220
|
-
📝 task_stage.md (0B)
|
|
221
|
-
📝 task_structure.md (112B)
|
|
222
|
-
📝 task_tools.md (0B)
|
|
223
|
-
📝 task_types.md (0B)
|
|
224
|
-
📝 task_web.md (0B)
|
|
225
|
-
📝 quick_start.md (4KB 973B)
|
|
226
|
-
📝 tutorial.md (0B)
|
|
227
|
-
📁 experiments (15KB 111B)
|
|
228
|
-
🐍 benchmark_datastructures.py (6KB 52B)
|
|
229
|
-
🐍 benchmark_hash.py (1KB 284B)
|
|
230
|
-
🐍 benchmark_queue.py (5KB 504B)
|
|
231
|
-
🐍 benchmark_tqdm.py (1KB 160B)
|
|
232
|
-
🐍 experiment_tqdm.py (1KB 135B)
|
|
233
|
-
📁 go_worker (6MB 968KB 38B)
|
|
234
|
-
📁 worker (5KB 746B)
|
|
235
|
-
🌀 parser.go (394B)
|
|
236
|
-
🌀 processor.go (2KB 612B)
|
|
237
|
-
🌀 types.go (299B)
|
|
238
|
-
🌀 worker.go (2KB 465B)
|
|
239
|
-
❓ go.mod (292B)
|
|
240
|
-
❓ go.sum (1KB 68B)
|
|
241
|
-
❓ go_worker.exe (6MB 960KB)
|
|
242
|
-
🌀 main.go (980B)
|
|
243
|
-
📁 img (4MB 700KB 695B)
|
|
244
|
-
📷 logo(old).png (836KB 542B)
|
|
245
|
-
📷 logo.png (122KB 747B)
|
|
246
|
-
📷 web_display.png (129KB 545B)
|
|
247
|
-
🎞️ web_ui.gif (3MB 635KB 909B)
|
|
248
|
-
📁 src (1MB 929KB 111B)
|
|
249
|
-
📁 celestialflow (1MB 911KB 486B)
|
|
250
|
-
📁 static (1MB 417KB 367B)
|
|
251
|
-
📁 css (31KB 621B)
|
|
252
|
-
🎨 base.css (6KB 181B)
|
|
253
|
-
🎨 dashboard.css (7KB 414B)
|
|
254
|
-
🎨 errors.css (5KB 216B)
|
|
255
|
-
🎨 inject.css (12KB 834B)
|
|
256
|
-
📁 js (33KB 672B)
|
|
257
|
-
📜 main.js (5KB 65B)
|
|
258
|
-
📜 task_errors.js (4KB 595B)
|
|
259
|
-
📜 task_injection.js (8KB 484B)
|
|
260
|
-
📜 task_statuses.js (8KB 51B)
|
|
261
|
-
📜 task_structure.js (4KB 709B)
|
|
262
|
-
📜 task_topology.js (1KB 375B)
|
|
263
|
-
📜 utils.js (1KB 441B)
|
|
264
|
-
❓ favicon.ico (1MB 352KB 98B)
|
|
265
|
-
📁 templates (13KB 174B)
|
|
266
|
-
🌐 index.html (13KB 174B)
|
|
267
|
-
📁 [1项排除的目录] (339KB 553B)
|
|
268
|
-
🐍 task_graph.py (26KB 700B)
|
|
269
|
-
🐍 task_logging.py (10KB 282B)
|
|
270
|
-
🐍 task_manage.py (36KB 110B)
|
|
271
|
-
🐍 task_nodes.py (10KB 814B)
|
|
272
|
-
🐍 task_progress.py (1KB 477B)
|
|
273
|
-
🐍 task_queue.py (10KB 948B)
|
|
274
|
-
🐍 task_report.py (4KB 828B)
|
|
275
|
-
🐍 task_stage.py (6KB 577B)
|
|
276
|
-
🐍 task_structure.py (7KB 255B)
|
|
277
|
-
🐍 task_tools.py (17KB 19B)
|
|
278
|
-
🐍 task_types.py (2KB 269B)
|
|
279
|
-
🐍 task_web.py (6KB 235B)
|
|
280
|
-
🐍 __init__.py (1KB 22B)
|
|
281
|
-
📁 [1项排除的目录] (17KB 649B)
|
|
282
|
-
📁 tests (126KB 724B)
|
|
283
|
-
📁 [1项排除的目录] (90KB 464B)
|
|
284
|
-
🐍 conftest.py (49B)
|
|
285
|
-
📝 README.md (6KB 324B)
|
|
286
|
-
🐍 test_graph.py (5KB 547B)
|
|
287
|
-
🐍 test_manage.py (3KB 90B)
|
|
288
|
-
🐍 test_nodes.py (11KB 218B)
|
|
289
|
-
🐍 test_structure.py (10KB 56B)
|
|
290
|
-
📁 [6项排除的目录] (210MB 685KB 450B)
|
|
291
|
-
❓ .env (88B)
|
|
292
|
-
❓ .gitignore (272B)
|
|
293
|
-
❓ LICENSE (1KB 65B)
|
|
294
|
-
❓ Makefile (501B)
|
|
295
|
-
❓ pyproject.toml (1KB 594B)
|
|
296
|
-
📝 README.md (16KB 3B)
|
|
297
|
-
(celestial-flow 3.0.9)
|
|
298
|
-
```
|
|
213
|
+
<p align="center">
|
|
214
|
+
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
215
|
+
<br/>
|
|
216
|
+
<em>celestial-flow 3.1.1</em>
|
|
217
|
+
</p>
|
|
299
218
|
|
|
300
|
-
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()
|
|
219
|
+
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
301
220
|
|
|
302
221
|
## 更新日志(Change Log)
|
|
303
222
|
|
|
@@ -316,17 +235,35 @@ flowchart TD
|
|
|
316
235
|
- 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
|
|
317
236
|
- 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
|
|
318
237
|
- 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
|
|
319
|
-
- 3.0.7: 将TaskStage从
|
|
238
|
+
- 3.0.7: 将TaskStage从TaskExecutor中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
|
|
320
239
|
- 3.0.8: 在ctree逻辑上将"任务重试"事件后的"任务成功/失败/重试"事件视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
|
|
321
|
-
- 3.0.9:
|
|
240
|
+
- 3.0.9:
|
|
322
241
|
- 更新前端mermaid显示中部分节点图标;
|
|
323
|
-
- 对ctree_client的大量修改;
|
|
242
|
+
- 对ctree_client进行匹配CelestialTree的大量修改;
|
|
324
243
|
- 将ctree_client移出为单独的project;
|
|
325
244
|
- 在前端中添加error_id的显示, 为之后显示provenance_tree做准备;
|
|
326
245
|
- 增加大量warning与error, 用于提醒不规范设置;
|
|
327
246
|
- 优化前后端中错误数据的传输方式, 在大量错误数据时减少内存消耗;
|
|
328
247
|
- 优化TaskLogger中log队列的准入机制;
|
|
329
|
-
- 修改部分Bug
|
|
248
|
+
- 修改部分Bug;
|
|
249
|
+
- 3.1.0:
|
|
250
|
+
- 新增:
|
|
251
|
+
- 优化web端仪表盘页面的"总体状态摘要"卡片, 新增"总重复任务"与"总剩余市场", 后者由节点间拓扑关系计算而成, 后续还需要优化;
|
|
252
|
+
- 修复:
|
|
253
|
+
- 3.0.9版本下当web端与celestialflow运行端不同时, error数据无法传递的问题;
|
|
254
|
+
- 修复NullTaskReporter使用问题;
|
|
255
|
+
- 3.1.1:
|
|
256
|
+
- 新增:
|
|
257
|
+
- [Important] CelestialTree中引入grpc, 这大大减少了emit操作的耗时;
|
|
258
|
+
- [Important] 为TerminationSignal添加id, 并可以像task一样通过CelestialTree进行跟踪;
|
|
259
|
+
- 优化task_graph与task_executor的log分级, 现在默认为"SUCCESS"级;
|
|
260
|
+
- 将go_worker部分分离为单独project: [celestialflow-goworker]https://github.com/Mr-xiaotian/celestialflow-goworker
|
|
261
|
+
- 在readme中使用svg图片来展示文件夹结构;
|
|
262
|
+
- 优化全局剩余时间的计算;
|
|
263
|
+
- 优化部分代码结构;
|
|
264
|
+
- 修复:
|
|
265
|
+
- 修复节点剩余时间在小于1s时显示0的问题(这很影响判断);
|
|
266
|
+
- 在task_graph中使用"staged"模式时会报错的问题;
|
|
330
267
|
|
|
331
268
|
## Star 历史趋势(Star History)
|
|
332
269
|
|
|
@@ -1,60 +1,66 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "celestialflow"
|
|
7
|
-
version = "3.
|
|
8
|
-
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
license = { text = "MIT" }
|
|
11
|
-
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
|
-
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
-
requires-python = ">=3.8"
|
|
14
|
-
|
|
15
|
-
dependencies = [
|
|
16
|
-
"tqdm",
|
|
17
|
-
"loguru",
|
|
18
|
-
"fastapi",
|
|
19
|
-
"uvicorn",
|
|
20
|
-
"requests",
|
|
21
|
-
"networkx",
|
|
22
|
-
"redis",
|
|
23
|
-
"jinja2",
|
|
24
|
-
"celestialtree",
|
|
25
|
-
]
|
|
26
|
-
|
|
27
|
-
classifiers = [
|
|
28
|
-
"Programming Language :: Python :: 3",
|
|
29
|
-
"License :: OSI Approved :: MIT License",
|
|
30
|
-
"Operating System :: OS Independent",
|
|
31
|
-
"Framework :: FastAPI",
|
|
32
|
-
"Topic :: Software Development :: Libraries",
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
[project.urls]
|
|
36
|
-
"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
|
|
37
|
-
"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
|
|
38
|
-
|
|
39
|
-
[project.scripts]
|
|
40
|
-
celestialflow-web = "celestialflow.task_web:main_entry"
|
|
41
|
-
|
|
42
|
-
[tool.setuptools]
|
|
43
|
-
license-files = []
|
|
44
|
-
|
|
45
|
-
[tool.setuptools.package-data]
|
|
46
|
-
"celestialflow" = ["templates/*.html", "static/**/*"]
|
|
47
|
-
|
|
48
|
-
[tool.pytest.ini_options]
|
|
49
|
-
addopts = "-s"
|
|
50
|
-
log_cli = true
|
|
51
|
-
log_cli_level = "INFO"
|
|
52
|
-
log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
|
|
53
|
-
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
54
|
-
|
|
55
|
-
filterwarnings = [
|
|
56
|
-
"ignore::DeprecationWarning",
|
|
57
|
-
"ignore::pytest.PytestDeprecationWarning",
|
|
58
|
-
]
|
|
59
|
-
|
|
60
|
-
asyncio_default_fixture_loop_scope = "function"
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "celestialflow"
|
|
7
|
+
version = "3.1.1"
|
|
8
|
+
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
|
+
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
+
requires-python = ">=3.8"
|
|
14
|
+
|
|
15
|
+
dependencies = [
|
|
16
|
+
"tqdm",
|
|
17
|
+
"loguru",
|
|
18
|
+
"fastapi",
|
|
19
|
+
"uvicorn",
|
|
20
|
+
"requests",
|
|
21
|
+
"networkx",
|
|
22
|
+
"redis",
|
|
23
|
+
"jinja2",
|
|
24
|
+
"celestialtree",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"License :: OSI Approved :: MIT License",
|
|
30
|
+
"Operating System :: OS Independent",
|
|
31
|
+
"Framework :: FastAPI",
|
|
32
|
+
"Topic :: Software Development :: Libraries",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
|
|
37
|
+
"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
celestialflow-web = "celestialflow.task_web:main_entry"
|
|
41
|
+
|
|
42
|
+
[tool.setuptools]
|
|
43
|
+
license-files = []
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.package-data]
|
|
46
|
+
"celestialflow" = ["templates/*.html", "static/**/*"]
|
|
47
|
+
|
|
48
|
+
[tool.pytest.ini_options]
|
|
49
|
+
addopts = "-s"
|
|
50
|
+
log_cli = true
|
|
51
|
+
log_cli_level = "INFO"
|
|
52
|
+
log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
|
|
53
|
+
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
54
|
+
|
|
55
|
+
filterwarnings = [
|
|
56
|
+
"ignore::DeprecationWarning",
|
|
57
|
+
"ignore::pytest.PytestDeprecationWarning",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
61
|
+
|
|
62
|
+
[dependency-groups]
|
|
63
|
+
dev = [
|
|
64
|
+
"build>=1.2.2.post1",
|
|
65
|
+
"twine>=6.1.0",
|
|
66
|
+
]
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from .task_graph import TaskGraph
|
|
2
|
-
from .
|
|
2
|
+
from .task_executor import TaskExecutor
|
|
3
3
|
from .task_stage import TaskStage
|
|
4
4
|
from .task_nodes import (
|
|
5
5
|
TaskSplitter,
|
|
@@ -34,7 +34,7 @@ __all__ = [
|
|
|
34
34
|
"TaskComplete",
|
|
35
35
|
"TaskWheel",
|
|
36
36
|
"TaskGrid",
|
|
37
|
-
"
|
|
37
|
+
"TaskExecutor",
|
|
38
38
|
"TaskStage",
|
|
39
39
|
"TaskSplitter",
|
|
40
40
|
"TaskRedisSink",
|
|
@@ -10,6 +10,40 @@
|
|
|
10
10
|
--gray-medium: #9ca3af;
|
|
11
11
|
--gray-dark: #4b5563;
|
|
12
12
|
--gray-slate: #6b7280;
|
|
13
|
+
|
|
14
|
+
/* ================================
|
|
15
|
+
Summary palette (light mode)
|
|
16
|
+
================================ */
|
|
17
|
+
--summary-blue-bg: #dbeafe;
|
|
18
|
+
--summary-orange-bg: #fed7aa;
|
|
19
|
+
--summary-yellow-bg: #fef3c7;
|
|
20
|
+
--summary-red-bg: #fee2e2;
|
|
21
|
+
--summary-green-bg: #d1fae5;
|
|
22
|
+
--summary-purple-bg: #e9d5ff;
|
|
23
|
+
|
|
24
|
+
--summary-blue-value: #2563eb;
|
|
25
|
+
--summary-orange-value: #ea580c;
|
|
26
|
+
--summary-yellow-value: #d97706;
|
|
27
|
+
--summary-red-value: #dc2626;
|
|
28
|
+
--summary-green-value: #059669;
|
|
29
|
+
--summary-purple-value: #7c3aed;
|
|
30
|
+
|
|
31
|
+
/* ================================
|
|
32
|
+
Summary palette (dark mode)
|
|
33
|
+
================================ */
|
|
34
|
+
--summary-blue-bg-dark: #1e3a8a;
|
|
35
|
+
--summary-orange-bg-dark: #7c2d12;
|
|
36
|
+
--summary-yellow-bg-dark: #78350f;
|
|
37
|
+
--summary-red-bg-dark: #7f1d1d;
|
|
38
|
+
--summary-green-bg-dark: #14532d;
|
|
39
|
+
--summary-purple-bg-dark: #4c1d95;
|
|
40
|
+
|
|
41
|
+
--summary-blue-value-dark: #60a5fa;
|
|
42
|
+
--summary-orange-value-dark: #fdba74;
|
|
43
|
+
--summary-yellow-value-dark: #facc15;
|
|
44
|
+
--summary-red-value-dark: #f87171;
|
|
45
|
+
--summary-green-value-dark: #34d399;
|
|
46
|
+
--summary-purple-value-dark: #c4b5fd;
|
|
13
47
|
}
|
|
14
48
|
|
|
15
49
|
/* ================================
|