celestialflow 3.0.6__tar.gz → 3.0.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {celestialflow-3.0.6 → celestialflow-3.0.8}/PKG-INFO +82 -54
  2. {celestialflow-3.0.6 → celestialflow-3.0.8}/README.md +81 -53
  3. {celestialflow-3.0.6 → celestialflow-3.0.8}/pyproject.toml +1 -1
  4. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/__init__.py +4 -0
  5. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/client.py +34 -17
  6. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/tools.py +6 -1
  7. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_errors.js +2 -2
  8. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_structure.js +1 -1
  9. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_topology.js +2 -2
  10. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_graph.py +124 -116
  11. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_logging.py +52 -30
  12. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_manage.py +165 -263
  13. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_nodes.py +113 -32
  14. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_queue.py +24 -19
  15. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_report.py +2 -2
  16. celestialflow-3.0.8/src/celestialflow/task_stage.py +176 -0
  17. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_structure.py +26 -23
  18. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_tools.py +38 -16
  19. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_types.py +7 -3
  20. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_web.py +1 -2
  21. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/PKG-INFO +82 -54
  22. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/SOURCES.txt +1 -0
  23. {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_graph.py +15 -14
  24. {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_nodes.py +62 -27
  25. {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_structure.py +75 -73
  26. {celestialflow-3.0.6 → celestialflow-3.0.8}/setup.cfg +0 -0
  27. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/__init__.py +0 -0
  28. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/__init__.py +0 -0
  29. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/base.css +0 -0
  30. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/dashboard.css +0 -0
  31. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/errors.css +0 -0
  32. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/inject.css +0 -0
  33. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/favicon.ico +0 -0
  34. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/main.js +0 -0
  35. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_injection.js +0 -0
  36. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_statuses.js +0 -0
  37. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/utils.js +0 -0
  38. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_progress.py +0 -0
  39. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/templates/index.html +0 -0
  40. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/dependency_links.txt +0 -0
  41. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/entry_points.txt +0 -0
  42. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/requires.txt +0 -0
  43. {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/top_level.txt +0 -0
  44. {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_manage.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: celestialflow
3
- Version: 3.0.6
3
+ Version: 3.0.8
4
4
  Summary: A flexible GRAPH-based task orchestration framework.
5
5
  Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
6
6
  License: MIT
@@ -173,7 +173,7 @@ if __name__ == "__main__":
173
173
  graph = TaskGraph([stage1])
174
174
 
175
175
  # 初始化任务并启动
176
- graph.start_graph({stage1.get_stage_tag(): [(1, 2), (3, 4), (5, 6)]})
176
+ graph.start_graph({stage1.get_tag(): [(1, 2), (3, 4), (5, 6)]})
177
177
  ```
178
178
 
179
179
  注意不要在.ipynb中运行。
@@ -244,79 +244,105 @@ flowchart TD
244
244
  ## 文件结构(File Structure)
245
245
 
246
246
  ```
247
- 📁 CelestialFlow (205MB 169KB 76B)
248
- 📁 experiments (14KB 536B)
249
- 🐍 benchmark_datastructures.py (5KB 796B)
247
+ 📁 CelestialFlow (213MB 633KB 545B)
248
+ 📁 docs (25KB 183B)
249
+ 📁 reference (20KB 175B)
250
+ 📝 go_worker.md (6KB 378B)
251
+ 📝 task_graph.md (4KB 989B)
252
+ 📝 task_logging.md (0B)
253
+ 📝 task_manage.md (7KB 7B)
254
+ 📝 task_nodes.md (1KB 737B)
255
+ 📝 task_progress.md (0B)
256
+ 📝 task_queue.md (0B)
257
+ 📝 task_report.md (0B)
258
+ 📝 task_structure.md (112B)
259
+ 📝 task_tools.md (0B)
260
+ 📝 task_types.md (0B)
261
+ 📝 task_web.md (0B)
262
+ 📝 quick_start.md (5KB 8B)
263
+ 📝 tutorial.md (0B)
264
+ 📁 experiments (15KB 111B)
265
+ 🐍 benchmark_datastructures.py (6KB 52B)
250
266
  🐍 benchmark_hash.py (1KB 284B)
251
- 🐍 benchmark_queue.py (5KB 185B)
267
+ 🐍 benchmark_queue.py (5KB 504B)
252
268
  🐍 benchmark_tqdm.py (1KB 160B)
253
269
  🐍 experiment_tqdm.py (1KB 135B)
254
- 📁 go_worker (6MB 967KB 64B)
255
- 📁 worker (5KB 684B)
270
+ 📁 go_worker (6MB 968KB 38B)
271
+ 📁 worker (5KB 746B)
256
272
  🌀 parser.go (394B)
257
273
  🌀 processor.go (2KB 612B)
258
- 🌀 types.go (237B)
274
+ 🌀 types.go (299B)
259
275
  🌀 worker.go (2KB 465B)
260
- ❓ go.mod (258B)
261
- ❓ go.sum (591B)
276
+ ❓ go.mod (292B)
277
+ ❓ go.sum (1KB 68B)
262
278
  ❓ go_worker.exe (6MB 960KB)
263
- 🌀 main.go (579B)
264
- 📁 img (966KB 63B)
265
- 📷 logo.png (836KB 542B)
279
+ 🌀 main.go (980B)
280
+ 📁 img (4MB 700KB 695B)
281
+ 📷 logo(old).png (836KB 542B)
282
+ 📷 logo.png (122KB 747B)
266
283
  📷 web_display.png (129KB 545B)
267
- 📁 src (1MB 884KB 224B)
268
- 📁 celestialflow (1MB 869KB 480B)
269
- 📁 static (1MB 419KB 510B)
270
- 📁 css (32KB 568B)
284
+ 🎞️ web_ui.gif (3MB 635KB 909B)
285
+ 📁 src (1MB 920KB 171B)
286
+ 📁 celestialflow (1MB 902KB 818B)
287
+ 📁 adapters (13KB 837B)
288
+ 📁 celestialtree (13KB 681B)
289
+ 📁 [1项排除的目录] (6KB 960B)
290
+ 🐍 client.py (5KB 486B)
291
+ 🐍 tools.py (1KB 182B)
292
+ 🐍 __init__.py (77B)
293
+ 📁 [1项排除的目录] (156B)
294
+ 🐍 __init__.py (0B)
295
+ 📁 static (1MB 417KB 101B)
296
+ 📁 css (31KB 410B)
271
297
  🎨 base.css (6KB 155B)
272
- 🎨 dashboard.css (8KB 435B)
298
+ 🎨 dashboard.css (7KB 277B)
273
299
  🎨 errors.css (5KB 168B)
274
300
  🎨 inject.css (12KB 834B)
275
- 📁 js (34KB 868B)
276
- 📜 main.js (4KB 973B)
277
- 📜 task_errors.js (4KB 544B)
301
+ 📁 js (33KB 617B)
302
+ 📜 main.js (5KB 77B)
303
+ 📜 task_errors.js (4KB 550B)
278
304
  📜 task_injection.js (8KB 491B)
279
305
  📜 task_statuses.js (8KB 63B)
280
- 📜 task_structure.js (7KB 143B)
281
- 📜 task_topology.js (261B)
306
+ 📜 task_structure.js (4KB 668B)
307
+ 📜 task_topology.js (1KB 375B)
282
308
  📜 utils.js (1KB 441B)
283
309
  ❓ favicon.ico (1MB 352KB 98B)
284
- 📁 templates (12KB 973B)
285
- 🌐 index.html (12KB 973B)
286
- 📁 [1项排除的目录] (314KB 710B)
287
- 📝 README.md (13KB 131B)
288
- 🐍 task_graph.py (22KB 530B)
289
- 🐍 task_logging.py (6KB 291B)
290
- 🐍 task_manage.py (32KB 66B)
291
- 🐍 task_nodes.py (4KB 953B)
310
+ 📁 templates (13KB 150B)
311
+ 🌐 index.html (13KB 150B)
312
+ 📁 [1项排除的目录] (331KB 149B)
313
+ 🐍 task_graph.py (23KB 835B)
314
+ 🐍 task_logging.py (8KB 903B)
315
+ 🐍 task_manage.py (33KB 577B)
316
+ 🐍 task_nodes.py (10KB 922B)
292
317
  🐍 task_progress.py (1KB 477B)
293
- 🐍 task_queue.py (7KB 896B)
294
- 🐍 task_report.py (5KB 511B)
295
- 🐍 task_structure.py (6KB 93B)
296
- 🐍 task_tools.py (15KB 182B)
297
- 🐍 task_types.py (1KB 461B)
298
- 🐍 task_web.py (5KB 47B)
299
- 🐍 __init__.py (817B)
300
- 📁 celestialflow.egg-info (14KB 768B)
301
- PKG-INFO (13KB 363B)
302
- 📄 [5项排除的文件] (1KB 405B)
303
- 📁 tests (113KB 510B)
304
- 📁 [1项排除的目录] (81KB 604B)
305
- 📝 README.md (7KB 231B)
306
- 🐍 test_graph.py (5KB 83B)
307
- 🐍 test_manage.py (1KB 785B)
308
- 🐍 test_nodes.py (7KB 864B)
309
- 🐍 test_structure.py (9KB 1015B)
310
- 📁 [6项排除的目录] (195MB 276KB 932B)
311
- ❓ .gitignore (264B)
318
+ 🐍 task_queue.py (8KB 570B)
319
+ 🐍 task_report.py (4KB 805B)
320
+ 🐍 task_stage.py (5KB 511B)
321
+ 🐍 task_structure.py (5KB 996B)
322
+ 🐍 task_tools.py (15KB 105B)
323
+ 🐍 task_types.py (2KB 265B)
324
+ 🐍 task_web.py (5KB 693B)
325
+ 🐍 __init__.py (1KB 114B)
326
+ 📁 [1项排除的目录] (17KB 377B)
327
+ 📁 tests (126KB 669B)
328
+ 📁 [1项排除的目录] (90KB 310B)
329
+ 📝 README.md (6KB 324B)
330
+ 🐍 test_graph.py (5KB 594B)
331
+ 🐍 test_manage.py (3KB 90B)
332
+ 🐍 test_nodes.py (11KB 272B)
333
+ 🐍 test_structure.py (10KB 103B)
334
+ 📁 [6项排除的目录] (199MB 931KB 238B)
335
+ .env (88B)
336
+ ❓ .gitignore (272B)
312
337
  ❓ LICENSE (1KB 65B)
313
338
  ❓ Makefile (501B)
314
339
  ❓ pyproject.toml (1KB 223B)
315
340
  ⚙️ pytest.ini (254B)
316
- 📝 README.md (15KB 536B)
341
+ 📝 README.md (15KB 109B)
342
+ (celestial-flow 3.0.8)
317
343
  ```
318
344
 
319
- (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file生成。)
345
+ (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。)
320
346
 
321
347
  ## 更新日志(Change Log)
322
348
 
@@ -326,7 +352,7 @@ flowchart TD
326
352
  - 12/12/2024-12/16/2024: 在原有链式结构基础上允许节点有复数下级节点, 实现Tree结构; 将原有TaskChain改名为TaskTree
327
353
  - 3/16/2025: 支持Web端任务完成情况可视化
328
354
  - 6/9/2025: 支持节点拥有复数上级节点, 脱离纯Tree结构, 为之后循环图做准备
329
- - 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁入
355
+ - 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁出
330
356
  - 6/12/2025: 支持循环图, 下级节点可指向上级节点
331
357
  - 6/13/2025: 支持loop结构, 即节点可指向自己
332
358
  - 6/14/2025: 支持forest结构, 即可有多个根节点
@@ -335,6 +361,8 @@ flowchart TD
335
361
  - 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
336
362
  - 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
337
363
  - 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
364
+ - 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
365
+ - 3.0.8: 在ctree关系上将"任务重试"后的"任务成功/失败/重试"视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
338
366
 
339
367
  ## Star 历史趋势(Star History)
340
368
 
@@ -148,7 +148,7 @@ if __name__ == "__main__":
148
148
  graph = TaskGraph([stage1])
149
149
 
150
150
  # 初始化任务并启动
151
- graph.start_graph({stage1.get_stage_tag(): [(1, 2), (3, 4), (5, 6)]})
151
+ graph.start_graph({stage1.get_tag(): [(1, 2), (3, 4), (5, 6)]})
152
152
  ```
153
153
 
154
154
  注意不要在.ipynb中运行。
@@ -219,79 +219,105 @@ flowchart TD
219
219
  ## 文件结构(File Structure)
220
220
 
221
221
  ```
222
- 📁 CelestialFlow (205MB 169KB 76B)
223
- 📁 experiments (14KB 536B)
224
- 🐍 benchmark_datastructures.py (5KB 796B)
222
+ 📁 CelestialFlow (213MB 633KB 545B)
223
+ 📁 docs (25KB 183B)
224
+ 📁 reference (20KB 175B)
225
+ 📝 go_worker.md (6KB 378B)
226
+ 📝 task_graph.md (4KB 989B)
227
+ 📝 task_logging.md (0B)
228
+ 📝 task_manage.md (7KB 7B)
229
+ 📝 task_nodes.md (1KB 737B)
230
+ 📝 task_progress.md (0B)
231
+ 📝 task_queue.md (0B)
232
+ 📝 task_report.md (0B)
233
+ 📝 task_structure.md (112B)
234
+ 📝 task_tools.md (0B)
235
+ 📝 task_types.md (0B)
236
+ 📝 task_web.md (0B)
237
+ 📝 quick_start.md (5KB 8B)
238
+ 📝 tutorial.md (0B)
239
+ 📁 experiments (15KB 111B)
240
+ 🐍 benchmark_datastructures.py (6KB 52B)
225
241
  🐍 benchmark_hash.py (1KB 284B)
226
- 🐍 benchmark_queue.py (5KB 185B)
242
+ 🐍 benchmark_queue.py (5KB 504B)
227
243
  🐍 benchmark_tqdm.py (1KB 160B)
228
244
  🐍 experiment_tqdm.py (1KB 135B)
229
- 📁 go_worker (6MB 967KB 64B)
230
- 📁 worker (5KB 684B)
245
+ 📁 go_worker (6MB 968KB 38B)
246
+ 📁 worker (5KB 746B)
231
247
  🌀 parser.go (394B)
232
248
  🌀 processor.go (2KB 612B)
233
- 🌀 types.go (237B)
249
+ 🌀 types.go (299B)
234
250
  🌀 worker.go (2KB 465B)
235
- ❓ go.mod (258B)
236
- ❓ go.sum (591B)
251
+ ❓ go.mod (292B)
252
+ ❓ go.sum (1KB 68B)
237
253
  ❓ go_worker.exe (6MB 960KB)
238
- 🌀 main.go (579B)
239
- 📁 img (966KB 63B)
240
- 📷 logo.png (836KB 542B)
254
+ 🌀 main.go (980B)
255
+ 📁 img (4MB 700KB 695B)
256
+ 📷 logo(old).png (836KB 542B)
257
+ 📷 logo.png (122KB 747B)
241
258
  📷 web_display.png (129KB 545B)
242
- 📁 src (1MB 884KB 224B)
243
- 📁 celestialflow (1MB 869KB 480B)
244
- 📁 static (1MB 419KB 510B)
245
- 📁 css (32KB 568B)
259
+ 🎞️ web_ui.gif (3MB 635KB 909B)
260
+ 📁 src (1MB 920KB 171B)
261
+ 📁 celestialflow (1MB 902KB 818B)
262
+ 📁 adapters (13KB 837B)
263
+ 📁 celestialtree (13KB 681B)
264
+ 📁 [1项排除的目录] (6KB 960B)
265
+ 🐍 client.py (5KB 486B)
266
+ 🐍 tools.py (1KB 182B)
267
+ 🐍 __init__.py (77B)
268
+ 📁 [1项排除的目录] (156B)
269
+ 🐍 __init__.py (0B)
270
+ 📁 static (1MB 417KB 101B)
271
+ 📁 css (31KB 410B)
246
272
  🎨 base.css (6KB 155B)
247
- 🎨 dashboard.css (8KB 435B)
273
+ 🎨 dashboard.css (7KB 277B)
248
274
  🎨 errors.css (5KB 168B)
249
275
  🎨 inject.css (12KB 834B)
250
- 📁 js (34KB 868B)
251
- 📜 main.js (4KB 973B)
252
- 📜 task_errors.js (4KB 544B)
276
+ 📁 js (33KB 617B)
277
+ 📜 main.js (5KB 77B)
278
+ 📜 task_errors.js (4KB 550B)
253
279
  📜 task_injection.js (8KB 491B)
254
280
  📜 task_statuses.js (8KB 63B)
255
- 📜 task_structure.js (7KB 143B)
256
- 📜 task_topology.js (261B)
281
+ 📜 task_structure.js (4KB 668B)
282
+ 📜 task_topology.js (1KB 375B)
257
283
  📜 utils.js (1KB 441B)
258
284
  ❓ favicon.ico (1MB 352KB 98B)
259
- 📁 templates (12KB 973B)
260
- 🌐 index.html (12KB 973B)
261
- 📁 [1项排除的目录] (314KB 710B)
262
- 📝 README.md (13KB 131B)
263
- 🐍 task_graph.py (22KB 530B)
264
- 🐍 task_logging.py (6KB 291B)
265
- 🐍 task_manage.py (32KB 66B)
266
- 🐍 task_nodes.py (4KB 953B)
285
+ 📁 templates (13KB 150B)
286
+ 🌐 index.html (13KB 150B)
287
+ 📁 [1项排除的目录] (331KB 149B)
288
+ 🐍 task_graph.py (23KB 835B)
289
+ 🐍 task_logging.py (8KB 903B)
290
+ 🐍 task_manage.py (33KB 577B)
291
+ 🐍 task_nodes.py (10KB 922B)
267
292
  🐍 task_progress.py (1KB 477B)
268
- 🐍 task_queue.py (7KB 896B)
269
- 🐍 task_report.py (5KB 511B)
270
- 🐍 task_structure.py (6KB 93B)
271
- 🐍 task_tools.py (15KB 182B)
272
- 🐍 task_types.py (1KB 461B)
273
- 🐍 task_web.py (5KB 47B)
274
- 🐍 __init__.py (817B)
275
- 📁 celestialflow.egg-info (14KB 768B)
276
- PKG-INFO (13KB 363B)
277
- 📄 [5项排除的文件] (1KB 405B)
278
- 📁 tests (113KB 510B)
279
- 📁 [1项排除的目录] (81KB 604B)
280
- 📝 README.md (7KB 231B)
281
- 🐍 test_graph.py (5KB 83B)
282
- 🐍 test_manage.py (1KB 785B)
283
- 🐍 test_nodes.py (7KB 864B)
284
- 🐍 test_structure.py (9KB 1015B)
285
- 📁 [6项排除的目录] (195MB 276KB 932B)
286
- ❓ .gitignore (264B)
293
+ 🐍 task_queue.py (8KB 570B)
294
+ 🐍 task_report.py (4KB 805B)
295
+ 🐍 task_stage.py (5KB 511B)
296
+ 🐍 task_structure.py (5KB 996B)
297
+ 🐍 task_tools.py (15KB 105B)
298
+ 🐍 task_types.py (2KB 265B)
299
+ 🐍 task_web.py (5KB 693B)
300
+ 🐍 __init__.py (1KB 114B)
301
+ 📁 [1项排除的目录] (17KB 377B)
302
+ 📁 tests (126KB 669B)
303
+ 📁 [1项排除的目录] (90KB 310B)
304
+ 📝 README.md (6KB 324B)
305
+ 🐍 test_graph.py (5KB 594B)
306
+ 🐍 test_manage.py (3KB 90B)
307
+ 🐍 test_nodes.py (11KB 272B)
308
+ 🐍 test_structure.py (10KB 103B)
309
+ 📁 [6项排除的目录] (199MB 931KB 238B)
310
+ .env (88B)
311
+ ❓ .gitignore (272B)
287
312
  ❓ LICENSE (1KB 65B)
288
313
  ❓ Makefile (501B)
289
314
  ❓ pyproject.toml (1KB 223B)
290
315
  ⚙️ pytest.ini (254B)
291
- 📝 README.md (15KB 536B)
316
+ 📝 README.md (15KB 109B)
317
+ (celestial-flow 3.0.8)
292
318
  ```
293
319
 
294
- (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file生成。)
320
+ (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。)
295
321
 
296
322
  ## 更新日志(Change Log)
297
323
 
@@ -301,7 +327,7 @@ flowchart TD
301
327
  - 12/12/2024-12/16/2024: 在原有链式结构基础上允许节点有复数下级节点, 实现Tree结构; 将原有TaskChain改名为TaskTree
302
328
  - 3/16/2025: 支持Web端任务完成情况可视化
303
329
  - 6/9/2025: 支持节点拥有复数上级节点, 脱离纯Tree结构, 为之后循环图做准备
304
- - 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁入
330
+ - 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁出
305
331
  - 6/12/2025: 支持循环图, 下级节点可指向上级节点
306
332
  - 6/13/2025: 支持loop结构, 即节点可指向自己
307
333
  - 6/14/2025: 支持forest结构, 即可有多个根节点
@@ -310,6 +336,8 @@ flowchart TD
310
336
  - 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
311
337
  - 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
312
338
  - 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
339
+ - 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
340
+ - 3.0.8: 在ctree关系上将"任务重试"后的"任务成功/失败/重试"视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
313
341
 
314
342
  ## Star 历史趋势(Star History)
315
343
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "celestialflow"
7
- version = "3.0.6"
7
+ version = "3.0.8"
8
8
  description = "A flexible GRAPH-based task orchestration framework."
9
9
  readme = "README.md"
10
10
  license = { text = "MIT" }
@@ -1,10 +1,12 @@
1
1
  from .task_graph import TaskGraph
2
2
  from .task_manage import TaskManager
3
+ from .task_stage import TaskStage
3
4
  from .task_nodes import (
4
5
  TaskSplitter,
5
6
  TaskRedisSink,
6
7
  TaskRedisSource,
7
8
  TaskRedisAck,
9
+ TaskRouter,
8
10
  )
9
11
  from .task_structure import (
10
12
  TaskChain,
@@ -33,10 +35,12 @@ __all__ = [
33
35
  "TaskWheel",
34
36
  "TaskGrid",
35
37
  "TaskManager",
38
+ "TaskStage",
36
39
  "TaskSplitter",
37
40
  "TaskRedisSink",
38
41
  "TaskRedisSource",
39
42
  "TaskRedisAck",
43
+ "TaskRouter",
40
44
  "TerminationSignal",
41
45
  "TaskWebServer",
42
46
  "CelestialTreeClient",
@@ -1,6 +1,7 @@
1
1
  import json
2
2
  import threading
3
3
  import requests
4
+ from multiprocessing import Value as MPValue, Lock as MPLock
4
5
  from typing import List, Optional, Dict, Any, Callable
5
6
 
6
7
 
@@ -9,8 +10,8 @@ class Client:
9
10
  Python client for CelestialTree HTTP API.
10
11
  """
11
12
 
12
- def __init__(self, base_url: str, timeout: float = 5.0):
13
- self.base_url = base_url.rstrip("/")
13
+ def __init__(self, host: str = "127.0.0.1", port: int = 7777, timeout: float = 5.0):
14
+ self.base_url = f"http://{host}:{port}"
14
15
  self.timeout = timeout
15
16
 
16
17
  def init_session(self):
@@ -60,7 +61,9 @@ class Client:
60
61
  data=json.dumps(body),
61
62
  timeout=self.timeout,
62
63
  )
63
- r.raise_for_status()
64
+
65
+ if not (200 <= r.status_code < 300):
66
+ raise RuntimeError(r.json()["error"])
64
67
  return r.json()["id"]
65
68
 
66
69
  def get_event(self, event_id: int) -> Dict[str, Any]:
@@ -70,7 +73,9 @@ class Client:
70
73
  f"{self.base_url}/event/{event_id}",
71
74
  timeout=self.timeout,
72
75
  )
73
- r.raise_for_status()
76
+
77
+ if not (200 <= r.status_code < 300):
78
+ raise RuntimeError(r.json()["error"])
74
79
  return r.json()
75
80
 
76
81
  def children(self, event_id: int) -> List[int]:
@@ -80,7 +85,9 @@ class Client:
80
85
  f"{self.base_url}/children/{event_id}",
81
86
  timeout=self.timeout,
82
87
  )
83
- r.raise_for_status()
88
+
89
+ if not (200 <= r.status_code < 300):
90
+ raise RuntimeError(r.json()["error"])
84
91
  return r.json()["children"]
85
92
 
86
93
  def descendants(self, event_id: int) -> Dict[str, Any]:
@@ -90,7 +97,9 @@ class Client:
90
97
  f"{self.base_url}/descendants/{event_id}",
91
98
  timeout=self.timeout,
92
99
  )
93
- r.raise_for_status()
100
+
101
+ if not (200 <= r.status_code < 300):
102
+ raise RuntimeError(r.json()["error"])
94
103
  return r.json()
95
104
 
96
105
  def heads(self) -> List[int]:
@@ -100,17 +109,21 @@ class Client:
100
109
  f"{self.base_url}/heads",
101
110
  timeout=self.timeout,
102
111
  )
103
- r.raise_for_status()
112
+
113
+ if not (200 <= r.status_code < 300):
114
+ raise RuntimeError(r.json()["error"])
104
115
  return r.json()["heads"]
105
116
 
106
117
  def health(self) -> bool:
107
118
  self.init_session()
108
-
109
- r = self.session.get(
110
- f"{self.base_url}/healthz",
111
- timeout=self.timeout,
112
- )
113
- return r.status_code == 200
119
+ try:
120
+ r = self.session.get(
121
+ f"{self.base_url}/healthz",
122
+ timeout=self.timeout,
123
+ )
124
+ return r.status_code == 200
125
+ except Exception:
126
+ return False
114
127
 
115
128
  def version(self) -> Dict[str, Any]:
116
129
  self.init_session()
@@ -119,7 +132,9 @@ class Client:
119
132
  f"{self.base_url}/version",
120
133
  timeout=self.timeout,
121
134
  )
122
- r.raise_for_status()
135
+
136
+ if not (200 <= r.status_code < 300):
137
+ raise RuntimeError(r.json()["error"])
123
138
  return r.json()
124
139
 
125
140
  # ---------- SSE Subscribe ----------
@@ -162,11 +177,13 @@ class Client:
162
177
 
163
178
 
164
179
  class NullClient:
165
- event_id = 0
180
+ def __init__(self, event_id=None):
181
+ self.event_id = event_id if event_id is not None else MPValue("i", 0)
166
182
 
167
183
  def emit(self, *args, **kwargs):
168
- self.event_id += 1
169
- return self.event_id
184
+ with self.event_id.get_lock():
185
+ self.event_id.value += 1
186
+ return self.event_id.value
170
187
 
171
188
  def get_event(self, *args, **kwargs):
172
189
  return None
@@ -10,7 +10,12 @@ def format_tree(node: dict, prefix: str = "", is_last: bool = True) -> str:
10
10
  lines = []
11
11
 
12
12
  connector = "└── " if is_last else "├── "
13
- lines.append(f"{prefix}{connector}{node['id']}")
13
+
14
+ label = str(node["id"])
15
+ if node.get("is_ref"):
16
+ label += " (ref)"
17
+
18
+ lines.append(f"{prefix}{connector}{label}")
14
19
 
15
20
  children = node.get("children", [])
16
21
  if children:
@@ -25,7 +25,7 @@ function renderErrors() {
25
25
  const matchNode = !filter || e.node === filter;
26
26
  const matchKeyword = !keyword ||
27
27
  (e.error && e.error.toLowerCase().includes(keyword)) ||
28
- (e.task_id && e.task_id.toLowerCase().includes(keyword));
28
+ (e.task_repr && e.task_repr.toLowerCase().includes(keyword));
29
29
  return matchNode && matchKeyword;
30
30
  });
31
31
 
@@ -48,7 +48,7 @@ function renderErrors() {
48
48
  row.innerHTML = `
49
49
  <td class="error-message">${e.error}</td>
50
50
  <td>${e.node}</td>
51
- <td>${e.task_id}</td>
51
+ <td>${e.task_repr}</td>
52
52
  <td>${formatTimestamp(e.timestamp)}</td>
53
53
  `;
54
54
  errorsTableBody.appendChild(row);
@@ -111,7 +111,7 @@ linkStyle default stroke:#999,stroke-width:1.5px;
111
111
 
112
112
  // let defs = [];
113
113
 
114
- // if (topologyData.layout_mode === "serial") {
114
+ // if (topologyData.schedule_mode=== "staged") {
115
115
  // const layers = topologyData.layers_dict || {};
116
116
  // const sortedLayerKeys = Object.keys(layers).sort((a, b) => Number(a) - Number(b));
117
117
  // for (const layer of sortedLayerKeys) {
@@ -21,7 +21,7 @@ function renderTopologyInfo() {
21
21
 
22
22
  const {
23
23
  isDAG,
24
- layout_mode,
24
+ schedule_mode,
25
25
  class_name,
26
26
  layers_dict = {},
27
27
  } = topologyData;
@@ -43,7 +43,7 @@ function renderTopologyInfo() {
43
43
 
44
44
  <div class="topology-row">
45
45
  <span class="label">调度模式</span>
46
- <span class="value">${layout_mode}</span>
46
+ <span class="value">${schedule_mode}</span>
47
47
  </div>
48
48
 
49
49
  <div class="topology-row">