celestialflow 3.1.4__tar.gz → 3.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {celestialflow-3.1.4 → celestialflow-3.1.6}/PKG-INFO +18 -20
  2. {celestialflow-3.1.4 → celestialflow-3.1.6}/README.md +17 -19
  3. {celestialflow-3.1.4 → celestialflow-3.1.6}/pyproject.toml +80 -75
  4. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/__init__.py +17 -17
  5. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/__init__.py +3 -3
  6. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/core_graph.py +39 -30
  7. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/core_structure.py +13 -6
  8. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/util_analysis.py +4 -4
  9. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/util_serialize.py +4 -4
  10. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/observability/__init__.py +1 -1
  11. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/observability/core_report.py +60 -50
  12. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/core_base.py +7 -6
  13. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/core_fail.py +5 -2
  14. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/core_log.py +19 -5
  15. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/util_jsonl.py +1 -1
  16. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/__init__.py +1 -1
  17. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_metrics.py +4 -4
  18. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_queue.py +14 -12
  19. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_runner.py +45 -27
  20. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_errors.py +1 -0
  21. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_estimators.py +10 -6
  22. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_factories.py +5 -7
  23. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_types.py +7 -5
  24. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/__init__.py +3 -3
  25. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/core_executor.py +52 -36
  26. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/core_stage.py +26 -14
  27. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/core_stages.py +36 -13
  28. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_benchmark.py +12 -1
  29. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_clone.py +1 -1
  30. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_format.py +9 -9
  31. celestialflow-3.1.6/src/celestialflow/web/config.json +35 -0
  32. celestialflow-3.1.4/src/celestialflow/web/server.py → celestialflow-3.1.6/src/celestialflow/web/core_server.py +148 -61
  33. celestialflow-3.1.6/src/celestialflow/web/static/css/_colors.css +135 -0
  34. celestialflow-3.1.6/src/celestialflow/web/static/css/base.css +320 -0
  35. celestialflow-3.1.6/src/celestialflow/web/static/css/dashboard.css +348 -0
  36. celestialflow-3.1.6/src/celestialflow/web/static/css/errors.css +286 -0
  37. celestialflow-3.1.6/src/celestialflow/web/static/css/inject.css +562 -0
  38. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/main.js +5 -24
  39. celestialflow-3.1.6/src/celestialflow/web/static/js/task_analysis.js +59 -0
  40. celestialflow-3.1.6/src/celestialflow/web/static/js/task_errors.js +162 -0
  41. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_history.js +23 -11
  42. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_injection.js +21 -16
  43. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_statuses.js +58 -44
  44. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_structure.js +18 -3
  45. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_summary.js +14 -8
  46. celestialflow-3.1.6/src/celestialflow/web/static/js/task_topology.js +59 -0
  47. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/utils.js +95 -14
  48. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/main.ts +6 -32
  49. celestialflow-3.1.6/src/celestialflow/web/static/ts/task_analysis.ts +67 -0
  50. celestialflow-3.1.6/src/celestialflow/web/static/ts/task_errors.ts +178 -0
  51. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_history.ts +24 -13
  52. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_injection.ts +384 -379
  53. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_statuses.ts +172 -149
  54. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_structure.ts +19 -4
  55. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_summary.ts +14 -9
  56. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/utils.ts +264 -169
  57. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/templates/index.html +39 -35
  58. celestialflow-3.1.6/src/celestialflow/web/util_cal.py +4 -0
  59. celestialflow-3.1.6/src/celestialflow/web/util_config.py +23 -0
  60. celestialflow-3.1.6/src/celestialflow/web/util_error.py +53 -0
  61. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/PKG-INFO +18 -20
  62. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/SOURCES.txt +9 -3
  63. celestialflow-3.1.6/src/celestialflow.egg-info/entry_points.txt +2 -0
  64. {celestialflow-3.1.4 → celestialflow-3.1.6}/tests/test_executor.py +5 -38
  65. {celestialflow-3.1.4 → celestialflow-3.1.6}/tests/test_stages.py +45 -121
  66. {celestialflow-3.1.4 → celestialflow-3.1.6}/tests/test_structure.py +48 -116
  67. celestialflow-3.1.6/tests/test_utils.py +234 -0
  68. celestialflow-3.1.4/src/celestialflow/web/static/css/_colors.css +0 -183
  69. celestialflow-3.1.4/src/celestialflow/web/static/css/base.css +0 -305
  70. celestialflow-3.1.4/src/celestialflow/web/static/css/dashboard.css +0 -368
  71. celestialflow-3.1.4/src/celestialflow/web/static/css/errors.css +0 -261
  72. celestialflow-3.1.4/src/celestialflow/web/static/css/inject.css +0 -638
  73. celestialflow-3.1.4/src/celestialflow/web/static/js/task_errors.js +0 -131
  74. celestialflow-3.1.4/src/celestialflow/web/static/js/task_topology.js +0 -53
  75. celestialflow-3.1.4/src/celestialflow/web/static/ts/task_errors.ts +0 -147
  76. celestialflow-3.1.4/src/celestialflow/web/static/ts/task_topology.ts +0 -62
  77. celestialflow-3.1.4/src/celestialflow.egg-info/entry_points.txt +0 -2
  78. {celestialflow-3.1.4 → celestialflow-3.1.6}/setup.cfg +0 -0
  79. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/__init__.py +0 -0
  80. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/util_constant.py +0 -0
  81. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_envelope.py +0 -0
  82. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_progress.py +0 -0
  83. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_hash.py +0 -0
  84. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_queue.py +0 -0
  85. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/__init__.py +0 -0
  86. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_collections.py +0 -0
  87. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_debug.py +0 -0
  88. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/favicon.ico +0 -0
  89. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_config.js +0 -0
  90. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/web_config.js +0 -0
  91. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/globals.d.ts +0 -0
  92. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/web_config.ts +0 -0
  93. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/dependency_links.txt +0 -0
  94. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/requires.txt +0 -0
  95. {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: celestialflow
3
- Version: 3.1.4
3
+ Version: 3.1.6
4
4
  Summary: A flexible GRAPH-based task orchestration framework.
5
5
  Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
6
6
  License: MIT
@@ -254,35 +254,33 @@ flowchart TD
254
254
  <p align="center">
255
255
  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
256
256
  <br/>
257
- <em>celestial-flow 3.1.4</em>
257
+ <em>celestial-flow 3.1.6</em>
258
258
  </p>
259
259
 
260
260
  (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
261
261
 
262
262
  ## 版本日志(Version Log)
263
- - 3.1.4:
264
- - feat:
265
- - 添加前端设置文件config.json, 包含主题(白天与黑夜), 刷新时间, 历史长度, 卡片种类, 仪表盘布局;
266
- - 完善对termination_signal在ctree上的事件管理;
267
- - 新添termination_*系日志, 同时优化部分原有日志;
268
- - 在前端的错误数字上(包括单个stage的卡片与summary卡片)绑定跳转事件, 可以跳转到ErrorLog页面, 并显示对应的错误;
269
- - 修复部分原有的文档错误, 并添加新的前端代码文档;
270
- - refactor:
271
- - fail_sinker.task_error中不必再传时间, 方法会自己补充;
272
- - 将所有counter放入TaskMetrics管理, 断绝对TAskExecutor的调用依赖;
273
- - 将run_*函数分离并移入TaskRunner类, 同时将pool管理也迁入;
274
- - 将TaskQueue分离为更具体的TaskInQueue与TaskOutQueue, 同时TAskInQueue只接受一个MPQueue以避免原有的轮询逻辑, 减少CPU运算消耗;
275
- - 前端代码换用ts;
276
- - 重命名所有代码文件, 现在用core_与util_前缀来区分核心代码与辅助代码;
277
- - 将history数据从status中移出, 使用单独的/api/*_history端口;
278
- - fix:
279
- - TaskRedisTransport节点在mermaid中没有展示为parallelogram;
263
+ - 3.1.6
264
+ - feat
265
+ - 前端现在只储存一页的error数据, 有效减少了运行大规模任务时前端内存飙升的问题;
266
+ - 优化任务数显示, 大于1*10^7时显示科学计数法, 否则显示英式计数;
267
+ - 优化小屏模式下表格的显示: 改为用卡片式显示;
268
+ - 在error表格中加入index项;
269
+ - 大幅调整任务颜色分配, 现在重复任务使用黄色系, 等待任务使用灰色系;
270
+ - 将节点卡片中的已消耗时间颜色拟合为下方进度条颜色;
271
+ - refactor
272
+ - 用mypy整理了一遍类型标注;
273
+ - 删除与整合部分css代码;
274
+ - 将字体等适合rem单位的地方全部从px替换为rem, 并且都部分size进行统一;
275
+ - fix
276
+ - 修复总体剩余时间在特殊情况下显示0的问题(这玩意真麻烦);
277
+ - 修复部分小屏下的显示问题, 但折线图不显示的问题不太好解决;
280
278
 
281
279
  ## Star 历史趋势(Star History)
282
280
 
283
281
  如果对项目感兴趣的话,欢迎star。如果有问题或者建议的话, 欢迎提交[Issues](https://github.com/Mr-xiaotian/CelestialFlow/issues)或者在[Discussion](https://github.com/Mr-xiaotian/CelestialFlow/discussions)中告诉我。
284
282
 
285
- [![Star History Chart](https://api.star-history.com/svg?repos=Mr-xiaotian/CelestialFlow&type=Date)](https://star-history.com/#Mr-xiaotian/CelestialFlow&Date)
283
+ ![Star History Chart](https://api.star-history.com/svg?repos=Mr-xiaotian/CelestialFlow&type=Date)
286
284
 
287
285
  ## 许可(License)
288
286
  This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
@@ -229,35 +229,33 @@ flowchart TD
229
229
  <p align="center">
230
230
  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
231
231
  <br/>
232
- <em>celestial-flow 3.1.4</em>
232
+ <em>celestial-flow 3.1.6</em>
233
233
  </p>
234
234
 
235
235
  (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
236
236
 
237
237
  ## 版本日志(Version Log)
238
- - 3.1.4:
239
- - feat:
240
- - 添加前端设置文件config.json, 包含主题(白天与黑夜), 刷新时间, 历史长度, 卡片种类, 仪表盘布局;
241
- - 完善对termination_signal在ctree上的事件管理;
242
- - 新添termination_*系日志, 同时优化部分原有日志;
243
- - 在前端的错误数字上(包括单个stage的卡片与summary卡片)绑定跳转事件, 可以跳转到ErrorLog页面, 并显示对应的错误;
244
- - 修复部分原有的文档错误, 并添加新的前端代码文档;
245
- - refactor:
246
- - fail_sinker.task_error中不必再传时间, 方法会自己补充;
247
- - 将所有counter放入TaskMetrics管理, 断绝对TAskExecutor的调用依赖;
248
- - 将run_*函数分离并移入TaskRunner类, 同时将pool管理也迁入;
249
- - 将TaskQueue分离为更具体的TaskInQueue与TaskOutQueue, 同时TAskInQueue只接受一个MPQueue以避免原有的轮询逻辑, 减少CPU运算消耗;
250
- - 前端代码换用ts;
251
- - 重命名所有代码文件, 现在用core_与util_前缀来区分核心代码与辅助代码;
252
- - 将history数据从status中移出, 使用单独的/api/*_history端口;
253
- - fix:
254
- - TaskRedisTransport节点在mermaid中没有展示为parallelogram;
238
+ - 3.1.6
239
+ - feat
240
+ - 前端现在只储存一页的error数据, 有效减少了运行大规模任务时前端内存飙升的问题;
241
+ - 优化任务数显示, 大于1*10^7时显示科学计数法, 否则显示英式计数;
242
+ - 优化小屏模式下表格的显示: 改为用卡片式显示;
243
+ - 在error表格中加入index项;
244
+ - 大幅调整任务颜色分配, 现在重复任务使用黄色系, 等待任务使用灰色系;
245
+ - 将节点卡片中的已消耗时间颜色拟合为下方进度条颜色;
246
+ - refactor
247
+ - 用mypy整理了一遍类型标注;
248
+ - 删除与整合部分css代码;
249
+ - 将字体等适合rem单位的地方全部从px替换为rem, 并且都部分size进行统一;
250
+ - fix
251
+ - 修复总体剩余时间在特殊情况下显示0的问题(这玩意真麻烦);
252
+ - 修复部分小屏下的显示问题, 但折线图不显示的问题不太好解决;
255
253
 
256
254
  ## Star 历史趋势(Star History)
257
255
 
258
256
  如果对项目感兴趣的话,欢迎star。如果有问题或者建议的话, 欢迎提交[Issues](https://github.com/Mr-xiaotian/CelestialFlow/issues)或者在[Discussion](https://github.com/Mr-xiaotian/CelestialFlow/discussions)中告诉我。
259
257
 
260
- [![Star History Chart](https://api.star-history.com/svg?repos=Mr-xiaotian/CelestialFlow&type=Date)](https://star-history.com/#Mr-xiaotian/CelestialFlow&Date)
258
+ ![Star History Chart](https://api.star-history.com/svg?repos=Mr-xiaotian/CelestialFlow&type=Date)
261
259
 
262
260
  ## 许可(License)
263
261
  This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
@@ -1,75 +1,80 @@
1
- [build-system]
2
- requires = ["setuptools>=61.0", "wheel"]
3
- build-backend = "setuptools.build_meta"
4
-
5
- [project]
6
- name = "celestialflow"
7
- version = "3.1.4"
8
- description = "A flexible GRAPH-based task orchestration framework."
9
- readme = "README.md"
10
- license = { text = "MIT" }
11
- authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
12
- keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
13
- requires-python = ">=3.10"
14
-
15
- dependencies = [
16
- "tqdm",
17
- "fastapi",
18
- "uvicorn",
19
- "requests",
20
- "networkx",
21
- "redis",
22
- "jinja2",
23
- "celestialtree>=0.1.2",
24
- ]
25
-
26
- classifiers = [
27
- "Programming Language :: Python :: 3",
28
- "License :: OSI Approved :: MIT License",
29
- "Operating System :: OS Independent",
30
- "Framework :: FastAPI",
31
- "Topic :: Software Development :: Libraries",
32
- ]
33
-
34
- [project.urls]
35
- "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
36
- "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
37
-
38
- [project.scripts]
39
- celestialflow-web = "celestialflow.web.server:main_entry"
40
-
41
- [tool.setuptools]
42
- license-files = []
43
-
44
- [tool.setuptools.package-data]
45
- celestialflow = [
46
- "web/templates/*.html",
47
- "web/templates/**/*.html",
48
- "web/static/*",
49
- "web/static/**/*",
50
- ]
51
-
52
- [tool.pytest.ini_options]
53
- addopts = "-s"
54
- log_cli = true
55
- log_cli_level = "INFO"
56
- log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
57
- log_cli_date_format = "%Y-%m-%d %H:%M:%S"
58
-
59
- filterwarnings = [
60
- "ignore::DeprecationWarning",
61
- "ignore::pytest.PytestDeprecationWarning",
62
- ]
63
-
64
- asyncio_default_fixture_loop_scope = "function"
65
-
66
- [dependency-groups]
67
- dev = [
68
- "build>=1.2.2.post1",
69
- "twine>=6.1.0",
70
- "pytest>=8.3.4",
71
- "pytest-asyncio>=0.25.3",
72
- "httpx>=0.28.1",
73
- "python-dotenv>=1.2.2",
74
- "black>=26.1.0",
75
- ]
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "celestialflow"
7
+ version = "3.1.6"
8
+ description = "A flexible GRAPH-based task orchestration framework."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
12
+ keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
13
+ requires-python = ">=3.10"
14
+
15
+ dependencies = [
16
+ "tqdm",
17
+ "fastapi",
18
+ "uvicorn",
19
+ "requests",
20
+ "networkx",
21
+ "redis",
22
+ "jinja2",
23
+ "celestialtree>=0.1.2",
24
+ ]
25
+
26
+ classifiers = [
27
+ "Programming Language :: Python :: 3",
28
+ "License :: OSI Approved :: MIT License",
29
+ "Operating System :: OS Independent",
30
+ "Framework :: FastAPI",
31
+ "Topic :: Software Development :: Libraries",
32
+ ]
33
+
34
+ [project.urls]
35
+ "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
36
+ "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
37
+
38
+ [project.scripts]
39
+ celestialflow-web = "celestialflow.web.core_server:main_entry"
40
+
41
+ [tool.setuptools]
42
+ license-files = []
43
+
44
+ [tool.setuptools.package-data]
45
+ celestialflow = [
46
+ "web/templates/**/*.html",
47
+ "web/static/**/*",
48
+ "web/config.json"
49
+ ]
50
+
51
+ [tool.pytest.ini_options]
52
+ addopts = "-s"
53
+ log_cli = true
54
+ log_cli_level = "INFO"
55
+ log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
56
+ log_cli_date_format = "%Y-%m-%d %H:%M:%S"
57
+
58
+ filterwarnings = [
59
+ "ignore::DeprecationWarning",
60
+ "ignore::pytest.PytestDeprecationWarning",
61
+ ]
62
+
63
+ asyncio_default_fixture_loop_scope = "function"
64
+
65
+ [dependency-groups]
66
+ dev = [
67
+ "build>=1.2.2.post1",
68
+ "twine>=6.1.0",
69
+ "pytest>=8.3.4",
70
+ "pytest-asyncio>=0.25.3",
71
+ "httpx>=0.28.1",
72
+ "python-dotenv>=1.2.2",
73
+ "dill",
74
+ "mypy>=1.19.1",
75
+ ]
76
+
77
+ [tool.mypy]
78
+ python_version = "3.10"
79
+ no_implicit_optional = false
80
+ disable_error_code = ["import-untyped"]
@@ -1,32 +1,32 @@
1
1
  # __init__.py
2
- from .stage import (
3
- TaskExecutor,
4
- TaskStage,
5
- TaskSplitter,
6
- TaskRedisTransport,
7
- TaskRedisSource,
8
- TaskRedisAck,
9
- TaskRouter,
10
- )
11
2
  from .graph import (
12
- TaskGraph,
13
3
  TaskChain,
14
- TaskLoop,
15
- TaskCross,
16
4
  TaskComplete,
17
- TaskWheel,
5
+ TaskCross,
6
+ TaskGraph,
18
7
  TaskGrid,
8
+ TaskLoop,
9
+ TaskWheel,
19
10
  )
20
11
  from .persistence.util_jsonl import (
21
12
  load_jsonl_logs,
22
- load_task_by_stage,
23
13
  load_task_by_error,
14
+ load_task_by_stage,
24
15
  )
25
- from .runtime.util_types import TerminationSignal
26
16
  from .runtime.util_hash import make_hashable
17
+ from .runtime.util_types import TerminationSignal
18
+ from .stage import (
19
+ TaskExecutor,
20
+ TaskRedisAck,
21
+ TaskRedisSource,
22
+ TaskRedisTransport,
23
+ TaskRouter,
24
+ TaskSplitter,
25
+ TaskStage,
26
+ )
27
+ from .utils.util_benchmark import benchmark_executor, benchmark_graph
27
28
  from .utils.util_format import format_table
28
- from .utils.util_benchmark import benchmark_graph, benchmark_executor
29
- from .web.server import TaskWebServer
29
+ from .web.core_server import TaskWebServer
30
30
 
31
31
  __all__ = [
32
32
  "TaskGraph",
@@ -2,11 +2,11 @@
2
2
  from .core_graph import TaskGraph
3
3
  from .core_structure import (
4
4
  TaskChain,
5
- TaskLoop,
6
- TaskCross,
7
5
  TaskComplete,
8
- TaskWheel,
6
+ TaskCross,
9
7
  TaskGrid,
8
+ TaskLoop,
9
+ TaskWheel,
10
10
  )
11
11
 
12
12
  __all__ = [
@@ -1,41 +1,46 @@
1
1
  # graph/core_graph.py
2
+ import multiprocessing
2
3
  import time
3
4
  import warnings
4
- import multiprocessing
5
5
  from collections import defaultdict, deque
6
+ from pathlib import Path
7
+ from typing import Any
6
8
  from multiprocessing import Queue as MPQueue
7
9
 
8
10
  from celestialtree import (
9
11
  Client as CelestialTreeClient,
12
+ )
13
+ from celestialtree import (
10
14
  NullClient as NullCelestialTreeClient,
15
+ )
16
+ from celestialtree import (
11
17
  format_descendants_forest,
12
18
  format_provenance_forest,
13
19
  )
20
+ from networkx import is_directed_acyclic_graph
14
21
 
15
- from ..runtime import TaskInQueue, TaskOutQueue, TaskEnvelope
22
+ from ..observability import NullTaskReporter, TaskReporter
23
+ from ..persistence import FailListener, FailSinker, LogListener, LogSinker
24
+ from ..persistence.util_jsonl import load_task_by_error, load_task_by_stage
25
+ from ..runtime import TaskEnvelope, TaskInQueue, TaskOutQueue
26
+ from ..runtime.util_errors import UnconsumedError
16
27
  from ..runtime.util_estimators import (
17
28
  calc_elapsed,
18
- calc_remaining,
19
29
  calc_global_remain_equal_pred,
30
+ calc_remaining,
20
31
  )
21
- from ..runtime.util_errors import UnconsumedError
22
32
  from ..runtime.util_types import (
33
+ NULL_PREV_STAGE,
34
+ STAGE_STYLE,
23
35
  StageStatus,
24
36
  TerminationSignal,
25
- NullPrevStage,
26
- STAGE_STYLE,
27
- NULL_PREV_STAGE,
28
37
  )
29
38
  from ..stage import TaskStage
30
- from ..observability import TaskReporter, NullTaskReporter
31
- from ..persistence import FailListener, FailSinker, LogListener, LogSinker
32
- from ..persistence.util_jsonl import load_task_by_stage, load_task_by_error
33
39
  from ..utils.util_collections import cluster_by_value_sorted
34
40
  from ..utils.util_format import format_avg_time
35
41
  from .util_analysis import (
36
- format_networkx_graph,
37
42
  compute_node_levels,
38
- is_directed_acyclic_graph,
43
+ format_networkx_graph,
39
44
  )
40
45
  from .util_serialize import build_structure_graph, format_structure_list_from_graph
41
46
 
@@ -103,15 +108,15 @@ class TaskGraph:
103
108
  # 用于保存所有子进程的引用
104
109
  self.processes: list[multiprocessing.Process] = []
105
110
  # 用于保存每个节点的运行信息
106
- self.stage_runtime_dict: dict[str, dict] = defaultdict(dict)
111
+ self.stage_runtime_dict: dict[str, dict[str, Any]] = defaultdict(dict)
107
112
  # 用于保存每个节点的上一次collect_runtime_snapshot()的状态信息
108
- self.status_dict: dict[str, dict] = defaultdict(dict)
113
+ self.status_dict: dict[str, dict[str, Any]] = defaultdict(dict)
109
114
  # 用于保存任务图的摘要信息
110
115
  self.graph_summary: dict[str, int | float] = {}
111
116
  # 用于保存每个节点的历史状态信息列表(仅保留最近20条)
112
117
  self.stage_history: dict[str, list[dict]] = {}
113
118
  # 用于保存每个节点的输入任务ID集合
114
- self.input_ids: dict[str, set] = defaultdict(set)
119
+ self.input_ids: dict[str, set[int]] = defaultdict(set)
115
120
 
116
121
  def init_listener(self) -> None:
117
122
  """
@@ -165,11 +170,11 @@ class TaskGraph:
165
170
  queue.extend(stage.next_stages)
166
171
 
167
172
  for stage_tag, stage_runtime in self.stage_runtime_dict.items():
168
- stage: TaskStage = stage_runtime["stage"]
173
+ current_stage: TaskStage = stage_runtime["stage"]
169
174
  in_queue: TaskInQueue = stage_runtime["in_queue"]
170
175
 
171
176
  # 遍历每个前驱,创建边队列
172
- for prev_stage in stage.prev_stages:
177
+ for prev_stage in current_stage.prev_stages:
173
178
  prev_stage_tag = prev_stage.get_tag()
174
179
  in_queue.add_source_tag(prev_stage_tag)
175
180
 
@@ -236,6 +241,7 @@ class TaskGraph:
236
241
  self._is_report = is_report
237
242
  self._report_host = host
238
243
  self._report_port = port
244
+ self.reporter: TaskReporter | NullTaskReporter
239
245
  if is_report:
240
246
  self.reporter = TaskReporter(
241
247
  host=host,
@@ -292,7 +298,7 @@ class TaskGraph:
292
298
  :param execution_mode: 节点内部执行模式, 可选值为 'serial' 或 'thread''
293
299
  """
294
300
 
295
- def set_subsequent_stage_mode(stage: TaskStage):
301
+ def set_subsequent_stage_mode(stage: TaskStage) -> None:
296
302
  stage.set_stage_mode(stage_mode)
297
303
  stage.set_execution_mode(execution_mode)
298
304
  visited_stages.add(stage)
@@ -302,7 +308,7 @@ class TaskGraph:
302
308
  continue
303
309
  set_subsequent_stage_mode(next_stage)
304
310
 
305
- visited_stages = set()
311
+ visited_stages: set[TaskStage] = set()
306
312
  for root_stage in self.root_stages:
307
313
  set_subsequent_stage_mode(root_stage)
308
314
  self.init_analysis()
@@ -495,11 +501,11 @@ class TaskGraph:
495
501
 
496
502
  # 收集并持久化每个 stage 中未消费的任务
497
503
  for stage_tag, stage_runtime in self.stage_runtime_dict.items():
498
- stage: TaskStage = stage_runtime["stage"]
504
+ current_stage: TaskStage = stage_runtime["stage"]
499
505
  in_queue: TaskInQueue = stage_runtime["in_queue"]
500
506
 
501
507
  remaining_sources = in_queue.drain()
502
- stage.metrics.add_error_count(len(remaining_sources))
508
+ current_stage.metrics.add_error_count(len(remaining_sources))
503
509
 
504
510
  # 持久化逻辑
505
511
  for source in remaining_sources:
@@ -508,7 +514,7 @@ class TaskGraph:
508
514
  error_id = self.ctree_client.emit(
509
515
  "task.error",
510
516
  [task_id],
511
- payload=stage.get_summary(),
517
+ payload=current_stage.get_summary(),
512
518
  )
513
519
 
514
520
  self.fail_sinker.task_error(
@@ -516,8 +522,8 @@ class TaskGraph:
516
522
  )
517
523
 
518
524
  self.log_sinker.task_error(
519
- stage.get_func_name(),
520
- stage.get_task_repr(task),
525
+ current_stage.get_func_name(),
526
+ current_stage.get_task_repr(task),
521
527
  UnconsumedError(),
522
528
  task_id,
523
529
  error_id,
@@ -585,9 +591,7 @@ class TaskGraph:
585
591
  start_time = stage_runtime.get("start_time", 0)
586
592
  last_elapsed = last_stage_status_dict.get("elapsed_time", 0)
587
593
  last_pending = last_stage_status_dict.get("tasks_pending", 0)
588
- elapsed = calc_elapsed(
589
- status, start_time, last_elapsed, last_pending, interval
590
- )
594
+ elapsed = calc_elapsed(status, last_elapsed, last_pending, interval)
591
595
 
592
596
  # 估算剩余时间
593
597
  remaining = calc_remaining(
@@ -647,6 +651,9 @@ class TaskGraph:
647
651
  def get_fail_by_error_dict(self) -> dict:
648
652
  return load_task_by_error(self.fail_listener.jsonl_path)
649
653
 
654
+ def get_total_error_num(self) -> int:
655
+ return self.fail_listener.total_error_num
656
+
650
657
  def get_status_dict(self) -> dict[str, dict]:
651
658
  """
652
659
  获取任务链的状态字典
@@ -663,9 +670,9 @@ class TaskGraph:
663
670
  """获取任务链的历史状态信息字典"""
664
671
  return self.stage_history
665
672
 
666
- def get_graph_topology(self) -> dict:
673
+ def get_graph_analysis(self) -> dict:
667
674
  """
668
- 获取任务图的拓扑信息
675
+ 获取任务图的分析信息
669
676
  """
670
677
  return {
671
678
  "isDAG": self.isDAG,
@@ -696,7 +703,9 @@ class TaskGraph:
696
703
  """
697
704
  获取失败任务的回退路径
698
705
  """
699
- return str(self.fail_listener.jsonl_path.resolve())
706
+ if self.fail_listener.jsonl_path is None:
707
+ return ""
708
+ return str(Path(self.fail_listener.jsonl_path).resolve())
700
709
 
701
710
  def get_stage_input_trace(self, stage_tag: str) -> str:
702
711
  """
@@ -17,6 +17,7 @@ class TaskChain(TaskGraph):
17
17
 
18
18
  :param stages: TaskStage 列表, 每个 TaskStage 节点将连接到下一个节点
19
19
  :param chain_mode: 控制任务链中各节点同时运行(process), 亦或者依次运行(serial)
20
+ :param log_level: 日志级别
20
21
  """
21
22
  for num, stage in enumerate(stages):
22
23
  stage_name = f"Stage {num + 1}"
@@ -24,7 +25,9 @@ class TaskChain(TaskGraph):
24
25
  stage.set_graph_context(next_stages, chain_mode, stage_name)
25
26
 
26
27
  root_stage = stages[0]
27
- super().__init__(root_stages=[root_stage], log_level=log_level)
28
+ super().__init__(
29
+ root_stages=[root_stage], schedule_mode="eager", log_level=log_level
30
+ )
28
31
 
29
32
  def start_chain(
30
33
  self, init_tasks_dict: dict, put_termination_signal: bool = True
@@ -53,8 +56,8 @@ class TaskCross(TaskGraph):
53
56
  :param layers:
54
57
  按层划分的任务节点列表。每个子列表代表一层,列表中的 TaskStage 将并行执行。
55
58
  相邻层之间的所有节点将建立全连接依赖(即每个上一层节点都连接到下一层所有节点)。
56
- :param schedule_mode:
57
- 控制任务图的调度布局模式
59
+ :param schedule_mode: 控制任务图的调度布局模式
60
+ :param log_level: 日志级别
58
61
  """
59
62
  for i in range(len(layers)):
60
63
  curr_layer = layers[i]
@@ -64,7 +67,7 @@ class TaskCross(TaskGraph):
64
67
  stage.set_graph_context(
65
68
  next_stages=next_layer,
66
69
  stage_mode="process",
67
- stage_name=f"Layer{i+1}-{index+1}",
70
+ stage_name=f"Layer{i + 1}-{index + 1}",
68
71
  )
69
72
  super().__init__(
70
73
  root_stages=layers[0], schedule_mode=schedule_mode, log_level=log_level
@@ -98,6 +101,7 @@ class TaskGrid(TaskGraph):
98
101
  任务网格,每个子列表代表一行,列表中的 TaskStage 将按行并行执行。
99
102
  每个节点将连接到其右侧和下方的节点。
100
103
  :param schedule_mode: 控制任务图的调度布局模式
104
+ :param log_level: 日志级别
101
105
  """
102
106
  rows, cols = len(grid), len(grid[0])
103
107
  for i in range(rows):
@@ -108,7 +112,7 @@ class TaskGrid(TaskGraph):
108
112
  nexts.append(grid[i + 1][j]) # down
109
113
  if j + 1 < cols:
110
114
  nexts.append(grid[i][j + 1]) # right
111
- curr.set_graph_context(nexts, "process", f"Grid-{i+1}-{j+1}")
115
+ curr.set_graph_context(nexts, "process", f"Grid-{i + 1}-{j + 1}")
112
116
  super().__init__(
113
117
  root_stages=[grid[0][0]], schedule_mode=schedule_mode, log_level=log_level
114
118
  ) # 起点为左上角
@@ -133,6 +137,7 @@ class TaskLoop(TaskGraph):
133
137
  由于环的结构特性, 强制使用 'eager' 节点模式
134
138
 
135
139
  :param stages: TaskStage 列表, 每个 TaskStage 节点将连接到下一个节点, 形成一个闭环
140
+ :param log_level: 日志级别
136
141
  """
137
142
  for num, stage in enumerate(stages):
138
143
  stage_name = f"Stage {num + 1}"
@@ -163,13 +168,14 @@ class TaskWheel(TaskGraph):
163
168
 
164
169
  :param center: 中心节点
165
170
  :param ring: 环节点
171
+ :param log_level: 日志级别
166
172
  """
167
173
  # 中心连向环
168
174
  center.set_graph_context(ring, "process", "Center")
169
175
  # 环相连(成闭环)
170
176
  for i, node in enumerate(ring):
171
177
  next_stage = ring[(i + 1) % len(ring)]
172
- node.set_graph_context([next_stage], "process", f"Ring-{i+1}")
178
+ node.set_graph_context([next_stage], "process", f"Ring-{i + 1}")
173
179
  super().__init__(root_stages=[center], log_level=log_level)
174
180
 
175
181
  def start_wheel(
@@ -190,6 +196,7 @@ class TaskComplete(TaskGraph):
190
196
  TaskComplete: 完全图结构,每个节点都连向除自己以外的所有其他节点
191
197
 
192
198
  :param stages: 所有 TaskStage 节点
199
+ :param log_level: 日志级别
193
200
  """
194
201
  for i, stage in enumerate(stages):
195
202
  next_stages = [s for j, s in enumerate(stages) if i != j]
@@ -1,8 +1,8 @@
1
1
  # graph/util_analysis.py
2
- import networkx as nx
3
- from networkx import is_directed_acyclic_graph
4
2
  from typing import Any
5
3
 
4
+ import networkx as nx
5
+
6
6
 
7
7
  # ======== (图论分析) ========
8
8
  def format_networkx_graph(structure_graph: list[dict[str, Any]]) -> nx.DiGraph:
@@ -15,11 +15,11 @@ def format_networkx_graph(structure_graph: list[dict[str, Any]]) -> nx.DiGraph:
15
15
  G = nx.DiGraph()
16
16
 
17
17
  def add_node_and_edges(node: dict[str, Any]):
18
- node_id = f'{node["name"]}[{node["func_name"]}]'
18
+ node_id = f"{node['name']}[{node['func_name']}]"
19
19
  G.add_node(node_id, **{"mode": node.get("stage_mode")})
20
20
 
21
21
  for child in node.get("next_stages", []):
22
- child_id = f'{child["name"]}[{child["func_name"]}]'
22
+ child_id = f"{child['name']}[{child['func_name']}]"
23
23
  G.add_edge(node_id, child_id)
24
24
  # 递归添加子节点
25
25
  add_node_and_edges(child)