celestialflow 3.1.4__tar.gz → 3.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celestialflow-3.1.4 → celestialflow-3.1.6}/PKG-INFO +18 -20
- {celestialflow-3.1.4 → celestialflow-3.1.6}/README.md +17 -19
- {celestialflow-3.1.4 → celestialflow-3.1.6}/pyproject.toml +80 -75
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/__init__.py +17 -17
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/__init__.py +3 -3
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/core_graph.py +39 -30
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/core_structure.py +13 -6
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/util_analysis.py +4 -4
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/graph/util_serialize.py +4 -4
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/observability/__init__.py +1 -1
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/observability/core_report.py +60 -50
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/core_base.py +7 -6
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/core_fail.py +5 -2
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/core_log.py +19 -5
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/util_jsonl.py +1 -1
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/__init__.py +1 -1
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_metrics.py +4 -4
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_queue.py +14 -12
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_runner.py +45 -27
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_errors.py +1 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_estimators.py +10 -6
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_factories.py +5 -7
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_types.py +7 -5
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/__init__.py +3 -3
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/core_executor.py +52 -36
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/core_stage.py +26 -14
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/stage/core_stages.py +36 -13
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_benchmark.py +12 -1
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_clone.py +1 -1
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_format.py +9 -9
- celestialflow-3.1.6/src/celestialflow/web/config.json +35 -0
- celestialflow-3.1.4/src/celestialflow/web/server.py → celestialflow-3.1.6/src/celestialflow/web/core_server.py +148 -61
- celestialflow-3.1.6/src/celestialflow/web/static/css/_colors.css +135 -0
- celestialflow-3.1.6/src/celestialflow/web/static/css/base.css +320 -0
- celestialflow-3.1.6/src/celestialflow/web/static/css/dashboard.css +348 -0
- celestialflow-3.1.6/src/celestialflow/web/static/css/errors.css +286 -0
- celestialflow-3.1.6/src/celestialflow/web/static/css/inject.css +562 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/main.js +5 -24
- celestialflow-3.1.6/src/celestialflow/web/static/js/task_analysis.js +59 -0
- celestialflow-3.1.6/src/celestialflow/web/static/js/task_errors.js +162 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_history.js +23 -11
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_injection.js +21 -16
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_statuses.js +58 -44
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_structure.js +18 -3
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_summary.js +14 -8
- celestialflow-3.1.6/src/celestialflow/web/static/js/task_topology.js +59 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/utils.js +95 -14
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/main.ts +6 -32
- celestialflow-3.1.6/src/celestialflow/web/static/ts/task_analysis.ts +67 -0
- celestialflow-3.1.6/src/celestialflow/web/static/ts/task_errors.ts +178 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_history.ts +24 -13
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_injection.ts +384 -379
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_statuses.ts +172 -149
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_structure.ts +19 -4
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/task_summary.ts +14 -9
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/utils.ts +264 -169
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/templates/index.html +39 -35
- celestialflow-3.1.6/src/celestialflow/web/util_cal.py +4 -0
- celestialflow-3.1.6/src/celestialflow/web/util_config.py +23 -0
- celestialflow-3.1.6/src/celestialflow/web/util_error.py +53 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/PKG-INFO +18 -20
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/SOURCES.txt +9 -3
- celestialflow-3.1.6/src/celestialflow.egg-info/entry_points.txt +2 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/tests/test_executor.py +5 -38
- {celestialflow-3.1.4 → celestialflow-3.1.6}/tests/test_stages.py +45 -121
- {celestialflow-3.1.4 → celestialflow-3.1.6}/tests/test_structure.py +48 -116
- celestialflow-3.1.6/tests/test_utils.py +234 -0
- celestialflow-3.1.4/src/celestialflow/web/static/css/_colors.css +0 -183
- celestialflow-3.1.4/src/celestialflow/web/static/css/base.css +0 -305
- celestialflow-3.1.4/src/celestialflow/web/static/css/dashboard.css +0 -368
- celestialflow-3.1.4/src/celestialflow/web/static/css/errors.css +0 -261
- celestialflow-3.1.4/src/celestialflow/web/static/css/inject.css +0 -638
- celestialflow-3.1.4/src/celestialflow/web/static/js/task_errors.js +0 -131
- celestialflow-3.1.4/src/celestialflow/web/static/js/task_topology.js +0 -53
- celestialflow-3.1.4/src/celestialflow/web/static/ts/task_errors.ts +0 -147
- celestialflow-3.1.4/src/celestialflow/web/static/ts/task_topology.ts +0 -62
- celestialflow-3.1.4/src/celestialflow.egg-info/entry_points.txt +0 -2
- {celestialflow-3.1.4 → celestialflow-3.1.6}/setup.cfg +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/__init__.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/persistence/util_constant.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_envelope.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/core_progress.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_hash.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/runtime/util_queue.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/__init__.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_collections.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/utils/util_debug.py +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/favicon.ico +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/task_config.js +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/js/web_config.js +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/globals.d.ts +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow/web/static/ts/web_config.ts +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/dependency_links.txt +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/requires.txt +0 -0
- {celestialflow-3.1.4 → celestialflow-3.1.6}/src/celestialflow.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celestialflow
|
|
3
|
-
Version: 3.1.
|
|
3
|
+
Version: 3.1.6
|
|
4
4
|
Summary: A flexible GRAPH-based task orchestration framework.
|
|
5
5
|
Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -254,35 +254,33 @@ flowchart TD
|
|
|
254
254
|
<p align="center">
|
|
255
255
|
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
256
256
|
<br/>
|
|
257
|
-
<em>celestial-flow 3.1.
|
|
257
|
+
<em>celestial-flow 3.1.6</em>
|
|
258
258
|
</p>
|
|
259
259
|
|
|
260
260
|
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
261
261
|
|
|
262
262
|
## 版本日志(Version Log)
|
|
263
|
-
- 3.1.
|
|
264
|
-
- feat
|
|
265
|
-
-
|
|
266
|
-
-
|
|
267
|
-
-
|
|
268
|
-
-
|
|
269
|
-
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
-
|
|
273
|
-
-
|
|
274
|
-
-
|
|
275
|
-
|
|
276
|
-
-
|
|
277
|
-
-
|
|
278
|
-
- fix:
|
|
279
|
-
- TaskRedisTransport节点在mermaid中没有展示为parallelogram;
|
|
263
|
+
- 3.1.6
|
|
264
|
+
- feat
|
|
265
|
+
- 前端现在只储存一页的error数据, 有效减少了运行大规模任务时前端内存飙升的问题;
|
|
266
|
+
- 优化任务数显示, 大于1*10^7时显示科学计数法, 否则显示英式计数;
|
|
267
|
+
- 优化小屏模式下表格的显示: 改为用卡片式显示;
|
|
268
|
+
- 在error表格中加入index项;
|
|
269
|
+
- 大幅调整任务颜色分配, 现在重复任务使用黄色系, 等待任务使用灰色系;
|
|
270
|
+
- 将节点卡片中的已消耗时间颜色拟合为下方进度条颜色;
|
|
271
|
+
- refactor
|
|
272
|
+
- 用mypy整理了一遍类型标注;
|
|
273
|
+
- 删除与整合部分css代码;
|
|
274
|
+
- 将字体等适合rem单位的地方全部从px替换为rem, 并且都部分size进行统一;
|
|
275
|
+
- fix
|
|
276
|
+
- 修复总体剩余时间在特殊情况下显示0的问题(这玩意真麻烦);
|
|
277
|
+
- 修复部分小屏下的显示问题, 但折线图不显示的问题不太好解决;
|
|
280
278
|
|
|
281
279
|
## Star 历史趋势(Star History)
|
|
282
280
|
|
|
283
281
|
如果对项目感兴趣的话,欢迎star。如果有问题或者建议的话, 欢迎提交[Issues](https://github.com/Mr-xiaotian/CelestialFlow/issues)或者在[Discussion](https://github.com/Mr-xiaotian/CelestialFlow/discussions)中告诉我。
|
|
284
282
|
|
|
285
|
-
|
|
283
|
+

|
|
286
284
|
|
|
287
285
|
## 许可(License)
|
|
288
286
|
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
@@ -229,35 +229,33 @@ flowchart TD
|
|
|
229
229
|
<p align="center">
|
|
230
230
|
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
231
231
|
<br/>
|
|
232
|
-
<em>celestial-flow 3.1.
|
|
232
|
+
<em>celestial-flow 3.1.6</em>
|
|
233
233
|
</p>
|
|
234
234
|
|
|
235
235
|
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
236
236
|
|
|
237
237
|
## 版本日志(Version Log)
|
|
238
|
-
- 3.1.
|
|
239
|
-
- feat
|
|
240
|
-
-
|
|
241
|
-
-
|
|
242
|
-
-
|
|
243
|
-
-
|
|
244
|
-
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
-
|
|
248
|
-
-
|
|
249
|
-
-
|
|
250
|
-
|
|
251
|
-
-
|
|
252
|
-
-
|
|
253
|
-
- fix:
|
|
254
|
-
- TaskRedisTransport节点在mermaid中没有展示为parallelogram;
|
|
238
|
+
- 3.1.6
|
|
239
|
+
- feat
|
|
240
|
+
- 前端现在只储存一页的error数据, 有效减少了运行大规模任务时前端内存飙升的问题;
|
|
241
|
+
- 优化任务数显示, 大于1*10^7时显示科学计数法, 否则显示英式计数;
|
|
242
|
+
- 优化小屏模式下表格的显示: 改为用卡片式显示;
|
|
243
|
+
- 在error表格中加入index项;
|
|
244
|
+
- 大幅调整任务颜色分配, 现在重复任务使用黄色系, 等待任务使用灰色系;
|
|
245
|
+
- 将节点卡片中的已消耗时间颜色拟合为下方进度条颜色;
|
|
246
|
+
- refactor
|
|
247
|
+
- 用mypy整理了一遍类型标注;
|
|
248
|
+
- 删除与整合部分css代码;
|
|
249
|
+
- 将字体等适合rem单位的地方全部从px替换为rem, 并且都部分size进行统一;
|
|
250
|
+
- fix
|
|
251
|
+
- 修复总体剩余时间在特殊情况下显示0的问题(这玩意真麻烦);
|
|
252
|
+
- 修复部分小屏下的显示问题, 但折线图不显示的问题不太好解决;
|
|
255
253
|
|
|
256
254
|
## Star 历史趋势(Star History)
|
|
257
255
|
|
|
258
256
|
如果对项目感兴趣的话,欢迎star。如果有问题或者建议的话, 欢迎提交[Issues](https://github.com/Mr-xiaotian/CelestialFlow/issues)或者在[Discussion](https://github.com/Mr-xiaotian/CelestialFlow/discussions)中告诉我。
|
|
259
257
|
|
|
260
|
-
|
|
258
|
+

|
|
261
259
|
|
|
262
260
|
## 许可(License)
|
|
263
261
|
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
@@ -1,75 +1,80 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "celestialflow"
|
|
7
|
-
version = "3.1.
|
|
8
|
-
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
license = { text = "MIT" }
|
|
11
|
-
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
|
-
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
-
requires-python = ">=3.10"
|
|
14
|
-
|
|
15
|
-
dependencies = [
|
|
16
|
-
"tqdm",
|
|
17
|
-
"fastapi",
|
|
18
|
-
"uvicorn",
|
|
19
|
-
"requests",
|
|
20
|
-
"networkx",
|
|
21
|
-
"redis",
|
|
22
|
-
"jinja2",
|
|
23
|
-
"celestialtree>=0.1.2",
|
|
24
|
-
]
|
|
25
|
-
|
|
26
|
-
classifiers = [
|
|
27
|
-
"Programming Language :: Python :: 3",
|
|
28
|
-
"License :: OSI Approved :: MIT License",
|
|
29
|
-
"Operating System :: OS Independent",
|
|
30
|
-
"Framework :: FastAPI",
|
|
31
|
-
"Topic :: Software Development :: Libraries",
|
|
32
|
-
]
|
|
33
|
-
|
|
34
|
-
[project.urls]
|
|
35
|
-
"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
|
|
36
|
-
"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
|
|
37
|
-
|
|
38
|
-
[project.scripts]
|
|
39
|
-
celestialflow-web = "celestialflow.web.
|
|
40
|
-
|
|
41
|
-
[tool.setuptools]
|
|
42
|
-
license-files = []
|
|
43
|
-
|
|
44
|
-
[tool.setuptools.package-data]
|
|
45
|
-
celestialflow = [
|
|
46
|
-
"web/templates
|
|
47
|
-
"web/
|
|
48
|
-
"web/
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
"ignore::
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
[
|
|
67
|
-
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"pytest>=
|
|
71
|
-
"
|
|
72
|
-
"
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
]
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "celestialflow"
|
|
7
|
+
version = "3.1.6"
|
|
8
|
+
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
|
+
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
|
|
15
|
+
dependencies = [
|
|
16
|
+
"tqdm",
|
|
17
|
+
"fastapi",
|
|
18
|
+
"uvicorn",
|
|
19
|
+
"requests",
|
|
20
|
+
"networkx",
|
|
21
|
+
"redis",
|
|
22
|
+
"jinja2",
|
|
23
|
+
"celestialtree>=0.1.2",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Framework :: FastAPI",
|
|
31
|
+
"Topic :: Software Development :: Libraries",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
|
|
36
|
+
"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
celestialflow-web = "celestialflow.web.core_server:main_entry"
|
|
40
|
+
|
|
41
|
+
[tool.setuptools]
|
|
42
|
+
license-files = []
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.package-data]
|
|
45
|
+
celestialflow = [
|
|
46
|
+
"web/templates/**/*.html",
|
|
47
|
+
"web/static/**/*",
|
|
48
|
+
"web/config.json"
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
addopts = "-s"
|
|
53
|
+
log_cli = true
|
|
54
|
+
log_cli_level = "INFO"
|
|
55
|
+
log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
|
|
56
|
+
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
57
|
+
|
|
58
|
+
filterwarnings = [
|
|
59
|
+
"ignore::DeprecationWarning",
|
|
60
|
+
"ignore::pytest.PytestDeprecationWarning",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
64
|
+
|
|
65
|
+
[dependency-groups]
|
|
66
|
+
dev = [
|
|
67
|
+
"build>=1.2.2.post1",
|
|
68
|
+
"twine>=6.1.0",
|
|
69
|
+
"pytest>=8.3.4",
|
|
70
|
+
"pytest-asyncio>=0.25.3",
|
|
71
|
+
"httpx>=0.28.1",
|
|
72
|
+
"python-dotenv>=1.2.2",
|
|
73
|
+
"dill",
|
|
74
|
+
"mypy>=1.19.1",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
[tool.mypy]
|
|
78
|
+
python_version = "3.10"
|
|
79
|
+
no_implicit_optional = false
|
|
80
|
+
disable_error_code = ["import-untyped"]
|
|
@@ -1,32 +1,32 @@
|
|
|
1
1
|
# __init__.py
|
|
2
|
-
from .stage import (
|
|
3
|
-
TaskExecutor,
|
|
4
|
-
TaskStage,
|
|
5
|
-
TaskSplitter,
|
|
6
|
-
TaskRedisTransport,
|
|
7
|
-
TaskRedisSource,
|
|
8
|
-
TaskRedisAck,
|
|
9
|
-
TaskRouter,
|
|
10
|
-
)
|
|
11
2
|
from .graph import (
|
|
12
|
-
TaskGraph,
|
|
13
3
|
TaskChain,
|
|
14
|
-
TaskLoop,
|
|
15
|
-
TaskCross,
|
|
16
4
|
TaskComplete,
|
|
17
|
-
|
|
5
|
+
TaskCross,
|
|
6
|
+
TaskGraph,
|
|
18
7
|
TaskGrid,
|
|
8
|
+
TaskLoop,
|
|
9
|
+
TaskWheel,
|
|
19
10
|
)
|
|
20
11
|
from .persistence.util_jsonl import (
|
|
21
12
|
load_jsonl_logs,
|
|
22
|
-
load_task_by_stage,
|
|
23
13
|
load_task_by_error,
|
|
14
|
+
load_task_by_stage,
|
|
24
15
|
)
|
|
25
|
-
from .runtime.util_types import TerminationSignal
|
|
26
16
|
from .runtime.util_hash import make_hashable
|
|
17
|
+
from .runtime.util_types import TerminationSignal
|
|
18
|
+
from .stage import (
|
|
19
|
+
TaskExecutor,
|
|
20
|
+
TaskRedisAck,
|
|
21
|
+
TaskRedisSource,
|
|
22
|
+
TaskRedisTransport,
|
|
23
|
+
TaskRouter,
|
|
24
|
+
TaskSplitter,
|
|
25
|
+
TaskStage,
|
|
26
|
+
)
|
|
27
|
+
from .utils.util_benchmark import benchmark_executor, benchmark_graph
|
|
27
28
|
from .utils.util_format import format_table
|
|
28
|
-
from .
|
|
29
|
-
from .web.server import TaskWebServer
|
|
29
|
+
from .web.core_server import TaskWebServer
|
|
30
30
|
|
|
31
31
|
__all__ = [
|
|
32
32
|
"TaskGraph",
|
|
@@ -1,41 +1,46 @@
|
|
|
1
1
|
# graph/core_graph.py
|
|
2
|
+
import multiprocessing
|
|
2
3
|
import time
|
|
3
4
|
import warnings
|
|
4
|
-
import multiprocessing
|
|
5
5
|
from collections import defaultdict, deque
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
6
8
|
from multiprocessing import Queue as MPQueue
|
|
7
9
|
|
|
8
10
|
from celestialtree import (
|
|
9
11
|
Client as CelestialTreeClient,
|
|
12
|
+
)
|
|
13
|
+
from celestialtree import (
|
|
10
14
|
NullClient as NullCelestialTreeClient,
|
|
15
|
+
)
|
|
16
|
+
from celestialtree import (
|
|
11
17
|
format_descendants_forest,
|
|
12
18
|
format_provenance_forest,
|
|
13
19
|
)
|
|
20
|
+
from networkx import is_directed_acyclic_graph
|
|
14
21
|
|
|
15
|
-
from ..
|
|
22
|
+
from ..observability import NullTaskReporter, TaskReporter
|
|
23
|
+
from ..persistence import FailListener, FailSinker, LogListener, LogSinker
|
|
24
|
+
from ..persistence.util_jsonl import load_task_by_error, load_task_by_stage
|
|
25
|
+
from ..runtime import TaskEnvelope, TaskInQueue, TaskOutQueue
|
|
26
|
+
from ..runtime.util_errors import UnconsumedError
|
|
16
27
|
from ..runtime.util_estimators import (
|
|
17
28
|
calc_elapsed,
|
|
18
|
-
calc_remaining,
|
|
19
29
|
calc_global_remain_equal_pred,
|
|
30
|
+
calc_remaining,
|
|
20
31
|
)
|
|
21
|
-
from ..runtime.util_errors import UnconsumedError
|
|
22
32
|
from ..runtime.util_types import (
|
|
33
|
+
NULL_PREV_STAGE,
|
|
34
|
+
STAGE_STYLE,
|
|
23
35
|
StageStatus,
|
|
24
36
|
TerminationSignal,
|
|
25
|
-
NullPrevStage,
|
|
26
|
-
STAGE_STYLE,
|
|
27
|
-
NULL_PREV_STAGE,
|
|
28
37
|
)
|
|
29
38
|
from ..stage import TaskStage
|
|
30
|
-
from ..observability import TaskReporter, NullTaskReporter
|
|
31
|
-
from ..persistence import FailListener, FailSinker, LogListener, LogSinker
|
|
32
|
-
from ..persistence.util_jsonl import load_task_by_stage, load_task_by_error
|
|
33
39
|
from ..utils.util_collections import cluster_by_value_sorted
|
|
34
40
|
from ..utils.util_format import format_avg_time
|
|
35
41
|
from .util_analysis import (
|
|
36
|
-
format_networkx_graph,
|
|
37
42
|
compute_node_levels,
|
|
38
|
-
|
|
43
|
+
format_networkx_graph,
|
|
39
44
|
)
|
|
40
45
|
from .util_serialize import build_structure_graph, format_structure_list_from_graph
|
|
41
46
|
|
|
@@ -103,15 +108,15 @@ class TaskGraph:
|
|
|
103
108
|
# 用于保存所有子进程的引用
|
|
104
109
|
self.processes: list[multiprocessing.Process] = []
|
|
105
110
|
# 用于保存每个节点的运行信息
|
|
106
|
-
self.stage_runtime_dict: dict[str, dict] = defaultdict(dict)
|
|
111
|
+
self.stage_runtime_dict: dict[str, dict[str, Any]] = defaultdict(dict)
|
|
107
112
|
# 用于保存每个节点的上一次collect_runtime_snapshot()的状态信息
|
|
108
|
-
self.status_dict: dict[str, dict] = defaultdict(dict)
|
|
113
|
+
self.status_dict: dict[str, dict[str, Any]] = defaultdict(dict)
|
|
109
114
|
# 用于保存任务图的摘要信息
|
|
110
115
|
self.graph_summary: dict[str, int | float] = {}
|
|
111
116
|
# 用于保存每个节点的历史状态信息列表(仅保留最近20条)
|
|
112
117
|
self.stage_history: dict[str, list[dict]] = {}
|
|
113
118
|
# 用于保存每个节点的输入任务ID集合
|
|
114
|
-
self.input_ids: dict[str, set] = defaultdict(set)
|
|
119
|
+
self.input_ids: dict[str, set[int]] = defaultdict(set)
|
|
115
120
|
|
|
116
121
|
def init_listener(self) -> None:
|
|
117
122
|
"""
|
|
@@ -165,11 +170,11 @@ class TaskGraph:
|
|
|
165
170
|
queue.extend(stage.next_stages)
|
|
166
171
|
|
|
167
172
|
for stage_tag, stage_runtime in self.stage_runtime_dict.items():
|
|
168
|
-
|
|
173
|
+
current_stage: TaskStage = stage_runtime["stage"]
|
|
169
174
|
in_queue: TaskInQueue = stage_runtime["in_queue"]
|
|
170
175
|
|
|
171
176
|
# 遍历每个前驱,创建边队列
|
|
172
|
-
for prev_stage in
|
|
177
|
+
for prev_stage in current_stage.prev_stages:
|
|
173
178
|
prev_stage_tag = prev_stage.get_tag()
|
|
174
179
|
in_queue.add_source_tag(prev_stage_tag)
|
|
175
180
|
|
|
@@ -236,6 +241,7 @@ class TaskGraph:
|
|
|
236
241
|
self._is_report = is_report
|
|
237
242
|
self._report_host = host
|
|
238
243
|
self._report_port = port
|
|
244
|
+
self.reporter: TaskReporter | NullTaskReporter
|
|
239
245
|
if is_report:
|
|
240
246
|
self.reporter = TaskReporter(
|
|
241
247
|
host=host,
|
|
@@ -292,7 +298,7 @@ class TaskGraph:
|
|
|
292
298
|
:param execution_mode: 节点内部执行模式, 可选值为 'serial' 或 'thread''
|
|
293
299
|
"""
|
|
294
300
|
|
|
295
|
-
def set_subsequent_stage_mode(stage: TaskStage):
|
|
301
|
+
def set_subsequent_stage_mode(stage: TaskStage) -> None:
|
|
296
302
|
stage.set_stage_mode(stage_mode)
|
|
297
303
|
stage.set_execution_mode(execution_mode)
|
|
298
304
|
visited_stages.add(stage)
|
|
@@ -302,7 +308,7 @@ class TaskGraph:
|
|
|
302
308
|
continue
|
|
303
309
|
set_subsequent_stage_mode(next_stage)
|
|
304
310
|
|
|
305
|
-
visited_stages = set()
|
|
311
|
+
visited_stages: set[TaskStage] = set()
|
|
306
312
|
for root_stage in self.root_stages:
|
|
307
313
|
set_subsequent_stage_mode(root_stage)
|
|
308
314
|
self.init_analysis()
|
|
@@ -495,11 +501,11 @@ class TaskGraph:
|
|
|
495
501
|
|
|
496
502
|
# 收集并持久化每个 stage 中未消费的任务
|
|
497
503
|
for stage_tag, stage_runtime in self.stage_runtime_dict.items():
|
|
498
|
-
|
|
504
|
+
current_stage: TaskStage = stage_runtime["stage"]
|
|
499
505
|
in_queue: TaskInQueue = stage_runtime["in_queue"]
|
|
500
506
|
|
|
501
507
|
remaining_sources = in_queue.drain()
|
|
502
|
-
|
|
508
|
+
current_stage.metrics.add_error_count(len(remaining_sources))
|
|
503
509
|
|
|
504
510
|
# 持久化逻辑
|
|
505
511
|
for source in remaining_sources:
|
|
@@ -508,7 +514,7 @@ class TaskGraph:
|
|
|
508
514
|
error_id = self.ctree_client.emit(
|
|
509
515
|
"task.error",
|
|
510
516
|
[task_id],
|
|
511
|
-
payload=
|
|
517
|
+
payload=current_stage.get_summary(),
|
|
512
518
|
)
|
|
513
519
|
|
|
514
520
|
self.fail_sinker.task_error(
|
|
@@ -516,8 +522,8 @@ class TaskGraph:
|
|
|
516
522
|
)
|
|
517
523
|
|
|
518
524
|
self.log_sinker.task_error(
|
|
519
|
-
|
|
520
|
-
|
|
525
|
+
current_stage.get_func_name(),
|
|
526
|
+
current_stage.get_task_repr(task),
|
|
521
527
|
UnconsumedError(),
|
|
522
528
|
task_id,
|
|
523
529
|
error_id,
|
|
@@ -585,9 +591,7 @@ class TaskGraph:
|
|
|
585
591
|
start_time = stage_runtime.get("start_time", 0)
|
|
586
592
|
last_elapsed = last_stage_status_dict.get("elapsed_time", 0)
|
|
587
593
|
last_pending = last_stage_status_dict.get("tasks_pending", 0)
|
|
588
|
-
elapsed = calc_elapsed(
|
|
589
|
-
status, start_time, last_elapsed, last_pending, interval
|
|
590
|
-
)
|
|
594
|
+
elapsed = calc_elapsed(status, last_elapsed, last_pending, interval)
|
|
591
595
|
|
|
592
596
|
# 估算剩余时间
|
|
593
597
|
remaining = calc_remaining(
|
|
@@ -647,6 +651,9 @@ class TaskGraph:
|
|
|
647
651
|
def get_fail_by_error_dict(self) -> dict:
|
|
648
652
|
return load_task_by_error(self.fail_listener.jsonl_path)
|
|
649
653
|
|
|
654
|
+
def get_total_error_num(self) -> int:
|
|
655
|
+
return self.fail_listener.total_error_num
|
|
656
|
+
|
|
650
657
|
def get_status_dict(self) -> dict[str, dict]:
|
|
651
658
|
"""
|
|
652
659
|
获取任务链的状态字典
|
|
@@ -663,9 +670,9 @@ class TaskGraph:
|
|
|
663
670
|
"""获取任务链的历史状态信息字典"""
|
|
664
671
|
return self.stage_history
|
|
665
672
|
|
|
666
|
-
def
|
|
673
|
+
def get_graph_analysis(self) -> dict:
|
|
667
674
|
"""
|
|
668
|
-
|
|
675
|
+
获取任务图的分析信息
|
|
669
676
|
"""
|
|
670
677
|
return {
|
|
671
678
|
"isDAG": self.isDAG,
|
|
@@ -696,7 +703,9 @@ class TaskGraph:
|
|
|
696
703
|
"""
|
|
697
704
|
获取失败任务的回退路径
|
|
698
705
|
"""
|
|
699
|
-
|
|
706
|
+
if self.fail_listener.jsonl_path is None:
|
|
707
|
+
return ""
|
|
708
|
+
return str(Path(self.fail_listener.jsonl_path).resolve())
|
|
700
709
|
|
|
701
710
|
def get_stage_input_trace(self, stage_tag: str) -> str:
|
|
702
711
|
"""
|
|
@@ -17,6 +17,7 @@ class TaskChain(TaskGraph):
|
|
|
17
17
|
|
|
18
18
|
:param stages: TaskStage 列表, 每个 TaskStage 节点将连接到下一个节点
|
|
19
19
|
:param chain_mode: 控制任务链中各节点同时运行(process), 亦或者依次运行(serial)
|
|
20
|
+
:param log_level: 日志级别
|
|
20
21
|
"""
|
|
21
22
|
for num, stage in enumerate(stages):
|
|
22
23
|
stage_name = f"Stage {num + 1}"
|
|
@@ -24,7 +25,9 @@ class TaskChain(TaskGraph):
|
|
|
24
25
|
stage.set_graph_context(next_stages, chain_mode, stage_name)
|
|
25
26
|
|
|
26
27
|
root_stage = stages[0]
|
|
27
|
-
super().__init__(
|
|
28
|
+
super().__init__(
|
|
29
|
+
root_stages=[root_stage], schedule_mode="eager", log_level=log_level
|
|
30
|
+
)
|
|
28
31
|
|
|
29
32
|
def start_chain(
|
|
30
33
|
self, init_tasks_dict: dict, put_termination_signal: bool = True
|
|
@@ -53,8 +56,8 @@ class TaskCross(TaskGraph):
|
|
|
53
56
|
:param layers:
|
|
54
57
|
按层划分的任务节点列表。每个子列表代表一层,列表中的 TaskStage 将并行执行。
|
|
55
58
|
相邻层之间的所有节点将建立全连接依赖(即每个上一层节点都连接到下一层所有节点)。
|
|
56
|
-
:param schedule_mode:
|
|
57
|
-
|
|
59
|
+
:param schedule_mode: 控制任务图的调度布局模式
|
|
60
|
+
:param log_level: 日志级别
|
|
58
61
|
"""
|
|
59
62
|
for i in range(len(layers)):
|
|
60
63
|
curr_layer = layers[i]
|
|
@@ -64,7 +67,7 @@ class TaskCross(TaskGraph):
|
|
|
64
67
|
stage.set_graph_context(
|
|
65
68
|
next_stages=next_layer,
|
|
66
69
|
stage_mode="process",
|
|
67
|
-
stage_name=f"Layer{i+1}-{index+1}",
|
|
70
|
+
stage_name=f"Layer{i + 1}-{index + 1}",
|
|
68
71
|
)
|
|
69
72
|
super().__init__(
|
|
70
73
|
root_stages=layers[0], schedule_mode=schedule_mode, log_level=log_level
|
|
@@ -98,6 +101,7 @@ class TaskGrid(TaskGraph):
|
|
|
98
101
|
任务网格,每个子列表代表一行,列表中的 TaskStage 将按行并行执行。
|
|
99
102
|
每个节点将连接到其右侧和下方的节点。
|
|
100
103
|
:param schedule_mode: 控制任务图的调度布局模式
|
|
104
|
+
:param log_level: 日志级别
|
|
101
105
|
"""
|
|
102
106
|
rows, cols = len(grid), len(grid[0])
|
|
103
107
|
for i in range(rows):
|
|
@@ -108,7 +112,7 @@ class TaskGrid(TaskGraph):
|
|
|
108
112
|
nexts.append(grid[i + 1][j]) # down
|
|
109
113
|
if j + 1 < cols:
|
|
110
114
|
nexts.append(grid[i][j + 1]) # right
|
|
111
|
-
curr.set_graph_context(nexts, "process", f"Grid-{i+1}-{j+1}")
|
|
115
|
+
curr.set_graph_context(nexts, "process", f"Grid-{i + 1}-{j + 1}")
|
|
112
116
|
super().__init__(
|
|
113
117
|
root_stages=[grid[0][0]], schedule_mode=schedule_mode, log_level=log_level
|
|
114
118
|
) # 起点为左上角
|
|
@@ -133,6 +137,7 @@ class TaskLoop(TaskGraph):
|
|
|
133
137
|
由于环的结构特性, 强制使用 'eager' 节点模式
|
|
134
138
|
|
|
135
139
|
:param stages: TaskStage 列表, 每个 TaskStage 节点将连接到下一个节点, 形成一个闭环
|
|
140
|
+
:param log_level: 日志级别
|
|
136
141
|
"""
|
|
137
142
|
for num, stage in enumerate(stages):
|
|
138
143
|
stage_name = f"Stage {num + 1}"
|
|
@@ -163,13 +168,14 @@ class TaskWheel(TaskGraph):
|
|
|
163
168
|
|
|
164
169
|
:param center: 中心节点
|
|
165
170
|
:param ring: 环节点
|
|
171
|
+
:param log_level: 日志级别
|
|
166
172
|
"""
|
|
167
173
|
# 中心连向环
|
|
168
174
|
center.set_graph_context(ring, "process", "Center")
|
|
169
175
|
# 环相连(成闭环)
|
|
170
176
|
for i, node in enumerate(ring):
|
|
171
177
|
next_stage = ring[(i + 1) % len(ring)]
|
|
172
|
-
node.set_graph_context([next_stage], "process", f"Ring-{i+1}")
|
|
178
|
+
node.set_graph_context([next_stage], "process", f"Ring-{i + 1}")
|
|
173
179
|
super().__init__(root_stages=[center], log_level=log_level)
|
|
174
180
|
|
|
175
181
|
def start_wheel(
|
|
@@ -190,6 +196,7 @@ class TaskComplete(TaskGraph):
|
|
|
190
196
|
TaskComplete: 完全图结构,每个节点都连向除自己以外的所有其他节点
|
|
191
197
|
|
|
192
198
|
:param stages: 所有 TaskStage 节点
|
|
199
|
+
:param log_level: 日志级别
|
|
193
200
|
"""
|
|
194
201
|
for i, stage in enumerate(stages):
|
|
195
202
|
next_stages = [s for j, s in enumerate(stages) if i != j]
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# graph/util_analysis.py
|
|
2
|
-
import networkx as nx
|
|
3
|
-
from networkx import is_directed_acyclic_graph
|
|
4
2
|
from typing import Any
|
|
5
3
|
|
|
4
|
+
import networkx as nx
|
|
5
|
+
|
|
6
6
|
|
|
7
7
|
# ======== (图论分析) ========
|
|
8
8
|
def format_networkx_graph(structure_graph: list[dict[str, Any]]) -> nx.DiGraph:
|
|
@@ -15,11 +15,11 @@ def format_networkx_graph(structure_graph: list[dict[str, Any]]) -> nx.DiGraph:
|
|
|
15
15
|
G = nx.DiGraph()
|
|
16
16
|
|
|
17
17
|
def add_node_and_edges(node: dict[str, Any]):
|
|
18
|
-
node_id = f
|
|
18
|
+
node_id = f"{node['name']}[{node['func_name']}]"
|
|
19
19
|
G.add_node(node_id, **{"mode": node.get("stage_mode")})
|
|
20
20
|
|
|
21
21
|
for child in node.get("next_stages", []):
|
|
22
|
-
child_id = f
|
|
22
|
+
child_id = f"{child['name']}[{child['func_name']}]"
|
|
23
23
|
G.add_edge(node_id, child_id)
|
|
24
24
|
# 递归添加子节点
|
|
25
25
|
add_node_and_edges(child)
|