celestialflow 3.2.1__tar.gz → 3.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celestialflow-3.2.1 → celestialflow-3.2.3}/PKG-INFO +55 -51
- {celestialflow-3.2.1 → celestialflow-3.2.3}/README.md +52 -48
- {celestialflow-3.2.1 → celestialflow-3.2.3}/pyproject.toml +4 -7
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/graph/core_graph.py +103 -103
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/graph/core_structure.py +22 -16
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/graph/util_analysis.py +3 -6
- celestialflow-3.2.3/src/celestialflow/graph/util_serialize.py +150 -0
- celestialflow-3.2.3/src/celestialflow/graph/util_types.py +27 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/observability/core_progress.py +1 -1
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/observability/core_report.py +25 -40
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/persistence/core_fail.py +29 -11
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/persistence/core_log.py +15 -18
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/persistence/core_success.py +7 -6
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/persistence/util_jsonl.py +14 -29
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/core_dispatch.py +45 -32
- celestialflow-3.2.3/src/celestialflow/runtime/core_envelope.py +75 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/core_metrics.py +50 -40
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/core_queue.py +20 -13
- celestialflow-3.2.3/src/celestialflow/runtime/util_constant.py +19 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/util_estimators.py +17 -21
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/util_types.py +47 -42
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/stage/core_executor.py +83 -113
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/stage/core_stage.py +58 -11
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/stage/core_stages.py +112 -82
- celestialflow-3.2.3/src/celestialflow/stage/util_types.py +6 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/utils/util_benchmark.py +5 -3
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/utils/util_clone.py +22 -15
- celestialflow-3.2.3/src/celestialflow/web/config.json +34 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/core_server.py +219 -145
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/routes/pull_routes.py +20 -31
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/routes/push_routes.py +171 -186
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/css/base.css +54 -43
- celestialflow-3.2.3/src/celestialflow/web/static/css/dashboard.css +175 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/dashboard_analysis.css +41 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/dashboard_history.css +89 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/dashboard_statuses.css +225 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/dashboard_structure.css +27 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/dashboard_summary.css +136 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/errors.css +364 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/injection_editor.css +223 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/injection_layout.css +119 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/injection_nodes.css +92 -0
- celestialflow-3.2.3/src/celestialflow/web/static/css/injection_preview.css +140 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/dashboard_analysis.js +27 -10
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/dashboard_history.js +56 -31
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/dashboard_statuses.js +64 -47
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/dashboard_structure.js +75 -42
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/dashboard_summary.js +9 -29
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/errors.js +69 -27
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/i18n.js +160 -81
- celestialflow-3.2.3/src/celestialflow/web/static/js/injection.js +675 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/layout_editor.js +64 -26
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/main.js +123 -25
- celestialflow-3.2.3/src/celestialflow/web/static/js/utils.js +177 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/js/web_config.js +271 -111
- celestialflow-3.2.3/src/celestialflow/web/static/ts/dashboard_analysis.ts +95 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/dashboard_history.ts +72 -49
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/dashboard_statuses.ts +85 -55
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/dashboard_structure.ts +86 -48
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/dashboard_summary.ts +9 -32
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/errors.ts +106 -32
- celestialflow-3.2.3/src/celestialflow/web/static/ts/globals.d.ts +155 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/i18n.ts +188 -85
- celestialflow-3.2.3/src/celestialflow/web/static/ts/injection.ts +787 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/layout_editor.ts +76 -37
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/ts/main.ts +137 -29
- celestialflow-3.2.3/src/celestialflow/web/static/ts/utils.ts +205 -0
- celestialflow-3.2.3/src/celestialflow/web/static/ts/web_config.ts +557 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/templates/index.html +258 -208
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/util_error.py +27 -10
- celestialflow-3.2.3/src/celestialflow/web/util_models.py +96 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow.egg-info/PKG-INFO +55 -51
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow.egg-info/SOURCES.txt +7 -2
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow.egg-info/requires.txt +1 -1
- celestialflow-3.2.1/src/celestialflow/graph/util_serialize.py +0 -151
- celestialflow-3.2.1/src/celestialflow/persistence/util_constant.py +0 -12
- celestialflow-3.2.1/src/celestialflow/runtime/core_envelope.py +0 -60
- celestialflow-3.2.1/src/celestialflow/web/config.json +0 -21
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard.css +0 -71
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_analysis.css +0 -39
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_history.css +0 -87
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_statuses.css +0 -238
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_structure.css +0 -25
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_summary.css +0 -134
- celestialflow-3.2.1/src/celestialflow/web/static/css/errors.css +0 -315
- celestialflow-3.2.1/src/celestialflow/web/static/css/injection.css +0 -615
- celestialflow-3.2.1/src/celestialflow/web/static/js/injection.js +0 -403
- celestialflow-3.2.1/src/celestialflow/web/static/js/utils.js +0 -117
- celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_analysis.ts +0 -70
- celestialflow-3.2.1/src/celestialflow/web/static/ts/globals.d.ts +0 -27
- celestialflow-3.2.1/src/celestialflow/web/static/ts/injection.ts +0 -454
- celestialflow-3.2.1/src/celestialflow/web/static/ts/utils.ts +0 -132
- celestialflow-3.2.1/src/celestialflow/web/static/ts/web_config.ts +0 -345
- celestialflow-3.2.1/src/celestialflow/web/util_models.py +0 -76
- {celestialflow-3.2.1 → celestialflow-3.2.3}/setup.cfg +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/funnel/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/funnel/core_inlet.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/funnel/core_spout.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/graph/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/observability/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/observability/core_observer.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/persistence/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/util_errors.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/runtime/util_hash.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/stage/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/utils/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/utils/util_collections.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/utils/util_format.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/routes/__init__.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/css/_colors.css +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/static/favicon.ico +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/util_cal.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow/web/util_config.py +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow.egg-info/dependency_links.txt +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow.egg-info/entry_points.txt +0 -0
- {celestialflow-3.2.1 → celestialflow-3.2.3}/src/celestialflow.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celestialflow
|
|
3
|
-
Version: 3.2.
|
|
3
|
+
Version: 3.2.3
|
|
4
4
|
Summary: A flexible GRAPH-based task orchestration framework.
|
|
5
5
|
Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Classifier: Framework :: FastAPI
|
|
14
14
|
Classifier: Topic :: Software Development :: Libraries
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
Requires-Dist: tqdm
|
|
18
18
|
Requires-Dist: fastapi
|
|
@@ -21,7 +21,7 @@ Requires-Dist: requests
|
|
|
21
21
|
Requires-Dist: networkx
|
|
22
22
|
Requires-Dist: redis
|
|
23
23
|
Requires-Dist: jinja2
|
|
24
|
-
Requires-Dist: celestialtree>=0.1.
|
|
24
|
+
Requires-Dist: celestialtree>=0.1.3
|
|
25
25
|
|
|
26
26
|
# CelestialFlow ——一个轻量级、可并行、基于图结构的 Python 任务调度框架
|
|
27
27
|
|
|
@@ -212,7 +212,7 @@ flowchart TD
|
|
|
212
212
|
|
|
213
213
|
以下三篇可以作为补充阅读:
|
|
214
214
|
|
|
215
|
-
- [runtime/
|
|
215
|
+
- [runtime/util_hash.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/util_hash.md)
|
|
216
216
|
- [runtime/util_types.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/util_types.md)
|
|
217
217
|
- [runtime/util_errors.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/util_errors.md)
|
|
218
218
|
- [persistence/core_fail.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/persistence/core_fail.md)
|
|
@@ -228,24 +228,24 @@ flowchart TD
|
|
|
228
228
|
|
|
229
229
|
你可以继续运行更多的演示代码,这里记录了各个演示文件与其中的演示函数说明:
|
|
230
230
|
|
|
231
|
-
[🎮demo/](https://github.com/Mr-xiaotian/CelestialFlow/
|
|
231
|
+
[🎮demo/](https://github.com/Mr-xiaotian/CelestialFlow/tree/main/docs/zh-CN/demo)
|
|
232
232
|
|
|
233
233
|
如果你想运行测试代码, 可以先查看如下文档内容:
|
|
234
234
|
|
|
235
|
-
[🧪tests/](https://github.com/Mr-xiaotian/CelestialFlow/
|
|
235
|
+
[🧪tests/](https://github.com/Mr-xiaotian/CelestialFlow/tree/main/docs/zh-CN/tests)
|
|
236
236
|
|
|
237
237
|
如果你想查看bench内容, 这里的数据成为框架中部分设计的决策依据:
|
|
238
238
|
|
|
239
|
-
[⚡bench/](https://github.com/Mr-xiaotian/CelestialFlow/
|
|
239
|
+
[⚡bench/](https://github.com/Mr-xiaotian/CelestialFlow/tree/main/docs/zh-CN/bench)
|
|
240
240
|
|
|
241
241
|
## 环境要求(Requirements)
|
|
242
242
|
|
|
243
|
-
**CelestialFlow** 基于 Python 3.
|
|
243
|
+
**CelestialFlow** 基于 Python 3.12+,并依赖以下核心组件。
|
|
244
244
|
请确保你的环境能够正常安装这些依赖(`pip install celestialflow` 会自动安装)。
|
|
245
245
|
|
|
246
246
|
| 依赖包 | 说明 |
|
|
247
247
|
| ----------------- | ---- |
|
|
248
|
-
| **Python ≥ 3.
|
|
248
|
+
| **Python ≥ 3.12** | 运行环境,建议使用 3.12 及以上版本 |
|
|
249
249
|
| **fastapi** | Web 服务接口框架(用于任务可视化与远程控制) |
|
|
250
250
|
| **uvicorn** | FastAPI 的高性能 ASGI 服务器 |
|
|
251
251
|
| **requests** | HTTP 客户端库,用于任务状态上报与远程调用 |
|
|
@@ -260,57 +260,61 @@ flowchart TD
|
|
|
260
260
|
<p align="center">
|
|
261
261
|
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
262
262
|
<br/>
|
|
263
|
-
<em>celestial-flow 3.2.
|
|
263
|
+
<em>celestial-flow 3.2.3</em>
|
|
264
264
|
</p>
|
|
265
265
|
|
|
266
266
|
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
267
267
|
|
|
268
268
|
## 版本日志(Version Log)
|
|
269
|
-
- 3.2.
|
|
269
|
+
- 3.2.3
|
|
270
270
|
- feat:
|
|
271
|
-
- **[
|
|
272
|
-
-
|
|
273
|
-
|
|
274
|
-
-
|
|
275
|
-
|
|
276
|
-
-
|
|
277
|
-
|
|
278
|
-
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
-
|
|
282
|
-
|
|
271
|
+
- **[IMPORTANT]** 前端仪表盘设置面板中新添 `节点等待使用全局估计` 开关, 开启后节点卡片中原 `等待` 量将被替换为 `全局等待`, 并在卡片中显示 `全局剩余时间`
|
|
272
|
+
- 这个功能非常有趣, 开启前后能看到数值大幅波动
|
|
273
|
+
- **[IMPORTANT]** 重写任务注入页面, 现在实用性远远高于之前版本
|
|
274
|
+
- 可以给每个节点单独配置注入任务列表, 在一起发送
|
|
275
|
+
- **[IMPORTANT]** 移除 `TaskExecutor` 中 `get_args` 与 `process_result` 两个方法, 以及 `unpack_task_args` 属性
|
|
276
|
+
- 这是为引入泛型必要的修改
|
|
277
|
+
- `process_result` 功能很简单, 对func输出的result进行再次处理, 但事实上在输入func前对齐进行包装也能达到一样的效果
|
|
278
|
+
- `get_args` 功能更为复杂, 可以直接将前一节点提供的 `result` 转为当前节点所需的 `task` 类型, 非常灵活; 但问题在于太过灵活, 导致使用心智负担很大
|
|
279
|
+
- `unpack_task_args` 就是 `get_args` 带来的一项心智负担, 默认 `get_args` 会把 `task` 进行 `(task, )` 包裹后发送给func, 开启 `unpack_task_args` 后则直接发送
|
|
280
|
+
- **[IMPORTANT]** `graph` 的init参数中添加 `name`, 以与 `executor` `stage` 一致
|
|
281
|
+
- 破坏接口破坏性更新
|
|
282
|
+
- 在日志的 `start_graph` `end_graph` 与web端的 `graph_anaylysis` 中都有显示
|
|
283
|
+
- 彻底移除前后端通信中的 `graph_summary` , 原本残余的 `全局剩余时间` 现在拆为各个节点的 `全局等待` 与 `全局剩余时间`
|
|
284
|
+
- 这里所说的 `全局` 意为根据图论关系, 由上游剩余的任务数估算下游总共能获得多少任务
|
|
285
|
+
- 例如: 图关系 `A -> B`, A已处理任务2, 未处理任务3, B已处理任务4, 未处理任务2. 这意味着A成功的2个任务为B带来总共了6个任务, 那么我们可以据此估计B总功能获取"3/2*6=9"个任务, 因此B的 `等待` 任务数量为2, 但 `全局等待` 任务数量为5
|
|
286
|
+
- 前端中添加部分提示气泡, 鼠标放上去后可以介绍相关信息, 例如本次新加入的节点 `全局等待` 的含义
|
|
287
|
+
- 移除前端中节点卡片的拖拽功能
|
|
288
|
+
- 这个功能是在最早加入web页面时添加的, 当时感觉很帅, 但现在有点玩腻了
|
|
289
|
+
- 前端错误日志页面添加 `任务注入` 按钮, 可以把选定任务直接添加到任务注入页面中节点的代注入任务列表中
|
|
290
|
+
- 在 `TaskSplitter` 中添加 `split_item` 方法, 可自由定义
|
|
291
|
+
- 原本的 `splitter` 非常依赖于 `get_args`, 现在通过 `split_item` 方法稍微弥补其缺失的灵活性
|
|
283
292
|
- refactor:
|
|
284
|
-
- **[
|
|
285
|
-
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
-
|
|
290
|
-
|
|
291
|
-
-
|
|
292
|
-
-
|
|
293
|
-
-
|
|
294
|
-
-
|
|
295
|
-
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
-
|
|
299
|
-
-
|
|
300
|
-
-
|
|
301
|
-
- `util_error` 中添加更多错误类型,保证项目中所有主动 raise 的错误都在当前的错误体系下
|
|
302
|
-
- 删除前端代码中所有的 `onclick`,改用 `data-*` 属性
|
|
303
|
-
- 清简 `config.json` 中的部分字段
|
|
304
|
-
- 删除部分不必要的代码
|
|
293
|
+
- **[IMPORTANT]** 引入泛型, 同时强制性要求py版本>=3.12
|
|
294
|
+
- 泛型的引入使得代码更加类型安全, 同时也提高了代码的可读性
|
|
295
|
+
- 3.12版本对泛型的表述非常直观
|
|
296
|
+
- 移除前端代码中所有对 `localStorage` 的使用
|
|
297
|
+
- 在已经有config配置文件的情况下意义不大, 反而会带来困扰
|
|
298
|
+
- 将任务队列的drain操作从graph层移至stage层
|
|
299
|
+
- 之前不能这样做是因为节点的 `stage_mode` 可能为 `process`, 此时在主进程持有的节点非真实运行的节点
|
|
300
|
+
- 算是3.2.0版本带来的持久影响之一
|
|
301
|
+
- 优化 `TaskMetric` 中对锁的使用
|
|
302
|
+
- 移除 `TaskEnvelope` 中的 `change_id()` 方法, 现在默认envelope不可变, 同时 `emit_retry_envelope` 中不再把原有的envelope的id修改后继续提交给worker, 而是直接使用新生成的envelope
|
|
303
|
+
- 移除错误日志中的 `error` `error_repr` `task_repr` 字段
|
|
304
|
+
- 修改前后端通信中任务注入数据的格式, 以方便同时提交多个节点的任务数据
|
|
305
|
+
- 前端代码中开启strict检查
|
|
306
|
+
- 将前端中 `injection.css` 文件拆分成多个文件
|
|
307
|
+
- 修改 `config.json` 的数据格式, 现在按照生效区域进行分类
|
|
308
|
+
- 收紧前端中的数据类型
|
|
309
|
+
- 添加 `ReportTaskGraph` 类型, 专门用于给reporter做类型声明
|
|
305
310
|
- fix:
|
|
306
|
-
-
|
|
311
|
+
- i18n本地化在部分字段上失效的问题
|
|
312
|
+
- 直接点击仪表盘中错误数字跳转到错误日志页后, 设置面板显示的还是仪表盘页面的设置
|
|
313
|
+
- 前端中为各项仪表盘请求添加 `RequestSeq`, 以避免前后发送两个请求, 但因为延迟关系, 先发送的请求的返回覆盖掉后发送请求的返回
|
|
314
|
+
- 修复前端中各项空字段需要在第一次refresh才显示的问题, 体感上会导致"加载"很慢
|
|
307
315
|
- chore:
|
|
308
|
-
-
|
|
309
|
-
|
|
310
|
-
- 主要是 `bench_execution_mode` 中的两种 Fibonacci 运算量级不同,无法正常进行模型 bench
|
|
311
|
-
- 添加 `.github/workflows`,实现 CI/CD
|
|
312
|
-
- 更新支持的 Python 版本为 3.11,以与部分库要求相符
|
|
313
|
-
- 添加两个 skill,分别用于更新文档与审查项目
|
|
316
|
+
- 将文档更新翻译为英/日两语
|
|
317
|
+
- 这项操作太耗token了
|
|
314
318
|
|
|
315
319
|
更多过往日志可看:
|
|
316
320
|
|
|
@@ -187,7 +187,7 @@ flowchart TD
|
|
|
187
187
|
|
|
188
188
|
以下三篇可以作为补充阅读:
|
|
189
189
|
|
|
190
|
-
- [runtime/
|
|
190
|
+
- [runtime/util_hash.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/util_hash.md)
|
|
191
191
|
- [runtime/util_types.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/util_types.md)
|
|
192
192
|
- [runtime/util_errors.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/util_errors.md)
|
|
193
193
|
- [persistence/core_fail.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/persistence/core_fail.md)
|
|
@@ -203,24 +203,24 @@ flowchart TD
|
|
|
203
203
|
|
|
204
204
|
你可以继续运行更多的演示代码,这里记录了各个演示文件与其中的演示函数说明:
|
|
205
205
|
|
|
206
|
-
[🎮demo/](https://github.com/Mr-xiaotian/CelestialFlow/
|
|
206
|
+
[🎮demo/](https://github.com/Mr-xiaotian/CelestialFlow/tree/main/docs/zh-CN/demo)
|
|
207
207
|
|
|
208
208
|
如果你想运行测试代码, 可以先查看如下文档内容:
|
|
209
209
|
|
|
210
|
-
[🧪tests/](https://github.com/Mr-xiaotian/CelestialFlow/
|
|
210
|
+
[🧪tests/](https://github.com/Mr-xiaotian/CelestialFlow/tree/main/docs/zh-CN/tests)
|
|
211
211
|
|
|
212
212
|
如果你想查看bench内容, 这里的数据成为框架中部分设计的决策依据:
|
|
213
213
|
|
|
214
|
-
[⚡bench/](https://github.com/Mr-xiaotian/CelestialFlow/
|
|
214
|
+
[⚡bench/](https://github.com/Mr-xiaotian/CelestialFlow/tree/main/docs/zh-CN/bench)
|
|
215
215
|
|
|
216
216
|
## 环境要求(Requirements)
|
|
217
217
|
|
|
218
|
-
**CelestialFlow** 基于 Python 3.
|
|
218
|
+
**CelestialFlow** 基于 Python 3.12+,并依赖以下核心组件。
|
|
219
219
|
请确保你的环境能够正常安装这些依赖(`pip install celestialflow` 会自动安装)。
|
|
220
220
|
|
|
221
221
|
| 依赖包 | 说明 |
|
|
222
222
|
| ----------------- | ---- |
|
|
223
|
-
| **Python ≥ 3.
|
|
223
|
+
| **Python ≥ 3.12** | 运行环境,建议使用 3.12 及以上版本 |
|
|
224
224
|
| **fastapi** | Web 服务接口框架(用于任务可视化与远程控制) |
|
|
225
225
|
| **uvicorn** | FastAPI 的高性能 ASGI 服务器 |
|
|
226
226
|
| **requests** | HTTP 客户端库,用于任务状态上报与远程调用 |
|
|
@@ -235,57 +235,61 @@ flowchart TD
|
|
|
235
235
|
<p align="center">
|
|
236
236
|
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
237
237
|
<br/>
|
|
238
|
-
<em>celestial-flow 3.2.
|
|
238
|
+
<em>celestial-flow 3.2.3</em>
|
|
239
239
|
</p>
|
|
240
240
|
|
|
241
241
|
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
242
242
|
|
|
243
243
|
## 版本日志(Version Log)
|
|
244
|
-
- 3.2.
|
|
244
|
+
- 3.2.3
|
|
245
245
|
- feat:
|
|
246
|
-
- **[
|
|
247
|
-
-
|
|
248
|
-
|
|
249
|
-
-
|
|
250
|
-
|
|
251
|
-
-
|
|
252
|
-
|
|
253
|
-
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
-
|
|
257
|
-
|
|
246
|
+
- **[IMPORTANT]** 前端仪表盘设置面板中新添 `节点等待使用全局估计` 开关, 开启后节点卡片中原 `等待` 量将被替换为 `全局等待`, 并在卡片中显示 `全局剩余时间`
|
|
247
|
+
- 这个功能非常有趣, 开启前后能看到数值大幅波动
|
|
248
|
+
- **[IMPORTANT]** 重写任务注入页面, 现在实用性远远高于之前版本
|
|
249
|
+
- 可以给每个节点单独配置注入任务列表, 在一起发送
|
|
250
|
+
- **[IMPORTANT]** 移除 `TaskExecutor` 中 `get_args` 与 `process_result` 两个方法, 以及 `unpack_task_args` 属性
|
|
251
|
+
- 这是为引入泛型必要的修改
|
|
252
|
+
- `process_result` 功能很简单, 对func输出的result进行再次处理, 但事实上在输入func前对齐进行包装也能达到一样的效果
|
|
253
|
+
- `get_args` 功能更为复杂, 可以直接将前一节点提供的 `result` 转为当前节点所需的 `task` 类型, 非常灵活; 但问题在于太过灵活, 导致使用心智负担很大
|
|
254
|
+
- `unpack_task_args` 就是 `get_args` 带来的一项心智负担, 默认 `get_args` 会把 `task` 进行 `(task, )` 包裹后发送给func, 开启 `unpack_task_args` 后则直接发送
|
|
255
|
+
- **[IMPORTANT]** `graph` 的init参数中添加 `name`, 以与 `executor` `stage` 一致
|
|
256
|
+
- 破坏接口破坏性更新
|
|
257
|
+
- 在日志的 `start_graph` `end_graph` 与web端的 `graph_anaylysis` 中都有显示
|
|
258
|
+
- 彻底移除前后端通信中的 `graph_summary` , 原本残余的 `全局剩余时间` 现在拆为各个节点的 `全局等待` 与 `全局剩余时间`
|
|
259
|
+
- 这里所说的 `全局` 意为根据图论关系, 由上游剩余的任务数估算下游总共能获得多少任务
|
|
260
|
+
- 例如: 图关系 `A -> B`, A已处理任务2, 未处理任务3, B已处理任务4, 未处理任务2. 这意味着A成功的2个任务为B带来总共了6个任务, 那么我们可以据此估计B总功能获取"3/2*6=9"个任务, 因此B的 `等待` 任务数量为2, 但 `全局等待` 任务数量为5
|
|
261
|
+
- 前端中添加部分提示气泡, 鼠标放上去后可以介绍相关信息, 例如本次新加入的节点 `全局等待` 的含义
|
|
262
|
+
- 移除前端中节点卡片的拖拽功能
|
|
263
|
+
- 这个功能是在最早加入web页面时添加的, 当时感觉很帅, 但现在有点玩腻了
|
|
264
|
+
- 前端错误日志页面添加 `任务注入` 按钮, 可以把选定任务直接添加到任务注入页面中节点的代注入任务列表中
|
|
265
|
+
- 在 `TaskSplitter` 中添加 `split_item` 方法, 可自由定义
|
|
266
|
+
- 原本的 `splitter` 非常依赖于 `get_args`, 现在通过 `split_item` 方法稍微弥补其缺失的灵活性
|
|
258
267
|
- refactor:
|
|
259
|
-
- **[
|
|
260
|
-
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
-
|
|
265
|
-
|
|
266
|
-
-
|
|
267
|
-
-
|
|
268
|
-
-
|
|
269
|
-
-
|
|
270
|
-
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
-
|
|
274
|
-
-
|
|
275
|
-
-
|
|
276
|
-
- `util_error` 中添加更多错误类型,保证项目中所有主动 raise 的错误都在当前的错误体系下
|
|
277
|
-
- 删除前端代码中所有的 `onclick`,改用 `data-*` 属性
|
|
278
|
-
- 清简 `config.json` 中的部分字段
|
|
279
|
-
- 删除部分不必要的代码
|
|
268
|
+
- **[IMPORTANT]** 引入泛型, 同时强制性要求py版本>=3.12
|
|
269
|
+
- 泛型的引入使得代码更加类型安全, 同时也提高了代码的可读性
|
|
270
|
+
- 3.12版本对泛型的表述非常直观
|
|
271
|
+
- 移除前端代码中所有对 `localStorage` 的使用
|
|
272
|
+
- 在已经有config配置文件的情况下意义不大, 反而会带来困扰
|
|
273
|
+
- 将任务队列的drain操作从graph层移至stage层
|
|
274
|
+
- 之前不能这样做是因为节点的 `stage_mode` 可能为 `process`, 此时在主进程持有的节点非真实运行的节点
|
|
275
|
+
- 算是3.2.0版本带来的持久影响之一
|
|
276
|
+
- 优化 `TaskMetric` 中对锁的使用
|
|
277
|
+
- 移除 `TaskEnvelope` 中的 `change_id()` 方法, 现在默认envelope不可变, 同时 `emit_retry_envelope` 中不再把原有的envelope的id修改后继续提交给worker, 而是直接使用新生成的envelope
|
|
278
|
+
- 移除错误日志中的 `error` `error_repr` `task_repr` 字段
|
|
279
|
+
- 修改前后端通信中任务注入数据的格式, 以方便同时提交多个节点的任务数据
|
|
280
|
+
- 前端代码中开启strict检查
|
|
281
|
+
- 将前端中 `injection.css` 文件拆分成多个文件
|
|
282
|
+
- 修改 `config.json` 的数据格式, 现在按照生效区域进行分类
|
|
283
|
+
- 收紧前端中的数据类型
|
|
284
|
+
- 添加 `ReportTaskGraph` 类型, 专门用于给reporter做类型声明
|
|
280
285
|
- fix:
|
|
281
|
-
-
|
|
286
|
+
- i18n本地化在部分字段上失效的问题
|
|
287
|
+
- 直接点击仪表盘中错误数字跳转到错误日志页后, 设置面板显示的还是仪表盘页面的设置
|
|
288
|
+
- 前端中为各项仪表盘请求添加 `RequestSeq`, 以避免前后发送两个请求, 但因为延迟关系, 先发送的请求的返回覆盖掉后发送请求的返回
|
|
289
|
+
- 修复前端中各项空字段需要在第一次refresh才显示的问题, 体感上会导致"加载"很慢
|
|
282
290
|
- chore:
|
|
283
|
-
-
|
|
284
|
-
|
|
285
|
-
- 主要是 `bench_execution_mode` 中的两种 Fibonacci 运算量级不同,无法正常进行模型 bench
|
|
286
|
-
- 添加 `.github/workflows`,实现 CI/CD
|
|
287
|
-
- 更新支持的 Python 版本为 3.11,以与部分库要求相符
|
|
288
|
-
- 添加两个 skill,分别用于更新文档与审查项目
|
|
291
|
+
- 将文档更新翻译为英/日两语
|
|
292
|
+
- 这项操作太耗token了
|
|
289
293
|
|
|
290
294
|
更多过往日志可看:
|
|
291
295
|
|
|
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "celestialflow"
|
|
7
|
-
version = "3.2.
|
|
7
|
+
version = "3.2.3"
|
|
8
8
|
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
11
11
|
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
12
|
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
-
requires-python = ">=3.
|
|
13
|
+
requires-python = ">=3.12"
|
|
14
14
|
|
|
15
15
|
dependencies = [
|
|
16
16
|
"tqdm",
|
|
@@ -20,7 +20,7 @@ dependencies = [
|
|
|
20
20
|
"networkx",
|
|
21
21
|
"redis",
|
|
22
22
|
"jinja2",
|
|
23
|
-
"celestialtree>=0.1.
|
|
23
|
+
"celestialtree>=0.1.3",
|
|
24
24
|
]
|
|
25
25
|
|
|
26
26
|
classifiers = [
|
|
@@ -79,14 +79,11 @@ include = ["src"]
|
|
|
79
79
|
ignore = ["tests", "**/tests/**", "typings", "demo", "bench", "experiments", "examples", "temp"]
|
|
80
80
|
|
|
81
81
|
typeCheckingMode = "strict"
|
|
82
|
-
reportExplicitAny = "none"
|
|
83
|
-
reportAny = "none"
|
|
84
|
-
reportUnannotatedClassAttribute = "none"
|
|
85
82
|
reportUnusedFunction = "none"
|
|
86
83
|
reportImplicitOverride = "none"
|
|
87
84
|
|
|
88
85
|
[tool.ruff]
|
|
89
|
-
target-version = "
|
|
86
|
+
target-version = "py312"
|
|
90
87
|
exclude = ["tests", "typings", "demo", "bench", "experiments", "examples", "temp"]
|
|
91
88
|
|
|
92
89
|
[tool.ruff.lint]
|