celestialflow 3.1.9__tar.gz → 3.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. {celestialflow-3.1.9 → celestialflow-3.2.1}/PKG-INFO +56 -29
  2. {celestialflow-3.1.9 → celestialflow-3.2.1}/README.md +54 -27
  3. {celestialflow-3.1.9 → celestialflow-3.2.1}/pyproject.toml +115 -81
  4. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/__init__.py +21 -16
  5. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/funnel/__init__.py +2 -2
  6. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/funnel/core_inlet.py +22 -22
  7. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/funnel/core_spout.py +20 -11
  8. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/graph/__init__.py +4 -4
  9. celestialflow-3.2.1/src/celestialflow/graph/core_graph.py +722 -0
  10. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/graph/core_structure.py +104 -62
  11. celestialflow-3.2.1/src/celestialflow/graph/util_analysis.py +73 -0
  12. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/graph/util_serialize.py +40 -34
  13. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/observability/__init__.py +6 -1
  14. celestialflow-3.2.1/src/celestialflow/observability/core_observer.py +67 -0
  15. celestialflow-3.2.1/src/celestialflow/observability/core_progress.py +61 -0
  16. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/observability/core_report.py +47 -73
  17. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/__init__.py +2 -2
  18. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/core_fail.py +50 -27
  19. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/core_log.py +256 -117
  20. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/core_success.py +2 -2
  21. celestialflow-3.2.1/src/celestialflow/persistence/util_jsonl.py +229 -0
  22. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/__init__.py +2 -2
  23. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/core_dispatch.py +59 -46
  24. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/core_envelope.py +15 -13
  25. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/core_metrics.py +26 -23
  26. celestialflow-3.2.1/src/celestialflow/runtime/core_queue.py +253 -0
  27. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/util_errors.py +122 -24
  28. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/util_estimators.py +12 -9
  29. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/util_hash.py +10 -9
  30. celestialflow-3.2.1/src/celestialflow/runtime/util_types.py +217 -0
  31. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/__init__.py +4 -4
  32. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/core_executor.py +207 -241
  33. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/core_stage.py +73 -88
  34. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/core_stages.py +119 -74
  35. celestialflow-3.2.1/src/celestialflow/utils/__init__.py +5 -0
  36. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_benchmark.py +48 -34
  37. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_clone.py +39 -30
  38. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_collections.py +1 -1
  39. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_format.py +32 -25
  40. celestialflow-3.2.1/src/celestialflow/web/__init__.py +11 -0
  41. celestialflow-3.2.1/src/celestialflow/web/config.json +21 -0
  42. celestialflow-3.2.1/src/celestialflow/web/core_server.py +145 -0
  43. celestialflow-3.2.1/src/celestialflow/web/routes/__init__.py +33 -0
  44. celestialflow-3.2.1/src/celestialflow/web/routes/pull_routes.py +148 -0
  45. celestialflow-3.2.1/src/celestialflow/web/routes/push_routes.py +186 -0
  46. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/static/css/_colors.css +136 -135
  47. celestialflow-3.2.1/src/celestialflow/web/static/css/base.css +805 -0
  48. celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard.css +71 -0
  49. celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_analysis.css +39 -0
  50. celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_history.css +87 -0
  51. celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_statuses.css +238 -0
  52. celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_structure.css +25 -0
  53. celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_summary.css +134 -0
  54. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/static/css/errors.css +30 -1
  55. celestialflow-3.1.9/src/celestialflow/web/static/css/inject.css → celestialflow-3.2.1/src/celestialflow/web/static/css/injection.css +57 -4
  56. celestialflow-3.1.9/src/celestialflow/web/static/js/task_analysis.js → celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_analysis.js +19 -12
  57. celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_history.js +356 -0
  58. celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_statuses.js +235 -0
  59. celestialflow-3.1.9/src/celestialflow/web/static/js/task_structure.js → celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_structure.js +23 -12
  60. celestialflow-3.1.9/src/celestialflow/web/static/js/task_summary.js → celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_summary.js +20 -5
  61. celestialflow-3.1.9/src/celestialflow/web/static/js/task_errors.js → celestialflow-3.2.1/src/celestialflow/web/static/js/errors.js +29 -19
  62. celestialflow-3.2.1/src/celestialflow/web/static/js/i18n.js +368 -0
  63. celestialflow-3.1.9/src/celestialflow/web/static/js/task_injection.js → celestialflow-3.2.1/src/celestialflow/web/static/js/injection.js +101 -35
  64. celestialflow-3.2.1/src/celestialflow/web/static/js/layout_editor.js +158 -0
  65. celestialflow-3.2.1/src/celestialflow/web/static/js/main.js +235 -0
  66. celestialflow-3.2.1/src/celestialflow/web/static/js/utils.js +117 -0
  67. celestialflow-3.2.1/src/celestialflow/web/static/js/web_config.js +296 -0
  68. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_analysis.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_analysis.ts +20 -17
  69. celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_history.ts +443 -0
  70. celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_statuses.ts +330 -0
  71. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_structure.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_structure.ts +35 -16
  72. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_summary.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_summary.ts +25 -12
  73. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_errors.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/errors.ts +33 -22
  74. celestialflow-3.2.1/src/celestialflow/web/static/ts/globals.d.ts +27 -0
  75. celestialflow-3.2.1/src/celestialflow/web/static/ts/i18n.ts +385 -0
  76. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_injection.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/injection.ts +111 -41
  77. celestialflow-3.2.1/src/celestialflow/web/static/ts/layout_editor.ts +184 -0
  78. celestialflow-3.2.1/src/celestialflow/web/static/ts/main.ts +267 -0
  79. celestialflow-3.2.1/src/celestialflow/web/static/ts/utils.ts +132 -0
  80. celestialflow-3.2.1/src/celestialflow/web/static/ts/web_config.ts +345 -0
  81. celestialflow-3.2.1/src/celestialflow/web/templates/index.html +659 -0
  82. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/util_config.py +10 -6
  83. celestialflow-3.2.1/src/celestialflow/web/util_models.py +76 -0
  84. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/PKG-INFO +56 -29
  85. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/SOURCES.txt +29 -28
  86. celestialflow-3.1.9/src/celestialflow/graph/core_graph.py +0 -895
  87. celestialflow-3.1.9/src/celestialflow/graph/util_analysis.py +0 -55
  88. celestialflow-3.1.9/src/celestialflow/observability/core_observer.py +0 -26
  89. celestialflow-3.1.9/src/celestialflow/observability/core_progress.py +0 -30
  90. celestialflow-3.1.9/src/celestialflow/persistence/util_jsonl.py +0 -183
  91. celestialflow-3.1.9/src/celestialflow/runtime/core_queue.py +0 -288
  92. celestialflow-3.1.9/src/celestialflow/runtime/util_factories.py +0 -95
  93. celestialflow-3.1.9/src/celestialflow/runtime/util_queue.py +0 -13
  94. celestialflow-3.1.9/src/celestialflow/runtime/util_types.py +0 -147
  95. celestialflow-3.1.9/src/celestialflow/utils/__init__.py +0 -1
  96. celestialflow-3.1.9/src/celestialflow/utils/util_debug.py +0 -16
  97. celestialflow-3.1.9/src/celestialflow/web/__init__.py +0 -6
  98. celestialflow-3.1.9/src/celestialflow/web/config.json +0 -35
  99. celestialflow-3.1.9/src/celestialflow/web/core_server.py +0 -522
  100. celestialflow-3.1.9/src/celestialflow/web/static/css/base.css +0 -320
  101. celestialflow-3.1.9/src/celestialflow/web/static/css/dashboard.css +0 -348
  102. celestialflow-3.1.9/src/celestialflow/web/static/js/main.js +0 -85
  103. celestialflow-3.1.9/src/celestialflow/web/static/js/task_config.js +0 -124
  104. celestialflow-3.1.9/src/celestialflow/web/static/js/task_history.js +0 -135
  105. celestialflow-3.1.9/src/celestialflow/web/static/js/task_statuses.js +0 -121
  106. celestialflow-3.1.9/src/celestialflow/web/static/js/task_topology.js +0 -59
  107. celestialflow-3.1.9/src/celestialflow/web/static/js/utils.js +0 -269
  108. celestialflow-3.1.9/src/celestialflow/web/static/js/web_config.js +0 -119
  109. celestialflow-3.1.9/src/celestialflow/web/static/ts/globals.d.ts +0 -6
  110. celestialflow-3.1.9/src/celestialflow/web/static/ts/main.ts +0 -99
  111. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_history.ts +0 -157
  112. celestialflow-3.1.9/src/celestialflow/web/static/ts/task_statuses.ts +0 -172
  113. celestialflow-3.1.9/src/celestialflow/web/static/ts/utils.ts +0 -314
  114. celestialflow-3.1.9/src/celestialflow/web/static/ts/web_config.ts +0 -144
  115. celestialflow-3.1.9/src/celestialflow/web/templates/index.html +0 -394
  116. celestialflow-3.1.9/tests/test_envelope.py +0 -51
  117. celestialflow-3.1.9/tests/test_executor.py +0 -382
  118. celestialflow-3.1.9/tests/test_graph.py +0 -579
  119. celestialflow-3.1.9/tests/test_metrics.py +0 -119
  120. celestialflow-3.1.9/tests/test_queue.py +0 -149
  121. celestialflow-3.1.9/tests/test_stage.py +0 -120
  122. celestialflow-3.1.9/tests/test_utils.py +0 -234
  123. {celestialflow-3.1.9 → celestialflow-3.2.1}/setup.cfg +0 -0
  124. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/util_constant.py +0 -0
  125. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/static/favicon.ico +0 -0
  126. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/util_cal.py +0 -0
  127. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/util_error.py +0 -0
  128. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/dependency_links.txt +0 -0
  129. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/entry_points.txt +0 -0
  130. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/requires.txt +0 -0
  131. {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: celestialflow
3
- Version: 3.1.9
3
+ Version: 3.2.1
4
4
  Summary: A flexible GRAPH-based task orchestration framework.
5
5
  Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
6
6
  License: MIT
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
12
12
  Classifier: Operating System :: OS Independent
13
13
  Classifier: Framework :: FastAPI
14
14
  Classifier: Topic :: Software Development :: Libraries
15
- Requires-Python: >=3.10
15
+ Requires-Python: >=3.11
16
16
  Description-Content-Type: text/markdown
17
17
  Requires-Dist: tqdm
18
18
  Requires-Dist: fastapi
@@ -44,7 +44,7 @@ Requires-Dist: celestialtree>=0.1.2
44
44
  </p>
45
45
 
46
46
  <p align="center">
47
- <a href="README.md">中文</a> | <a href="docs/en/README.md">English</a> | <a href="docs/ja/README.md">日本語</a>
47
+ <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/README.md">中文</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/en/README.md">English</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/ja/README.md">日本語</a>
48
48
  </p>
49
49
 
50
50
  **CelestialFlow** 是一个轻量级但功能完全的任务流框架,适合需要 **复杂依赖关系**、**灵活执行模型**、**跨设备运行**与**实时可视化监控** 的中/大型 Python 任务系统。
@@ -64,11 +64,10 @@ TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示
64
64
 
65
65
  TaskStage 的任务执行模式同样包含三种,与TaskExecutor中一致。
66
66
 
67
- 在图级别上,每个 Stage 支持三种上下文模式:
67
+ 在图级别上,每个 Stage 支持两种上下文模式:
68
68
 
69
69
  * **线性执行(serial layout)**:当前节点执行完毕再启动下一节点(下游节点可提前接收任务但不会立即执行)。
70
70
  * **线程执行(thread layout)**:当前节点在主进程的独立线程中启动,适合 I/O 密集型任务和不可 pickle 的函数(如 lambda)。
71
- * **并行执行(process layout)**:当前节点启动后立刻前去启动下一节点。
72
71
 
73
72
  TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达 **树形(Tree)**、**环形(loop)** 乃至于 **完全图(Complete Graph)** 形式的任务依赖。
74
73
 
@@ -150,16 +149,16 @@ def square(x):
150
149
 
151
150
  if __name__ == "__main__":
152
151
  # 定义两个任务节点
153
- stage1 = TaskStage(name="Adder", func=add, execution_mode="thread", unpack_task_args=True, stage_mode="process")
154
- stage2 = TaskStage(name="Squarer"func=square, execution_mode="thread", stage_mode="process")
152
+ stage1 = TaskStage(name="Adder", func=add, stage_mode="thread", execution_mode="thread", unpack_task_args=True)
153
+ stage2 = TaskStage(name="Squarer", func=square, stage_mode="thread", execution_mode="thread")
155
154
 
156
155
  # 构建任务图结构
157
156
  graph = TaskGraph()
158
- graph.set_stages(root_stages=[stage1], stages=[stage2])
157
+ graph.set_stages(stages=[stage1, stage2])
159
158
  graph.connect([stage1], [stage2])
160
159
 
161
160
  # 初始化任务并启动
162
- graph.start_graph({stage1.get_tag(): [(1, 2), (3, 4), (5, 6)]})
161
+ graph.start_graph({stage1.get_name(): [(1, 2), (3, 4), (5, 6)]})
163
162
  ```
164
163
 
165
164
  注意不要在.ipynb中运行。
@@ -241,12 +240,12 @@ flowchart TD
241
240
 
242
241
  ## 环境要求(Requirements)
243
242
 
244
- **CelestialFlow** 基于 Python 3.10+,并依赖以下核心组件。
243
+ **CelestialFlow** 基于 Python 3.11+,并依赖以下核心组件。
245
244
  请确保你的环境能够正常安装这些依赖(`pip install celestialflow` 会自动安装)。
246
245
 
247
246
  | 依赖包 | 说明 |
248
247
  | ----------------- | ---- |
249
- | **Python ≥ 3.10** | 运行环境,建议使用 3.10 及以上版本 |
248
+ | **Python ≥ 3.11** | 运行环境,建议使用 3.11 及以上版本 |
250
249
  | **fastapi** | Web 服务接口框架(用于任务可视化与远程控制) |
251
250
  | **uvicorn** | FastAPI 的高性能 ASGI 服务器 |
252
251
  | **requests** | HTTP 客户端库,用于任务状态上报与远程调用 |
@@ -261,33 +260,61 @@ flowchart TD
261
260
  <p align="center">
262
261
  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
263
262
  <br/>
264
- <em>celestial-flow 3.1.9</em>
263
+ <em>celestial-flow 3.2.1</em>
265
264
  </p>
266
265
 
267
266
  (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
268
267
 
269
268
  ## 版本日志(Version Log)
270
- - 3.1.9
271
- - feat:
272
- - [Important] 在TaskStage中允许使用 `async` executor_mode;
273
- - 3.1.8版本中其实已经可以实现, 在3.1.9版本中彻底测试通过;
274
- - [Important] 创建基类BaseObservor, 现在可以从外界接入自定义Observor来检测运行状况;
275
- - 同时创建轻量观察者CallbackObserver;
276
- - 在TaskStage中添加init字段`log_level`, 自此stage init字段集实现对executor init字段集的包含关系, 仅多出`stage_mode`字段;
277
- - escapeHtml中添加更多替换项;
278
- - 在dispatch_thread中添加对futures的清理, 避免无用future的堆积;
279
- - 基于bench\bench_futures_memory.py, 在十万级任务下futures内存开销为177.01 MB, 在定期清理后则为0.48 MB
280
- - refactor:
281
- - 重构TaskProgress, 使其继承BaseObservor;
282
- - 接口破坏性重构, TaskExecutor的show_progress参数已删除, 现在只有显性的add_observer TaskProgress后才会使用progress
283
- - 具体使用参考demo\demo_executor.py
284
- - 重构TaskEnvelope, 不再使用wrap与unwrap, 同时hash值惰性求解;
269
+ - 3.2.1:
270
+ - feat:
271
+ - **[Important]** 大幅强化配置按钮功能
272
+ - 现在可以配置界面语言,并支持中、日、英三语言切换
273
+ - 可以配置错误日志每页日志条数
274
+ - 可以配置结构图中是否显示增量,不过现在样式上有点丑
275
+ - 以及最重要的,可以直接编辑仪表盘页的卡片布局
276
+ - 另外,所有设置保存成功后会有提示
277
+ - 折线图中可以选择数据类型
278
+ - 现在包括"数据累计"、"等待队列"和"数据变化率"
279
+ - 对 structure 的一些调整
280
+ - 所有 structure 的参数中添加 `stage_mode`,用于统一控制节点模式
281
+ - struct 中节点不再重新命名
282
+ - 在部分 Spout 中增加缓冲机制
283
+ - refactor:
284
+ - **[Important]** 删除 executor 中的 `tag` 属性,并全面使用 `name` 属性进行替代
285
+ - `tag` 的出现是为了适应早期无法区分节点而设计的,但现在 `name` 已经强制唯一性,继续使用冗长的 `tag` 会导致不便
286
+ - **[Important]** 将 executor 中 `task_queue` 与 `result_queue` 的定义提前到 `__init__` 中
287
+ - 对于 executor 来说区别不大,但对于 stage 而言这是更理想且优雅的形式,只是由于原先 `stage_mode="process"` 时不允许执行函数内带有 `MPQueue`,所以才使用 graph 中统一定义,然后通过 process 的 args 注入的形式
288
+ - **[Important]** 重构 `core_server`,将其分解出 `routes/` 与 `util_models`,实现结构优化
289
+ - 进一步地,将 `log_queue` 和 `fail_queue` 的注入也提前到 `graph.set_stages` 中,实现 `stage.start_stage` 完全无输入参数
290
+ - 同时在 executor 中添加 `put_task` 与 `put_signal`,并在 graph 中进行复用
291
+ - 之前不能这样做是因为 stage 中此时还没有定义 queue
292
+ - 移除 `core_graph` 中的 `StageRuntime`
293
+ - 将类型标注中的部分 `Any` 改为更明确的类型
294
+ - 删除 HTML 中的仪表盘部分的 card,现在交给 `web-config` 来定义
295
+ - 删除部分没必要的前后端通信,优化通信负载
296
+ - 移除后端传递的节点历史信息,由前端进行维护
297
+ - 移除总体统计中除"总剩余时间"的后端数据,由前端进行计算
298
+ - 优化错误存储的 JSONL 数据结构,现在保存更细致的错误信息
299
+ - 调整 pyright 规则,并实现 pyright 0 error / 0 warning
300
+ - 给 `web/` 下几乎所有 `.ts` 与 `.css` 文件重命名,并整理文件内容边界
301
+ - `util_error` 中添加更多错误类型,保证项目中所有主动 raise 的错误都在当前的错误体系下
302
+ - 删除前端代码中所有的 `onclick`,改用 `data-*` 属性
303
+ - 清简 `config.json` 中的部分字段
304
+ - 删除部分不必要的代码
305
+ - fix:
306
+ - 对部分太宽泛的错误捕捉进行收窄
285
307
  - chore:
286
- - 删除部分无用代码以提高代码质量;
308
+ - 添加更多测试代码,并整理 `tests/` 结构
309
+ - 修复 `bench/` 中的一些问题
310
+ - 主要是 `bench_execution_mode` 中的两种 Fibonacci 运算量级不同,无法正常进行模型 bench
311
+ - 添加 `.github/workflows`,实现 CI/CD
312
+ - 更新支持的 Python 版本为 3.11,以与部分库要求相符
313
+ - 添加两个 skill,分别用于更新文档与审查项目
287
314
 
288
315
  更多过往日志可看:
289
316
 
290
- [change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/change_log.md )
317
+ [change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/change_log.md )
291
318
 
292
319
  ## Star 历史趋势(Star History)
293
320
 
@@ -19,7 +19,7 @@
19
19
  </p>
20
20
 
21
21
  <p align="center">
22
- <a href="README.md">中文</a> | <a href="docs/en/README.md">English</a> | <a href="docs/ja/README.md">日本語</a>
22
+ <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/README.md">中文</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/en/README.md">English</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/ja/README.md">日本語</a>
23
23
  </p>
24
24
 
25
25
  **CelestialFlow** 是一个轻量级但功能完全的任务流框架,适合需要 **复杂依赖关系**、**灵活执行模型**、**跨设备运行**与**实时可视化监控** 的中/大型 Python 任务系统。
@@ -39,11 +39,10 @@ TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示
39
39
 
40
40
  TaskStage 的任务执行模式同样包含三种,与TaskExecutor中一致。
41
41
 
42
- 在图级别上,每个 Stage 支持三种上下文模式:
42
+ 在图级别上,每个 Stage 支持两种上下文模式:
43
43
 
44
44
  * **线性执行(serial layout)**:当前节点执行完毕再启动下一节点(下游节点可提前接收任务但不会立即执行)。
45
45
  * **线程执行(thread layout)**:当前节点在主进程的独立线程中启动,适合 I/O 密集型任务和不可 pickle 的函数(如 lambda)。
46
- * **并行执行(process layout)**:当前节点启动后立刻前去启动下一节点。
47
46
 
48
47
  TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达 **树形(Tree)**、**环形(loop)** 乃至于 **完全图(Complete Graph)** 形式的任务依赖。
49
48
 
@@ -125,16 +124,16 @@ def square(x):
125
124
 
126
125
  if __name__ == "__main__":
127
126
  # 定义两个任务节点
128
- stage1 = TaskStage(name="Adder", func=add, execution_mode="thread", unpack_task_args=True, stage_mode="process")
129
- stage2 = TaskStage(name="Squarer"func=square, execution_mode="thread", stage_mode="process")
127
+ stage1 = TaskStage(name="Adder", func=add, stage_mode="thread", execution_mode="thread", unpack_task_args=True)
128
+ stage2 = TaskStage(name="Squarer", func=square, stage_mode="thread", execution_mode="thread")
130
129
 
131
130
  # 构建任务图结构
132
131
  graph = TaskGraph()
133
- graph.set_stages(root_stages=[stage1], stages=[stage2])
132
+ graph.set_stages(stages=[stage1, stage2])
134
133
  graph.connect([stage1], [stage2])
135
134
 
136
135
  # 初始化任务并启动
137
- graph.start_graph({stage1.get_tag(): [(1, 2), (3, 4), (5, 6)]})
136
+ graph.start_graph({stage1.get_name(): [(1, 2), (3, 4), (5, 6)]})
138
137
  ```
139
138
 
140
139
  注意不要在.ipynb中运行。
@@ -216,12 +215,12 @@ flowchart TD
216
215
 
217
216
  ## 环境要求(Requirements)
218
217
 
219
- **CelestialFlow** 基于 Python 3.10+,并依赖以下核心组件。
218
+ **CelestialFlow** 基于 Python 3.11+,并依赖以下核心组件。
220
219
  请确保你的环境能够正常安装这些依赖(`pip install celestialflow` 会自动安装)。
221
220
 
222
221
  | 依赖包 | 说明 |
223
222
  | ----------------- | ---- |
224
- | **Python ≥ 3.10** | 运行环境,建议使用 3.10 及以上版本 |
223
+ | **Python ≥ 3.11** | 运行环境,建议使用 3.11 及以上版本 |
225
224
  | **fastapi** | Web 服务接口框架(用于任务可视化与远程控制) |
226
225
  | **uvicorn** | FastAPI 的高性能 ASGI 服务器 |
227
226
  | **requests** | HTTP 客户端库,用于任务状态上报与远程调用 |
@@ -236,33 +235,61 @@ flowchart TD
236
235
  <p align="center">
237
236
  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
238
237
  <br/>
239
- <em>celestial-flow 3.1.9</em>
238
+ <em>celestial-flow 3.2.1</em>
240
239
  </p>
241
240
 
242
241
  (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
243
242
 
244
243
  ## 版本日志(Version Log)
245
- - 3.1.9
246
- - feat:
247
- - [Important] 在TaskStage中允许使用 `async` executor_mode;
248
- - 3.1.8版本中其实已经可以实现, 在3.1.9版本中彻底测试通过;
249
- - [Important] 创建基类BaseObservor, 现在可以从外界接入自定义Observor来检测运行状况;
250
- - 同时创建轻量观察者CallbackObserver;
251
- - 在TaskStage中添加init字段`log_level`, 自此stage init字段集实现对executor init字段集的包含关系, 仅多出`stage_mode`字段;
252
- - escapeHtml中添加更多替换项;
253
- - 在dispatch_thread中添加对futures的清理, 避免无用future的堆积;
254
- - 基于bench\bench_futures_memory.py, 在十万级任务下futures内存开销为177.01 MB, 在定期清理后则为0.48 MB
255
- - refactor:
256
- - 重构TaskProgress, 使其继承BaseObservor;
257
- - 接口破坏性重构, TaskExecutor的show_progress参数已删除, 现在只有显性的add_observer TaskProgress后才会使用progress
258
- - 具体使用参考demo\demo_executor.py
259
- - 重构TaskEnvelope, 不再使用wrap与unwrap, 同时hash值惰性求解;
244
+ - 3.2.1:
245
+ - feat:
246
+ - **[Important]** 大幅强化配置按钮功能
247
+ - 现在可以配置界面语言,并支持中、日、英三语言切换
248
+ - 可以配置错误日志每页日志条数
249
+ - 可以配置结构图中是否显示增量,不过现在样式上有点丑
250
+ - 以及最重要的,可以直接编辑仪表盘页的卡片布局
251
+ - 另外,所有设置保存成功后会有提示
252
+ - 折线图中可以选择数据类型
253
+ - 现在包括"数据累计"、"等待队列"和"数据变化率"
254
+ - 对 structure 的一些调整
255
+ - 所有 structure 的参数中添加 `stage_mode`,用于统一控制节点模式
256
+ - struct 中节点不再重新命名
257
+ - 在部分 Spout 中增加缓冲机制
258
+ - refactor:
259
+ - **[Important]** 删除 executor 中的 `tag` 属性,并全面使用 `name` 属性进行替代
260
+ - `tag` 的出现是为了适应早期无法区分节点而设计的,但现在 `name` 已经强制唯一性,继续使用冗长的 `tag` 会导致不便
261
+ - **[Important]** 将 executor 中 `task_queue` 与 `result_queue` 的定义提前到 `__init__` 中
262
+ - 对于 executor 来说区别不大,但对于 stage 而言这是更理想且优雅的形式,只是由于原先 `stage_mode="process"` 时不允许执行函数内带有 `MPQueue`,所以才使用 graph 中统一定义,然后通过 process 的 args 注入的形式
263
+ - **[Important]** 重构 `core_server`,将其分解出 `routes/` 与 `util_models`,实现结构优化
264
+ - 进一步地,将 `log_queue` 和 `fail_queue` 的注入也提前到 `graph.set_stages` 中,实现 `stage.start_stage` 完全无输入参数
265
+ - 同时在 executor 中添加 `put_task` 与 `put_signal`,并在 graph 中进行复用
266
+ - 之前不能这样做是因为 stage 中此时还没有定义 queue
267
+ - 移除 `core_graph` 中的 `StageRuntime`
268
+ - 将类型标注中的部分 `Any` 改为更明确的类型
269
+ - 删除 HTML 中的仪表盘部分的 card,现在交给 `web-config` 来定义
270
+ - 删除部分没必要的前后端通信,优化通信负载
271
+ - 移除后端传递的节点历史信息,由前端进行维护
272
+ - 移除总体统计中除"总剩余时间"的后端数据,由前端进行计算
273
+ - 优化错误存储的 JSONL 数据结构,现在保存更细致的错误信息
274
+ - 调整 pyright 规则,并实现 pyright 0 error / 0 warning
275
+ - 给 `web/` 下几乎所有 `.ts` 与 `.css` 文件重命名,并整理文件内容边界
276
+ - `util_error` 中添加更多错误类型,保证项目中所有主动 raise 的错误都在当前的错误体系下
277
+ - 删除前端代码中所有的 `onclick`,改用 `data-*` 属性
278
+ - 清简 `config.json` 中的部分字段
279
+ - 删除部分不必要的代码
280
+ - fix:
281
+ - 对部分太宽泛的错误捕捉进行收窄
260
282
  - chore:
261
- - 删除部分无用代码以提高代码质量;
283
+ - 添加更多测试代码,并整理 `tests/` 结构
284
+ - 修复 `bench/` 中的一些问题
285
+ - 主要是 `bench_execution_mode` 中的两种 Fibonacci 运算量级不同,无法正常进行模型 bench
286
+ - 添加 `.github/workflows`,实现 CI/CD
287
+ - 更新支持的 Python 版本为 3.11,以与部分库要求相符
288
+ - 添加两个 skill,分别用于更新文档与审查项目
262
289
 
263
290
  更多过往日志可看:
264
291
 
265
- [change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/change_log.md )
292
+ [change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/change_log.md )
266
293
 
267
294
  ## Star 历史趋势(Star History)
268
295
 
@@ -1,81 +1,115 @@
1
- [build-system]
2
- requires = ["setuptools>=61.0", "wheel"]
3
- build-backend = "setuptools.build_meta"
4
-
5
- [project]
6
- name = "celestialflow"
7
- version = "3.1.9"
8
- description = "A flexible GRAPH-based task orchestration framework."
9
- readme = "README.md"
10
- license = { text = "MIT" }
11
- authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
12
- keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
13
- requires-python = ">=3.10"
14
-
15
- dependencies = [
16
- "tqdm",
17
- "fastapi",
18
- "uvicorn",
19
- "requests",
20
- "networkx",
21
- "redis",
22
- "jinja2",
23
- "celestialtree>=0.1.2",
24
- ]
25
-
26
- classifiers = [
27
- "Programming Language :: Python :: 3",
28
- "License :: OSI Approved :: MIT License",
29
- "Operating System :: OS Independent",
30
- "Framework :: FastAPI",
31
- "Topic :: Software Development :: Libraries",
32
- ]
33
-
34
- [project.urls]
35
- "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
36
- "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
37
-
38
- [project.scripts]
39
- celestialflow-web = "celestialflow.web.core_server:main_entry"
40
-
41
- [tool.setuptools]
42
- license-files = []
43
-
44
- [tool.setuptools.package-data]
45
- celestialflow = [
46
- "web/templates/**/*.html",
47
- "web/static/**/*",
48
- "web/config.json"
49
- ]
50
-
51
- [tool.pytest.ini_options]
52
- addopts = "-s"
53
- log_cli = true
54
- log_cli_level = "INFO"
55
- log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
56
- log_cli_date_format = "%Y-%m-%d %H:%M:%S"
57
-
58
- filterwarnings = [
59
- "ignore::DeprecationWarning",
60
- "ignore::pytest.PytestDeprecationWarning",
61
- ]
62
-
63
- asyncio_default_fixture_loop_scope = "function"
64
-
65
- [dependency-groups]
66
- dev = [
67
- "build>=1.2.2.post1",
68
- "twine>=6.1.0",
69
- "pytest>=8.3.4",
70
- "pytest-asyncio>=0.25.3",
71
- "httpx>=0.28.1",
72
- "python-dotenv>=1.2.2",
73
- "dill",
74
- "mypy>=1.19.1",
75
- "ruff",
76
- ]
77
-
78
- [tool.mypy]
79
- python_version = "3.10"
80
- no_implicit_optional = false
81
- disable_error_code = ["import-untyped"]
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "celestialflow"
7
+ version = "3.2.1"
8
+ description = "A flexible GRAPH-based task orchestration framework."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
12
+ keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
13
+ requires-python = ">=3.11"
14
+
15
+ dependencies = [
16
+ "tqdm",
17
+ "fastapi",
18
+ "uvicorn",
19
+ "requests",
20
+ "networkx",
21
+ "redis",
22
+ "jinja2",
23
+ "celestialtree>=0.1.2",
24
+ ]
25
+
26
+ classifiers = [
27
+ "Programming Language :: Python :: 3",
28
+ "License :: OSI Approved :: MIT License",
29
+ "Operating System :: OS Independent",
30
+ "Framework :: FastAPI",
31
+ "Topic :: Software Development :: Libraries",
32
+ ]
33
+
34
+ [project.urls]
35
+ "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
36
+ "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
37
+
38
+ [project.scripts]
39
+ celestialflow-web = "celestialflow.web.core_server:main_entry"
40
+
41
+ [tool.setuptools]
42
+ license-files = []
43
+
44
+ [tool.setuptools.package-data]
45
+ celestialflow = [
46
+ "web/templates/**/*.html",
47
+ "web/static/**/*",
48
+ "web/config.json"
49
+ ]
50
+
51
+ [tool.pytest.ini_options]
52
+ addopts = "-s"
53
+ log_cli = true
54
+ log_cli_level = "INFO"
55
+ log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
56
+ log_cli_date_format = "%Y-%m-%d %H:%M:%S"
57
+
58
+ filterwarnings = [
59
+ "ignore::DeprecationWarning",
60
+ "ignore::pytest.PytestDeprecationWarning",
61
+ ]
62
+
63
+ asyncio_default_fixture_loop_scope = "function"
64
+
65
+ [dependency-groups]
66
+ dev = [
67
+ "build>=1.2.2.post1",
68
+ "twine>=6.1.0",
69
+ "pytest>=8.3.4",
70
+ "pytest-asyncio>=0.25.3",
71
+ "httpx>=0.28.1",
72
+ "python-dotenv>=1.2.2",
73
+ "ruff>=0.15.0",
74
+ "pyright>=1.1.409",
75
+ ]
76
+
77
+ [tool.pyright]
78
+ include = ["src"]
79
+ ignore = ["tests", "**/tests/**", "typings", "demo", "bench", "experiments", "examples", "temp"]
80
+
81
+ typeCheckingMode = "strict"
82
+ reportExplicitAny = "none"
83
+ reportAny = "none"
84
+ reportUnannotatedClassAttribute = "none"
85
+ reportUnusedFunction = "none"
86
+ reportImplicitOverride = "none"
87
+
88
+ [tool.ruff]
89
+ target-version = "py311"
90
+ exclude = ["tests", "typings", "demo", "bench", "experiments", "examples", "temp"]
91
+
92
+ [tool.ruff.lint]
93
+ select = [
94
+ "E", "F", # pycodestyle errors + pyflakes
95
+ "I", # isort (import order)
96
+ "UP", # pyupgrade
97
+ "B", # flake8-bugbear
98
+ "SIM", # flake8-simplify
99
+ "C4", # flake8-comprehensions
100
+ "RUF", # ruff-specific rules
101
+ ]
102
+ ignore = [
103
+ "E501", # line too long(交给 formatter)
104
+ "RUF001", # 字符串内 Unicode 字符(中文内容)
105
+ "RUF002", # 中文标点(项目使用中文 docstring)
106
+ "RUF003", # 中文注释标点
107
+ ]
108
+
109
+ [tool.ruff.format]
110
+ quote-style = "double"
111
+ indent-style = "space"
112
+ skip-magic-trailing-comma = false
113
+
114
+ [tool.ruff.lint.isort]
115
+ known-first-party = ["celestialflow"]
@@ -1,4 +1,9 @@
1
1
  # __init__.py
2
+ """CelestialFlow — 基于图结构的轻量级异步任务编排框架。
3
+
4
+ 提供任务图构建、执行调度、实时监控、Web 可视化和持久化等核心能力。
5
+ """
6
+
2
7
  from .graph import (
3
8
  TaskChain,
4
9
  TaskComplete,
@@ -30,30 +35,30 @@ from .utils.util_format import format_table
30
35
  from .web import TaskWebServer
31
36
 
32
37
  __all__ = [
33
- "TaskGraph",
38
+ "BaseObserver",
39
+ "CallbackObserver",
34
40
  "TaskChain",
35
- "TaskLoop",
36
- "TaskCross",
37
41
  "TaskComplete",
38
- "TaskWheel",
42
+ "TaskCross",
43
+ "TaskExecutor",
44
+ "TaskGraph",
39
45
  "TaskGrid",
40
- "BaseObserver",
41
- "CallbackObserver",
46
+ "TaskLoop",
42
47
  "TaskProgress",
43
- "TaskExecutor",
44
- "TaskStage",
45
- "TaskSplitter",
46
- "TaskRedisTransport",
47
- "TaskRedisSource",
48
48
  "TaskRedisAck",
49
+ "TaskRedisSource",
50
+ "TaskRedisTransport",
49
51
  "TaskRouter",
50
- "TerminationSignal",
52
+ "TaskSplitter",
53
+ "TaskStage",
51
54
  "TaskWebServer",
55
+ "TaskWheel",
56
+ "TerminationSignal",
57
+ "benchmark_executor",
58
+ "benchmark_graph",
59
+ "format_table",
52
60
  "load_jsonl_logs",
53
- "load_task_by_stage",
54
61
  "load_task_by_error",
62
+ "load_task_by_stage",
55
63
  "make_hashable",
56
- "format_table",
57
- "benchmark_graph",
58
- "benchmark_executor",
59
64
  ]
@@ -1,8 +1,8 @@
1
1
  # funnel/__init__.py
2
- from .core_spout import BaseSpout
3
2
  from .core_inlet import BaseInlet
3
+ from .core_spout import BaseSpout
4
4
 
5
5
  __all__ = [
6
- "BaseSpout",
7
6
  "BaseInlet",
7
+ "BaseSpout",
8
8
  ]
@@ -1,22 +1,22 @@
1
- # funnel/core_inlet.py
2
- from __future__ import annotations
3
-
4
- from typing import Any
5
-
6
-
7
- class BaseInlet:
8
- """数据收集器基类,负责将记录通过队列发送到对应的监听器。"""
9
-
10
- def __init__(self, queue: Any) -> None:
11
- """初始化收集器
12
-
13
- :param queue: 记录队列
14
- """
15
- self.queue: Any = queue
16
-
17
- def _funnel(self, record: Any) -> None:
18
- """将记录放入队列
19
-
20
- :param record: 待发送的记录
21
- """
22
- self.queue.put(record)
1
+ # funnel/core_inlet.py
2
+ from __future__ import annotations
3
+
4
+ from typing import Any
5
+
6
+
7
+ class BaseInlet:
8
+ """数据收集器基类,负责将记录通过队列发送到对应的监听器。"""
9
+
10
+ def __init__(self, queue: Any) -> None:
11
+ """初始化收集器
12
+
13
+ :param queue: 记录队列
14
+ """
15
+ self.queue: Any = queue
16
+
17
+ def _funnel(self, record: Any) -> None:
18
+ """将记录放入队列
19
+
20
+ :param record: 待发送的记录
21
+ """
22
+ self.queue.put(record)