celestialflow 3.0.6__tar.gz → 3.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celestialflow-3.0.6 → celestialflow-3.0.8}/PKG-INFO +82 -54
- {celestialflow-3.0.6 → celestialflow-3.0.8}/README.md +81 -53
- {celestialflow-3.0.6 → celestialflow-3.0.8}/pyproject.toml +1 -1
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/__init__.py +4 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/client.py +34 -17
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/tools.py +6 -1
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_errors.js +2 -2
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_structure.js +1 -1
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_topology.js +2 -2
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_graph.py +124 -116
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_logging.py +52 -30
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_manage.py +165 -263
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_nodes.py +113 -32
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_queue.py +24 -19
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_report.py +2 -2
- celestialflow-3.0.8/src/celestialflow/task_stage.py +176 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_structure.py +26 -23
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_tools.py +38 -16
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_types.py +7 -3
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_web.py +1 -2
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/PKG-INFO +82 -54
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/SOURCES.txt +1 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_graph.py +15 -14
- {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_nodes.py +62 -27
- {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_structure.py +75 -73
- {celestialflow-3.0.6 → celestialflow-3.0.8}/setup.cfg +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/__init__.py +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/__init__.py +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/base.css +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/dashboard.css +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/errors.css +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/css/inject.css +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/favicon.ico +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/main.js +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_injection.js +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/task_statuses.js +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/static/js/utils.js +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/task_progress.py +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/templates/index.html +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/dependency_links.txt +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/entry_points.txt +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/requires.txt +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow.egg-info/top_level.txt +0 -0
- {celestialflow-3.0.6 → celestialflow-3.0.8}/tests/test_manage.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celestialflow
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.8
|
|
4
4
|
Summary: A flexible GRAPH-based task orchestration framework.
|
|
5
5
|
Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -173,7 +173,7 @@ if __name__ == "__main__":
|
|
|
173
173
|
graph = TaskGraph([stage1])
|
|
174
174
|
|
|
175
175
|
# 初始化任务并启动
|
|
176
|
-
graph.start_graph({stage1.
|
|
176
|
+
graph.start_graph({stage1.get_tag(): [(1, 2), (3, 4), (5, 6)]})
|
|
177
177
|
```
|
|
178
178
|
|
|
179
179
|
注意不要在.ipynb中运行。
|
|
@@ -244,79 +244,105 @@ flowchart TD
|
|
|
244
244
|
## 文件结构(File Structure)
|
|
245
245
|
|
|
246
246
|
```
|
|
247
|
-
📁 CelestialFlow (
|
|
248
|
-
📁
|
|
249
|
-
|
|
247
|
+
📁 CelestialFlow (213MB 633KB 545B)
|
|
248
|
+
📁 docs (25KB 183B)
|
|
249
|
+
📁 reference (20KB 175B)
|
|
250
|
+
📝 go_worker.md (6KB 378B)
|
|
251
|
+
📝 task_graph.md (4KB 989B)
|
|
252
|
+
📝 task_logging.md (0B)
|
|
253
|
+
📝 task_manage.md (7KB 7B)
|
|
254
|
+
📝 task_nodes.md (1KB 737B)
|
|
255
|
+
📝 task_progress.md (0B)
|
|
256
|
+
📝 task_queue.md (0B)
|
|
257
|
+
📝 task_report.md (0B)
|
|
258
|
+
📝 task_structure.md (112B)
|
|
259
|
+
📝 task_tools.md (0B)
|
|
260
|
+
📝 task_types.md (0B)
|
|
261
|
+
📝 task_web.md (0B)
|
|
262
|
+
📝 quick_start.md (5KB 8B)
|
|
263
|
+
📝 tutorial.md (0B)
|
|
264
|
+
📁 experiments (15KB 111B)
|
|
265
|
+
🐍 benchmark_datastructures.py (6KB 52B)
|
|
250
266
|
🐍 benchmark_hash.py (1KB 284B)
|
|
251
|
-
🐍 benchmark_queue.py (5KB
|
|
267
|
+
🐍 benchmark_queue.py (5KB 504B)
|
|
252
268
|
🐍 benchmark_tqdm.py (1KB 160B)
|
|
253
269
|
🐍 experiment_tqdm.py (1KB 135B)
|
|
254
|
-
📁 go_worker (6MB
|
|
255
|
-
📁 worker (5KB
|
|
270
|
+
📁 go_worker (6MB 968KB 38B)
|
|
271
|
+
📁 worker (5KB 746B)
|
|
256
272
|
🌀 parser.go (394B)
|
|
257
273
|
🌀 processor.go (2KB 612B)
|
|
258
|
-
🌀 types.go (
|
|
274
|
+
🌀 types.go (299B)
|
|
259
275
|
🌀 worker.go (2KB 465B)
|
|
260
|
-
❓ go.mod (
|
|
261
|
-
❓ go.sum (
|
|
276
|
+
❓ go.mod (292B)
|
|
277
|
+
❓ go.sum (1KB 68B)
|
|
262
278
|
❓ go_worker.exe (6MB 960KB)
|
|
263
|
-
🌀 main.go (
|
|
264
|
-
📁 img (
|
|
265
|
-
📷 logo.png
|
|
279
|
+
🌀 main.go (980B)
|
|
280
|
+
📁 img (4MB 700KB 695B)
|
|
281
|
+
📷 logo(old).png (836KB 542B)
|
|
282
|
+
📷 logo.png (122KB 747B)
|
|
266
283
|
📷 web_display.png (129KB 545B)
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
284
|
+
🎞️ web_ui.gif (3MB 635KB 909B)
|
|
285
|
+
📁 src (1MB 920KB 171B)
|
|
286
|
+
📁 celestialflow (1MB 902KB 818B)
|
|
287
|
+
📁 adapters (13KB 837B)
|
|
288
|
+
📁 celestialtree (13KB 681B)
|
|
289
|
+
📁 [1项排除的目录] (6KB 960B)
|
|
290
|
+
🐍 client.py (5KB 486B)
|
|
291
|
+
🐍 tools.py (1KB 182B)
|
|
292
|
+
🐍 __init__.py (77B)
|
|
293
|
+
📁 [1项排除的目录] (156B)
|
|
294
|
+
🐍 __init__.py (0B)
|
|
295
|
+
📁 static (1MB 417KB 101B)
|
|
296
|
+
📁 css (31KB 410B)
|
|
271
297
|
🎨 base.css (6KB 155B)
|
|
272
|
-
🎨 dashboard.css (
|
|
298
|
+
🎨 dashboard.css (7KB 277B)
|
|
273
299
|
🎨 errors.css (5KB 168B)
|
|
274
300
|
🎨 inject.css (12KB 834B)
|
|
275
|
-
📁 js (
|
|
276
|
-
📜 main.js (
|
|
277
|
-
📜 task_errors.js (4KB
|
|
301
|
+
📁 js (33KB 617B)
|
|
302
|
+
📜 main.js (5KB 77B)
|
|
303
|
+
📜 task_errors.js (4KB 550B)
|
|
278
304
|
📜 task_injection.js (8KB 491B)
|
|
279
305
|
📜 task_statuses.js (8KB 63B)
|
|
280
|
-
📜 task_structure.js (
|
|
281
|
-
📜 task_topology.js (
|
|
306
|
+
📜 task_structure.js (4KB 668B)
|
|
307
|
+
📜 task_topology.js (1KB 375B)
|
|
282
308
|
📜 utils.js (1KB 441B)
|
|
283
309
|
❓ favicon.ico (1MB 352KB 98B)
|
|
284
|
-
📁 templates (
|
|
285
|
-
🌐 index.html (
|
|
286
|
-
📁 [1项排除的目录] (
|
|
287
|
-
|
|
288
|
-
🐍
|
|
289
|
-
🐍
|
|
290
|
-
🐍
|
|
291
|
-
🐍 task_nodes.py (4KB 953B)
|
|
310
|
+
📁 templates (13KB 150B)
|
|
311
|
+
🌐 index.html (13KB 150B)
|
|
312
|
+
📁 [1项排除的目录] (331KB 149B)
|
|
313
|
+
🐍 task_graph.py (23KB 835B)
|
|
314
|
+
🐍 task_logging.py (8KB 903B)
|
|
315
|
+
🐍 task_manage.py (33KB 577B)
|
|
316
|
+
🐍 task_nodes.py (10KB 922B)
|
|
292
317
|
🐍 task_progress.py (1KB 477B)
|
|
293
|
-
🐍 task_queue.py (
|
|
294
|
-
🐍 task_report.py (
|
|
295
|
-
🐍
|
|
296
|
-
🐍
|
|
297
|
-
🐍
|
|
298
|
-
🐍
|
|
299
|
-
🐍
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
🐍
|
|
307
|
-
🐍
|
|
308
|
-
🐍
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
❓ .gitignore (
|
|
318
|
+
🐍 task_queue.py (8KB 570B)
|
|
319
|
+
🐍 task_report.py (4KB 805B)
|
|
320
|
+
🐍 task_stage.py (5KB 511B)
|
|
321
|
+
🐍 task_structure.py (5KB 996B)
|
|
322
|
+
🐍 task_tools.py (15KB 105B)
|
|
323
|
+
🐍 task_types.py (2KB 265B)
|
|
324
|
+
🐍 task_web.py (5KB 693B)
|
|
325
|
+
🐍 __init__.py (1KB 114B)
|
|
326
|
+
📁 [1项排除的目录] (17KB 377B)
|
|
327
|
+
📁 tests (126KB 669B)
|
|
328
|
+
📁 [1项排除的目录] (90KB 310B)
|
|
329
|
+
📝 README.md (6KB 324B)
|
|
330
|
+
🐍 test_graph.py (5KB 594B)
|
|
331
|
+
🐍 test_manage.py (3KB 90B)
|
|
332
|
+
🐍 test_nodes.py (11KB 272B)
|
|
333
|
+
🐍 test_structure.py (10KB 103B)
|
|
334
|
+
📁 [6项排除的目录] (199MB 931KB 238B)
|
|
335
|
+
❓ .env (88B)
|
|
336
|
+
❓ .gitignore (272B)
|
|
312
337
|
❓ LICENSE (1KB 65B)
|
|
313
338
|
❓ Makefile (501B)
|
|
314
339
|
❓ pyproject.toml (1KB 223B)
|
|
315
340
|
⚙️ pytest.ini (254B)
|
|
316
|
-
📝 README.md (15KB
|
|
341
|
+
📝 README.md (15KB 109B)
|
|
342
|
+
(celestial-flow 3.0.8)
|
|
317
343
|
```
|
|
318
344
|
|
|
319
|
-
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file生成。)
|
|
345
|
+
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。)
|
|
320
346
|
|
|
321
347
|
## 更新日志(Change Log)
|
|
322
348
|
|
|
@@ -326,7 +352,7 @@ flowchart TD
|
|
|
326
352
|
- 12/12/2024-12/16/2024: 在原有链式结构基础上允许节点有复数下级节点, 实现Tree结构; 将原有TaskChain改名为TaskTree
|
|
327
353
|
- 3/16/2025: 支持Web端任务完成情况可视化
|
|
328
354
|
- 6/9/2025: 支持节点拥有复数上级节点, 脱离纯Tree结构, 为之后循环图做准备
|
|
329
|
-
- 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task
|
|
355
|
+
- 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁出
|
|
330
356
|
- 6/12/2025: 支持循环图, 下级节点可指向上级节点
|
|
331
357
|
- 6/13/2025: 支持loop结构, 即节点可指向自己
|
|
332
358
|
- 6/14/2025: 支持forest结构, 即可有多个根节点
|
|
@@ -335,6 +361,8 @@ flowchart TD
|
|
|
335
361
|
- 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
|
|
336
362
|
- 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
|
|
337
363
|
- 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
|
|
364
|
+
- 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
|
|
365
|
+
- 3.0.8: 在ctree关系上将"任务重试"后的"任务成功/失败/重试"视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
|
|
338
366
|
|
|
339
367
|
## Star 历史趋势(Star History)
|
|
340
368
|
|
|
@@ -148,7 +148,7 @@ if __name__ == "__main__":
|
|
|
148
148
|
graph = TaskGraph([stage1])
|
|
149
149
|
|
|
150
150
|
# 初始化任务并启动
|
|
151
|
-
graph.start_graph({stage1.
|
|
151
|
+
graph.start_graph({stage1.get_tag(): [(1, 2), (3, 4), (5, 6)]})
|
|
152
152
|
```
|
|
153
153
|
|
|
154
154
|
注意不要在.ipynb中运行。
|
|
@@ -219,79 +219,105 @@ flowchart TD
|
|
|
219
219
|
## 文件结构(File Structure)
|
|
220
220
|
|
|
221
221
|
```
|
|
222
|
-
📁 CelestialFlow (
|
|
223
|
-
📁
|
|
224
|
-
|
|
222
|
+
📁 CelestialFlow (213MB 633KB 545B)
|
|
223
|
+
📁 docs (25KB 183B)
|
|
224
|
+
📁 reference (20KB 175B)
|
|
225
|
+
📝 go_worker.md (6KB 378B)
|
|
226
|
+
📝 task_graph.md (4KB 989B)
|
|
227
|
+
📝 task_logging.md (0B)
|
|
228
|
+
📝 task_manage.md (7KB 7B)
|
|
229
|
+
📝 task_nodes.md (1KB 737B)
|
|
230
|
+
📝 task_progress.md (0B)
|
|
231
|
+
📝 task_queue.md (0B)
|
|
232
|
+
📝 task_report.md (0B)
|
|
233
|
+
📝 task_structure.md (112B)
|
|
234
|
+
📝 task_tools.md (0B)
|
|
235
|
+
📝 task_types.md (0B)
|
|
236
|
+
📝 task_web.md (0B)
|
|
237
|
+
📝 quick_start.md (5KB 8B)
|
|
238
|
+
📝 tutorial.md (0B)
|
|
239
|
+
📁 experiments (15KB 111B)
|
|
240
|
+
🐍 benchmark_datastructures.py (6KB 52B)
|
|
225
241
|
🐍 benchmark_hash.py (1KB 284B)
|
|
226
|
-
🐍 benchmark_queue.py (5KB
|
|
242
|
+
🐍 benchmark_queue.py (5KB 504B)
|
|
227
243
|
🐍 benchmark_tqdm.py (1KB 160B)
|
|
228
244
|
🐍 experiment_tqdm.py (1KB 135B)
|
|
229
|
-
📁 go_worker (6MB
|
|
230
|
-
📁 worker (5KB
|
|
245
|
+
📁 go_worker (6MB 968KB 38B)
|
|
246
|
+
📁 worker (5KB 746B)
|
|
231
247
|
🌀 parser.go (394B)
|
|
232
248
|
🌀 processor.go (2KB 612B)
|
|
233
|
-
🌀 types.go (
|
|
249
|
+
🌀 types.go (299B)
|
|
234
250
|
🌀 worker.go (2KB 465B)
|
|
235
|
-
❓ go.mod (
|
|
236
|
-
❓ go.sum (
|
|
251
|
+
❓ go.mod (292B)
|
|
252
|
+
❓ go.sum (1KB 68B)
|
|
237
253
|
❓ go_worker.exe (6MB 960KB)
|
|
238
|
-
🌀 main.go (
|
|
239
|
-
📁 img (
|
|
240
|
-
📷 logo.png
|
|
254
|
+
🌀 main.go (980B)
|
|
255
|
+
📁 img (4MB 700KB 695B)
|
|
256
|
+
📷 logo(old).png (836KB 542B)
|
|
257
|
+
📷 logo.png (122KB 747B)
|
|
241
258
|
📷 web_display.png (129KB 545B)
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
259
|
+
🎞️ web_ui.gif (3MB 635KB 909B)
|
|
260
|
+
📁 src (1MB 920KB 171B)
|
|
261
|
+
📁 celestialflow (1MB 902KB 818B)
|
|
262
|
+
📁 adapters (13KB 837B)
|
|
263
|
+
📁 celestialtree (13KB 681B)
|
|
264
|
+
📁 [1项排除的目录] (6KB 960B)
|
|
265
|
+
🐍 client.py (5KB 486B)
|
|
266
|
+
🐍 tools.py (1KB 182B)
|
|
267
|
+
🐍 __init__.py (77B)
|
|
268
|
+
📁 [1项排除的目录] (156B)
|
|
269
|
+
🐍 __init__.py (0B)
|
|
270
|
+
📁 static (1MB 417KB 101B)
|
|
271
|
+
📁 css (31KB 410B)
|
|
246
272
|
🎨 base.css (6KB 155B)
|
|
247
|
-
🎨 dashboard.css (
|
|
273
|
+
🎨 dashboard.css (7KB 277B)
|
|
248
274
|
🎨 errors.css (5KB 168B)
|
|
249
275
|
🎨 inject.css (12KB 834B)
|
|
250
|
-
📁 js (
|
|
251
|
-
📜 main.js (
|
|
252
|
-
📜 task_errors.js (4KB
|
|
276
|
+
📁 js (33KB 617B)
|
|
277
|
+
📜 main.js (5KB 77B)
|
|
278
|
+
📜 task_errors.js (4KB 550B)
|
|
253
279
|
📜 task_injection.js (8KB 491B)
|
|
254
280
|
📜 task_statuses.js (8KB 63B)
|
|
255
|
-
📜 task_structure.js (
|
|
256
|
-
📜 task_topology.js (
|
|
281
|
+
📜 task_structure.js (4KB 668B)
|
|
282
|
+
📜 task_topology.js (1KB 375B)
|
|
257
283
|
📜 utils.js (1KB 441B)
|
|
258
284
|
❓ favicon.ico (1MB 352KB 98B)
|
|
259
|
-
📁 templates (
|
|
260
|
-
🌐 index.html (
|
|
261
|
-
📁 [1项排除的目录] (
|
|
262
|
-
|
|
263
|
-
🐍
|
|
264
|
-
🐍
|
|
265
|
-
🐍
|
|
266
|
-
🐍 task_nodes.py (4KB 953B)
|
|
285
|
+
📁 templates (13KB 150B)
|
|
286
|
+
🌐 index.html (13KB 150B)
|
|
287
|
+
📁 [1项排除的目录] (331KB 149B)
|
|
288
|
+
🐍 task_graph.py (23KB 835B)
|
|
289
|
+
🐍 task_logging.py (8KB 903B)
|
|
290
|
+
🐍 task_manage.py (33KB 577B)
|
|
291
|
+
🐍 task_nodes.py (10KB 922B)
|
|
267
292
|
🐍 task_progress.py (1KB 477B)
|
|
268
|
-
🐍 task_queue.py (
|
|
269
|
-
🐍 task_report.py (
|
|
270
|
-
🐍
|
|
271
|
-
🐍
|
|
272
|
-
🐍
|
|
273
|
-
🐍
|
|
274
|
-
🐍
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
🐍
|
|
282
|
-
🐍
|
|
283
|
-
🐍
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
❓ .gitignore (
|
|
293
|
+
🐍 task_queue.py (8KB 570B)
|
|
294
|
+
🐍 task_report.py (4KB 805B)
|
|
295
|
+
🐍 task_stage.py (5KB 511B)
|
|
296
|
+
🐍 task_structure.py (5KB 996B)
|
|
297
|
+
🐍 task_tools.py (15KB 105B)
|
|
298
|
+
🐍 task_types.py (2KB 265B)
|
|
299
|
+
🐍 task_web.py (5KB 693B)
|
|
300
|
+
🐍 __init__.py (1KB 114B)
|
|
301
|
+
📁 [1项排除的目录] (17KB 377B)
|
|
302
|
+
📁 tests (126KB 669B)
|
|
303
|
+
📁 [1项排除的目录] (90KB 310B)
|
|
304
|
+
📝 README.md (6KB 324B)
|
|
305
|
+
🐍 test_graph.py (5KB 594B)
|
|
306
|
+
🐍 test_manage.py (3KB 90B)
|
|
307
|
+
🐍 test_nodes.py (11KB 272B)
|
|
308
|
+
🐍 test_structure.py (10KB 103B)
|
|
309
|
+
📁 [6项排除的目录] (199MB 931KB 238B)
|
|
310
|
+
❓ .env (88B)
|
|
311
|
+
❓ .gitignore (272B)
|
|
287
312
|
❓ LICENSE (1KB 65B)
|
|
288
313
|
❓ Makefile (501B)
|
|
289
314
|
❓ pyproject.toml (1KB 223B)
|
|
290
315
|
⚙️ pytest.ini (254B)
|
|
291
|
-
📝 README.md (15KB
|
|
316
|
+
📝 README.md (15KB 109B)
|
|
317
|
+
(celestial-flow 3.0.8)
|
|
292
318
|
```
|
|
293
319
|
|
|
294
|
-
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file生成。)
|
|
320
|
+
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。)
|
|
295
321
|
|
|
296
322
|
## 更新日志(Change Log)
|
|
297
323
|
|
|
@@ -301,7 +327,7 @@ flowchart TD
|
|
|
301
327
|
- 12/12/2024-12/16/2024: 在原有链式结构基础上允许节点有复数下级节点, 实现Tree结构; 将原有TaskChain改名为TaskTree
|
|
302
328
|
- 3/16/2025: 支持Web端任务完成情况可视化
|
|
303
329
|
- 6/9/2025: 支持节点拥有复数上级节点, 脱离纯Tree结构, 为之后循环图做准备
|
|
304
|
-
- 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task
|
|
330
|
+
- 6/11/2025: 自[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)项目instances.inst_task迁出
|
|
305
331
|
- 6/12/2025: 支持循环图, 下级节点可指向上级节点
|
|
306
332
|
- 6/13/2025: 支持loop结构, 即节点可指向自己
|
|
307
333
|
- 6/14/2025: 支持forest结构, 即可有多个根节点
|
|
@@ -310,6 +336,8 @@ flowchart TD
|
|
|
310
336
|
- 3.0.4: 新增一个抽象结构TaskQueue, 用于表示节点的所有"入边"与"出边"; 恢复未消费任务的保存功能
|
|
311
337
|
- 3.0.5: 删除原有的TaskRedisTransfer节点, 并增添三种新的redis交互节点TaskRedisSink TaskRedisSource TaskRedisAck, 用于跨语言 跨进程 跨设备处理任务; 并在Web页面添加展示拓扑信息的卡片
|
|
312
338
|
- 3.0.6: 添加对[CelestialTree](https://github.com/Mr-xiaotian/CelestialTree)系统的支持, 现在可以追踪单个任务的流向
|
|
339
|
+
- 3.0.7: 将TaskStage从TaskManager中单独抽出来作为一个子类; 增加新节点TaskRouter, 可以将传入的任务选择的传给不同的下游节点, 而不是进行广播
|
|
340
|
+
- 3.0.8: 在ctree关系上将"任务重试"后的"任务成功/失败/重试"视为因果关系, 而非之前的并行关系; 重构错误搜集部分逻辑; 修复大量3.0.6与3.07版本引入的bug; 优化部分log表现
|
|
313
341
|
|
|
314
342
|
## Star 历史趋势(Star History)
|
|
315
343
|
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
from .task_graph import TaskGraph
|
|
2
2
|
from .task_manage import TaskManager
|
|
3
|
+
from .task_stage import TaskStage
|
|
3
4
|
from .task_nodes import (
|
|
4
5
|
TaskSplitter,
|
|
5
6
|
TaskRedisSink,
|
|
6
7
|
TaskRedisSource,
|
|
7
8
|
TaskRedisAck,
|
|
9
|
+
TaskRouter,
|
|
8
10
|
)
|
|
9
11
|
from .task_structure import (
|
|
10
12
|
TaskChain,
|
|
@@ -33,10 +35,12 @@ __all__ = [
|
|
|
33
35
|
"TaskWheel",
|
|
34
36
|
"TaskGrid",
|
|
35
37
|
"TaskManager",
|
|
38
|
+
"TaskStage",
|
|
36
39
|
"TaskSplitter",
|
|
37
40
|
"TaskRedisSink",
|
|
38
41
|
"TaskRedisSource",
|
|
39
42
|
"TaskRedisAck",
|
|
43
|
+
"TaskRouter",
|
|
40
44
|
"TerminationSignal",
|
|
41
45
|
"TaskWebServer",
|
|
42
46
|
"CelestialTreeClient",
|
{celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/client.py
RENAMED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import threading
|
|
3
3
|
import requests
|
|
4
|
+
from multiprocessing import Value as MPValue, Lock as MPLock
|
|
4
5
|
from typing import List, Optional, Dict, Any, Callable
|
|
5
6
|
|
|
6
7
|
|
|
@@ -9,8 +10,8 @@ class Client:
|
|
|
9
10
|
Python client for CelestialTree HTTP API.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
|
-
def __init__(self,
|
|
13
|
-
self.base_url =
|
|
13
|
+
def __init__(self, host: str = "127.0.0.1", port: int = 7777, timeout: float = 5.0):
|
|
14
|
+
self.base_url = f"http://{host}:{port}"
|
|
14
15
|
self.timeout = timeout
|
|
15
16
|
|
|
16
17
|
def init_session(self):
|
|
@@ -60,7 +61,9 @@ class Client:
|
|
|
60
61
|
data=json.dumps(body),
|
|
61
62
|
timeout=self.timeout,
|
|
62
63
|
)
|
|
63
|
-
|
|
64
|
+
|
|
65
|
+
if not (200 <= r.status_code < 300):
|
|
66
|
+
raise RuntimeError(r.json()["error"])
|
|
64
67
|
return r.json()["id"]
|
|
65
68
|
|
|
66
69
|
def get_event(self, event_id: int) -> Dict[str, Any]:
|
|
@@ -70,7 +73,9 @@ class Client:
|
|
|
70
73
|
f"{self.base_url}/event/{event_id}",
|
|
71
74
|
timeout=self.timeout,
|
|
72
75
|
)
|
|
73
|
-
|
|
76
|
+
|
|
77
|
+
if not (200 <= r.status_code < 300):
|
|
78
|
+
raise RuntimeError(r.json()["error"])
|
|
74
79
|
return r.json()
|
|
75
80
|
|
|
76
81
|
def children(self, event_id: int) -> List[int]:
|
|
@@ -80,7 +85,9 @@ class Client:
|
|
|
80
85
|
f"{self.base_url}/children/{event_id}",
|
|
81
86
|
timeout=self.timeout,
|
|
82
87
|
)
|
|
83
|
-
|
|
88
|
+
|
|
89
|
+
if not (200 <= r.status_code < 300):
|
|
90
|
+
raise RuntimeError(r.json()["error"])
|
|
84
91
|
return r.json()["children"]
|
|
85
92
|
|
|
86
93
|
def descendants(self, event_id: int) -> Dict[str, Any]:
|
|
@@ -90,7 +97,9 @@ class Client:
|
|
|
90
97
|
f"{self.base_url}/descendants/{event_id}",
|
|
91
98
|
timeout=self.timeout,
|
|
92
99
|
)
|
|
93
|
-
|
|
100
|
+
|
|
101
|
+
if not (200 <= r.status_code < 300):
|
|
102
|
+
raise RuntimeError(r.json()["error"])
|
|
94
103
|
return r.json()
|
|
95
104
|
|
|
96
105
|
def heads(self) -> List[int]:
|
|
@@ -100,17 +109,21 @@ class Client:
|
|
|
100
109
|
f"{self.base_url}/heads",
|
|
101
110
|
timeout=self.timeout,
|
|
102
111
|
)
|
|
103
|
-
|
|
112
|
+
|
|
113
|
+
if not (200 <= r.status_code < 300):
|
|
114
|
+
raise RuntimeError(r.json()["error"])
|
|
104
115
|
return r.json()["heads"]
|
|
105
116
|
|
|
106
117
|
def health(self) -> bool:
|
|
107
118
|
self.init_session()
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
119
|
+
try:
|
|
120
|
+
r = self.session.get(
|
|
121
|
+
f"{self.base_url}/healthz",
|
|
122
|
+
timeout=self.timeout,
|
|
123
|
+
)
|
|
124
|
+
return r.status_code == 200
|
|
125
|
+
except Exception:
|
|
126
|
+
return False
|
|
114
127
|
|
|
115
128
|
def version(self) -> Dict[str, Any]:
|
|
116
129
|
self.init_session()
|
|
@@ -119,7 +132,9 @@ class Client:
|
|
|
119
132
|
f"{self.base_url}/version",
|
|
120
133
|
timeout=self.timeout,
|
|
121
134
|
)
|
|
122
|
-
|
|
135
|
+
|
|
136
|
+
if not (200 <= r.status_code < 300):
|
|
137
|
+
raise RuntimeError(r.json()["error"])
|
|
123
138
|
return r.json()
|
|
124
139
|
|
|
125
140
|
# ---------- SSE Subscribe ----------
|
|
@@ -162,11 +177,13 @@ class Client:
|
|
|
162
177
|
|
|
163
178
|
|
|
164
179
|
class NullClient:
|
|
165
|
-
event_id
|
|
180
|
+
def __init__(self, event_id=None):
|
|
181
|
+
self.event_id = event_id if event_id is not None else MPValue("i", 0)
|
|
166
182
|
|
|
167
183
|
def emit(self, *args, **kwargs):
|
|
168
|
-
self.event_id
|
|
169
|
-
|
|
184
|
+
with self.event_id.get_lock():
|
|
185
|
+
self.event_id.value += 1
|
|
186
|
+
return self.event_id.value
|
|
170
187
|
|
|
171
188
|
def get_event(self, *args, **kwargs):
|
|
172
189
|
return None
|
{celestialflow-3.0.6 → celestialflow-3.0.8}/src/celestialflow/adapters/celestialtree/tools.py
RENAMED
|
@@ -10,7 +10,12 @@ def format_tree(node: dict, prefix: str = "", is_last: bool = True) -> str:
|
|
|
10
10
|
lines = []
|
|
11
11
|
|
|
12
12
|
connector = "└── " if is_last else "├── "
|
|
13
|
-
|
|
13
|
+
|
|
14
|
+
label = str(node["id"])
|
|
15
|
+
if node.get("is_ref"):
|
|
16
|
+
label += " (ref)"
|
|
17
|
+
|
|
18
|
+
lines.append(f"{prefix}{connector}{label}")
|
|
14
19
|
|
|
15
20
|
children = node.get("children", [])
|
|
16
21
|
if children:
|
|
@@ -25,7 +25,7 @@ function renderErrors() {
|
|
|
25
25
|
const matchNode = !filter || e.node === filter;
|
|
26
26
|
const matchKeyword = !keyword ||
|
|
27
27
|
(e.error && e.error.toLowerCase().includes(keyword)) ||
|
|
28
|
-
(e.
|
|
28
|
+
(e.task_repr && e.task_repr.toLowerCase().includes(keyword));
|
|
29
29
|
return matchNode && matchKeyword;
|
|
30
30
|
});
|
|
31
31
|
|
|
@@ -48,7 +48,7 @@ function renderErrors() {
|
|
|
48
48
|
row.innerHTML = `
|
|
49
49
|
<td class="error-message">${e.error}</td>
|
|
50
50
|
<td>${e.node}</td>
|
|
51
|
-
<td>${e.
|
|
51
|
+
<td>${e.task_repr}</td>
|
|
52
52
|
<td>${formatTimestamp(e.timestamp)}</td>
|
|
53
53
|
`;
|
|
54
54
|
errorsTableBody.appendChild(row);
|
|
@@ -111,7 +111,7 @@ linkStyle default stroke:#999,stroke-width:1.5px;
|
|
|
111
111
|
|
|
112
112
|
// let defs = [];
|
|
113
113
|
|
|
114
|
-
// if (topologyData.
|
|
114
|
+
// if (topologyData.schedule_mode=== "staged") {
|
|
115
115
|
// const layers = topologyData.layers_dict || {};
|
|
116
116
|
// const sortedLayerKeys = Object.keys(layers).sort((a, b) => Number(a) - Number(b));
|
|
117
117
|
// for (const layer of sortedLayerKeys) {
|
|
@@ -21,7 +21,7 @@ function renderTopologyInfo() {
|
|
|
21
21
|
|
|
22
22
|
const {
|
|
23
23
|
isDAG,
|
|
24
|
-
|
|
24
|
+
schedule_mode,
|
|
25
25
|
class_name,
|
|
26
26
|
layers_dict = {},
|
|
27
27
|
} = topologyData;
|
|
@@ -43,7 +43,7 @@ function renderTopologyInfo() {
|
|
|
43
43
|
|
|
44
44
|
<div class="topology-row">
|
|
45
45
|
<span class="label">调度模式</span>
|
|
46
|
-
<span class="value">${
|
|
46
|
+
<span class="value">${schedule_mode}</span>
|
|
47
47
|
</div>
|
|
48
48
|
|
|
49
49
|
<div class="topology-row">
|