celestialflow 3.2.7__tar.gz → 3.2.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {celestialflow-3.2.7/src/celestialflow.egg-info → celestialflow-3.2.8}/PKG-INFO +93 -66
  2. {celestialflow-3.2.7 → celestialflow-3.2.8}/README.md +92 -65
  3. {celestialflow-3.2.7 → celestialflow-3.2.8}/pyproject.toml +7 -1
  4. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/__init__.py +4 -3
  5. celestialflow-3.2.8/src/celestialflow/benchmark/__init__.py +6 -0
  6. {celestialflow-3.2.7/src/celestialflow/utils → celestialflow-3.2.8/src/celestialflow/benchmark}/util_benchmark.py +20 -17
  7. {celestialflow-3.2.7/src/celestialflow/utils → celestialflow-3.2.8/src/celestialflow/benchmark}/util_clone.py +34 -12
  8. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/funnel/core_spout.py +4 -1
  9. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/graph/core_graph.py +295 -284
  10. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/graph/core_structure.py +40 -21
  11. {celestialflow-3.2.7/src/celestialflow/runtime → celestialflow-3.2.8/src/celestialflow/graph}/util_estimators.py +15 -65
  12. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/graph/util_graph.py +2 -3
  13. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/graph/util_serialize.py +7 -7
  14. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/observability/__init__.py +2 -1
  15. celestialflow-3.2.8/src/celestialflow/observability/core_observer.py +112 -0
  16. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/observability/core_report.py +42 -34
  17. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/observability/util_types.py +14 -7
  18. celestialflow-3.2.8/src/celestialflow/persistence/__init__.py +31 -0
  19. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/persistence/core_fallback.py +23 -10
  20. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/persistence/core_log.py +82 -57
  21. celestialflow-3.2.8/src/celestialflow/persistence/core_scope.py +41 -0
  22. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/persistence/util_payload.py +2 -1
  23. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/persistence/util_sqlite.py +10 -8
  24. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/__init__.py +1 -4
  25. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/core_envelope.py +11 -11
  26. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/core_metrics.py +36 -13
  27. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/core_queue.py +20 -25
  28. celestialflow-3.2.8/src/celestialflow/runtime/util_config.py +40 -0
  29. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/util_errors.py +207 -285
  30. celestialflow-3.2.8/src/celestialflow/runtime/util_estimators.py +68 -0
  31. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/util_event.py +63 -63
  32. {celestialflow-3.2.7/src/celestialflow/utils → celestialflow-3.2.8/src/celestialflow/runtime}/util_format.py +10 -18
  33. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/util_hash.py +3 -3
  34. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/util_types.py +2 -6
  35. {celestialflow-3.2.7/src/celestialflow/runtime → celestialflow-3.2.8/src/celestialflow/stage}/core_dispatch.py +68 -60
  36. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/stage/core_executor.py +692 -670
  37. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/stage/core_stage.py +46 -111
  38. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/stage/core_stages.py +23 -15
  39. {celestialflow-3.2.7 → celestialflow-3.2.8/src/celestialflow.egg-info}/PKG-INFO +93 -66
  40. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow.egg-info/SOURCES.txt +9 -7
  41. celestialflow-3.2.7/src/celestialflow/observability/core_observer.py +0 -51
  42. celestialflow-3.2.7/src/celestialflow/persistence/__init__.py +0 -15
  43. celestialflow-3.2.7/src/celestialflow/utils/__init__.py +0 -5
  44. celestialflow-3.2.7/src/celestialflow/utils/util_collections.py +0 -16
  45. {celestialflow-3.2.7 → celestialflow-3.2.8}/LICENSE +0 -0
  46. {celestialflow-3.2.7 → celestialflow-3.2.8}/setup.cfg +0 -0
  47. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/funnel/__init__.py +0 -0
  48. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/funnel/core_inlet.py +0 -0
  49. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/funnel/util_count.py +0 -0
  50. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/graph/__init__.py +0 -0
  51. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/runtime/util_constant.py +0 -0
  52. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/stage/__init__.py +0 -0
  53. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/stage/util_callable.py +0 -0
  54. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/stage/util_types.py +0 -0
  55. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow.egg-info/dependency_links.txt +0 -0
  56. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow.egg-info/requires.txt +0 -0
  57. {celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow.egg-info/top_level.txt +0 -0
{celestialflow-3.2.7/src/celestialflow.egg-info → celestialflow-3.2.8}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: celestialflow
3
- Version: 3.2.7
3
+ Version: 3.2.8
4
4
  Summary: A flexible GRAPH-based task orchestration framework.
5
5
  Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
6
6
  License: MIT
@@ -258,98 +258,125 @@ flowchart TD
258
258
  ## 文件结构（File Structure）
259
259
 
260
260
  ```
261
- 📁 CelestialFlow (471MB 982KB 22B)
262
- 📁 .agents (55KB 152B)
263
- 📁 skills[已折叠] (55KB 152B)
264
- 📁 .claude (99B)
265
- 🗄️ settings.local.json (99B)
266
- 📁 .github (601B)
267
- 📁 workflows[已折叠] (601B)
268
- 📁 bench (150KB 552B)
269
- 📁 [1项排除的目录] (47KB 905B)
261
+ 📁 CelestialFlow (587MB 684KB 313B)
262
+ 📁 bench (296KB 842B)
263
+ 📁 [1项排除的目录] (194KB 222B)
270
264
  🐍 bench_datastructures.py (6KB 690B)
271
- 🐍 bench_execution_mode.py (2KB 889B)
272
- 🐍 bench_futures_memory.py (2KB 267B)
273
- 🐍 bench_gil_vs_nogil.py (10KB 159B)
274
- 🐍 bench_graph_mode.py (7KB 173B)
275
- 🐍 bench_hash.py (7KB 39B)
265
+ 🐍 bench_execution_mode.py (2KB 831B)
266
+ 🐍 bench_futures_memory.py (2KB 269B)
267
+ 🐍 bench_gil_vs_nogil.py (10KB 7B)
268
+ 🐍 bench_graph_mode.py (7KB 450B)
269
+ 🐍 bench_hash.py (7KB 67B)
276
270
  🐍 bench_hash_container.py (3KB 1009B)
277
- 🐍 bench_hash_memory.py (3KB 613B)
278
- 🐍 bench_http_grpc.py (2KB 883B)
279
- 🐍 bench_ipc_queue.py (7KB 74B)
271
+ 🐍 bench_hash_memory.py (3KB 642B)
272
+ 🐍 bench_http_grpc.py (2KB 608B)
273
+ 🐍 bench_ipc_queue.py (7KB 104B)
280
274
  🐍 bench_lock_overhead.py (9KB 421B)
281
- 🐍 bench_mpqueue_vs_shared_memory.py (13KB 106B)
282
- 🐍 bench_observer.py (7KB 899B)
283
- 🐍 bench_persistence_spout.py (4KB 193B)
275
+ 🐍 bench_mpqueue_vs_shared_memory.py (13KB 127B)
276
+ 🐍 bench_observer.py (7KB 860B)
277
+ 🐍 bench_persistence_spout.py (4KB 279B)
284
278
  🐍 bench_queue.py (5KB 857B)
285
279
  🐍 bench_requests.py (6KB 813B)
286
280
  🐍 bench_tqdm.py (1KB 235B)
287
281
  🐍 bench_utils.py (543B)
288
- 📁 demo (105KB 440B)
289
- 📁 [1项排除的目录] (61KB 883B)
290
- 🐍 demo_executor.py (1KB 504B)
282
+ 📁 demo (147KB 87B)
283
+ 📁 [1项排除的目录] (100KB 9B)
284
+ 🐍 demo_executor.py (1KB 495B)
291
285
  🐍 demo_funnel.py (2KB 289B)
292
- 🐍 demo_graph.py (3KB 615B)
293
- 🐍 demo_observer.py (4KB 293B)
294
- 🐍 demo_redis.py (9KB 201B)
295
- 🐍 demo_stages.py (4KB 519B)
296
- 🐍 demo_structure.py (12KB 8B)
297
- 🐍 demo_utils.py (6KB 200B)
298
- 📁 dist (145KB 290B)
286
+ 🐍 demo_graph.py (3KB 565B)
287
+ 🐍 demo_network.py (3KB 879B)
288
+ 🐍 demo_observer.py (4KB 270B)
289
+ 🐍 demo_redis.py (9KB 144B)
290
+ 🐍 demo_stages.py (4KB 336B)
291
+ 🐍 demo_structure.py (12KB 24B)
292
+ 🐍 demo_utils.py (6KB 148B)
293
+ 📁 dist (147KB 48B)
299
294
  ❓ .gitignore (1B)
300
- ❓ celestialflow-3.2.6-py3-none-any.whl (78KB 975B)
301
- 📦 celestialflow-3.2.6.tar.gz (66KB 338B)
302
- 📁 docs (1MB 712KB 884B)
303
- 📁 en[已折叠] (569KB 485B)
304
- 📁 ja[已折叠] (642KB 959B)
305
- 📁 zh-CN[已折叠] (524KB 464B)
306
- 📁 experiments (2KB 1015B)
295
+ ❓ celestialflow-3.2.7-py3-none-any.whl (79KB 235B)
296
+ 📦 celestialflow-3.2.7.tar.gz (67KB 836B)
297
+ 📁 docs (1MB 722KB 21B)
298
+ 📁 en[已折叠] (568KB 991B)
299
+ 📁 ja[已折叠] (642KB 383B)
300
+ 📁 zh-CN[已折叠] (534KB 695B)
301
+ 📁 experiments (2KB 1015B)
307
302
  🐍 experiment_networkx.py (1KB 878B)
308
303
  🐍 experiment_tqdm.py (1KB 137B)
309
- 📁 img (5MB 871KB 242B)
304
+ 📁 img (5MB 871KB 242B)
310
305
  📷 file_structure.svg (4MB 918KB 1000B)
311
306
  📷 logo(old).png (836KB 542B)
312
307
  📷 logo.png (122KB 747B)
313
308
  📷 scc_condensation.svg (17KB 1B)
314
- 📁 src (1MB 767KB 296B)
315
- 📁 celestialflow[已折叠] (1MB 752KB 195B)
316
- 📁 celestialflow.egg-info[已折叠] (15KB 101B)
317
- 📁 tests (3MB 902KB 217B)
318
- 📁 funnel[已折叠] (96KB 481B)
319
- 📁 graph[已折叠] (631KB 532B)
320
- 📁 observability[已折叠] (157KB 435B)
321
- 📁 persistence[已折叠] (359KB 68B)
322
- 📁 runtime[已折叠] (1MB 286KB 487B)
323
- 📁 stage[已折叠] (281KB 752B)
324
- 📁 utils[已折叠] (648KB 931B)
309
+ 📁 src (1MB 858KB 89B)
310
+ 📁 celestialflow[已折叠] (1MB 839KB 629B)
311
+ 📁 celestialflow.egg-info[已折叠] (18KB 484B)
312
+ 📁 tests (4MB 167KB 552B)
313
+ 📁 benchmark[已折叠] (38KB 197B)
314
+ 📁 funnel[已折叠] (96KB 363B)
315
+ 📁 graph[已折叠] (721KB 564B)
316
+ 📁 observability[已折叠] (189KB 236B)
317
+ 📁 persistence[已折叠] (388KB 426B)
318
+ 📁 runtime[已折叠] (1MB 295KB 529B)
319
+ 📁 stage[已折叠] (382KB 155B)
320
+ 📁 utils[已折叠] (639KB 527B)
325
321
  📁 [1项排除的目录] (487KB 589B)
326
322
  🐍 conftest.py (1KB 38B)
327
323
  🐍 __init__.py (0B)
328
- 📁 [9项排除的目录] (458MB 223KB 525B)
329
- ❓ .env (468B)
330
- ❓ .gitignore (1KB 313B)
331
- 📝 AGENTS.md (971B)
332
- ❓ LICENSE (1KB 65B)
333
- ⚙️ pyproject.toml (2KB 555B)
334
- 📝 README.md (12KB 272B)
335
- 🔒 uv.lock (98KB 257B)
324
+ 📁 [12项排除的目录] (573MB 420KB 160B)
325
+ ❓ .env (468B)
326
+ ❓ .gitignore (1KB 314B)
327
+ 📝 AGENTS.md (1KB 173B)
328
+ ❓ LICENSE (1KB 65B)
329
+ ❓ Makefile (155B)
330
+ ⚙️ pyproject.toml (2KB 668B)
331
+ 📝 README.md (19KB 24B)
332
+ 🔒 uv.lock (97KB 510B)
336
333
  ```
337
334
  <p align="center">
338
- <em>celestial-flow 3.2.7</em>
335
+ <em>celestial-flow 3.2.8</em>
339
336
  </p>
340
337
 
341
338
  (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
342
339
 
343
340
  ## 版本日志（Version Log）
344
- - 3.2.7
341
+ - 3.2.8
345
342
  - feat:
346
- - `TaskInQueue` 中添加 `maxsize` 参数, 用于限制队列最大长度
347
- - 在 `graph.connect` 中添加对两个stage输入输出参数类型的检验, 如果不通过, 会有pyright报错
343
+ - [IMPORTANT] 添加 `TaskGraph.run_async`, 现在可以直接进行图级别的异步
344
+ - 根据 `bench_graph_mode.py` 最新测试, 在I/O密集型任务中, `serial`+`async`（6.05s），比 `serial`+`serial`（69.04s）快 **11.4x**
345
+ - 在 `core_structure` 中添加报错校验
346
+ - 不再允许设置 `execution_mode=async` 时调用 `TashExecutor.start`(如今的`TashExecutor.run`), 只能调用 `TashExecutor.run_async`
347
+ - 另外调用 `TashExecutor.run_async` 时也会进行模式检查, 不是 `execution_mode=async` 就会抛出异常
348
+ - 在 `BaseObserver` 中添加 `observer_error`, 用于处理 `BaseObserver.on_*` 函数的报错
349
+ - 在 `TaskGraph._finish_start_graph` `TaskExecutor._finish_start` 中对所有执行步骤进行 `try-except`, 以使收尾步骤尽量全部完成
350
+ - 添加 `core_scope`, 用于独立控制 `funnel` 的生命周期, 并用于 `TaskGraph.run/run_async` `TaskExecutor.run/run_async`
348
351
  - refactor:
349
- - 移除对 `tqdm` 的依赖
350
- - 根据 `bench\bench_observer.py`, tqdm对轻量级任务影响有限
351
- - 这次移除主要是为了完成尽量零第三方库依赖的目标
352
- - 将 `observer` 的管理从 `executor` 转到 `metric`
352
+ - [IMPORTANT] 原 `TaskGraph.start_graph/start_graph_async` `TaskExecutor.start/start_async` 已被重命名为 `TaskGraph.run/run_async` `TaskExecutor.run/run_async`
353
+ - 破坏性更新
354
+ - 其底层缘由是为了合并 `TaskStage.start_stage/start_stage_async` 与 `TaskExecutor.start/start_async` 而做出的一系列重构之一
355
+ - 现在的 `TaskGraph.start/start_async` `TaskExecutor.start/start_async` 不再接受任务, 而是专注于处理现有任务列表中的任务
356
+ - [IMPORTANT] 将funnel从 `TaskGraph` `TaskExecutor` 中的显性调用与显性传递, 改为所有使用端均从独立文件中import
357
+ - 一来是因为原先的传递链太丑了, 二来是为了简化 `TaskExecutor.start/start_async` 逻辑, 为其与 `TaskStage.start_stage/start_stage_async` 的合并做准备
358
+ - 移动部分文件以解决部分模块级的循环引用问题
359
+ - 原先并非文件级循环引用, 在使用上并无问题
360
+ - 修改 `TaskDispatch.dispatch_thread` 中处理已完成future的逻辑, 避免cpu浪费
361
+ - 删除 `util_errors` 中一些不必要的错误类
362
+ - 将 `TaskGraph._finalize_stages` 进行拆分, 删除不必要的机制, 将剩余机制移至其他方法
363
+ - 修改 `TaskGraph.set_reporter` 的逻辑, 使其与 `set_ctree` 保持一致
364
+ - 将 `TaskOutQueue` 中的 `queue_list|target_name|_name_to_idx` 改为 `_queues`, 并删除 `put_channel`
365
+ - 我有些困惑为什么最初我没有这么做
366
+ - 在 `TaskExecutor` 中合并 `_get_task_repr` and `_get_result_repr`
367
+ - 这两个方法原先差异巨大, 但后来经过多次其他部分的重构, 现在逻辑已经一致
368
+ - 将 `TaskStage` 中的 `_status` 交给 `TaskMetrics` 维护
369
+ - 依旧是为了合并 `TaskExecutor.start/start_async` 与 `TaskStage.start_stage/start_stage_async`
370
+ - 移除 `TaskGraph` 中的 `put_stage_queue`
371
+ - 任务输入完全使用 `TaskExecutor` 中的 `put_task` `put_signal` 方法
372
+ - fix:
373
+ - 在 `TaskDispatch.worker/worker_async` 中添加错误捕捉, 捕捉为 `CRITICAL` 级错误
374
+ - 避免 `worker` 级出错导致计数错误, 永远无法退出
375
+ - 在 `spout` 与 `reporter` 的 `stop` 操作中添加对于线程没有成功 `join` 的报错
376
+ - 修复 `TaskGraph.restore_db` 与 `TaskExecutor.restore_db` 中开启 `filter_by_error_type=Ture` 时跳过 `status=pending` 任务记录的问题
377
+ - 修复 `TaskGraph` 中部分 `get_*` 方法依赖 `_build_analysis` 的产物, 但 `_build_analysis` 未执行导致的问题
378
+ - 修复 `TaskMetrics` 中 `get_counts` 与 `is_tasks_finished` 中可能导致死锁的问题
379
+ - 修复 `TaskReporter` 中 `stop` 里进行的最后一次 `_refresh_all` 没有错误捕捉, 导致后续收尾未完成的问题
353
380
 
354
381
  更多过往日志可看:
355
382
 
{celestialflow-3.2.7 → celestialflow-3.2.8}/README.md RENAMED
@@ -239,98 +239,125 @@ flowchart TD
239
239
  ## 文件结构（File Structure）
240
240
 
241
241
  ```
242
- 📁 CelestialFlow (471MB 982KB 22B)
243
- 📁 .agents (55KB 152B)
244
- 📁 skills[已折叠] (55KB 152B)
245
- 📁 .claude (99B)
246
- 🗄️ settings.local.json (99B)
247
- 📁 .github (601B)
248
- 📁 workflows[已折叠] (601B)
249
- 📁 bench (150KB 552B)
250
- 📁 [1项排除的目录] (47KB 905B)
242
+ 📁 CelestialFlow (587MB 684KB 313B)
243
+ 📁 bench (296KB 842B)
244
+ 📁 [1项排除的目录] (194KB 222B)
251
245
  🐍 bench_datastructures.py (6KB 690B)
252
- 🐍 bench_execution_mode.py (2KB 889B)
253
- 🐍 bench_futures_memory.py (2KB 267B)
254
- 🐍 bench_gil_vs_nogil.py (10KB 159B)
255
- 🐍 bench_graph_mode.py (7KB 173B)
256
- 🐍 bench_hash.py (7KB 39B)
246
+ 🐍 bench_execution_mode.py (2KB 831B)
247
+ 🐍 bench_futures_memory.py (2KB 269B)
248
+ 🐍 bench_gil_vs_nogil.py (10KB 7B)
249
+ 🐍 bench_graph_mode.py (7KB 450B)
250
+ 🐍 bench_hash.py (7KB 67B)
257
251
  🐍 bench_hash_container.py (3KB 1009B)
258
- 🐍 bench_hash_memory.py (3KB 613B)
259
- 🐍 bench_http_grpc.py (2KB 883B)
260
- 🐍 bench_ipc_queue.py (7KB 74B)
252
+ 🐍 bench_hash_memory.py (3KB 642B)
253
+ 🐍 bench_http_grpc.py (2KB 608B)
254
+ 🐍 bench_ipc_queue.py (7KB 104B)
261
255
  🐍 bench_lock_overhead.py (9KB 421B)
262
- 🐍 bench_mpqueue_vs_shared_memory.py (13KB 106B)
263
- 🐍 bench_observer.py (7KB 899B)
264
- 🐍 bench_persistence_spout.py (4KB 193B)
256
+ 🐍 bench_mpqueue_vs_shared_memory.py (13KB 127B)
257
+ 🐍 bench_observer.py (7KB 860B)
258
+ 🐍 bench_persistence_spout.py (4KB 279B)
265
259
  🐍 bench_queue.py (5KB 857B)
266
260
  🐍 bench_requests.py (6KB 813B)
267
261
  🐍 bench_tqdm.py (1KB 235B)
268
262
  🐍 bench_utils.py (543B)
269
- 📁 demo (105KB 440B)
270
- 📁 [1项排除的目录] (61KB 883B)
271
- 🐍 demo_executor.py (1KB 504B)
263
+ 📁 demo (147KB 87B)
264
+ 📁 [1项排除的目录] (100KB 9B)
265
+ 🐍 demo_executor.py (1KB 495B)
272
266
  🐍 demo_funnel.py (2KB 289B)
273
- 🐍 demo_graph.py (3KB 615B)
274
- 🐍 demo_observer.py (4KB 293B)
275
- 🐍 demo_redis.py (9KB 201B)
276
- 🐍 demo_stages.py (4KB 519B)
277
- 🐍 demo_structure.py (12KB 8B)
278
- 🐍 demo_utils.py (6KB 200B)
279
- 📁 dist (145KB 290B)
267
+ 🐍 demo_graph.py (3KB 565B)
268
+ 🐍 demo_network.py (3KB 879B)
269
+ 🐍 demo_observer.py (4KB 270B)
270
+ 🐍 demo_redis.py (9KB 144B)
271
+ 🐍 demo_stages.py (4KB 336B)
272
+ 🐍 demo_structure.py (12KB 24B)
273
+ 🐍 demo_utils.py (6KB 148B)
274
+ 📁 dist (147KB 48B)
280
275
  ❓ .gitignore (1B)
281
- ❓ celestialflow-3.2.6-py3-none-any.whl (78KB 975B)
282
- 📦 celestialflow-3.2.6.tar.gz (66KB 338B)
283
- 📁 docs (1MB 712KB 884B)
284
- 📁 en[已折叠] (569KB 485B)
285
- 📁 ja[已折叠] (642KB 959B)
286
- 📁 zh-CN[已折叠] (524KB 464B)
287
- 📁 experiments (2KB 1015B)
276
+ ❓ celestialflow-3.2.7-py3-none-any.whl (79KB 235B)
277
+ 📦 celestialflow-3.2.7.tar.gz (67KB 836B)
278
+ 📁 docs (1MB 722KB 21B)
279
+ 📁 en[已折叠] (568KB 991B)
280
+ 📁 ja[已折叠] (642KB 383B)
281
+ 📁 zh-CN[已折叠] (534KB 695B)
282
+ 📁 experiments (2KB 1015B)
288
283
  🐍 experiment_networkx.py (1KB 878B)
289
284
  🐍 experiment_tqdm.py (1KB 137B)
290
- 📁 img (5MB 871KB 242B)
285
+ 📁 img (5MB 871KB 242B)
291
286
  📷 file_structure.svg (4MB 918KB 1000B)
292
287
  📷 logo(old).png (836KB 542B)
293
288
  📷 logo.png (122KB 747B)
294
289
  📷 scc_condensation.svg (17KB 1B)
295
- 📁 src (1MB 767KB 296B)
296
- 📁 celestialflow[已折叠] (1MB 752KB 195B)
297
- 📁 celestialflow.egg-info[已折叠] (15KB 101B)
298
- 📁 tests (3MB 902KB 217B)
299
- 📁 funnel[已折叠] (96KB 481B)
300
- 📁 graph[已折叠] (631KB 532B)
301
- 📁 observability[已折叠] (157KB 435B)
302
- 📁 persistence[已折叠] (359KB 68B)
303
- 📁 runtime[已折叠] (1MB 286KB 487B)
304
- 📁 stage[已折叠] (281KB 752B)
305
- 📁 utils[已折叠] (648KB 931B)
290
+ 📁 src (1MB 858KB 89B)
291
+ 📁 celestialflow[已折叠] (1MB 839KB 629B)
292
+ 📁 celestialflow.egg-info[已折叠] (18KB 484B)
293
+ 📁 tests (4MB 167KB 552B)
294
+ 📁 benchmark[已折叠] (38KB 197B)
295
+ 📁 funnel[已折叠] (96KB 363B)
296
+ 📁 graph[已折叠] (721KB 564B)
297
+ 📁 observability[已折叠] (189KB 236B)
298
+ 📁 persistence[已折叠] (388KB 426B)
299
+ 📁 runtime[已折叠] (1MB 295KB 529B)
300
+ 📁 stage[已折叠] (382KB 155B)
301
+ 📁 utils[已折叠] (639KB 527B)
306
302
  📁 [1项排除的目录] (487KB 589B)
307
303
  🐍 conftest.py (1KB 38B)
308
304
  🐍 __init__.py (0B)
309
- 📁 [9项排除的目录] (458MB 223KB 525B)
310
- ❓ .env (468B)
311
- ❓ .gitignore (1KB 313B)
312
- 📝 AGENTS.md (971B)
313
- ❓ LICENSE (1KB 65B)
314
- ⚙️ pyproject.toml (2KB 555B)
315
- 📝 README.md (12KB 272B)
316
- 🔒 uv.lock (98KB 257B)
305
+ 📁 [12项排除的目录] (573MB 420KB 160B)
306
+ ❓ .env (468B)
307
+ ❓ .gitignore (1KB 314B)
308
+ 📝 AGENTS.md (1KB 173B)
309
+ ❓ LICENSE (1KB 65B)
310
+ ❓ Makefile (155B)
311
+ ⚙️ pyproject.toml (2KB 668B)
312
+ 📝 README.md (19KB 24B)
313
+ 🔒 uv.lock (97KB 510B)
317
314
  ```
318
315
  <p align="center">
319
- <em>celestial-flow 3.2.7</em>
316
+ <em>celestial-flow 3.2.8</em>
320
317
  </p>
321
318
 
322
319
  (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
323
320
 
324
321
  ## 版本日志（Version Log）
325
- - 3.2.7
322
+ - 3.2.8
326
323
  - feat:
327
- - `TaskInQueue` 中添加 `maxsize` 参数, 用于限制队列最大长度
328
- - 在 `graph.connect` 中添加对两个stage输入输出参数类型的检验, 如果不通过, 会有pyright报错
324
+ - [IMPORTANT] 添加 `TaskGraph.run_async`, 现在可以直接进行图级别的异步
325
+ - 根据 `bench_graph_mode.py` 最新测试, 在I/O密集型任务中, `serial`+`async`（6.05s），比 `serial`+`serial`（69.04s）快 **11.4x**
326
+ - 在 `core_structure` 中添加报错校验
327
+ - 不再允许设置 `execution_mode=async` 时调用 `TashExecutor.start`(如今的`TashExecutor.run`), 只能调用 `TashExecutor.run_async`
328
+ - 另外调用 `TashExecutor.run_async` 时也会进行模式检查, 不是 `execution_mode=async` 就会抛出异常
329
+ - 在 `BaseObserver` 中添加 `observer_error`, 用于处理 `BaseObserver.on_*` 函数的报错
330
+ - 在 `TaskGraph._finish_start_graph` `TaskExecutor._finish_start` 中对所有执行步骤进行 `try-except`, 以使收尾步骤尽量全部完成
331
+ - 添加 `core_scope`, 用于独立控制 `funnel` 的生命周期, 并用于 `TaskGraph.run/run_async` `TaskExecutor.run/run_async`
329
332
  - refactor:
330
- - 移除对 `tqdm` 的依赖
331
- - 根据 `bench\bench_observer.py`, tqdm对轻量级任务影响有限
332
- - 这次移除主要是为了完成尽量零第三方库依赖的目标
333
- - 将 `observer` 的管理从 `executor` 转到 `metric`
333
+ - [IMPORTANT] 原 `TaskGraph.start_graph/start_graph_async` `TaskExecutor.start/start_async` 已被重命名为 `TaskGraph.run/run_async` `TaskExecutor.run/run_async`
334
+ - 破坏性更新
335
+ - 其底层缘由是为了合并 `TaskStage.start_stage/start_stage_async` 与 `TaskExecutor.start/start_async` 而做出的一系列重构之一
336
+ - 现在的 `TaskGraph.start/start_async` `TaskExecutor.start/start_async` 不再接受任务, 而是专注于处理现有任务列表中的任务
337
+ - [IMPORTANT] 将funnel从 `TaskGraph` `TaskExecutor` 中的显性调用与显性传递, 改为所有使用端均从独立文件中import
338
+ - 一来是因为原先的传递链太丑了, 二来是为了简化 `TaskExecutor.start/start_async` 逻辑, 为其与 `TaskStage.start_stage/start_stage_async` 的合并做准备
339
+ - 移动部分文件以解决部分模块级的循环引用问题
340
+ - 原先并非文件级循环引用, 在使用上并无问题
341
+ - 修改 `TaskDispatch.dispatch_thread` 中处理已完成future的逻辑, 避免cpu浪费
342
+ - 删除 `util_errors` 中一些不必要的错误类
343
+ - 将 `TaskGraph._finalize_stages` 进行拆分, 删除不必要的机制, 将剩余机制移至其他方法
344
+ - 修改 `TaskGraph.set_reporter` 的逻辑, 使其与 `set_ctree` 保持一致
345
+ - 将 `TaskOutQueue` 中的 `queue_list|target_name|_name_to_idx` 改为 `_queues`, 并删除 `put_channel`
346
+ - 我有些困惑为什么最初我没有这么做
347
+ - 在 `TaskExecutor` 中合并 `_get_task_repr` and `_get_result_repr`
348
+ - 这两个方法原先差异巨大, 但后来经过多次其他部分的重构, 现在逻辑已经一致
349
+ - 将 `TaskStage` 中的 `_status` 交给 `TaskMetrics` 维护
350
+ - 依旧是为了合并 `TaskExecutor.start/start_async` 与 `TaskStage.start_stage/start_stage_async`
351
+ - 移除 `TaskGraph` 中的 `put_stage_queue`
352
+ - 任务输入完全使用 `TaskExecutor` 中的 `put_task` `put_signal` 方法
353
+ - fix:
354
+ - 在 `TaskDispatch.worker/worker_async` 中添加错误捕捉, 捕捉为 `CRITICAL` 级错误
355
+ - 避免 `worker` 级出错导致计数错误, 永远无法退出
356
+ - 在 `spout` 与 `reporter` 的 `stop` 操作中添加对于线程没有成功 `join` 的报错
357
+ - 修复 `TaskGraph.restore_db` 与 `TaskExecutor.restore_db` 中开启 `filter_by_error_type=Ture` 时跳过 `status=pending` 任务记录的问题
358
+ - 修复 `TaskGraph` 中部分 `get_*` 方法依赖 `_build_analysis` 的产物, 但 `_build_analysis` 未执行导致的问题
359
+ - 修复 `TaskMetrics` 中 `get_counts` 与 `is_tasks_finished` 中可能导致死锁的问题
360
+ - 修复 `TaskReporter` 中 `stop` 里进行的最后一次 `_refresh_all` 没有错误捕捉, 导致后续收尾未完成的问题
334
361
 
335
362
  更多过往日志可看:
336
363
 
{celestialflow-3.2.7 → celestialflow-3.2.8}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "celestialflow"
7
- version = "3.2.7"
7
+ version = "3.2.8"
8
8
  description = "A flexible GRAPH-based task orchestration framework."
9
9
  readme = "README.md"
10
10
  license = { text = "MIT" }
@@ -27,6 +27,9 @@ classifiers = [
27
27
  "Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
28
28
  "Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
29
29
 
30
+ [tool.uv]
31
+ index-url = "https://pypi.tuna.tsinghua.edu.cn/simple"
32
+
30
33
  [tool.pytest.ini_options]
31
34
  addopts = "-s"
32
35
  log_cli = true
@@ -56,6 +59,9 @@ dev = [
56
59
  "tqdm",
57
60
  ]
58
61
 
62
+ [tool.celestialflow]
63
+ log_level = "INFO"
64
+
59
65
  [tool.pyright]
60
66
  include = ["src"]
61
67
  ignore = ["tests", "**/tests/**", "typings", "demo", "bench", "experiments", "examples", "temp"]
{celestialflow-3.2.7 → celestialflow-3.2.8}/src/celestialflow/__init__.py RENAMED
@@ -4,6 +4,7 @@
4
4
  提供任务图构建、执行调度、实时监控与持久化等核心能力。
5
5
  """
6
6
 
7
+ from .benchmark.util_benchmark import benchmark_executor, benchmark_graph
7
8
  from .funnel import BaseInlet, BaseSpout
8
9
  from .graph import (
9
10
  TaskChain,
@@ -14,11 +15,12 @@ from .graph import (
14
15
  TaskLoop,
15
16
  TaskWheel,
16
17
  )
17
- from .observability import BaseObserver
18
+ from .observability import BaseObserver, TaskReporter
18
19
  from .persistence.util_sqlite import (
19
20
  load_records,
20
21
  load_tasks_grouped_by_stage,
21
22
  )
23
+ from .runtime.util_format import format_table
22
24
  from .runtime.util_hash import make_hashable
23
25
  from .runtime.util_types import TerminationSignal
24
26
  from .stage import (
@@ -27,8 +29,6 @@ from .stage import (
27
29
  TaskSplitter,
28
30
  TaskStage,
29
31
  )
30
- from .utils.util_benchmark import benchmark_executor, benchmark_graph
31
- from .utils.util_format import format_table
32
32
 
33
33
  __all__ = [
34
34
  "BaseInlet",
@@ -41,6 +41,7 @@ __all__ = [
41
41
  "TaskGraph",
42
42
  "TaskGrid",
43
43
  "TaskLoop",
44
+ "TaskReporter",
44
45
  "TaskRouter",
45
46
  "TaskSplitter",
46
47
  "TaskStage",
celestialflow-3.2.8/src/celestialflow/benchmark/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ # benchmark/__init__.py
2
+ """CelestialFlow 基准测试与克隆工具模块。
3
+
4
+ 提供执行器/任务图的克隆（clone）与基准测试（benchmark）能力。
5
+ 该模块位于依赖链顶层，可依赖其他模块，但不应被其他模块依赖。
6
+ """
{celestialflow-3.2.7/src/celestialflow/utils → celestialflow-3.2.8/src/celestialflow/benchmark}/util_benchmark.py RENAMED
@@ -1,4 +1,4 @@
1
- # utils/util_benchmark.py
1
+ # benchmark/util_benchmark.py
2
2
  from __future__ import annotations
3
3
 
4
4
  import time
@@ -6,9 +6,9 @@ from collections.abc import Iterable, Mapping
6
6
  from typing import Any
7
7
 
8
8
  from ..graph import TaskGraph
9
+ from ..runtime.util_format import format_table
9
10
  from ..stage import TaskExecutor
10
11
  from .util_clone import clone_executor, clone_graph
11
- from .util_format import format_table
12
12
 
13
13
  type AnyTaskExecutor = TaskExecutor[Any, Any]
14
14
 
@@ -41,7 +41,8 @@ async def benchmark_executor(
41
41
  cloned_executor.set_execution_mode(mode)
42
42
 
43
43
  start: float = time.perf_counter()
44
- cloned_executor.start(task_list)
44
+ cloned_executor.put_tasks(task_list)
45
+ cloned_executor.start()
45
46
  use_time.append([time.perf_counter() - start])
46
47
  results.append(cloned_executor.get_success_pairs())
47
48
 
@@ -50,7 +51,8 @@ async def benchmark_executor(
50
51
  cloned_executor.set_execution_mode(mode)
51
52
 
52
53
  start = time.perf_counter()
53
- await cloned_executor.start_async(task_list)
54
+ cloned_executor.put_tasks(task_list)
55
+ await cloned_executor.start_async()
54
56
  use_time.append([time.perf_counter() - start])
55
57
  results.append(cloned_executor.get_success_pairs())
56
58
 
@@ -68,7 +70,7 @@ async def benchmark_executor(
68
70
  }
69
71
 
70
72
 
71
- def benchmark_graph(
73
+ async def benchmark_graph(
72
74
  sync_graph: TaskGraph,
73
75
  async_graph: TaskGraph,
74
76
  init_tasks_dict: Mapping[str, Iterable[Any]],
@@ -88,38 +90,39 @@ def benchmark_graph(
88
90
  :return: 包含测试结果的字典
89
91
  """
90
92
  stage_modes = stage_modes or ["serial", "thread"]
91
- sync_modes = execution_sync_modes or ["serial", "thread"]
92
- async_modes = execution_async_modes or ["async"]
93
+ execution_sync_modes = execution_sync_modes or ["serial", "thread"]
94
+ execution_async_modes = execution_async_modes or ["async"]
93
95
 
94
96
  base_tasks: dict[str, list[Any]] = {
95
97
  stage_name: list(tasks) for stage_name, tasks in init_tasks_dict.items()
96
98
  }
97
- execution_modes: list[str] = sync_modes + async_modes
99
+ execution_modes: list[str] = execution_sync_modes + execution_async_modes
98
100
 
99
101
  test_table_list: list[list[float]] = []
100
102
 
101
103
  for stage_mode in stage_modes:
102
104
  time_list: list[float] = []
103
- for execution_mode in sync_modes:
105
+
106
+ for execution_mode in execution_sync_modes:
104
107
  cloned_graph: TaskGraph = clone_graph(sync_graph)
105
108
  cloned_graph.set_graph_mode(stage_mode, execution_mode)
106
109
 
107
- run_tasks: dict[str, list[Any]] = {
110
+ sync_run_tasks: dict[str, Iterable[Any]] = {
108
111
  stage_name: list(tasks) for stage_name, tasks in base_tasks.items()
109
112
  }
110
113
  start_time: float = time.perf_counter()
111
- cloned_graph.start_graph(run_tasks)
114
+ cloned_graph.run(sync_run_tasks)
112
115
  time_list.append(time.perf_counter() - start_time)
113
116
 
114
- for execution_mode in async_modes:
117
+ for execution_mode in execution_async_modes:
115
118
  cloned_graph = clone_graph(async_graph)
116
119
  cloned_graph.set_graph_mode(stage_mode, execution_mode)
117
120
 
118
- run_tasks = {
121
+ async_run_tasks: dict[str, Iterable[Any]] = {
119
122
  stage_name: list(tasks) for stage_name, tasks in base_tasks.items()
120
123
  }
121
124
  start_time = time.perf_counter()
122
- cloned_graph.start_graph(run_tasks)
125
+ await cloned_graph.run_async(async_run_tasks)
123
126
  time_list.append(time.perf_counter() - start_time)
124
127
 
125
128
  test_table_list.append(time_list)
@@ -128,12 +131,12 @@ def benchmark_graph(
128
131
  test_table_list,
129
132
  stage_modes,
130
133
  execution_modes,
131
- r"stage\execution",
134
+ "stage/execution",
132
135
  )
133
136
  print(f"Time table:\n{time_table}")
134
137
  return {
135
138
  "table": time_table,
136
139
  "stage_modes": stage_modes,
137
- "sync_modes": sync_modes,
138
- "async_modes": async_modes if async_graph else [],
140
+ "sync_modes": execution_sync_modes,
141
+ "async_modes": execution_async_modes if async_graph else [],
139
142
  }