celestialflow 3.2.6__tar.gz → 3.2.7__tar.gz

{celestialflow-3.2.6/src/celestialflow.egg-info → celestialflow-3.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: celestialflow
-Version: 3.2.6
+Version: 3.2.7
 Summary: A flexible GRAPH-based task orchestration framework.
 Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
 License: MIT
@@ -14,7 +14,6 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: tqdm
 Requires-Dist: requests
 Dynamic: license-file
 
@@ -134,16 +133,27 @@ uv sync --group dev
 ```python
 from celestialflow import TaskStage, TaskGraph
 
-def add(x, y): 
+
+def add(x, y):
     return x + y
 
-def square(x): 
-    return x ** 2
+
+def square(x):
+    return x**2
+
 
 if __name__ == "__main__":
     # 定义两个任务节点
-    stage1 = TaskStage(name="Adder", func=add, stage_mode="thread", execution_mode="thread", unpack_task_args=True)
-    stage2 = TaskStage(name="Squarer", func=square, stage_mode="thread", execution_mode="thread")
+    stage1 = TaskStage(
+        name="Adder",
+        func=add,
+        stage_mode="thread",
+        execution_mode="thread",
+        unpack_task_args=True,
+    )
+    stage2 = TaskStage(
+        name="Squarer", func=square, stage_mode="thread", execution_mode="thread"
+    )
 
     # 构建任务图结构
     graph = TaskGraph(name="DemoGraph")
@@ -165,12 +175,12 @@ if __name__ == "__main__":
 - [TaskExecutor.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/stage/core_executor.md)
 - [TaskStage.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/stage/core_stage.md)
 - [TaskGraph.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/graph/core_graph.md)
-- [TaskProgress.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/observability/core_progress.md)
 - [TaskMetrics.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/core_metrics.md)
 - [TaskQueue.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/core_queue.md)
 - [TaskStages.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/stage/core_stages.md)
 - [TaskReport.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/observability/core_report.md)
 - [TaskStructure.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/graph/core_structure.md)
+- [BaseObserver.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/observability/core_observer.md)
 - [Go Worker.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/other/go_worker.md)
 
 推荐阅读顺序:
@@ -183,7 +193,7 @@ flowchart TD
     classDef execution fill:#f3e8ff,stroke:#a855f7,color:#581c87;
 
     TM[TaskExecutor.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
-    TM --> TP[TaskProgress.md]
+    TM --> OB[BaseObserver.md]
     TM --> TME[TaskMetrics.md]
 
     TG --> TQ[TaskQueue.md]
@@ -247,26 +257,99 @@ flowchart TD
 
 ## 文件结构（File Structure）
 
+```
+📁 CelestialFlow	(471MB 982KB 22B)
+    📁 .agents        	(55KB 152B)
+        📁 skills[已折叠]	(55KB 152B)
+    📁 .claude        	(99B)
+        🗄️ settings.local.json	(99B)
+    📁 .github        	(601B)
+        📁 workflows[已折叠]	(601B)
+    📁 bench          	(150KB 552B)
+        📁 [1项排除的目录]                  	(47KB 905B)
+        🐍 bench_datastructures.py          	(6KB 690B)
+        🐍 bench_execution_mode.py          	(2KB 889B)
+        🐍 bench_futures_memory.py          	(2KB 267B)
+        🐍 bench_gil_vs_nogil.py            	(10KB 159B)
+        🐍 bench_graph_mode.py              	(7KB 173B)
+        🐍 bench_hash.py                    	(7KB 39B)
+        🐍 bench_hash_container.py          	(3KB 1009B)
+        🐍 bench_hash_memory.py             	(3KB 613B)
+        🐍 bench_http_grpc.py               	(2KB 883B)
+        🐍 bench_ipc_queue.py               	(7KB 74B)
+        🐍 bench_lock_overhead.py           	(9KB 421B)
+        🐍 bench_mpqueue_vs_shared_memory.py	(13KB 106B)
+        🐍 bench_observer.py                	(7KB 899B)
+        🐍 bench_persistence_spout.py       	(4KB 193B)
+        🐍 bench_queue.py                   	(5KB 857B)
+        🐍 bench_requests.py                	(6KB 813B)
+        🐍 bench_tqdm.py                    	(1KB 235B)
+        🐍 bench_utils.py                   	(543B)
+    📁 demo           	(105KB 440B)
+        📁 [1项排除的目录]  	(61KB 883B)
+        🐍 demo_executor.py 	(1KB 504B)
+        🐍 demo_funnel.py   	(2KB 289B)
+        🐍 demo_graph.py    	(3KB 615B)
+        🐍 demo_observer.py 	(4KB 293B)
+        🐍 demo_redis.py    	(9KB 201B)
+        🐍 demo_stages.py   	(4KB 519B)
+        🐍 demo_structure.py	(12KB 8B)
+        🐍 demo_utils.py    	(6KB 200B)
+    📁 dist           	(145KB 290B)
+        ❓ .gitignore                          	(1B)
+        ❓ celestialflow-3.2.6-py3-none-any.whl	(78KB 975B)
+        📦 celestialflow-3.2.6.tar.gz          	(66KB 338B)
+    📁 docs           	(1MB 712KB 884B)
+        📁 en[已折叠]   	(569KB 485B)
+        📁 ja[已折叠]   	(642KB 959B)
+        📁 zh-CN[已折叠]	(524KB 464B)
+    📁 experiments    	(2KB 1015B)
+        🐍 experiment_networkx.py	(1KB 878B)
+        🐍 experiment_tqdm.py    	(1KB 137B)
+    📁 img            	(5MB 871KB 242B)
+        📷 file_structure.svg  	(4MB 918KB 1000B)
+        📷 logo(old).png       	(836KB 542B)
+        📷 logo.png            	(122KB 747B)
+        📷 scc_condensation.svg	(17KB 1B)
+    📁 src            	(1MB 767KB 296B)
+        📁 celestialflow[已折叠]         	(1MB 752KB 195B)
+        📁 celestialflow.egg-info[已折叠]	(15KB 101B)
+    📁 tests          	(3MB 902KB 217B)
+        📁 funnel[已折叠]       	(96KB 481B)
+        📁 graph[已折叠]        	(631KB 532B)
+        📁 observability[已折叠]	(157KB 435B)
+        📁 persistence[已折叠]  	(359KB 68B)
+        📁 runtime[已折叠]      	(1MB 286KB 487B)
+        📁 stage[已折叠]        	(281KB 752B)
+        📁 utils[已折叠]        	(648KB 931B)
+        📁 [1项排除的目录]      	(487KB 589B)
+        🐍 conftest.py          	(1KB 38B)
+        🐍 __init__.py          	(0B)
+    📁 [9项排除的目录]	(458MB 223KB 525B)
+    ❓ .env           	(468B)
+    ❓ .gitignore     	(1KB 313B)
+    📝 AGENTS.md      	(971B)
+    ❓ LICENSE        	(1KB 65B)
+    ⚙️ pyproject.toml 	(2KB 555B)
+    📝 README.md      	(12KB 272B)
+    🔒 uv.lock        	(98KB 257B)
+```
 <p align="center">
-  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
-  <br/>
-  <em>celestial-flow 3.2.6</em>
+  <em>celestial-flow 3.2.7</em>
 </p>
 
 (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
 
 ## 版本日志（Version Log）
-- 3.2.6
+- 3.2.7
   - feat:
-    - 在 `graph` 与 `stage` 中添加直接读取db文件并重试/继续任务的方法
-      - 分别为 `start_graph_db` 与 `start_db`
+    - `TaskInQueue` 中添加 `maxsize` 参数, 用于限制队列最大长度
+    - 在 `graph.connect` 中添加对两个stage输入输出参数类型的检验, 如果不通过, 会有pyright报错
   - refactor:
-    - **[IMPORTANT]** 将web部分移至单独项目 [celestialflow-web](https://github.com/Mr-xiaotian/celestialflow-web)
-      - 考虑许久的决定, 当前的web端代码与项目整体风格差异极大, 已经不再适合继续合并一处
-      - 当然这意味着 `celestialflow-web` 命令在当前项目下失效, 需要单独安装 `pip install celestialflow-web`
-  - chore:
-    - 全量更新文档, 并翻译为en/ja双语言
-    - 移除 docs/zh-CN 下的 `README.md`, 现在中文readme只保留根目录下的这一份
+    - 移除对 `tqdm` 的依赖
+      - 根据 `bench\bench_observer.py`, tqdm对轻量级任务影响有限
+      - 这次移除主要是为了完成尽量零第三方库依赖的目标
+    - 将 `observer` 的管理从 `executor` 转到 `metric`
 
 更多过往日志可看:
 

{celestialflow-3.2.6 → celestialflow-3.2.7}/README.md

@@ -114,16 +114,27 @@ uv sync --group dev
 ```python
 from celestialflow import TaskStage, TaskGraph
 
-def add(x, y): 
+
+def add(x, y):
     return x + y
 
-def square(x): 
-    return x ** 2
+
+def square(x):
+    return x**2
+
 
 if __name__ == "__main__":
     # 定义两个任务节点
-    stage1 = TaskStage(name="Adder", func=add, stage_mode="thread", execution_mode="thread", unpack_task_args=True)
-    stage2 = TaskStage(name="Squarer", func=square, stage_mode="thread", execution_mode="thread")
+    stage1 = TaskStage(
+        name="Adder",
+        func=add,
+        stage_mode="thread",
+        execution_mode="thread",
+        unpack_task_args=True,
+    )
+    stage2 = TaskStage(
+        name="Squarer", func=square, stage_mode="thread", execution_mode="thread"
+    )
 
     # 构建任务图结构
     graph = TaskGraph(name="DemoGraph")
@@ -145,12 +156,12 @@ if __name__ == "__main__":
 - [TaskExecutor.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/stage/core_executor.md)
 - [TaskStage.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/stage/core_stage.md)
 - [TaskGraph.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/graph/core_graph.md)
-- [TaskProgress.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/observability/core_progress.md)
 - [TaskMetrics.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/core_metrics.md)
 - [TaskQueue.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/runtime/core_queue.md)
 - [TaskStages.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/stage/core_stages.md)
 - [TaskReport.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/observability/core_report.md)
 - [TaskStructure.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/graph/core_structure.md)
+- [BaseObserver.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/src/observability/core_observer.md)
 - [Go Worker.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/other/go_worker.md)
 
 推荐阅读顺序:
@@ -163,7 +174,7 @@ flowchart TD
     classDef execution fill:#f3e8ff,stroke:#a855f7,color:#581c87;
 
     TM[TaskExecutor.md] --> TS[TaskStage.md] --> TG[TaskGraph.md]
-    TM --> TP[TaskProgress.md]
+    TM --> OB[BaseObserver.md]
     TM --> TME[TaskMetrics.md]
 
     TG --> TQ[TaskQueue.md]
@@ -227,26 +238,99 @@ flowchart TD
 
 ## 文件结构（File Structure）
 
+```
+📁 CelestialFlow	(471MB 982KB 22B)
+    📁 .agents        	(55KB 152B)
+        📁 skills[已折叠]	(55KB 152B)
+    📁 .claude        	(99B)
+        🗄️ settings.local.json	(99B)
+    📁 .github        	(601B)
+        📁 workflows[已折叠]	(601B)
+    📁 bench          	(150KB 552B)
+        📁 [1项排除的目录]                  	(47KB 905B)
+        🐍 bench_datastructures.py          	(6KB 690B)
+        🐍 bench_execution_mode.py          	(2KB 889B)
+        🐍 bench_futures_memory.py          	(2KB 267B)
+        🐍 bench_gil_vs_nogil.py            	(10KB 159B)
+        🐍 bench_graph_mode.py              	(7KB 173B)
+        🐍 bench_hash.py                    	(7KB 39B)
+        🐍 bench_hash_container.py          	(3KB 1009B)
+        🐍 bench_hash_memory.py             	(3KB 613B)
+        🐍 bench_http_grpc.py               	(2KB 883B)
+        🐍 bench_ipc_queue.py               	(7KB 74B)
+        🐍 bench_lock_overhead.py           	(9KB 421B)
+        🐍 bench_mpqueue_vs_shared_memory.py	(13KB 106B)
+        🐍 bench_observer.py                	(7KB 899B)
+        🐍 bench_persistence_spout.py       	(4KB 193B)
+        🐍 bench_queue.py                   	(5KB 857B)
+        🐍 bench_requests.py                	(6KB 813B)
+        🐍 bench_tqdm.py                    	(1KB 235B)
+        🐍 bench_utils.py                   	(543B)
+    📁 demo           	(105KB 440B)
+        📁 [1项排除的目录]  	(61KB 883B)
+        🐍 demo_executor.py 	(1KB 504B)
+        🐍 demo_funnel.py   	(2KB 289B)
+        🐍 demo_graph.py    	(3KB 615B)
+        🐍 demo_observer.py 	(4KB 293B)
+        🐍 demo_redis.py    	(9KB 201B)
+        🐍 demo_stages.py   	(4KB 519B)
+        🐍 demo_structure.py	(12KB 8B)
+        🐍 demo_utils.py    	(6KB 200B)
+    📁 dist           	(145KB 290B)
+        ❓ .gitignore                          	(1B)
+        ❓ celestialflow-3.2.6-py3-none-any.whl	(78KB 975B)
+        📦 celestialflow-3.2.6.tar.gz          	(66KB 338B)
+    📁 docs           	(1MB 712KB 884B)
+        📁 en[已折叠]   	(569KB 485B)
+        📁 ja[已折叠]   	(642KB 959B)
+        📁 zh-CN[已折叠]	(524KB 464B)
+    📁 experiments    	(2KB 1015B)
+        🐍 experiment_networkx.py	(1KB 878B)
+        🐍 experiment_tqdm.py    	(1KB 137B)
+    📁 img            	(5MB 871KB 242B)
+        📷 file_structure.svg  	(4MB 918KB 1000B)
+        📷 logo(old).png       	(836KB 542B)
+        📷 logo.png            	(122KB 747B)
+        📷 scc_condensation.svg	(17KB 1B)
+    📁 src            	(1MB 767KB 296B)
+        📁 celestialflow[已折叠]         	(1MB 752KB 195B)
+        📁 celestialflow.egg-info[已折叠]	(15KB 101B)
+    📁 tests          	(3MB 902KB 217B)
+        📁 funnel[已折叠]       	(96KB 481B)
+        📁 graph[已折叠]        	(631KB 532B)
+        📁 observability[已折叠]	(157KB 435B)
+        📁 persistence[已折叠]  	(359KB 68B)
+        📁 runtime[已折叠]      	(1MB 286KB 487B)
+        📁 stage[已折叠]        	(281KB 752B)
+        📁 utils[已折叠]        	(648KB 931B)
+        📁 [1项排除的目录]      	(487KB 589B)
+        🐍 conftest.py          	(1KB 38B)
+        🐍 __init__.py          	(0B)
+    📁 [9项排除的目录]	(458MB 223KB 525B)
+    ❓ .env           	(468B)
+    ❓ .gitignore     	(1KB 313B)
+    📝 AGENTS.md      	(971B)
+    ❓ LICENSE        	(1KB 65B)
+    ⚙️ pyproject.toml 	(2KB 555B)
+    📝 README.md      	(12KB 272B)
+    🔒 uv.lock        	(98KB 257B)
+```
 <p align="center">
-  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
-  <br/>
-  <em>celestial-flow 3.2.6</em>
+  <em>celestial-flow 3.2.7</em>
 </p>
 
 (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
 
 ## 版本日志（Version Log）
-- 3.2.6
+- 3.2.7
   - feat:
-    - 在 `graph` 与 `stage` 中添加直接读取db文件并重试/继续任务的方法
-      - 分别为 `start_graph_db` 与 `start_db`
+    - `TaskInQueue` 中添加 `maxsize` 参数, 用于限制队列最大长度
+    - 在 `graph.connect` 中添加对两个stage输入输出参数类型的检验, 如果不通过, 会有pyright报错
   - refactor:
-    - **[IMPORTANT]** 将web部分移至单独项目 [celestialflow-web](https://github.com/Mr-xiaotian/celestialflow-web)
-      - 考虑许久的决定, 当前的web端代码与项目整体风格差异极大, 已经不再适合继续合并一处
-      - 当然这意味着 `celestialflow-web` 命令在当前项目下失效, 需要单独安装 `pip install celestialflow-web`
-  - chore:
-    - 全量更新文档, 并翻译为en/ja双语言
-    - 移除 docs/zh-CN 下的 `README.md`, 现在中文readme只保留根目录下的这一份
+    - 移除对 `tqdm` 的依赖
+      - 根据 `bench\bench_observer.py`, tqdm对轻量级任务影响有限
+      - 这次移除主要是为了完成尽量零第三方库依赖的目标
+    - 将 `observer` 的管理从 `executor` 转到 `metric`
 
 更多过往日志可看:
 

{celestialflow-3.2.6 → celestialflow-3.2.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "celestialflow"
-version = "3.2.6"
+version = "3.2.7"
 description = "A flexible GRAPH-based task orchestration framework."
 readme = "README.md"
 license = { text = "MIT" }
@@ -13,7 +13,6 @@ keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
 requires-python = ">=3.12"
 
 dependencies = [
-    "tqdm",
     "requests",
 ]
 
@@ -54,12 +53,14 @@ dev = [
     "pyright>=1.1.409",
     "redis",
     "celestialtree>=0.1.3",
+    "tqdm",
 ]
 
 [tool.pyright]
 include = ["src"]
 ignore = ["tests", "**/tests/**", "typings", "demo", "bench", "experiments", "examples", "temp"]
 
+pythonVersion = "3.14"
 typeCheckingMode = "strict"
 reportUnusedFunction = "none"
 reportImplicitOverride = "none"

{celestialflow-3.2.6 → celestialflow-3.2.7}/src/celestialflow/__init__.py

@@ -14,7 +14,7 @@ from .graph import (
     TaskLoop,
     TaskWheel,
 )
-from .observability import BaseObserver, TaskProgress
+from .observability import BaseObserver
 from .persistence.util_sqlite import (
     load_records,
     load_tasks_grouped_by_stage,
@@ -41,7 +41,6 @@ __all__ = [
     "TaskGraph",
     "TaskGrid",
     "TaskLoop",
-    "TaskProgress",
     "TaskRouter",
     "TaskSplitter",
     "TaskStage",

{celestialflow-3.2.6 → celestialflow-3.2.7}/src/celestialflow/graph/core_graph.py

@@ -27,6 +27,7 @@ from ..runtime.util_estimators import (
 )
 from ..runtime.util_event import EventClient, LocalEventClient
 from ..runtime.util_types import TerminationSignal
+from ..stage.core_stage import TaskStage
 from ..stage.util_types import AnyTaskStage
 from ..utils.util_collections import cluster_by_value_sorted
 from ..utils.util_format import format_avg_time
@@ -103,20 +104,27 @@ class TaskGraph:
         """
         # 用于保存所有子线程的引用
         self.threads: list[threading.Thread] = []
+
         # 用于保存每个节点的运行信息
         self.stage_dict: dict[str, AnyTaskStage] = {}
+
         # 用于保存每个节点的上一次collect_runtime_snapshot()的状态信息
         self.status_dict: dict[str, dict[str, Any]] = defaultdict(dict)
+
         # 用于保存最近一次状态快照对应的统一时间戳
         self.status_timestamp: float = 0.0
+
         # 用于保存每个节点的输入任务ID集合
         self.input_ids: dict[str, set[int]] = defaultdict(set)
+
         # 用于保存源节点列表（由 _build_analysis 自动计算）
         self.source_stages: list[AnyTaskStage] = []
+
         # 用于保存图结构的邻接表
         self.out_edges: dict[str, list[str]] = defaultdict(list)
         self.in_edges: dict[str, list[str]] = defaultdict(list)
         self.order_graph = OrderGraph()
+
         # 用于保存任务图启动时间
         self.start_time: float = 0.0
 
@@ -157,10 +165,10 @@ class TaskGraph:
             stage.set_ctree(self.ctree_client)
             stage.set_inlet(self.fallback_inlet, self.log_inlet)
 
-    def connect(
+    def connect[R](
         self,
-        from_stages: list[AnyTaskStage],
-        to_stages: list[AnyTaskStage],
+        from_stages: list[TaskStage[Any, R]],
+        to_stages: list[TaskStage[R, Any]],
     ) -> None:
         """
         建立超边连接：from_stages 中的每个节点连接到 to_stages 中的每个节点。

{celestialflow-3.2.6 → celestialflow-3.2.7}/src/celestialflow/observability/__init__.py

@@ -5,12 +5,10 @@
 """
 
 from .core_observer import BaseObserver
-from .core_progress import TaskProgress
 from .core_report import NullTaskReporter, TaskReporter
 
 __all__ = [
     "BaseObserver",
     "NullTaskReporter",
-    "TaskProgress",
     "TaskReporter",
 ]

{celestialflow-3.2.6 → celestialflow-3.2.7}/src/celestialflow/runtime/core_dispatch.py

@@ -210,9 +210,14 @@ class TaskDispatch[T, R]:
                 if self._pool is None:
                     raise InitializationError("execution pool has not been initialized")
 
-                futures.append(self._pool.submit(self._worker, envelope))
-                if len(futures) >= self.max_workers * 2:
+                # 等待在途任务数低于水位线
+                while True:
+                    # 清理已完成的任务
                     futures = [f for f in futures if not f.done()]
+                    if len(futures) < self.max_workers:
+                        break
+                    time.sleep(0.001)  # 简短让出
+                futures.append(self._pool.submit(self._worker, envelope))
 
             # 等待当前批次的所有任务完成
             for future in futures:

{celestialflow-3.2.6 → celestialflow-3.2.7}/src/celestialflow/runtime/core_metrics.py

@@ -2,9 +2,13 @@
 from __future__ import annotations
 
 from threading import Lock
+from typing import TYPE_CHECKING
 
 from .util_types import SumCounter, ValueWrapper
 
+if TYPE_CHECKING:
+    from ..observability import BaseObserver
+
 
 class TaskMetrics:
     """
@@ -35,6 +39,7 @@ class TaskMetrics:
         """
         self.enable_duplicate_check = enable_duplicate_check
         self.retry_exceptions = ()
+        self._observers: list[BaseObserver] = []
 
         self.lock = Lock()
         self._init_counter()
@@ -51,6 +56,7 @@ class TaskMetrics:
         self.error_counter = ValueWrapper(value=0, lock=self.lock)
         self.duplicate_counter = ValueWrapper(value=0, lock=self.lock)
 
+    # ==== 重置 ====
     def reset_counter(self) -> None:
         """
         重置计数器
@@ -69,6 +75,23 @@ class TaskMetrics:
         """
         self.processed_set = set()  # task_hash
 
+    # ==== 观察者 ====
+    def add_observer(self, observer: BaseObserver) -> None:
+        """
+        注册观察者。
+
+        :param observer: 要注册的观察者实例
+        """
+        self._observers.append(observer)
+
+    def remove_observer(self, observer: BaseObserver) -> None:
+        """
+        移除观察者。
+
+        :param observer: 要移除的观察者实例
+        """
+        self._observers.remove(observer)
+
     # ==== 去重 ====
     def is_duplicate(self, task_hash: bytes) -> bool:
         """
@@ -125,6 +148,8 @@ class TaskMetrics:
         :param add_count: 增加的任务数
         """
         self.task_counter.add(add_count)
+        for observer in self._observers:
+            observer.on_tasks_added(add_count)
 
     def add_success_count(self, count: int = 1) -> None:
         """
@@ -135,8 +160,10 @@ class TaskMetrics:
         :param count: 增加的成功任务数量，默认值为 1。
         """
         self.success_counter.add(count)
+        for observer in self._observers:
+            observer.on_task_success(count)
 
-    def add_error_count(self, count: int = 1) -> None:
+    def add_fail_count(self, count: int = 1) -> None:
         """
         更新失败任务计数器
 
@@ -145,6 +172,8 @@ class TaskMetrics:
         :param count: 增加的失败任务数量，默认值为 1。
         """
         self.error_counter.add(count)
+        for observer in self._observers:
+            observer.on_task_fail(count)
 
     def add_duplicate_count(self, count: int = 1) -> None:
         """
@@ -155,6 +184,17 @@ class TaskMetrics:
         :param count: 增加的重复任务数量，默认值为 1。
         """
         self.duplicate_counter.add(count)
+        for observer in self._observers:
+            observer.on_task_duplicate(count)
+
+    # ==== 启动与结束 ====
+    def on_start(self, _name: str, _total: int) -> None:
+        for observer in self._observers:
+            observer.on_start(_name, _total)
+
+    def on_finish(self) -> None:
+        for observer in self._observers:
+            observer.on_finish()
 
     # ==== 查询 ====
     def is_tasks_finished(self) -> bool:

{celestialflow-3.2.6 → celestialflow-3.2.7}/src/celestialflow/runtime/core_queue.py

@@ -1,12 +1,11 @@
 # runtime/core_queue.py
 from __future__ import annotations
 
-from queue import Empty
+from queue import Empty, Queue
 from typing import Any
 
 from .core_envelope import TaskEnvelope
 from .util_errors import (
-    ConfigurationError,
     DuplicateNodeError,
     TerminationMergeError,
     UnknownNodeError,
@@ -18,29 +17,27 @@ from .util_types import TerminationIdPool, TerminationSignal
 class TaskInQueue[T]:
     """任务输入队列，聚合多个上游来源的任务和终止信号。"""
 
+    out_name: str
     queue: Any
     source_names: list[str]
-    out_name: str
     termination_dict: dict[str, int]
 
     # ==== 初始化 ====
     def __init__(
         self,
-        queue: Any,
-        source_names: list[str],
         out_name: str,
+        maxsize: int = 0,
     ) -> None:
         """
         初始化任务入队
 
-        :param queue: 队列对象
-        :param source_names: 上游节点名称列表
         :param out_name: 当前节点唯一名称
+        :param maxsize: 队列最大容量，默认为 0（无限制）
         """
-        self.queue = queue
-        self.source_names = source_names
         self.out_name = out_name
+        self.queue = Queue(maxsize=maxsize)
 
+        self.source_names = []
         self.termination_dict = {}
 
     # ==== 添加 ====
@@ -181,36 +178,26 @@ class TaskInQueue[T]:
 class TaskOutQueue[T]:
     """任务输出队列，将任务广播到一个或多个下游队列通道。"""
 
+    in_name: str
     queue_list: list[Any]
     target_names: list[str]
-    in_name: str
     _name_to_idx: dict[str, int]
 
     # ==== 初始化 ====
     def __init__(
         self,
-        queue_list: list[Any],
-        target_names: list[str],
         in_name: str,
     ) -> None:
         """
         任务输出队列类，用于管理多个输出队列
 
-        :param queue_list: 输出队列列表，每个元素为一个线程队列或异步队列
-        :param target_names: 下游节点名称列表，用于标识每个队列
         :param in_name: 当前节点唯一名称，用于记录日志
-        :raises ConfigurationError: 如果队列列表和目标名称列表长度不一致
         """
-        if len(queue_list) != len(target_names):
-            raise ConfigurationError(
-                "queue_list and target_names must have the same length"
-            )
-
-        self.queue_list = queue_list
-        self.target_names = target_names
         self.in_name = in_name
 
-        self._name_to_idx = {name: i for i, name in enumerate(target_names)}
+        self.queue_list = []
+        self.target_names = []
+        self._name_to_idx = {}
 
     # ==== put ====