celestialflow 3.0.3__tar.gz → 3.0.4__tar.gz

{celestialflow-3.0.3 → celestialflow-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: celestialflow
-Version: 3.0.3
+Version: 3.0.4
 Summary: A flexible GRAPH-based task orchestration framework.
 Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
 License: MIT
@@ -26,7 +26,7 @@ Requires-Dist: jinja2
 # CelestialFlow ——一个轻量级、可并行、基于图结构的 Python 任务调度框架
 
 <p align="center">
-  <img src="img/startup.png" width="1080" alt="CelestialFlow Logo">
+  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/logo.png" width="1080" alt="CelestialFlow Logo">
 </p>
 
 <p align="center">
@@ -111,7 +111,7 @@ python src/celestialflow/task_web.py 5005
 
 可查看任务结构、执行状态、错误日志、以及实时注入任务等功能。
 
-![](img/web_display.png)
+![web_display.png](https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/web_display.png)
 
 ### 运行测试示例
 

{celestialflow-3.0.3 → celestialflow-3.0.4}/README.md

@@ -1,7 +1,7 @@
 # CelestialFlow ——一个轻量级、可并行、基于图结构的 Python 任务调度框架
 
 <p align="center">
-  <img src="img/startup.png" width="1080" alt="CelestialFlow Logo">
+  <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/logo.png" width="1080" alt="CelestialFlow Logo">
 </p>
 
 <p align="center">
@@ -86,7 +86,7 @@ python src/celestialflow/task_web.py 5005
 
 可查看任务结构、执行状态、错误日志、以及实时注入任务等功能。
 
-![](img/web_display.png)
+![web_display.png](https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/web_display.png)
 
 ### 运行测试示例
 

{celestialflow-3.0.3 → celestialflow-3.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "celestialflow"                   
-version = "3.0.3"                         
+version = "3.0.4"                         
 description = "A flexible GRAPH-based task orchestration framework."
 readme = "README.md"               
 license = { text = "MIT" }    

{celestialflow-3.0.3 → celestialflow-3.0.4}/src/celestialflow/__init__.py

@@ -10,7 +10,12 @@ from .task_structure import (
     TaskGrid,
 )
 from .task_types import TerminationSignal
-from .task_tools import load_task_by_stage, load_task_by_error, make_hashable, format_table
+from .task_tools import (
+    load_task_by_stage,
+    load_task_by_error,
+    make_hashable,
+    format_table,
+)
 from .task_web import TaskWebServer
 
 __all__ = [

{celestialflow-3.0.3 → celestialflow-3.0.4}/src/celestialflow/task_graph.py

@@ -2,21 +2,14 @@ import time
 import multiprocessing
 from collections import defaultdict, deque
 from datetime import datetime
-from multiprocessing import Value as MPValue, Lock as MPLock
 from multiprocessing import Queue as MPQueue
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List
 
 from .task_manage import TaskManager
-from .task_nodes import TaskSplitter
 from .task_report import TaskReporter
 from .task_logging import LogListener, TaskLogger
-from .task_types import (
-    StageStatus, 
-    ValueWrapper, 
-    SumCounter, 
-    TerminationSignal, 
-    TERMINATION_SIGNAL
-)
+from .task_queue import TaskQueue
+from .task_types import StageStatus, TerminationSignal, TERMINATION_SIGNAL
 from .task_tools import (
     format_duration,
     format_timestamp,
@@ -29,6 +22,8 @@ from .task_tools import (
     is_directed_acyclic_graph,
     compute_node_levels,
     cluster_by_value_sorted,
+    load_task_by_stage,
+    load_task_by_error
 )
 
 
@@ -73,8 +68,8 @@ class TaskGraph:
         self.processes: List[multiprocessing.Process] = []
 
         self.init_dict()
-        self.init_resources()
         self.init_log()
+        self.init_resources()
 
     def init_dict(self):
         """
@@ -83,25 +78,11 @@ class TaskGraph:
         self.stages_status_dict: Dict[str, dict] = defaultdict(
             dict
         )  # 用于保存每个节点的状态信息
-        self.stage_extra_stats = defaultdict(dict)  # 用于保存每个阶段的额外统计信息
-        self.last_status_dict = {}  # 用于保存每个节点的最后状态信息
-
-        self.edge_queue_map: Dict[Tuple[str, str], MPQueue] = (
-            {}
-        )  # 用于保存每个节点到下一个节点的队列
-
-        self.stage_locks = {}  # 锁，用于控制每个阶段success_counter的并发
-        self.stage_task_counter = {}  # 用于保存每个阶段处理的任务数
-        self.stage_success_counter = {}  # 用于保存每个阶段成功处理的任务数
-        self.stage_error_counter = {}  # 用于保存每个阶段失败处理的任务数
-        self.stage_duplicate_counter = {}  # 用于保存每个阶段重复处理的任务数
-
-        self.error_timeline_dict: Dict[str, list] = defaultdict(
-            list
-        )  # 用于保存错误到出现该错误任务的映射
-        self.all_stage_error_dict: Dict[str, dict] = defaultdict(
+        self.last_status_dict: Dict[str, dict] = defaultdict(
             dict
-        )  # 用于保存节点到节点失败任务的映射
+        )  # 用于保存每个节点的上一次状态信息
+
+        self.error_data: List[dict] = [] 
 
     def init_resources(self):
         """
@@ -110,66 +91,56 @@ class TaskGraph:
         self.fail_queue = MPQueue()
 
         visited_stages = set()
-        queue = deque(self.root_stages)  # BFS 用队列代替递归
+        queue = deque(self.root_stages)
 
+        # BFS 连接
         while queue:
             stage = queue.popleft()
             stage_tag = stage.get_stage_tag()
             if stage_tag in visited_stages:
                 continue
 
+            # 刷新所有 counter
+            stage.reset_counter()
+
             # 记录节点
             self.stages_status_dict[stage_tag]["stage"] = stage
-
-            # 初始化 counters（全部用 MPValue）
-            self.stage_task_counter[stage_tag] = SumCounter()
-            self.stage_success_counter[stage_tag] = self.stage_success_counter.get(
-                stage_tag, MPValue("i", 0)
+            self.stages_status_dict[stage_tag]["in_queue"] = TaskQueue(
+                queue_list=[],
+                queue_tag=[],
+                logger_queue=self.log_listener.get_queue(),
+                stage_tag=stage_tag,
+                direction="in",
             )
-            self.stage_error_counter[stage_tag] = MPValue("i", 0)
-            self.stage_duplicate_counter[stage_tag] = MPValue("i", 0)
-            self.stage_locks[stage_tag] = MPLock()
 
-            self.stage_extra_stats[stage_tag] = self.stage_extra_stats.get(
-                stage_tag, {}
+            self.stages_status_dict[stage_tag]["out_queue"] = TaskQueue(
+                queue_list=[],
+                queue_tag=[],
+                logger_queue=self.log_listener.get_queue(),
+                stage_tag=stage_tag,
+                direction="out",
             )
-            if isinstance(stage, TaskSplitter):
-                self.stage_extra_stats[stage_tag].setdefault(
-                    "split_output_count", MPValue("i", 0)
-                )
+            visited_stages.add(stage_tag)
 
-            # 为每个边 (prev -> stage) 创建队列
-            for prev_stage in stage.prev_stages:
-                prev_tag = prev_stage.get_stage_tag() if prev_stage else None
-                self.edge_queue_map[(prev_tag, stage_tag)] = MPQueue()
+            queue.extend(stage.next_stages)
 
-                if isinstance(prev_stage, TaskSplitter):
-                    self.stage_extra_stats[prev_tag] = self.stage_extra_stats.get(
-                        prev_tag, {}
-                    )
-                    self.stage_extra_stats[prev_tag].setdefault(
-                        "split_output_count", MPValue("i", 0)
-                    )
-                    self.stage_task_counter[stage_tag].add_counter(
-                        self.stage_extra_stats[prev_tag]["split_output_count"]
-                    )
-                else:
-                    # 确保上游 success_counter 已存在
-                    self.stage_success_counter[prev_tag] = (
-                        self.stage_success_counter.get(prev_tag, MPValue("i", 0))
-                    )
-                    self.stage_task_counter[stage_tag].add_counter(
-                        self.stage_success_counter[prev_tag]
-                    )
+        for stage_tag in self.stages_status_dict:
+            stage: TaskManager = self.stages_status_dict[stage_tag]["stage"]
+            in_queue: TaskQueue = self.stages_status_dict[stage_tag]["in_queue"]
 
-            if not stage.prev_stages:
-                # 起点节点
-                self.edge_queue_map[(None, stage_tag)] = MPQueue()
+            # 遍历每个前驱，创建边队列
+            for prev_stage in stage.prev_stages:
+                prev_stage_tag = prev_stage.get_stage_tag() if prev_stage else None
+                q = MPQueue()
 
-            visited_stages.add(stage_tag)
+                # sink side
+                in_queue.add_queue(q, prev_stage_tag)
 
-            for next_stage in stage.next_stages:
-                queue.append(next_stage)
+                # source side
+                if prev_stage is not None:
+                    self.stages_status_dict[prev_stage_tag]["out_queue"].add_queue(
+                        q, stage_tag
+                    )
 
     def init_log(self, level="INFO"):
         """
@@ -184,7 +155,7 @@ class TaskGraph:
         """
         初始化任务图结构
         """
-        self.structure_graph = build_structure_graph(self.root_stages)
+        self.structure_json = build_structure_graph(self.root_stages)
 
     def set_root_stages(self, root_stages: List[TaskManager]):
         """
@@ -242,19 +213,6 @@ class TaskGraph:
             set_subsequent_stage_mode(root_stage)
         self.init_structure_graph()
 
-    def put_termination(self, tag):
-        """
-        放入终止信号
-
-        :param tag: 阶段标签
-        """
-        preg_stages: List[TaskManager] = self.stages_status_dict[tag]["stage"].prev_stages
-        
-        for prev_stage in preg_stages:
-            prev_tag = prev_stage.get_stage_tag() if prev_stage else None
-            self.edge_queue_map[(prev_tag, tag)].put(TERMINATION_SIGNAL)
-            self.task_logger._log("TRACE", f"TERMINATION_SIGNAL put into {(prev_tag, tag)}")
-
     def put_stage_queue(self, tasks_dict: dict, put_termination_signal=True):
         """
         将任务放入队列
@@ -263,26 +221,24 @@ class TaskGraph:
         :param put_termination_signal: 是否放入终止信号
         """
         for tag, tasks in tasks_dict.items():
-            prev_stage: TaskManager = self.stages_status_dict[tag]["stage"].prev_stages[
-                0
-            ]
-            prev_tag = prev_stage.get_stage_tag() if prev_stage else None
+            stage: TaskManager = self.stages_status_dict[tag]["stage"]
+            in_queue: TaskQueue = self.stages_status_dict[tag]["in_queue"]
+
             for task in tasks:
                 if isinstance(task, TerminationSignal):
-                    self.put_termination(tag)
+                    in_queue.put(TERMINATION_SIGNAL)
                     continue
 
-                self.edge_queue_map[(prev_tag, tag)].put(make_hashable(task))
-                self.task_logger._log("TRACE", f"{task} put into {(prev_tag, tag)}")
-                self.stage_task_counter[tag] = self.stage_task_counter.get(
-                    tag, SumCounter()
-                )
-                self.stage_task_counter[tag].add_init_value(1)
+                in_queue.put_first(make_hashable(task))
+                stage.task_counter.add_init_value(1)
 
         if put_termination_signal:
             for root_stage in self.root_stages:
                 root_stage_tag = root_stage.get_stage_tag()
-                self.put_termination(root_stage_tag)
+                root_in_queue: TaskQueue = self.stages_status_dict[root_stage_tag][
+                    "in_queue"
+                ]
+                root_in_queue.put(TERMINATION_SIGNAL)
 
     def start_graph(self, init_tasks_dict: dict, put_termination_signal: bool = True):
         """
@@ -347,40 +303,20 @@ class TaskGraph:
     def _execute_stage(self, stage: TaskManager):
         """
         执行单个节点
-        
+
         :param stage: 节点
         """
         stage_tag = stage.get_stage_tag()
 
-        # 输入输出队列
-        input_queues = [
-            self.edge_queue_map[(prev.get_stage_tag() if prev else None, stage_tag)]
-            for prev in stage.prev_stages
-        ]
-        output_queues = (
-            [
-                self.edge_queue_map[(stage_tag, next_stage.get_stage_tag())]
-                for next_stage in stage.next_stages
-            ]
-            if stage.next_stages
-            else []
-        )
-
         logger_queue = self.log_listener.get_queue()
 
+        # 输入输出队列
+        input_queues = self.stages_status_dict[stage_tag]["in_queue"]
+        output_queues = self.stages_status_dict[stage_tag]["out_queue"]
+
         self.stages_status_dict[stage_tag]["status"] = StageStatus.RUNNING
         self.stages_status_dict[stage_tag]["start_time"] = time.time()
 
-        # counter 都在 init_resources 里初始化完了，这里直接用
-        stage.init_counter(
-            self.stage_task_counter[stage_tag],
-            self.stage_success_counter[stage_tag],
-            self.stage_error_counter[stage_tag],
-            self.stage_duplicate_counter[stage_tag],
-            self.stage_locks[stage_tag],
-            self.stage_extra_stats[stage_tag],
-        )
-
         if stage.stage_mode == "process":
             p = multiprocessing.Process(
                 target=stage.start_stage,
@@ -416,16 +352,23 @@ class TaskGraph:
             stage_status["status"] = StageStatus.STOPPED  # 已停止
 
         # 3️⃣ 收集并持久化每个 stage 中未消费的任务
-        # for stage_tag, stage_status in self.stages_status_dict.items():
-        #     queue: MPQueue = stage_status["task_queue"]
-        #     while not queue.empty():
-        #         try:
-        #             task = queue.get_nowait()
-        #             self.task_logger._log("DEBUG", f"获取 {stage_tag} 剩余任务: {task}")
+        for stage_tag, stage_status in self.stages_status_dict.items():
+            in_queue: TaskQueue = stage_status["in_queue"]
+
+            # 用你刚才统一的 drain() 提取当前剩余任务
+            remaining_sources = in_queue.drain()
+
+            # 如无剩余，跳过
+            if not remaining_sources:
+                continue
+
+            # 持久化逻辑（写日志 / 存储到全局 structure）
+            for source in remaining_sources:
+                task_str = str(source)
+                error_info = f"(UnconsumeError)"
+                timestamp = time.time()
 
-        #             self._persist_unconsumed_task(stage_tag, task)
-        #         except Exception as e:
-        #             self.task_logger._log("WARNING", f"获取 {stage_tag} 剩余任务失败: {e}")
+                self._persist_single_failure(task_str, error_info, stage_tag, timestamp)
 
     def release_resources(self):
         """
@@ -446,13 +389,13 @@ class TaskGraph:
             task_str = item["task"]
             error_info = item["error_info"]
             timestamp = item["timestamp"]
-            error_key = (error_info, stage_tag)
 
-            if task_str not in self.error_timeline_dict[error_key]:
-                self.error_timeline_dict[error_key].append((task_str, timestamp))
-
-            if task_str not in self.all_stage_error_dict[stage_tag]:
-                self.all_stage_error_dict[stage_tag][task_str] = error_key
+            self.error_data.append({
+                "timestamp": timestamp,
+                "node": stage_tag,
+                "error": error_info,
+                "task_id": task_str if len(task_str) < 100 else task_str[:100] + "...",
+            })
 
             self._persist_single_failure(task_str, error_info, stage_tag, timestamp)
 
@@ -460,17 +403,21 @@ class TaskGraph:
         """
         在运行开始时写入任务结构元信息到 jsonl 文件
         """
+        date_str = datetime.fromtimestamp(self.start_time).strftime("%Y-%m-%d")
+        time_str = datetime.fromtimestamp(self.start_time).strftime("%H-%M-%S-%f")[:-3]
+        self.error_jsonl_path = f"./fallback/{date_str}/realtime_errors({time_str}).jsonl"
+
         log_item = {
             "timestamp": datetime.now().isoformat(),
             "structure": self.get_structure_json(),
         }
         append_jsonl_log(
-            log_item, self.start_time, "./fallback", "realtime_errors", self.task_logger
+            log_item, self.error_jsonl_path, self.task_logger
         )
 
     def _persist_single_failure(self, task_str, error_info, stage_tag, timestamp):
         """
-        增量写入单条错误日志到每日文件中
+        增量写入单条错误日志到 jsonl 文件中
 
         :param task_str: 任务字符串
         :param error_info: 错误信息
@@ -484,48 +431,20 @@ class TaskGraph:
             "task": task_str,
         }
         append_jsonl_log(
-            log_item, self.start_time, "./fallback", "realtime_errors", self.task_logger
+            log_item, self.error_jsonl_path, self.task_logger
         )
 
-    def _persist_unconsumed_task(self, stage_tag, task):
+    def get_error_data(self):
         """
-        写入单个未消费任务到 JSONL 文件
-
-        :param stage_tag: 阶段标签
-        :param task: 任务对象
+        返回错误数据
         """
-        log_item = {
-            "timestamp": datetime.now().isoformat(),
-            "stage": stage_tag,
-            "task": str(task),
-        }
-        append_jsonl_log(
-            log_item, self.start_time, "./fallback", "leftover_tasks", self.task_logger
-        )
+        return self.error_data
 
-    def get_error_timeline_dict(self):
-        """
-        返回最终错误字典
-        """
-        return dict(self.error_timeline_dict)
-
-    def get_all_stage_error_dict(self):
-        """
-        返回最终失败字典
-        """
-        return dict(self.all_stage_error_dict)
+    def get_fail_by_stage_dict(self):
+        return load_task_by_stage(self.error_jsonl_path)
 
     def get_fail_by_error_dict(self):
-        return {
-            key: [a for a, _ in tuple_list]
-            for key, tuple_list in self.get_error_timeline_dict().items()
-        }
-
-    def get_fail_by_stage_dict(self):
-        return {
-            stage: list(inner_dict.keys())
-            for stage, inner_dict in self.get_all_stage_error_dict().items()
-        }
+        return load_task_by_error(self.error_jsonl_path)
 
     def get_status_dict(self) -> Dict[str, dict]:
         """
@@ -543,10 +462,10 @@ class TaskGraph:
 
             status = stage_status_dict.get("status", StageStatus.NOT_STARTED)
 
-            input = self.stage_task_counter.get(tag, ValueWrapper()).value
-            successed = self.stage_success_counter.get(tag, ValueWrapper()).value
-            failed = self.stage_error_counter.get(tag, ValueWrapper()).value
-            duplicated = self.stage_duplicate_counter.get(tag, ValueWrapper()).value
+            input = stage.task_counter.value
+            successed = stage.success_counter.value
+            failed = stage.error_counter.value
+            duplicated = stage.duplicate_counter.value
             processed = successed + failed + duplicated
             pending = max(0, input - processed)
 
@@ -633,13 +552,13 @@ class TaskGraph:
         }
 
     def get_structure_json(self):
-        return self.structure_graph
+        return self.structure_json
 
     def get_structure_list(self):
-        return format_structure_list_from_graph(self.structure_graph)
+        return format_structure_list_from_graph(self.structure_json)
 
     def get_networkx_graph(self):
-        return format_networkx_graph(self.structure_graph)
+        return format_networkx_graph(self.structure_json)
 
     def analyze_graph(self):
         """
@@ -650,8 +569,8 @@ class TaskGraph:
 
         self.isDAG = is_directed_acyclic_graph(networkx_graph)
         if self.isDAG:
-            self.stage_level_dict = compute_node_levels(networkx_graph)
-            self.layers_dict = cluster_by_value_sorted(self.stage_level_dict)
+            stage_level_dict = compute_node_levels(networkx_graph)
+            self.layers_dict = cluster_by_value_sorted(stage_level_dict)
 
     def test_methods(
         self,
@@ -681,10 +600,10 @@ class TaskGraph:
                 self.init_env()
                 self.set_graph_mode(stage_mode, execution_mode)
                 self.start_graph(init_tasks_dict)
+                fail_by_stage_dict.update(self.get_fail_by_stage_dict())
+                fail_by_error_dict.update(self.get_fail_by_error_dict())
 
                 time_list.append(time.time() - start_time)
-                fail_by_error_dict.update(self.get_fail_by_error_dict())
-                fail_by_stage_dict.update(self.get_fail_by_stage_dict())
 
             test_table_list.append(time_list)
 
@@ -694,6 +613,6 @@ class TaskGraph:
             execution_modes,
             r"stage\execution",
         )
-        results["Fail error dict"] = fail_by_error_dict
         results["Fail stage dict"] = fail_by_stage_dict
+        results["Fail error dict"] = fail_by_error_dict
         return results

{celestialflow-3.0.3 → celestialflow-3.0.4}/src/celestialflow/task_logging.py

@@ -13,6 +13,7 @@ class LogListener:
     """
     日志监听进程，用于将日志写入文件
     """
+
     def __init__(self, level="INFO"):
         now = strftime("%Y-%m-%d", localtime())
         self.log_path = f"logs/task_logger({now}).log"
@@ -66,7 +67,7 @@ class TaskLogger:
 
     # ==== manager ====
     def start_manager(self, func_name, task_num, execution_mode, worker_limit):
-        text = f"'{func_name}' start {task_num} tasks by {execution_mode}"
+        text = f"'Manager[{func_name}]' start {task_num} tasks by {execution_mode}"
         text += f"({worker_limit} workers)." if execution_mode != "serial" else "."
         self._log("INFO", text)
 
@@ -81,20 +82,19 @@ class TaskLogger:
     ):
         self._log(
             "INFO",
-            f"'{func_name}' end tasks by {execution_mode}. Use {use_time:.2f} second. "
+            f"'Manager[{func_name}]' end tasks by {execution_mode}. Use {use_time:.2f} second. "
             f"{success_num} tasks successed, {failed_num} tasks failed, {duplicated_num} tasks duplicated.",
         )
 
     # ==== stage ====
-    def start_stage(self, stage_name, func_name, execution_mode, worker_limit):
-        text = f"The {stage_name} in '{func_name}' start tasks by {execution_mode}"
+    def start_stage(self, stage_tag, execution_mode, worker_limit):
+        text = f"'{stage_tag}' start tasks by {execution_mode}"
         text += f"({worker_limit} workers)." if execution_mode != "serial" else "."
         self._log("INFO", text)
 
     def end_stage(
         self,
-        stage_name,
-        func_name,
+        stage_tag,
         execution_mode,
         use_time,
         success_num,
@@ -103,7 +103,7 @@ class TaskLogger:
     ):
         self._log(
             "INFO",
-            f"The {stage_name} in '{func_name}' end tasks by {execution_mode}. Use {use_time:.2f} second. "
+            f"'{stage_tag}' end tasks by {execution_mode}. Use {use_time:.2f} second. "
             f"{success_num} tasks successed, {failed_num} tasks failed, {duplicated_num} tasks duplicated.",
         )
 
@@ -152,3 +152,31 @@ class TaskLogger:
             "SUCCESS",
             f"In '{func_name}', Task {task_info} has split into {split_count} parts. Used {use_time:.2f} seconds.",
         )
+
+    # ==== queue ====
+    def put_source(self, source, queue_tag, stage_tag, direction):
+        if isinstance(source, TerminationSignal):
+            source = "TerminationSignal"
+
+        edge = f"'{queue_tag}' -> '{stage_tag}'" if direction == "in" else f"'{stage_tag}' -> '{queue_tag}'"
+        self._log(
+            "TRACE", 
+            f"Put {source} into Edge({edge})."
+        )
+
+    def get_source(self, source, queue_tag, stage_tag):
+        if isinstance(source, TerminationSignal):
+            source = "TerminationSignal"
+
+        edge = f"'{queue_tag}' -> '{stage_tag}'"
+        self._log(
+            "TRACE", 
+            f"Get {source} from Edge({edge})"
+        )
+
+    def get_source_error(self, queue_tag, stage_tag, exception):
+        exception_text = str(exception).replace("\n", " ")
+        self._log(
+            "WARNING",
+            f"Error get from Edge({queue_tag} -> {stage_tag}): ({type(exception).__name__}){exception_text}",
+        )