celestialflow 3.1.5__tar.gz → 3.1.6__tar.gz

{celestialflow-3.1.5 → celestialflow-3.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: celestialflow
-Version: 3.1.5
+Version: 3.1.6
 Summary: A flexible GRAPH-based task orchestration framework.
 Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
 License: MIT
@@ -254,22 +254,27 @@ flowchart TD
 <p align="center">
   <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
   <br/>
-  <em>celestial-flow 3.1.5</em>
+  <em>celestial-flow 3.1.6</em>
 </p>
 
 (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
 
 ## 版本日志（Version Log）
-- 3.1.5
+- 3.1.6
   - feat
-    - 大幅修改节点状态卡片的色彩视觉设计. 包括: 取消原本的悬浮设计, 改为平面化设计; 让进度条更直接的显示不同任务完成状态的比例; 使用左边框来显示节点是否在运行中. 而非原本的badge;
-    - 前端的error数据拉取不再每次都全量拉取, 而是只拉取自己当前没有的数据;
+    - 前端现在只储存一页的error数据, 有效减少了运行大规模任务时前端内存飙升的问题;
+    - 优化任务数显示, 大于1*10^7时显示科学计数法, 否则显示英式计数;
+    - 优化小屏模式下表格的显示: 改为用卡片式显示;
+    - 在error表格中加入index项;
+    - 大幅调整任务颜色分配, 现在重复任务使用黄色系, 等待任务使用灰色系;
+    - 将节点卡片中的已消耗时间颜色拟合为下方进度条颜色;
   - refactor
-    - 大幅重构前端代码中的色彩管理, 现在色彩的一致性更好;
-    - 修改前端代码中的数据更新判断, 现在相关判断交给serve.py;
+    - 用mypy整理了一遍类型标注;
+    - 删除与整合部分css代码;
+    - 将字体等适合rem单位的地方全部从px替换为rem, 并且都部分size进行统一;
   - fix
-    - 修复节点已运行时间在节点完成后显示为0的问题
-    - 修复在最新fastapi版本下TemplateResponse参数改变导致的问题
+    - 修复总体剩余时间在特殊情况下显示0的问题(这玩意真麻烦);
+    - 修复部分小屏下的显示问题, 但折线图不显示的问题不太好解决;
 
 ## Star 历史趋势（Star History）
 

{celestialflow-3.1.5 → celestialflow-3.1.6}/README.md

@@ -229,22 +229,27 @@ flowchart TD
 <p align="center">
   <img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
   <br/>
-  <em>celestial-flow 3.1.5</em>
+  <em>celestial-flow 3.1.6</em>
 </p>
 
 (该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
 
 ## 版本日志（Version Log）
-- 3.1.5
+- 3.1.6
   - feat
-    - 大幅修改节点状态卡片的色彩视觉设计. 包括: 取消原本的悬浮设计, 改为平面化设计; 让进度条更直接的显示不同任务完成状态的比例; 使用左边框来显示节点是否在运行中. 而非原本的badge;
-    - 前端的error数据拉取不再每次都全量拉取, 而是只拉取自己当前没有的数据;
+    - 前端现在只储存一页的error数据, 有效减少了运行大规模任务时前端内存飙升的问题;
+    - 优化任务数显示, 大于1*10^7时显示科学计数法, 否则显示英式计数;
+    - 优化小屏模式下表格的显示: 改为用卡片式显示;
+    - 在error表格中加入index项;
+    - 大幅调整任务颜色分配, 现在重复任务使用黄色系, 等待任务使用灰色系;
+    - 将节点卡片中的已消耗时间颜色拟合为下方进度条颜色;
   - refactor
-    - 大幅重构前端代码中的色彩管理, 现在色彩的一致性更好;
-    - 修改前端代码中的数据更新判断, 现在相关判断交给serve.py;
+    - 用mypy整理了一遍类型标注;
+    - 删除与整合部分css代码;
+    - 将字体等适合rem单位的地方全部从px替换为rem, 并且都部分size进行统一;
   - fix
-    - 修复节点已运行时间在节点完成后显示为0的问题
-    - 修复在最新fastapi版本下TemplateResponse参数改变导致的问题
+    - 修复总体剩余时间在特殊情况下显示0的问题(这玩意真麻烦);
+    - 修复部分小屏下的显示问题, 但折线图不显示的问题不太好解决;
 
 ## Star 历史趋势（Star History）
 

{celestialflow-3.1.5 → celestialflow-3.1.6}/pyproject.toml

@@ -1,74 +1,80 @@
-[build-system]
-requires = ["setuptools>=61.0", "wheel"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "celestialflow"                   
-version = "3.1.5"
-description = "A flexible GRAPH-based task orchestration framework."
-readme = "README.md"               
-license = { text = "MIT" }    
-authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
-keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
-requires-python = ">=3.10"
-
-dependencies = [
-    "tqdm",
-    "fastapi",
-    "uvicorn",
-    "requests",
-    "networkx",
-    "redis",
-    "jinja2",
-    "celestialtree>=0.1.2",
-]
-
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-    "Framework :: FastAPI",
-    "Topic :: Software Development :: Libraries",
-]
-
-[project.urls]
-"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
-"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
-
-[project.scripts]
-celestialflow-web = "celestialflow.web.server:main_entry"
-
-[tool.setuptools]
-license-files = []
-
-[tool.setuptools.package-data]
-celestialflow = [
-    "web/templates/**/*.html",
-    "web/static/**/*",
-    "web/config.json"
-]
-
-[tool.pytest.ini_options]
-addopts = "-s"
-log_cli = true
-log_cli_level = "INFO"
-log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
-log_cli_date_format = "%Y-%m-%d %H:%M:%S"
-
-filterwarnings = [
-    "ignore::DeprecationWarning",
-    "ignore::pytest.PytestDeprecationWarning",
-]
-
-asyncio_default_fixture_loop_scope = "function"
-
-[dependency-groups]
-dev = [
-    "build>=1.2.2.post1",
-    "twine>=6.1.0",
-    "pytest>=8.3.4",
-    "pytest-asyncio>=0.25.3",
-    "httpx>=0.28.1",
-    "python-dotenv>=1.2.2",
-    "black>=26.1.0",
-]
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "celestialflow"                   
+version = "3.1.6"
+description = "A flexible GRAPH-based task orchestration framework."
+readme = "README.md"               
+license = { text = "MIT" }    
+authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
+keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
+requires-python = ">=3.10"
+
+dependencies = [
+    "tqdm",
+    "fastapi",
+    "uvicorn",
+    "requests",
+    "networkx",
+    "redis",
+    "jinja2",
+    "celestialtree>=0.1.2",
+]
+
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Framework :: FastAPI",
+    "Topic :: Software Development :: Libraries",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
+"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
+
+[project.scripts]
+celestialflow-web = "celestialflow.web.core_server:main_entry"
+
+[tool.setuptools]
+license-files = []
+
+[tool.setuptools.package-data]
+celestialflow = [
+    "web/templates/**/*.html",
+    "web/static/**/*",
+    "web/config.json"
+]
+
+[tool.pytest.ini_options]
+addopts = "-s"
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+
+filterwarnings = [
+    "ignore::DeprecationWarning",
+    "ignore::pytest.PytestDeprecationWarning",
+]
+
+asyncio_default_fixture_loop_scope = "function"
+
+[dependency-groups]
+dev = [
+    "build>=1.2.2.post1",
+    "twine>=6.1.0",
+    "pytest>=8.3.4",
+    "pytest-asyncio>=0.25.3",
+    "httpx>=0.28.1",
+    "python-dotenv>=1.2.2",
+    "dill",
+    "mypy>=1.19.1",
+]
+
+[tool.mypy]
+python_version = "3.10"
+no_implicit_optional = false
+disable_error_code = ["import-untyped"]

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/__init__.py

@@ -1,32 +1,32 @@
 # __init__.py
-from .stage import (
-    TaskExecutor,
-    TaskStage,
-    TaskSplitter,
-    TaskRedisTransport,
-    TaskRedisSource,
-    TaskRedisAck,
-    TaskRouter,
-)
 from .graph import (
-    TaskGraph,
     TaskChain,
-    TaskLoop,
-    TaskCross,
     TaskComplete,
-    TaskWheel,
+    TaskCross,
+    TaskGraph,
     TaskGrid,
+    TaskLoop,
+    TaskWheel,
 )
 from .persistence.util_jsonl import (
     load_jsonl_logs,
-    load_task_by_stage,
     load_task_by_error,
+    load_task_by_stage,
 )
-from .runtime.util_types import TerminationSignal
 from .runtime.util_hash import make_hashable
+from .runtime.util_types import TerminationSignal
+from .stage import (
+    TaskExecutor,
+    TaskRedisAck,
+    TaskRedisSource,
+    TaskRedisTransport,
+    TaskRouter,
+    TaskSplitter,
+    TaskStage,
+)
+from .utils.util_benchmark import benchmark_executor, benchmark_graph
 from .utils.util_format import format_table
-from .utils.util_benchmark import benchmark_graph, benchmark_executor
-from .web.server import TaskWebServer
+from .web.core_server import TaskWebServer
 
 __all__ = [
     "TaskGraph",

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/graph/__init__.py

@@ -2,11 +2,11 @@
 from .core_graph import TaskGraph
 from .core_structure import (
     TaskChain,
-    TaskLoop,
-    TaskCross,
     TaskComplete,
-    TaskWheel,
+    TaskCross,
     TaskGrid,
+    TaskLoop,
+    TaskWheel,
 )
 
 __all__ = [

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/graph/core_graph.py

@@ -1,40 +1,46 @@
 # graph/core_graph.py
+import multiprocessing
 import time
 import warnings
-import multiprocessing
 from collections import defaultdict, deque
+from pathlib import Path
+from typing import Any
 from multiprocessing import Queue as MPQueue
 
 from celestialtree import (
     Client as CelestialTreeClient,
+)
+from celestialtree import (
     NullClient as NullCelestialTreeClient,
+)
+from celestialtree import (
     format_descendants_forest,
     format_provenance_forest,
 )
+from networkx import is_directed_acyclic_graph
 
-from ..runtime import TaskInQueue, TaskOutQueue, TaskEnvelope
+from ..observability import NullTaskReporter, TaskReporter
+from ..persistence import FailListener, FailSinker, LogListener, LogSinker
+from ..persistence.util_jsonl import load_task_by_error, load_task_by_stage
+from ..runtime import TaskEnvelope, TaskInQueue, TaskOutQueue
+from ..runtime.util_errors import UnconsumedError
 from ..runtime.util_estimators import (
     calc_elapsed,
-    calc_remaining,
     calc_global_remain_equal_pred,
+    calc_remaining,
 )
-from ..runtime.util_errors import UnconsumedError
 from ..runtime.util_types import (
+    NULL_PREV_STAGE,
+    STAGE_STYLE,
     StageStatus,
     TerminationSignal,
-    STAGE_STYLE,
-    NULL_PREV_STAGE,
 )
 from ..stage import TaskStage
-from ..observability import TaskReporter, NullTaskReporter
-from ..persistence import FailListener, FailSinker, LogListener, LogSinker
-from ..persistence.util_jsonl import load_task_by_stage, load_task_by_error
 from ..utils.util_collections import cluster_by_value_sorted
 from ..utils.util_format import format_avg_time
 from .util_analysis import (
-    format_networkx_graph,
     compute_node_levels,
-    is_directed_acyclic_graph,
+    format_networkx_graph,
 )
 from .util_serialize import build_structure_graph, format_structure_list_from_graph
 
@@ -102,15 +108,15 @@ class TaskGraph:
         # 用于保存所有子进程的引用
         self.processes: list[multiprocessing.Process] = []
         # 用于保存每个节点的运行信息
-        self.stage_runtime_dict: dict[str, dict] = defaultdict(dict)
+        self.stage_runtime_dict: dict[str, dict[str, Any]] = defaultdict(dict)
         # 用于保存每个节点的上一次collect_runtime_snapshot()的状态信息
-        self.status_dict: dict[str, dict] = defaultdict(dict)
+        self.status_dict: dict[str, dict[str, Any]] = defaultdict(dict)
         # 用于保存任务图的摘要信息
         self.graph_summary: dict[str, int | float] = {}
         # 用于保存每个节点的历史状态信息列表（仅保留最近20条）
         self.stage_history: dict[str, list[dict]] = {}
         # 用于保存每个节点的输入任务ID集合
-        self.input_ids: dict[str, set] = defaultdict(set)
+        self.input_ids: dict[str, set[int]] = defaultdict(set)
 
     def init_listener(self) -> None:
         """
@@ -164,11 +170,11 @@ class TaskGraph:
             queue.extend(stage.next_stages)
 
         for stage_tag, stage_runtime in self.stage_runtime_dict.items():
-            stage: TaskStage = stage_runtime["stage"]
+            current_stage: TaskStage = stage_runtime["stage"]
             in_queue: TaskInQueue = stage_runtime["in_queue"]
 
             # 遍历每个前驱，创建边队列
-            for prev_stage in stage.prev_stages:
+            for prev_stage in current_stage.prev_stages:
                 prev_stage_tag = prev_stage.get_tag()
                 in_queue.add_source_tag(prev_stage_tag)
 
@@ -235,6 +241,7 @@ class TaskGraph:
         self._is_report = is_report
         self._report_host = host
         self._report_port = port
+        self.reporter: TaskReporter | NullTaskReporter
         if is_report:
             self.reporter = TaskReporter(
                 host=host,
@@ -291,7 +298,7 @@ class TaskGraph:
         :param execution_mode: 节点内部执行模式, 可选值为 'serial' 或 'thread''
         """
 
-        def set_subsequent_stage_mode(stage: TaskStage):
+        def set_subsequent_stage_mode(stage: TaskStage) -> None:
             stage.set_stage_mode(stage_mode)
             stage.set_execution_mode(execution_mode)
             visited_stages.add(stage)
@@ -301,7 +308,7 @@ class TaskGraph:
                     continue
                 set_subsequent_stage_mode(next_stage)
 
-        visited_stages = set()
+        visited_stages: set[TaskStage] = set()
         for root_stage in self.root_stages:
             set_subsequent_stage_mode(root_stage)
         self.init_analysis()
@@ -494,11 +501,11 @@ class TaskGraph:
 
         # 收集并持久化每个 stage 中未消费的任务
         for stage_tag, stage_runtime in self.stage_runtime_dict.items():
-            stage: TaskStage = stage_runtime["stage"]
+            current_stage: TaskStage = stage_runtime["stage"]
             in_queue: TaskInQueue = stage_runtime["in_queue"]
 
             remaining_sources = in_queue.drain()
-            stage.metrics.add_error_count(len(remaining_sources))
+            current_stage.metrics.add_error_count(len(remaining_sources))
 
             # 持久化逻辑
             for source in remaining_sources:
@@ -507,7 +514,7 @@ class TaskGraph:
                 error_id = self.ctree_client.emit(
                     "task.error",
                     [task_id],
-                    payload=stage.get_summary(),
+                    payload=current_stage.get_summary(),
                 )
 
                 self.fail_sinker.task_error(
@@ -515,8 +522,8 @@ class TaskGraph:
                 )
 
                 self.log_sinker.task_error(
-                    stage.get_func_name(),
-                    stage.get_task_repr(task),
+                    current_stage.get_func_name(),
+                    current_stage.get_task_repr(task),
                     UnconsumedError(),
                     task_id,
                     error_id,
@@ -644,6 +651,9 @@ class TaskGraph:
     def get_fail_by_error_dict(self) -> dict:
         return load_task_by_error(self.fail_listener.jsonl_path)
 
+    def get_total_error_num(self) -> int:
+        return self.fail_listener.total_error_num
+
     def get_status_dict(self) -> dict[str, dict]:
         """
         获取任务链的状态字典
@@ -660,9 +670,9 @@ class TaskGraph:
         """获取任务链的历史状态信息字典"""
         return self.stage_history
 
-    def get_graph_topology(self) -> dict:
+    def get_graph_analysis(self) -> dict:
         """
-        获取任务图的拓扑信息
+        获取任务图的分析信息
         """
         return {
             "isDAG": self.isDAG,
@@ -693,7 +703,9 @@ class TaskGraph:
         """
         获取失败任务的回退路径
         """
-        return str(self.fail_listener.jsonl_path.resolve())
+        if self.fail_listener.jsonl_path is None:
+            return ""
+        return str(Path(self.fail_listener.jsonl_path).resolve())
 
     def get_stage_input_trace(self, stage_tag: str) -> str:
         """

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/graph/core_structure.py

@@ -67,7 +67,7 @@ class TaskCross(TaskGraph):
                 stage.set_graph_context(
                     next_stages=next_layer,
                     stage_mode="process",
-                    stage_name=f"Layer{i+1}-{index+1}",
+                    stage_name=f"Layer{i + 1}-{index + 1}",
                 )
         super().__init__(
             root_stages=layers[0], schedule_mode=schedule_mode, log_level=log_level
@@ -112,7 +112,7 @@ class TaskGrid(TaskGraph):
                     nexts.append(grid[i + 1][j])  # down
                 if j + 1 < cols:
                     nexts.append(grid[i][j + 1])  # right
-                curr.set_graph_context(nexts, "process", f"Grid-{i+1}-{j+1}")
+                curr.set_graph_context(nexts, "process", f"Grid-{i + 1}-{j + 1}")
         super().__init__(
             root_stages=[grid[0][0]], schedule_mode=schedule_mode, log_level=log_level
         )  # 起点为左上角
@@ -175,7 +175,7 @@ class TaskWheel(TaskGraph):
         # 环相连（成闭环）
         for i, node in enumerate(ring):
             next_stage = ring[(i + 1) % len(ring)]
-            node.set_graph_context([next_stage], "process", f"Ring-{i+1}")
+            node.set_graph_context([next_stage], "process", f"Ring-{i + 1}")
         super().__init__(root_stages=[center], log_level=log_level)
 
     def start_wheel(

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/graph/util_analysis.py

@@ -1,8 +1,8 @@
 # graph/util_analysis.py
-import networkx as nx
-from networkx import is_directed_acyclic_graph
 from typing import Any
 
+import networkx as nx
+
 
 # ======== (图论分析) ========
 def format_networkx_graph(structure_graph: list[dict[str, Any]]) -> nx.DiGraph:
@@ -15,11 +15,11 @@ def format_networkx_graph(structure_graph: list[dict[str, Any]]) -> nx.DiGraph:
     G = nx.DiGraph()
 
     def add_node_and_edges(node: dict[str, Any]):
-        node_id = f'{node["name"]}[{node["func_name"]}]'
+        node_id = f"{node['name']}[{node['func_name']}]"
         G.add_node(node_id, **{"mode": node.get("stage_mode")})
 
         for child in node.get("next_stages", []):
-            child_id = f'{child["name"]}[{child["func_name"]}]'
+            child_id = f"{child['name']}[{child['func_name']}]"
             G.add_edge(node_id, child_id)
             # 递归添加子节点
             add_node_and_edges(child)

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/graph/util_serialize.py

@@ -1,5 +1,5 @@
 # graph/util_serialize.py
-from typing import TYPE_CHECKING, Dict, Any, List, Set
+from typing import TYPE_CHECKING, Any, Dict, List, Set
 
 if TYPE_CHECKING:
     from ..stage import TaskStage
@@ -49,7 +49,7 @@ def _build_structure_subgraph(
     return node
 
 
-def format_structure_list_from_graph(root_roots: List[Dict] = None) -> List[str]:
+def format_structure_list_from_graph(root_roots: List[Dict] | None = None) -> List[str]:
     """
     从多个 JSON 图结构生成格式化任务结构文本列表（带边框）
 
@@ -64,7 +64,7 @@ def format_structure_list_from_graph(root_roots: List[Dict] = None) -> List[str]
         S = node.get("stage_mode", "?")  # S
         E = node.get("execution_mode", "?")  # E
 
-        return f"{N}::{F} " f"(S:{S}, E:{E})" f"{visited_note}"
+        return f"{N}::{F} (S:{S}, E:{E}){visited_note}"
 
     # 只渲染“子节点”（有父节点）——保证一定画连接符
     def build_child_lines(node: Dict, prefix: str, is_last: bool) -> List[str]:
@@ -88,7 +88,7 @@ def format_structure_list_from_graph(root_roots: List[Dict] = None) -> List[str]
             lines.extend(build_child_lines(child, "", i == len(next_stages) - 1))
         return lines
 
-    all_lines = []
+    all_lines: list[str] = []
     for root in root_roots or []:
         if all_lines:
             all_lines.append("")  # 根之间留空行

{celestialflow-3.1.5 → celestialflow-3.1.6}/src/celestialflow/observability/__init__.py

@@ -1,5 +1,5 @@
 # observability/__init__.py
-from .core_report import TaskReporter, NullTaskReporter
+from .core_report import NullTaskReporter, TaskReporter
 
 __all__ = [
     "TaskReporter",