mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

msprobe/core/monitor/db_utils.py

@@ -1,23 +1,25 @@
-# Copyright (c) 2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
 from collections import OrderedDict
 from collections.abc import Iterable
 from typing import Dict, List, Optional, Set, Tuple
 
-from msprobe.core.common.const import MonitorConst
-from msprobe.core.common.db_manager import DBManager
+from msprobe.core.common.const import MonitorConst, Const, Data2DBConst
+from msprobe.core.common.db_manager import DBManager, TrendSql
 
 
 def update_ordered_dict(main_dict: OrderedDict, new_list: List) -> OrderedDict:
@@ -28,151 +30,57 @@ def update_ordered_dict(main_dict: OrderedDict, new_list: List) -> OrderedDict:
     return main_dict
 
 
-def get_ordered_stats(stats: Iterable) -> List[str]:
-    """Get statistics in predefined order"""
-    if not isinstance(stats, Iterable):
+def get_ordered_list(data: Iterable, order: Iterable) -> List[str]:
+    if not isinstance(data, Iterable) or not isinstance(order, Iterable):
         return []
-    return [stat for stat in MonitorConst.OP_MONVIS_SUPPORTED if stat in stats]
-
-
-class MonitorSql:
-    """数据库表参数类"""
+    return [value for value in order if value in data]
 
-    @staticmethod
-    def create_monitoring_targets_table():
-        """监控目标表"""
-        return """
-        CREATE TABLE IF NOT EXISTS monitoring_targets (
-            target_id INTEGER PRIMARY KEY AUTOINCREMENT,
-            target_name TEXT NOT NULL,
-            vpp_stage INTEGER NOT NULL,
-            micro_step INTEGER NOT NULL DEFAULT 0,
-            UNIQUE(target_name, vpp_stage, micro_step) 
-        )"""
-
-    @staticmethod
-    def create_monitoring_metrics_table():
-        """监控指标表"""
-        return """
-        CREATE TABLE IF NOT EXISTS monitoring_metrics (
-            metric_id INTEGER PRIMARY KEY AUTOINCREMENT,
-            metric_name TEXT UNIQUE NOT NULL
-        )"""
+def get_tags_from_target(target_name):
+    """
+    full target name to tags
+    
+    :param target_name: e.g. layers.22.self_attention.linear_qkv.layer_norm_weight
+    :return tags: set of tags
+    """
+    tags = set()
+    parts = target_name.split(Const.SEP)[::-1]
+    max_idx = len(parts) - 1
+    for i, part in enumerate(parts):
+        if not part.isdigit():
+            tags.add((part, Data2DBConst.TAG_MODULE))
+            continue
+        if i == max_idx or parts[i+1].isdigit():  # 确保数字前面有非数字部分, 组成layer标签
+            continue
+        layer_tag = (f"{parts[i+1]}.{parts[i]}", Data2DBConst.TAG_LAYER)
+        tags.add(layer_tag)
+    return tags
     
-    @staticmethod
-    def get_metric_mapping_sql():
-        return """
-        SELECT m.metric_id, m.metric_name, GROUP_CONCAT(ms.stat_name) as stats 
-        FROM monitoring_metrics m 
-        LEFT JOIN metric_stats ms ON m.metric_id = ms.metric_id
-        GROUP BY m.metric_id
-        """
-
-    @staticmethod
-    def create_metric_stats_table():
-        """指标统计表"""
-        return """
-        CREATE TABLE IF NOT EXISTS metric_stats (
-            metric_id INTEGER NOT NULL,
-            stat_name TEXT NOT NULL,
-            PRIMARY KEY (metric_id, stat_name),
-            FOREIGN KEY (metric_id) REFERENCES monitoring_metrics(metric_id)
-        ) WITHOUT ROWID"""
-
-    @staticmethod
-    def create_global_stat_table():
-        return """
-        CREATE TABLE IF NOT EXISTS global_stats (
-            stat_name TEXT PRIMARY KEY,
-            stat_value INTEGER NOT NULL
-        ) WITHOUT ROWID"""
-
-    @classmethod
-    def get_table_definition(cls, table_name=""):
-        """
-        获取表定义SQL
-        :param table_name: 表名
-        :return: 建表SQL语句
-        :raises ValueError: 当表名不存在时
-        """
-        table_creators = {
-            "monitoring_targets": cls.create_monitoring_targets_table,
-            "monitoring_metrics": cls.create_monitoring_metrics_table,
-            "metric_stats": cls.create_metric_stats_table,
-            "global_stats": cls.create_global_stat_table,
-        }
-        if not table_name:
-            return [table_creators.get(table, lambda x: "")() for table in table_creators]
-        if table_name not in table_creators:
-            raise ValueError(f"Unsupported table name: {table_name}")
-        return table_creators[table_name]()
-
-    @classmethod
-    def get_metric_table_definition(cls, table_name, stats, patition=None):
-        stat_columns = [f"{stat} REAL DEFAULT NULL" for stat in stats]
-        if patition and len(patition) == 2:
-            partition_start_step, partition_end_step = patition
-            step_column = f"""step INTEGER NOT NULL CHECK(step BETWEEN {partition_start_step} 
-                    AND {partition_end_step}),"""
-        else:
-            step_column = "step INTEGER NOT NULL"
-        create_sql = f"""
-            CREATE TABLE {table_name} (
-                rank INTEGER NOT NULL,
-                {step_column}
-                target_id INTEGER NOT NULL,
-                {', '.join(stat_columns)},
-                PRIMARY KEY (rank, step, target_id),
-                FOREIGN KEY (target_id) REFERENCES monitoring_targets(target_id)
-            ) WITHOUT ROWID
-            """
-        return create_sql
-
 
 class MonitorDB:
     """Main class for monitoring database operations"""
 
-    def __init__(self, db_path: str, step_partition_size: int = 500):
+    def __init__(self, db_path: str):
         self.db_path = db_path
         self.db_manager = DBManager(db_path)
-        self.step_partition_size = step_partition_size
-
-    def get_metric_table_name(self, metric_id: int, step: int) -> str:
-        """Generate metric table name"""
-        step_start = (
-            step // self.step_partition_size) * self.step_partition_size
-        step_end = step_start + self.step_partition_size - 1
-        return f"metric_{metric_id}_step_{step_start}_{step_end}", step_start, step_end
 
     def init_schema(self) -> None:
         """Initialize database schema"""
-        self.db_manager.execute_multi_sql(MonitorSql.get_table_definition())
-
-        # Insert initial global stats
-        global_stats = [
-            ('max_rank', 0),
-            ('min_step', 0),
-            ('max_step', 0),
-            ('step_partition_size', self.step_partition_size)
-        ]
-        self.db_manager.insert_data("global_stats", global_stats)
+        self.db_manager.execute_multi_sql(TrendSql.get_table_definition())
 
     def insert_dimensions(
         self,
-        targets: OrderedDict,
-        metrics: Set[str],
-        metric_stats: Dict[str, Set[str]],
-        min_step: Optional[int] = None,
-        max_step: int = None,
+        targets: Dict[str,OrderedDict],
     ) -> None:
         """Insert dimension data into database"""
         # Insert targets
-        self.db_manager.insert_data(
-            "monitoring_targets",
-            [(name, vpp_stage, micro_step)
-             for (name, vpp_stage, micro_step) in targets],
-            key_list=["target_name", "vpp_stage", "micro_step"]
-        )
+        metrics = get_ordered_list(set(targets.keys()), MonitorConst.METRICS_TRENDVIS_SUPPORTED)
+        for metric_name in metrics:
+            self.db_manager.insert_data(
+                "monitoring_targets",
+                [(name, vpp_stage, micro_step)
+                for (name, vpp_stage, micro_step) in targets[metric_name]],
+                key_list=["target_name", "vpp_stage", "micro_step"]
+            )
 
         # Insert metrics
         self.db_manager.insert_data(
@@ -181,79 +89,53 @@ class MonitorDB:
             key_list=["metric_name"]
         )
 
-        # Insert metric-stat relationships
-        for metric, stats in metric_stats.items():
-            metric_id = self._get_metric_id(metric)
-            ordered_stats = get_ordered_stats(stats)
-
-            self.db_manager.insert_data(
-                "metric_stats",
-                [(metric_id, stat) for stat in ordered_stats],
-                key_list=["metric_id", "stat_name"]
-            )
-
-            # Create metric tables for each partition
-            if min_step is not None and max_step is not None:
-                first_partition = min_step // self.step_partition_size
-                last_partition = max_step // self.step_partition_size
-
-                for partition in range(first_partition, last_partition + 1):
-                    step_start = partition * self.step_partition_size
-                    self.create_metric_table(
-                        metric_id, step_start, ordered_stats)
-
-    def insert_rows(self, table_name, rows):
-        if not self.db_manager.table_exists(table_name):
-            raise RuntimeError(f"{table_name} not existed in {self.db_path}")
+    def insert_rows(self, rows):
+        table_name = "trend_data"
         inserted = self.db_manager.insert_data(table_name, rows)
         inserted = 0 if inserted is None else inserted
         return inserted
 
-    def create_metric_table(self, metric_id: int, step: int, stats: List[str]) -> str:
-        """Create metric table for a specific partition"""
-        table_name, partition_start_step, partition_end_step = self.get_metric_table_name(
-            metric_id,
-            step
-        )
-        if self.db_manager.table_exists(table_name):
-            return table_name
-
-        create_sql = MonitorSql.get_metric_table_definition(
-            table_name, stats, patition=(
-                partition_start_step, partition_end_step)
+    def init_global_stats_data(self, config: Dict):
+        """初始化全局统计表数据"""
+        # 准备插入数据
+        self.db_manager.execute_sql(
+            TrendSql.create_global_stats_table(config)
         )
-        self.db_manager.execute_sql(create_sql)
-        return table_name
-
-    def update_global_stats(self, max_rank: int = None, min_step: Optional[int] = None, max_step: int = None) -> None:
-        """Update global statistics"""
-        updates = [
-            ("max_rank", max_rank),
-            ("min_step", min_step),
-            ("max_step", max_step)
-        ]
-        for stat_name, value in updates:
-            if not value:
-                continue
-            self.db_manager.update_data(
-                table_name="global_stats",
-                updates={"stat_value": value},
-                where={"stat_name": stat_name}
-            )
+        
+        column_values = []
+        for _, column_value in config.items():
+            if isinstance(column_value, (list, set)):
+                column_values.append(",".join(column_value))
+            elif isinstance(column_value, (int, float)):
+                column_values.append(column_value)
+        self.db_manager.insert_data("global_stats", [column_values,])
 
     def get_metric_mapping(self) -> Dict[str, Tuple[int, List[str]]]:
-        """Get metric name to ID mapping with statistics"""
-        results = self.db_manager.execute_sql(
-            MonitorSql.get_metric_mapping_sql()
+        """获取metric名称到ID的映射及对应的统计信息"""
+        metric_results = self.db_manager.execute_sql(
+            TrendSql.get_metric_mapping_sql()
+        )        
+        global_stats_result = self.db_manager.execute_sql(
+            TrendSql.get_global_stats_sql()
         )
-
-        return {
-            row["metric_name"]: (
-                row["metric_id"],
-                get_ordered_stats(row["stats"].split(",")
-                                  ) if row["stats"] else []
-            ) for row in results
-        }
+        if not global_stats_result:
+            return {}
+        
+        global_stats_config = global_stats_result[0]
+        # 构建映射字典
+        metric_mapping = {}
+        for row in metric_results:
+            metric_name = row["metric_name"]
+            metric_id = row["metric_id"]
+            
+            # 从global_stats中获取该metric对应的统计信息
+            stats_value = global_stats_config.get(metric_name)
+            ordered_stats = []
+            if stats_value:
+                stats_list = stats_value.split(",") if isinstance(stats_value, str) else []
+                ordered_stats = get_ordered_list(stats_list, MonitorConst.OP_MONVIS_SUPPORTED)                
+            metric_mapping[metric_name] = (metric_id, ordered_stats)
+        return metric_mapping
 
     def get_target_mapping(self) -> Dict[Tuple[str, int, int], int]:
         """Get target mapping dictionary"""
@@ -268,6 +150,9 @@ class MonitorDB:
             for row in results
         }
 
+    def create_trend_data(self, stats):
+        self.db_manager.execute_sql(TrendSql.create_trend_table(stats))
+
     def _get_metric_id(self, metric_name: str) -> Optional[int]:
         """Get metric ID by name"""
         result = self.db_manager.select_data(
@@ -276,3 +161,53 @@ class MonitorDB:
             where={"metric_name": metric_name}
         )
         return result[0]["metric_id"] if result else None
+
+    def extract_tags_from_processed_targets(self, targets:dict , metric_id_dict: dict, target_dict: dict):
+        """从已处理的targets中提取标签并建立映射关系"""
+
+        for metric_name in targets:
+            tag_info_to_id = {}
+            mapping_data = []
+            new_tags_data = set()
+            target_to_tags = {}
+            metric_id = metric_id_dict.get(metric_name, [None,])[0]
+            for target in targets[metric_name]:
+                target_id = target_dict[target]
+                name, _, _ = target
+                tags = get_tags_from_target(name)
+                new_tags_data = new_tags_data.union(tags)
+                target_to_tags[target_id] = tags
+
+            # 插入新标签
+            if not new_tags_data:
+                continue
+            self.db_manager.insert_data(
+                "monitoring_tags",
+                [(tag_name, category, metric_id)
+                    for tag_name, category in new_tags_data],
+                key_list=["tag_name", "category", "metric_id"]
+            )
+
+            # 重新获取tag_id用于映射
+            tag_records = self.db_manager.select_data(
+                "monitoring_tags",
+                columns=["tag_id", "tag_name", "category", "metric_id"]
+            )
+            for row in tag_records:
+                tag_info_to_id[(row["tag_name"], row["category"],
+                                row["metric_id"])] = row["tag_id"]
+
+            for target_id, tags_list in target_to_tags.items():
+                for tag_name, category in tags_list:
+                    tag_id = tag_info_to_id.get(
+                        (tag_name, category, metric_id))
+                    if tag_id:
+                        mapping_data.append((target_id, tag_id))
+
+            # 插入映射关系
+            if mapping_data:
+                self.db_manager.insert_data(
+                    "tag_target_mapping",
+                    mapping_data,
+                    key_list=["target_id", "tag_id"]
+                )

msprobe/core/monitor/utils.py

@@ -1,17 +1,19 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
 from collections import namedtuple
 from datetime import timezone, timedelta
 from functools import wraps
@@ -24,7 +26,6 @@ from msprobe.core.common.log import logger
 from msprobe.core.common.utils import is_int
 from msprobe.core.common.file_utils import check_file_or_directory_path, recursive_chmod
 
-
 beijing_tz = timezone(timedelta(hours=8))
 MVResult = namedtuple('MVResult', ("exp_avg", "exp_avg_sq", "update", "ratio"))
 
@@ -96,7 +97,7 @@ def validate_targets(targets):
             raise TypeError('key of targets should be module_name[str] in config.json')
         if not isinstance(field, dict):
             raise TypeError('values of targets should be cared filed e.g. {"input": "tensor"} in config.json')
-        
+
 
 def validate_l2_targets(targets):
     if not isinstance(targets, dict):
@@ -114,15 +115,15 @@ def validate_l2_targets(targets):
 def validate_recording_l2_features(recording_l2_features):
     if not isinstance(recording_l2_features, bool):
         raise TypeError("recording_l2_features should be a bool")
-    
+
 
 def validate_sa_order(sa_order):
     if isinstance(sa_order, str):
         sa_order = sa_order.replace(' ', '')
     if sa_order not in MonitorConst.SA_ORDERS:
         raise TypeError(f'sa_order must be in {MonitorConst.SA_ORDERS}, got {sa_order}')
-    
-    
+
+
 def validate_print_struct(print_struct):
     if not isinstance(print_struct, bool):
         raise TypeError("print_struct should be a bool")
@@ -246,7 +247,7 @@ def validate_config(config):
 
     recording_l2_features = config.get("recording_l2_features", False)
     validate_recording_l2_features(recording_l2_features)
-    
+
     sa_order = config.get("sa_order", "s,b,h,d")
     validate_sa_order(sa_order)
 

msprobe/core/monitor_v2/__init__.py

@@ -0,0 +1,20 @@
+"""
+monitor_v2: next-generation monitoring core for unifying
+PyTorch / MindSpore monitor logic behind a single, clean abstraction.
+
+Goal:
+  - keep production behavior stable while iterating;
+  - migrate mature logic from existing monitors in small steps;
+  - enforce clearer layering and performance constraints.
+
+Current structure:
+  base.py
+    - BaseMonitorV2: minimal shared monitor scaffold.
+  trainer.py
+    - TrainerMonitorV2: orchestrator for step gating and writer integration.
+  factory.py
+    - MonitorFactory: registry of framework-specific monitor classes.
+  writer.py
+    - CSVWriterV2: CSV output for per-step monitor data.
+"""
+from .trainer import TrainerMonitorV2

msprobe/core/monitor_v2/base.py

@@ -0,0 +1,83 @@
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterable, List, Optional
+
+from msprobe.core.common.log import logger
+
+
+class BaseMonitorV2(ABC):
+    """
+    Minimal base class for v2 monitors.
+
+    Compared to core.monitor.BaseMonitor, this class is intentionally kept
+    small so that higher-level policies live in dedicated components
+    (framework, trainer, backends) instead of being mixed into the base.
+    """
+
+    DEFAULT_OPS = ["min", "max", "mean", "norm", "nans"]
+
+    def __init__(self) -> None:
+        self.config: Dict[str, Any] = {}
+        self._context: Dict[str, Any] = {}
+        self._ops: List[str] = []
+        # Common per-step rows buffer; subclasses can populate this and
+        # rely on the default collect() implementation.
+        self._rows: List[Dict[str, Any]] = []
+
+    def set_config(self, config: Dict[str, Any]) -> None:
+        self.config = config
+        self._configure_ops(config.get("ops"))
+
+    def set_context(self, **context: Any) -> None:
+        self._context.update(context)
+
+    def collect(self) -> Optional[Dict[str, Any]]:
+        """
+        Default collect implementation used by most monitors:
+        return per-step rows and reset the internal buffer.
+        """
+        if not self._rows:
+            return None
+        data = {"rows": self._rows}
+        self._rows = []
+        return data
+
+    @abstractmethod
+    def start(self, *args: Any, **kwargs: Any) -> None:
+        ...
+
+    @abstractmethod
+    def stop(self) -> None:
+        ...
+
+    def _configure_ops(self, ops: Optional[Iterable[str]] = None) -> None:
+        if ops is None or not isinstance(ops, Iterable):
+            self._ops = list(self.DEFAULT_OPS)
+            return
+        requested = [ops] if isinstance(ops, str) else list(ops)
+        invalid_ops = [op for op in requested if op not in self.DEFAULT_OPS]
+        if invalid_ops:
+            logger.warning(
+                f"[monitor_v2] Unsupported ops {invalid_ops}; supported: {self.DEFAULT_OPS}."
+            )
+        filtered = [op for op in requested if op in self.DEFAULT_OPS]
+        if not filtered:
+            logger.warning("[monitor_v2] No valid ops provided; fallback to default ops.")
+        self._ops = filtered if filtered else list(self.DEFAULT_OPS)