mindstudio-probe 1.2.2__py3-none-any.whl → 8.1.0__py3-none-any.whl

msprobe/mindspore/free_benchmark/self_check_tool_factory.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from msprobe.mindspore.common.const import Const
+from msprobe.core.common.log import logger
 from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
 from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelfCheck
 
@@ -41,8 +42,10 @@ class SelfCheckToolFactory:
     def create(config: DebuggerConfig):
         tool = SelfCheckToolFactory.tools.get(config.level)
         if not tool:
-            raise Exception(f"{config.level} is not supported.")
+            logger.error(f"{config.level} is not supported.")
+            raise ValueError
         tool = tool.get(config.execution_mode)
         if not tool:
-            raise Exception(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.")
+            logger.error(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.")
+            raise ValueError
         return tool(config)

msprobe/mindspore/grad_probe/global_context.py

@@ -16,6 +16,7 @@
 import os
 import threading
 from typing import Dict, Union, Tuple
+import time
 
 from msprobe.core.common.utils import is_int
 from msprobe.core.common.file_utils import create_directory, check_path_before_create
@@ -40,8 +41,12 @@ class GlobalContext:
     def __new__(cls, *args, **kwargs):
         if cls._instance is None:
             cls._instance_lock.acquire()
-            cls._instance = object.__new__(cls)
-            cls._instance_lock.release()
+            try:
+                cls._instance = object.__new__(cls)
+            except Exception as e:
+                raise RuntimeError("grad_probe global context init failed") from e
+            finally:
+                cls._instance_lock.release()
         return cls._instance
 
     def init_context(self, config_dict: Dict):
@@ -69,6 +74,8 @@ class GlobalContext:
         else:
             logger.warning("The output_path exists, the data will be covered.")
 
+        self._setting[GradConst.TIME_STAMP] = str(int(time.time()))
+
     def get_context(self, key: str):
         if key not in self._setting:
             logger.warning(f"Unrecognized {key}.")

msprobe/mindspore/grad_probe/grad_analyzer.py

@@ -111,7 +111,8 @@ class CSVGenerator(Process):
         output_path = context.get_context(GradConst.OUTPUT_PATH)
         self.level = context.get_context(GradConst.LEVEL)
         self.bounds = context.get_context(GradConst.BOUNDS)
-        self.dump_dir = f"{output_path}/rank{rank_id}/Dump/"
+        time_stamp = context.get_context(GradConst.TIME_STAMP)
+        self.dump_dir = f"{output_path}/rank{rank_id}/Dump{time_stamp}/"
         self.save_dir = f"{output_path}/rank{rank_id}/"
         self.current_step = None
         self.stop_event = multiprocessing.Event()

msprobe/mindspore/grad_probe/grad_stat_csv.py

@@ -15,6 +15,7 @@
 
 import hashlib
 from abc import ABC, abstractmethod
+import zlib
 
 import mindspore
 from mindspore import ops
@@ -76,8 +77,8 @@ class CsvMd5(CsvItem):
     def generate_csv_content(csv_input):
         grad = csv_input.grad
         tensor_bytes = grad.float().numpy().tobytes()
-        md5_hash = hashlib.md5(tensor_bytes)
-        return [md5_hash.hexdigest()]
+        md5_hash = f"{zlib.crc32(tensor_bytes):08x}"
+        return [md5_hash]
 
 
 @register_csv_item(GradConst.DISTRIBUTION)

msprobe/mindspore/grad_probe/hook.py

@@ -49,12 +49,10 @@ class HookInput:
         self.param_list = grad_context.get_context(GradConst.PARAM_LIST)
         self.rank_id = get_rank_id()
         output_path = grad_context.get_context(GradConst.OUTPUT_PATH)
-        self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump")
+        time_stamp = grad_context.get_context(GradConst.TIME_STAMP)
+        self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", f"Dump{time_stamp}")
         self.save_dir = os.path.join(output_path, f"rank{self.rank_id}")
         self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH)
-        if os.path.exists(self.save_dir):
-            logger.warning(f"Delete existing path {self.save_dir}.")
-            remove_path(self.save_dir)
         self.level = grad_context.get_context(GradConst.LEVEL)
         self.bounds = grad_context.get_context(GradConst.BOUNDS)
         self.mode = mindspore.get_context("mode")

msprobe/mindspore/mindspore_service.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+import mindspore as ms
+from mindspore.ops.primitive import Primitive
+
+from msprobe.core.common.utils import Const
+from msprobe.core.service import BaseService
+from msprobe.mindspore.cell_processor import CellProcessor
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.common.utils import (
+    get_rank_if_initialized,
+    is_mindtorch,
+    get_cells_and_names_with_index
+)
+from msprobe.mindspore.dump.hook_cell.api_register import get_api_register, ApiTemplate
+from msprobe.mindspore.dump.hook_cell.ms_hook_manager import MindsproeHookManager
+from msprobe.mindspore.dump.hook_cell.primitive_hooks import PrimitiveHookService
+from msprobe.mindspore.dump.jit_dump import JitDump
+
+try:
+    from mindspore.common._pijit_context import PIJitCaptureContext
+except ImportError:
+    pijit_label = False
+else:
+    pijit_label = True
+
+
+class MindsporeService(BaseService):    
+    @property
+    def _get_framework_type(self):
+        return Const.MT_FRAMEWORK if is_mindtorch() else Const.MS_FRAMEWORK
+    
+    @staticmethod
+    def _get_current_rank():
+        return get_rank_if_initialized()
+
+    def empty(self, *args, **kwargs):
+        pass
+
+    def _init_specific_components(self):
+        self.logger = logger
+        self.api_register = get_api_register()
+        self.primitive_hook_service = PrimitiveHookService(self)
+        self.cell_processor = CellProcessor(self.data_collector.scope)
+        self.hook_manager = MindsproeHookManager(self.data_collector, self.config)
+        self._setup_jit_context()
+        self.api_template = ApiTemplate
+
+    def _setup_jit_context(self):
+        if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L1]:
+            JitDump.set_config(self.config)
+            JitDump.set_data_collector(self.data_collector)
+            if hasattr(ms.common.api, "_MindsporeFunctionExecutor"):
+                ms.common.api._MindsporeFunctionExecutor = JitDump
+            else:
+                ms.common.api._JitExecutor = JitDump
+            ms.common.api._PyNativeExecutor.grad = JitDump.grad
+            if pijit_label:
+                PIJitCaptureContext.__enter__ = self.empty
+                PIJitCaptureContext.__exit__ = self.empty
+
+    def _register_module_hook(self):
+        self.cell_processor.register_cell_hook(self.model, self.build_hook, self.config)
+        self.logger.info_on_rank_0(f"The module {self.config.task} hook function is successfully mounted to the model.")
+
+    def _register_hook(self):
+        self._register_primitive_hook()
+
+    def _register_primitive_hook(self):
+        if self.config.level not in [Const.LEVEL_MIX, Const.LEVEL_L1]:
+            return
+        if not self.model or self.config.task not in Const.DUMP_DATA_COLLECTION_LIST:
+            return
+
+        primitive_set = set()
+        cells_and_names_with_index, _ = get_cells_and_names_with_index(self.model)
+        for cells_and_names in cells_and_names_with_index.values():
+            for _, cell in cells_and_names:
+                for attribute, value in vars(cell).items():
+                    if isinstance(value, Primitive):
+                        primitive_set.add((attribute, value))
+
+        for pname, primitive in primitive_set:
+            primitive_class_name = primitive.__class__.__name__
+            primitive_combined_name = pname + Const.SEP + primitive_class_name
+            new_primitive = type('NewPrimitive', (primitive.__class__,),
+                                    {'__call__': self.primitive_hook_service.wrap_primitive(primitive.__call__,
+                                                                                            primitive_combined_name)})
+            primitive.__class__ = new_primitive
+
+    def _reset_status(self):
+        super()._reset_status()
+        self.primitive_hook_service.primitive_counters.clear()
+        JitDump.jit_count = defaultdict(int)
+
+    def _change_jit_switch(self, status):
+        JitDump.jit_dump_switch = status

msprobe/mindspore/monitor/common_func.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from mindspore import nn
+from mindspore import communication
+from msprobe.mindspore.monitor.utils import logger
+from msprobe.mindspore.common.utils import is_mindtorch
+if is_mindtorch():
+    import torch
+
+
+def is_valid_instance(model):
+    return isinstance(model, torch.nn.Module) if is_mindtorch() else isinstance(model, nn.Cell)
+
+
+def get_submodules(model):
+    if not is_valid_instance(model):
+        logger.info("Counter invalid model, nothing to hook")
+        return {}
+    return model.named_modules() if is_mindtorch() else model.cells_and_names()
+
+
+def get_parameters(model):
+    if not is_valid_instance(model):
+        return {}
+    if is_mindtorch():
+        return model.named_parameters()
+    else:
+        return model.parameters_and_names()
+
+
+def get_rank():
+    if comm_is_initialized():
+        return communication.get_rank()
+    return 0
+
+
+def comm_is_initialized():
+    return communication.GlobalComm.INITED

msprobe/mindspore/monitor/data_writers.py

@@ -0,0 +1,237 @@
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import os
+from dataclasses import dataclass
+from collections import defaultdict
+
+import pandas as pd
+from mindspore import ops
+from mindspore import Tensor
+from mindspore import _no_grad
+
+from msprobe.core.common.log import logger
+from msprobe.core.common.file_utils import change_mode, create_directory, write_df_to_csv
+from msprobe.core.monitor.anomaly_processor import AnomalyDataFactory, AnomalyTurbulence, AnomalyScanner
+from msprobe.core.common.const import FileCheckConst, MonitorConst
+
+
+class BCOLORS:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+@dataclass
+class WriterInput:
+    path: str
+    ad_rules: list
+    job_id: str
+    anomaly_factory: AnomalyDataFactory = None
+    ndigits: int = 6
+    step_count_per_record: int = 1
+
+
+class BaseWriterWithAD:
+    def __init__(self, writer_input: WriterInput):
+        self.tag2scalars = {}
+        self.ad_rules = writer_input.ad_rules
+        self.job_id = writer_input.job_id
+        self.anomaly_factory = writer_input.anomaly_factory
+        self.anomalies = []
+        self.ndigits = writer_input.ndigits
+        self.beta = 0.99
+
+    @staticmethod
+    def stack_tensors(tensor_list):
+        """
+        Torch not support stack cpu and xpu tensors. Group the tensors into cpu_group and xpu_group,
+        stack them separately, migrate xpu_group to cpu, and then restore in the order of input.
+
+        :param tensor_list: [tensor(-1.6165), tensor(-1.0985), tensor(-1.7777), tensor(-1.8408, device='npu:0')]
+        :return: result: list of float
+        """
+        cpu_tensors = []
+        xpu_tensors = []
+
+        for tensor in tensor_list:
+            if isinstance(tensor, Tensor):
+                # 将device上的tensor先stack后to cpu
+                xpu_tensors.append(tensor)
+            else:
+                cpu_tensors.append(tensor)
+
+        xpu_stack = ops.stack(xpu_tensors).tolist() if xpu_tensors else ops.tensor([])
+
+        # 按照输入的顺序恢复
+        result = []
+        cpu_tensors_idx, xpu_tensors_idx = 0, 0
+        for tensor in tensor_list:
+            if isinstance(tensor, Tensor):
+                result.append(xpu_stack[xpu_tensors_idx])
+                xpu_tensors_idx += 1
+            else:
+                result.append(cpu_tensors[cpu_tensors_idx])
+                cpu_tensors_idx += 1
+
+        return result
+
+    def get_anomalies(self):
+        """返回已检测到的异常列表
+        """
+        return self.anomalies
+
+    def clear_anomalies(self):
+        self.anomalies.clear()
+
+    def add_scalar(self, tag, scalar_value, global_step=None, need_explain=False):
+        """If an anomaly is detected, the anomaly information is recorded and added to self.anomalies.
+        Args:
+            tag (tuple): tuple of tag_name and tag like ('0:1.post_attention_norm.weight/rank0/pre_grad', 'min').
+            scalar_value (float): scalar_value.
+            global_step (int): global_step.
+        Returns:
+            None
+        """
+        if not self.ad_rules or tag[-1] in ["shape", "dtype"]:
+            return
+        if isinstance(scalar_value, Tensor):
+            scalar_value = scalar_value.item()
+        avg = self._update_tag2scalars(tag, scalar_value)
+        detected, rule_name = self._ad(scalar_value, history=avg)
+        if detected:
+            if rule_name == AnomalyTurbulence.name and tag[-1] not in ["norm", "mean"]:
+                return
+            exception_message = (f"Rule {rule_name} reports anomaly signal in {tag} at step {global_step}, "
+                                 f"current value {scalar_value}, history mean {avg}.")
+            logger.info(f"{BCOLORS.WARNING}> {exception_message}{BCOLORS.ENDC}")
+            # append to self.anomalies for dump
+            if self.anomaly_factory:
+                self.anomalies.append(self.anomaly_factory.create(tag, exception_message, global_step))
+
+    def write_metrics(self, op_list, metric_value, step, prefix='', need_explain=False):
+        if not metric_value:
+            return
+        tensors = []
+        tags = list(itertools.product(metric_value.keys(), op_list))
+        for op2tensor in metric_value.values():
+            tensors.extend(op2tensor.values())
+
+        if not tensors:
+            return
+
+        with _no_grad():
+            metric_list = self.stack_tensors(tensors)
+        for tag, metric in zip(tags, metric_list):
+            self.add_scalar(tag, metric, step, need_explain)
+
+    def _ad(self, scalar_value, history):
+        return AnomalyScanner.scan(self.ad_rules, history, cur=scalar_value)
+
+    def _update_tag2scalars(self, tag, scalar_value):
+        """Update the average and count of a scalar value associated with a tag.
+
+        This method is used to maintain a running average of scalar values for each tag.
+
+
+        Args:
+            tag (str): The tag identifier.
+            scalar_value (float): The scalar value to be added.
+
+        Returns:
+            float: The average value before update.
+        """
+        abs_scalar_value = abs(scalar_value)
+        if tag not in self.tag2scalars:
+            self.tag2scalars[tag] = {'avg': abs_scalar_value, 'count': 0}
+        avg = self.tag2scalars[tag]['avg']
+        self.tag2scalars[tag]['avg'] = self.beta * avg + (1 - self.beta) * abs_scalar_value
+        self.tag2scalars[tag]['count'] += 1
+        return avg
+
+
+class CSVWriterWithAD(BaseWriterWithAD):
+    def __init__(self, writer_input: WriterInput):
+        super().__init__(writer_input)
+
+        path = writer_input.path
+        self.log_dir = path
+        create_directory(path)
+        change_mode(path, FileCheckConst.DATA_DIR_AUTHORITY)
+        self.context_dict = defaultdict(list)
+        self.header = []
+        self.step_count_per_record = writer_input.step_count_per_record
+
+    def get_step_interval(self, step):
+        count = step // self.step_count_per_record
+        return count * self.step_count_per_record, (count + 1) * self.step_count_per_record - 1
+
+    def write_csv(self, prefix, step):
+        """
+        Args:
+            prefix[str]: prefix of output csv file e.g. grad_unreduced
+            step[int]
+        """
+        if len(self.context_dict) == 0:
+            return
+
+        ster_start, step_end = self.get_step_interval(step)
+        filepath = os.path.join(self.log_dir, f'{prefix}_{ster_start}-{step_end}.csv')
+        if not os.path.exists(filepath):
+            data_frame = pd.DataFrame(columns=self.header)
+            write_df_to_csv(data_frame, filepath)
+
+        new_data = []
+        for name, metric_value in self.context_dict.items():
+            new_line = name.split(MonitorConst.NAME_SEP) + metric_value
+            new_line.insert(2, step)
+            new_data.append(new_line)
+        new_data = pd.DataFrame(new_data).round(self.ndigits).fillna("nan")
+        write_df_to_csv(new_data, filepath, mode='a+', header=False)
+        self.context_dict = defaultdict(list)
+
+    def add_scalar(self, tag, scalar_value, global_step, need_explain=False):
+        """
+        ('0:1.post_attention_norm.weight/rank0/pre_grad', 'min')
+        """
+        super().add_scalar(tag, scalar_value, global_step, need_explain=False)
+        split_name = tag[0].split('/')
+        name = split_name[0]
+        if need_explain:
+            if 'pre' in split_name[-1]:
+                name += '.input'
+            if 'post' in split_name[-1]:
+                name += '.output'
+        self.context_dict[name].append(scalar_value)
+
+    def write_metrics(self, op_list, metric_value, step, prefix='', need_explain=False, **kwargs):
+        need_explain = prefix == 'other'
+        super().write_metrics(op_list, metric_value, step, prefix='', need_explain=need_explain)
+
+        if prefix in [MonitorConst.ACTV, MonitorConst.ACTVGRAD] or kwargs.get("use_micro_step"):
+            self.header = MonitorConst.CSV_HEADER_MICRO_STEP + op_list
+        else:
+            self.header = MonitorConst.CSV_HEADER + op_list
+        self.write_csv(prefix, step)
+
+    def close(self):
+        pass

msprobe/mindspore/monitor/distributed/wrap_distributed.py

@@ -281,7 +281,7 @@ def create_hooks(context, monitor):
     global RANK
     pre_hooks = []
     hooks = []
-    RANK = str(get_rank())
+    RANK = get_rank()
     if communication.GlobalComm.INITED and RANK not in monitor.module_rank_list and monitor.module_rank_list != []:
         return [pre_hooks, hooks]
 

msprobe/mindspore/monitor/features.py

@@ -46,6 +46,8 @@ def get_max(x: Tensor):
 
 @_no_grad()
 def get_zeros(x: Tensor, eps: float):
+    if x.numel() == 0:
+        return Tensor(float('nan'))
     return mint.sum(mint.abs(x) < eps) / x.numel()
 
 
@@ -54,10 +56,20 @@ def get_nans(t):
     return ops.isnan(t.astype(mstype.float32)).sum()
 
 
+def get_shape(t):
+    return t.shape
+
+
+def get_dtype(t):
+    return t.dtype
+
+
 FUNC_MAP = {"min"  : get_min,
             "max"  : get_max,
             "mean" : get_mean,
             "norm" : get_norm,
             "nans" : get_nans,
-            "zeros": get_zeros
+            "zeros": get_zeros,
+            "shape": get_shape,
+            "dtype": get_dtype
            }