mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/core/data_dump/scope.py

@@ -1,40 +1,70 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import ABC, abstractmethod
-from msprobe.core.common.exceptions import ScopeException
+import re
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import ScopeException
 
 
-def build_scope(scope_class, scope=None, api_list=None):
-    if not scope and not api_list:
-        return None
-    if scope is None:
-        scope = []
-    if api_list is None:
-        api_list = []
-    if scope_class:
-        return scope_class(scope, api_list)
-    return build_range_scope_according_to_scope_name(scope, api_list)
-
-
-def build_range_scope_according_to_scope_name(scope, api_list):
-    api_range_scope = APIRangeScope(scope, api_list)
-    module_range_scope = ModuleRangeScope(scope, api_list)
-    if not scope:  # 如果没有scope参数则用哪类scope都一样
-        return api_range_scope
-    if api_range_scope.is_valid and module_range_scope.is_valid:
-        raise ScopeException(ScopeException.InvalidScope, f"scope={scope}.")
-    elif api_range_scope.is_valid:
-        return api_range_scope
-    elif module_range_scope.is_valid:
-        return module_range_scope
-    else:
-        raise ScopeException(ScopeException.InvalidScope, f"scope={scope}")
+class ScopeFactory:
+    def __init__(self, config):
+        self.task = config.task
+        self.level = config.level
+        self.scope = config.scope
+        self.api_list = config.list
+
+    def build_scope(self):
+        if not self.scope and not self.api_list:
+            return None
+        if self.scope is None:
+            self.scope = []
+        if self.api_list is None:
+            self.api_list = []
+        if self.task == Const.FREE_BENCHMARK:
+            return ListScope(self.scope, self.api_list)
+        return self._build_range_scope()
+
+    def _build_range_scope(self):
+        api_range_scope = APIRangeScope(self.scope, self.api_list, self.level)
+        module_range_scope = ModuleRangeScope(self.scope, self.api_list, self.level)
+        mix_range_scope = MixRangeScope(self.scope, self.api_list, self.level)
+
+        if self.level == Const.LEVEL_MIX:
+            return mix_range_scope
+        
+        if not self.scope:
+            return api_range_scope
+        if api_range_scope.is_valid and module_range_scope.is_valid:
+            raise ScopeException(ScopeException.InvalidScope, f"scope={self.scope}.")
+        elif api_range_scope.is_valid:
+            return api_range_scope
+        elif module_range_scope.is_valid:
+            return module_range_scope
+        else:
+            raise ScopeException(ScopeException.InvalidScope, f"scope={self.scope}")
 
 
 class BaseScope(ABC):
     Module_Type_Module = "Module"
     Module_Type_API = "api"
+    module_type = ["Module", "Cell"]
 
-    def __init__(self, scope, api_list):
+    def __init__(self, scope, api_list, level=None):
+        self.level = level
         scope, api_list = self.rectify_args(scope, api_list)
         self.scope = scope
         self.api_list = api_list
@@ -81,9 +111,9 @@ class ListScope(BaseScope):
                 f"scope和api_list不可以同时配置，实际配置为scope={scope}, api_list={api_list}.")
         return super(ListScope, ListScope).rectify_args(scope, api_list)
 
-    def check(self, module_name):
-        if not self.scope or module_name in self.scope:
-            return self.check_api_list(module_name)
+    def check(self, name):
+        if not self.scope or name in self.scope:
+            return self.check_api_list(name)
         return False
 
 
@@ -92,19 +122,36 @@ class RangeScope(BaseScope, ABC):
     def __init__(self, *args):
         super().__init__(*args)
         self.in_scope = False
+        self.in_list = False
         self.is_valid = self.check_scope_is_valid()
 
+    def check_name_pattern(self, name):
+        options_pattern = "|".join(re.escape(option) for option in Const.DUMP_PREFIX)
+        api_pattern = rf"^({options_pattern})\..*\.\d+\.(forward|backward)$"
+        module_pattern = r"^(Cell|Module)\..*\.(forward|backward)\.\d+$"
 
-    @staticmethod
-    def rectify_args(scope, api_list):
-        scope, api_list = super(RangeScope, RangeScope).rectify_args(scope, api_list)
-        if isinstance(scope, list):
-            if len(scope) == 1:
-                scope.append(scope[0])
-            elif len(scope) > 2:
+        if self.level == Const.LEVEL_L1:
+            if not re.match(api_pattern, name):
                 raise ScopeException(ScopeException.InvalidScope,
-                    f"scope参数指定区间断点，须传入长度为1或2的列表，实际长度为{len(scope)}.")
+                                    f"scope参数格式错误，要求格式为api完整命名，实际为{name}.")
+        
+        if self.level == Const.LEVEL_L0:
+            if not re.match(module_pattern, name):
+                raise ScopeException(ScopeException.InvalidScope,
+                                    f"scope参数格式错误，要求格式为模块完整命名，实际为{name}.")
+
+        if self.level == Const.LEVEL_MIX:
+            if not re.match(api_pattern, name) and not re.match(module_pattern, name):
+                raise ScopeException(ScopeException.InvalidScope,
+                                    f"scope参数格式错误，要求格式为api或模块完整命名，实际为{name}.")
 
+    def rectify_args(self, scope, api_list):
+        scope, api_list = super(RangeScope, RangeScope).rectify_args(scope, api_list)
+        if scope and len(scope) != 2:
+            raise ScopeException(ScopeException.InvalidScope,
+                f"scope参数指定区间断点，须传入长度为2的列表，实际长度为{len(scope)}.")
+        for name in scope:
+            self.check_name_pattern(name)
         return scope, api_list
 
     @abstractmethod
@@ -123,23 +170,23 @@ class APIRangeScope(RangeScope):
         if not self.scope:
             return True
         scope_start_type = self.scope[0].split(Const.SEP)[0]
-        if scope_start_type == BaseScope.Module_Type_Module:
+        if scope_start_type in BaseScope.module_type:
             return False
         scope_stop_type = self.scope[1].split(Const.SEP)[0]
-        if scope_stop_type == BaseScope.Module_Type_Module:
+        if scope_stop_type in BaseScope.module_type:
             return False
         return True
 
-    def check(self, api_name):
-        if self.scope and api_name == self.scope[0]:
+    def check(self, name):
+        if self.scope and name == self.scope[0]:
             self.in_scope = True
 
         if not self.scope or self.in_scope:
-            result = self.check_api_list(api_name)
+            result = self.check_api_list(name)
         else:
             result = False
 
-        if self.scope and api_name == self.scope[1]:
+        if self.scope and name == self.scope[1]:
             self.in_scope = False
         return result
 
@@ -150,13 +197,14 @@ class ModuleRangeScope(RangeScope):
         需要用pre_hook和full_backward_hook来精确控制module的开始和结束，
         在这些hook触发时调用begin_module和end_module做区间控制
     """
+
     def check_scope_is_valid(self):
         if not self.scope:
             return True
         scope_start_type = self.scope[0].split(Const.SEP)[0]
         scope_stop_type = self.scope[1].split(Const.SEP)[0]
-        if scope_start_type == BaseScope.Module_Type_Module and \
-                scope_stop_type == BaseScope.Module_Type_Module:
+        if scope_start_type in BaseScope.module_type and \
+                scope_stop_type in BaseScope.module_type:
             return True
         return False
 
@@ -172,7 +220,54 @@ class ModuleRangeScope(RangeScope):
         if module_name == self.scope[1]:
             self.in_scope = False
 
-    def check(self, module_name):
+    def check(self, name):
         if not self.scope or self.in_scope:
-            return self.check_api_list(module_name)
+            return self.check_api_list(name)
         return False
+
+
+class MixRangeScope(RangeScope):
+    def check_scope_is_valid(self):
+        return True if self.scope else False
+        
+    def begin_module(self, module_name):
+        if self.scope and module_name == self.scope[0]:
+            self.in_scope = True
+        for name in self.api_list:
+            if name in module_name:
+                self.in_list = True
+
+    def end_module(self, module_name):
+        if self.scope and module_name == self.scope[1]:
+            self.in_scope = False
+        for name in self.api_list:
+            if name in module_name:
+                self.in_list = False
+
+    def check_api_list(self, api_name):
+        if not self.api_list:
+            return True
+        
+        for name in self.api_list:
+            if name in api_name:
+                return True
+        return False
+    
+    def check(self, name):
+        """
+        dump时调用的接口，根据scope和api_list判断是否需要dump
+        """
+        result = False
+        if self.scope and name == self.scope[0]:
+            self.in_scope = True
+
+        if not self.scope or self.in_scope:
+            if self.in_list:
+                result = True
+            else:
+                result = self.check_api_list(name)
+
+        if self.scope and name == self.scope[1]:
+            self.in_scope = False
+        return result
+    

msprobe/core/grad_probe/constant.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 class GradConst:
 
@@ -33,6 +47,10 @@ class GradConst:
     # direction suffix
     DIR_SUFFIX = "dir.npy"
 
+    # bounds safety
+    BOUNDS_MINIMUM = -2**63
+    BOUNDS_MAXIMUM = 2**63 - 1
+
     # file safty
     DATA_DIR_AUTHORITY = 0o750
     DATA_FILE_AUTHORITY = 0o640
@@ -56,16 +74,16 @@ class GradConst:
     NORM = "norm"
 
 level_adp = {
-        "L0": {
-            "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": False
-        },
-        "L1": {
-            "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": True
-        },
-        "L2": {
-            "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": True
-        },
-    }
+    "L0": {
+        "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+        "have_grad_direction": False
+    },
+    "L1": {
+        "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+        "have_grad_direction": True
+    },
+    "L2": {
+        "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+        "have_grad_direction": True
+    },
+}

msprobe/core/grad_probe/grad_compare.py

@@ -1,13 +1,27 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 from typing import List
 
 from tqdm import tqdm
-import pandas as pd
 import matplotlib.pyplot as plt
 
-from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path
+from msprobe.core.common.file_utils import create_directory, check_file_or_directory_path
 from msprobe.core.common.log import logger
-from msprobe.core.common.file_utils import remove_path, load_npy, write_csv
+from msprobe.core.common.file_utils import remove_path, load_npy, write_csv, read_csv
 from msprobe.core.grad_probe.constant import GradConst
 from msprobe.core.grad_probe.utils import plt_savefig
 
@@ -21,7 +35,7 @@ class GradComparator:
                 continue
             if not os.path.exists(os.path.join(path2, summary_file)):
                 continue
-            summary_csv = pd.read_csv(os.path.join(path1, summary_file))
+            summary_csv = read_csv(os.path.join(path1, summary_file))
             return summary_csv["param_name"]
         raise RuntimeError("no matched grad_summary.csv for comparison, please dump data in same configuration")
     
@@ -34,6 +48,8 @@ class GradComparator:
 
     @classmethod
     def compare_distributed(cls, path1: str, path2: str, output_dir: str):
+        check_file_or_directory_path(path1, isdir=True)
+        check_file_or_directory_path(path2, isdir=True)
         ranks = cls._get_matched_dirs(path1, path2, "rank")
         logger.info(f"the following ranks will be compared: {ranks}")
         if not ranks:

msprobe/core/grad_probe/utils.py

@@ -1,8 +1,24 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import re
 from msprobe.core.grad_probe.constant import GradConst
 from msprobe.core.common.log import logger
 from msprobe.core.common.file_utils import write_csv, check_path_before_create, change_mode
 from msprobe.core.common.const import FileCheckConst
+from msprobe.core.common.utils import is_int
 import matplotlib.pyplot as plt
 
 
@@ -20,12 +36,37 @@ def check_numeral_list_ascend(lst):
 def check_param(param_name):
     if not re.match(GradConst.PARAM_VALID_PATTERN, param_name):
         raise RuntimeError("The parameter name contains special characters.")
-    
+
 
 def check_str(string, variable_name):
     if not isinstance(string, str):
         raise ValueError(f'The variable: "{variable_name}" is not a string.')
-    
+
+
+def check_bounds_element(bound):
+    return GradConst.BOUNDS_MINIMUM <= bound <= GradConst.BOUNDS_MAXIMUM
+
+
+def check_param_element(param):
+    if not re.match(GradConst.PARAM_VALID_PATTERN, param):
+        return False
+    else:
+        return True
+
+
+def check_bounds(bounds):
+    if not isinstance(bounds, list):
+        raise Exception(f"bounds must be a list")
+    prev = GradConst.BOUNDS_MINIMUM - 1
+    for element in bounds:
+        if not is_int(element) and not isinstance(element, float):
+            raise Exception("bounds element is not int or float")
+        if not check_bounds_element(element):
+            raise Exception("bounds element is out of int64 range")
+        if prev >= element:
+            raise Exception("bounds list is not ascending")
+        prev = element
+
 
 class ListCache(list):
     threshold = 1000
@@ -50,7 +91,7 @@ class ListCache(list):
         list.append(self, data)
         if len(self) >= ListCache.threshold:
             self.flush()
-    
+
     def set_output_file(self, output_file):
         self._output_file = output_file
 

msprobe/core/overflow_check/abnormal_scene.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Dict, Union, Any
+
+import numpy as np
+
+from msprobe.core.overflow_check.api_info import APIInfo
+from msprobe.core.overflow_check.level import OverflowLevel
+from msprobe.core.overflow_check.utils import has_nan_inf
+
+
+class AnomalyScene:
+    """异常场景的基类"""
+
+    def __init__(self, api_info: APIInfo):
+        self.api_name = api_info.api_name
+        self.api_data = api_info
+
+    @property
+    def rank(self) -> OverflowLevel:
+        """获取异常等级"""
+        raise NotImplementedError
+
+    @staticmethod
+    def _has_anomaly(data: Union[Dict, Any]) -> bool:
+        """检查张量是否包含异常值"""
+        return has_nan_inf(data)
+
+    def get_details(self) -> Dict:
+        """获取异常详情"""
+        return {
+            'api_name': self.api_name,
+            'rank': self.rank.value,
+            'scene_type': self.__class__.__name__,
+            'input_args_anomaly_indices': self._get_anomaly_indices_from_list(self.api_data.input_args),
+            'input_kwargs_anomaly_keys': self._get_anomaly_keys_from_dict(self.api_data.input_kwargs),
+            'output_anomaly_indices': self._get_anomaly_indices_from_list(self.api_data.output_data)
+        }
+
+    def matches(self) -> bool:
+        """
+            待子类实现对应匹配逻辑
+        Returns:
+
+        """
+        raise NotImplementedError
+
+    def _get_anomaly_indices_from_list(self, data_list: List[Dict]) -> List[int]:
+        return [i for i, data in enumerate(data_list) if self._has_anomaly(data)]
+
+    def _get_anomaly_keys_from_dict(self, data_dict: Dict) -> List[str]:
+        return [key for key, data in data_dict.items() if self._has_anomaly(data)]
+
+
+class InputOutputAnomalyScene(AnomalyScene):
+    """输入输出异常检测的基类"""
+    def has_input_anomaly(self) -> bool:
+        """检查输入是否有异常（包括args和kwargs）"""
+        # args
+        args_anomaly = any(self._has_anomaly(x) for x in self.api_data.input_args if isinstance(x, dict))
+        # kwargs
+        kwargs_anomaly = any(self._has_anomaly(x) for x in self.api_data.input_kwargs.values() if isinstance(x, dict))
+        return args_anomaly or kwargs_anomaly
+
+    def has_output_anomaly(self) -> bool:
+        """检查输出是否有异常"""
+        return any(self._has_anomaly(x) for x in self.api_data.output_data if isinstance(x, dict))
+
+    def matches(self) -> bool:
+        """判断是否匹配该场景"""
+        raise NotImplementedError
+
+
+class InputAnomalyOutputNormalScene(InputOutputAnomalyScene):
+    """输入异常，输出正常场景"""
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.MEDIUM
+
+    def matches(self) -> bool:
+        return self.has_input_anomaly() and not self.has_output_anomaly()
+
+
+class InputAnomalyOutputAnomalyScene(InputOutputAnomalyScene):
+    """输入异常，输出异常场景"""
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.HIGH
+
+    def matches(self) -> bool:
+        return self.has_input_anomaly() and self.has_output_anomaly()
+
+
+class InputNormalOutputAnomalyScene(InputOutputAnomalyScene):
+    """输入正常，输出异常场景"""
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.CRITICAL
+
+    def matches(self) -> bool:
+        return not self.has_input_anomaly() and self.has_output_anomaly()
+
+
+class NumericalMutationScene(AnomalyScene):
+    """
+        检查数值突变，统计输入args、kwargs中norm值，同时统计输出的norm最大值，计算差异，大于 threshold 则认为是异常情况
+    """
+    def __init__(self, api_info: APIInfo, threshold: float = 100000.0):
+        super().__init__(api_info)
+        self.threshold = threshold
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.HIGH
+
+    @staticmethod
+    def _get_tensor_norms(data_list: List[Dict]) -> List[float]:
+        norms = []
+        for data in data_list:
+            if isinstance(data, dict) and data.get('type') == 'torch.Tensor':
+                norm = data.get('Norm')
+                if norm is not None and not np.isnan(norm):
+                    norms.append(norm)
+        return norms
+
+    @staticmethod
+    def _get_kwargs_norms(data_dict: Dict) -> List[float]:
+        """
+            获取kwargs中张量的范数列表
+        Args:
+            data_dict:
+        Returns:
+        """
+        norms = []
+        for data in data_dict.values():
+            if isinstance(data, dict) and data.get('type') == 'torch.Tensor':
+                norm = data.get('Norm')
+                if norm is not None and not np.isnan(norm):
+                    norms.append(norm)
+        return norms
+
+    def matches(self) -> bool:
+        """
+            继承父类函数，实现数值突变检查
+        Returns:
+        """
+        # 收集所有输入的范数
+        input_norms = (self._get_tensor_norms(self.api_data.input_args) +
+                       self._get_kwargs_norms(self.api_data.input_kwargs))
+        # 收集所有输出的范数
+        output_norms = self._get_tensor_norms(self.api_data.output_data)
+
+        if not input_norms or not output_norms:
+            return False
+
+        max_input = max(input_norms)
+        max_output = max(output_norms)
+
+        if max_input == 0:
+            return max_output > self.threshold
+        return max_output / max_input > self.threshold
+
+    def get_details(self) -> Dict:
+        details = super().get_details()
+        details.update({
+            'threshold': self.threshold,
+            'scale_change_detected': self.matches()
+        })
+        return details

msprobe/core/overflow_check/api_info.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+
+from typing import Dict, List
+
+from msprobe.core.common.const import Const
+
+
+@dataclass
+class APIInfo:
+    api_name: str
+    torch_api_name: str
+    input_args: List[Dict]
+    input_kwargs: Dict
+    output_data: List[Dict]
+
+    def __init__(self, api_name, input_args=None, input_kwargs=None, output_data=None):
+        self.api_name = api_name
+        self.input_args = input_args
+        self.input_kwargs = input_kwargs
+        self.output_data = output_data
+        self.torch_api_name = self.extract_torch_api(self.api_name)
+
+    @staticmethod
+    def extract_torch_api(api_name) -> str:
+        """
+        Process tensor api name to extract first two fields in lowercase.
+        """
+        # Empty string checking
+        if not api_name.strip():
+            return ""
+
+        parts = api_name.split(Const.SEP)
+
+        # Handle different cases based on number of parts
+        if len(parts) == 0:
+            return ""
+        elif len(parts) == 1:
+            return parts[0].lower()
+        else:
+            return Const.SEP.join(parts[:2]).lower()