mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/core/data_dump/scope.py

@@ -14,36 +14,48 @@
 # limitations under the License.
 
 from abc import ABC, abstractmethod
+import re
 
 from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import ScopeException
 
 
-def build_scope(scope_class, scope=None, api_list=None):
-    if not scope and not api_list:
-        return None
-    if scope is None:
-        scope = []
-    if api_list is None:
-        api_list = []
-    if scope_class:
-        return scope_class(scope, api_list)
-    return build_range_scope_according_to_scope_name(scope, api_list)
-
-
-def build_range_scope_according_to_scope_name(scope, api_list):
-    api_range_scope = APIRangeScope(scope, api_list)
-    module_range_scope = ModuleRangeScope(scope, api_list)
-    if not scope:  # 如果没有scope参数则用哪类scope都一样
-        return api_range_scope
-    if api_range_scope.is_valid and module_range_scope.is_valid:
-        raise ScopeException(ScopeException.InvalidScope, f"scope={scope}.")
-    elif api_range_scope.is_valid:
-        return api_range_scope
-    elif module_range_scope.is_valid:
-        return module_range_scope
-    else:
-        raise ScopeException(ScopeException.InvalidScope, f"scope={scope}")
+class ScopeFactory:
+    def __init__(self, config):
+        self.task = config.task
+        self.level = config.level
+        self.scope = config.scope
+        self.api_list = config.list
+
+    def build_scope(self):
+        if not self.scope and not self.api_list:
+            return None
+        if self.scope is None:
+            self.scope = []
+        if self.api_list is None:
+            self.api_list = []
+        if self.task == Const.FREE_BENCHMARK:
+            return ListScope(self.scope, self.api_list)
+        return self._build_range_scope()
+
+    def _build_range_scope(self):
+        api_range_scope = APIRangeScope(self.scope, self.api_list, self.level)
+        module_range_scope = ModuleRangeScope(self.scope, self.api_list, self.level)
+        mix_range_scope = MixRangeScope(self.scope, self.api_list, self.level)
+
+        if self.level == Const.LEVEL_MIX:
+            return mix_range_scope
+
+        if not self.scope:
+            return api_range_scope
+        if api_range_scope.is_valid and module_range_scope.is_valid:
+            raise ScopeException(ScopeException.InvalidScope, f"scope={self.scope}.")
+        elif api_range_scope.is_valid:
+            return api_range_scope
+        elif module_range_scope.is_valid:
+            return module_range_scope
+        else:
+            raise ScopeException(ScopeException.InvalidScope, f"scope={self.scope}")
 
 
 class BaseScope(ABC):
@@ -51,7 +63,8 @@ class BaseScope(ABC):
     Module_Type_API = "api"
     module_type = ["Module", "Cell"]
 
-    def __init__(self, scope, api_list):
+    def __init__(self, scope, api_list, level=None):
+        self.level = level
         scope, api_list = self.rectify_args(scope, api_list)
         self.scope = scope
         self.api_list = api_list
@@ -60,21 +73,21 @@ class BaseScope(ABC):
     def rectify_args(scope, api_list):
         if not isinstance(api_list, list):
             raise ScopeException(ScopeException.InvalidApiStr,
-                f"api_list参数须配置为列表，实际类型为{type(api_list)}.")
+                                 f"api_list参数须配置为列表，实际类型为{type(api_list)}.")
         for api in api_list:
             if not isinstance(api, str):
                 raise ScopeException(ScopeException.InvalidApiStr,
-                    f"api_list中的元素须配置为字符串，实际类型为{type(api)}.")
+                                     f"api_list中的元素须配置为字符串，实际类型为{type(api)}.")
         if isinstance(scope, str):
             scope = [scope]
             return scope, api_list
         if not isinstance(scope, list):
             raise ScopeException(ScopeException.InvalidScope,
-                f"scope参数须配置为字符串或列表，实际类型为{type(scope)}.")
+                                 f"scope参数须配置为字符串或列表，实际类型为{type(scope)}.")
         for s in scope:
             if not isinstance(s, str):
                 raise ScopeException(ScopeException.InvalidScope,
-                f"scope列表元素要求类型为字符串，实际类型为{type(s)}.")
+                                     f"scope列表元素要求类型为字符串，实际类型为{type(s)}.")
         return scope, api_list
 
     @abstractmethod
@@ -95,7 +108,7 @@ class ListScope(BaseScope):
     def rectify_args(scope, api_list):
         if scope and api_list:
             raise ScopeException(ScopeException.ArgConflict,
-                f"scope和api_list不可以同时配置，实际配置为scope={scope}, api_list={api_list}.")
+                                 f"scope和api_list不可以同时配置，实际配置为scope={scope}, api_list={api_list}.")
         return super(ListScope, ListScope).rectify_args(scope, api_list)
 
     def check(self, name):
@@ -109,17 +122,37 @@ class RangeScope(BaseScope, ABC):
     def __init__(self, *args):
         super().__init__(*args)
         self.in_scope = False
+        self.in_list = False
+        self.start_name_set = set()
         self.is_valid = self.check_scope_is_valid()
 
-    @staticmethod
-    def rectify_args(scope, api_list):
-        scope, api_list = super(RangeScope, RangeScope).rectify_args(scope, api_list)
-        if isinstance(scope, list):
-            if len(scope) == 1:
-                scope.append(scope[0])
-            elif len(scope) > 2:
+    def check_name_pattern(self, name):
+        options_pattern = "|".join(re.escape(option) for option in Const.DUMP_PREFIX)
+        api_pattern = rf"^({options_pattern})\..*\.\d+\.(forward|backward)$"
+        module_pattern = r"^(Cell|Module)\..*\.(forward|backward)\.\d+$"
+
+        if self.level == Const.LEVEL_L1:
+            if not re.match(api_pattern, name):
+                raise ScopeException(ScopeException.InvalidScope,
+                                     f"scope参数格式错误，要求格式为api完整命名，实际为{name}.")
+
+        if self.level == Const.LEVEL_L0:
+            if not re.match(module_pattern, name):
+                raise ScopeException(ScopeException.InvalidScope,
+                                     f"scope参数格式错误，要求格式为模块完整命名，实际为{name}.")
+
+        if self.level == Const.LEVEL_MIX:
+            if not re.match(api_pattern, name) and not re.match(module_pattern, name):
                 raise ScopeException(ScopeException.InvalidScope,
-                    f"scope参数指定区间断点，须传入长度为1或2的列表，实际长度为{len(scope)}.")
+                                     f"scope参数格式错误，要求格式为api或模块完整命名，实际为{name}.")
+
+    def rectify_args(self, scope, api_list):
+        scope, api_list = super(RangeScope, RangeScope).rectify_args(scope, api_list)
+        if scope and len(scope) != 2:
+            raise ScopeException(ScopeException.InvalidScope,
+                                 f"scope参数指定区间断点，须传入长度为2的列表，实际长度为{len(scope)}.")
+        for name in scope:
+            self.check_name_pattern(name)
         return scope, api_list
 
     @abstractmethod
@@ -192,3 +225,50 @@ class ModuleRangeScope(RangeScope):
         if not self.scope or self.in_scope:
             return self.check_api_list(name)
         return False
+
+
+class MixRangeScope(RangeScope):
+    def check_scope_is_valid(self):
+        return True if self.scope else False
+
+    def begin_module(self, module_name):
+        if self.scope and module_name == self.scope[0]:
+            self.in_scope = True
+        for name in self.api_list:
+            if name in module_name:
+                self.in_list = True
+                self.start_name_set.add(module_name)  # 记录每一个开启in_list的module_name
+
+    def end_module(self, module_name):
+        if self.scope and module_name == self.scope[1]:
+            self.in_scope = False
+        self.start_name_set.discard(module_name)  # 从集合中删除每一个module_name
+        if not self.start_name_set:  # 如果集合为空，说明当前module_name是最后一个开启in_list的module_name
+            self.in_list = False  # 关闭in_list
+
+    def check_api_list(self, api_name):
+        if not self.api_list:
+            return True
+
+        for name in self.api_list:
+            if name in api_name:
+                return True
+        return False
+
+    def check(self, name):
+        """
+        dump时调用的接口，根据scope和api_list判断是否需要dump
+        """
+        result = False
+        if self.scope and name == self.scope[0]:
+            self.in_scope = True
+
+        if not self.scope or self.in_scope:
+            if self.in_list:
+                result = True
+            else:
+                result = self.check_api_list(name)
+
+        if self.scope and name == self.scope[1]:
+            self.in_scope = False
+        return result

msprobe/core/grad_probe/constant.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 class GradConst:
 
@@ -60,16 +74,16 @@ class GradConst:
     NORM = "norm"
 
 level_adp = {
-        "L0": {
-            "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": False
-        },
-        "L1": {
-            "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": True
-        },
-        "L2": {
-            "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": True
-        },
-    }
+    "L0": {
+        "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+        "have_grad_direction": False
+    },
+    "L1": {
+        "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+        "have_grad_direction": True
+    },
+    "L2": {
+        "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+        "have_grad_direction": True
+    },
+}

msprobe/core/grad_probe/grad_compare.py

@@ -1,10 +1,25 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 from typing import List
 
 from tqdm import tqdm
 import matplotlib.pyplot as plt
 
-from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path
+from msprobe.core.common.file_utils import create_directory, check_file_or_directory_path
 from msprobe.core.common.log import logger
 from msprobe.core.common.file_utils import remove_path, load_npy, write_csv, read_csv
 from msprobe.core.grad_probe.constant import GradConst
@@ -33,6 +48,8 @@ class GradComparator:
 
     @classmethod
     def compare_distributed(cls, path1: str, path2: str, output_dir: str):
+        check_file_or_directory_path(path1, isdir=True)
+        check_file_or_directory_path(path2, isdir=True)
         ranks = cls._get_matched_dirs(path1, path2, "rank")
         logger.info(f"the following ranks will be compared: {ranks}")
         if not ranks:

msprobe/core/grad_probe/utils.py

@@ -1,8 +1,24 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import re
 from msprobe.core.grad_probe.constant import GradConst
 from msprobe.core.common.log import logger
 from msprobe.core.common.file_utils import write_csv, check_path_before_create, change_mode
 from msprobe.core.common.const import FileCheckConst
+from msprobe.core.common.utils import is_int
 import matplotlib.pyplot as plt
 
 
@@ -26,13 +42,24 @@ def check_str(string, variable_name):
     if not isinstance(string, str):
         raise ValueError(f'The variable: "{variable_name}" is not a string.')
 
+
 def check_bounds_element(bound):
-    return GradConst.BOUNDS_MINIMUM <= bound and bound <= GradConst.BOUNDS_MAXIMUM
+    return GradConst.BOUNDS_MINIMUM <= bound <= GradConst.BOUNDS_MAXIMUM
+
+
+def check_param_element(param):
+    if not re.match(GradConst.PARAM_VALID_PATTERN, param):
+        return False
+    else:
+        return True
+
 
 def check_bounds(bounds):
+    if not isinstance(bounds, list):
+        raise Exception(f"bounds must be a list")
     prev = GradConst.BOUNDS_MINIMUM - 1
     for element in bounds:
-        if not isinstance(element, (int, float)):
+        if not is_int(element) and not isinstance(element, float):
             raise Exception("bounds element is not int or float")
         if not check_bounds_element(element):
             raise Exception("bounds element is out of int64 range")
@@ -40,6 +67,7 @@ def check_bounds(bounds):
             raise Exception("bounds list is not ascending")
         prev = element
 
+
 class ListCache(list):
     threshold = 1000
 

msprobe/core/overflow_check/abnormal_scene.py

@@ -0,0 +1,189 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Dict, Union, Any
+
+import numpy as np
+
+from msprobe.core.overflow_check.api_info import APIInfo
+from msprobe.core.overflow_check.level import OverflowLevel
+from msprobe.core.overflow_check.utils import has_nan_inf
+
+
+class AnomalyScene:
+    """异常场景的基类"""
+
+    def __init__(self, api_info: APIInfo):
+        self.api_name = api_info.api_name
+        self.api_data = api_info
+
+    @property
+    def rank(self) -> OverflowLevel:
+        """获取异常等级"""
+        raise NotImplementedError
+
+    @staticmethod
+    def _has_anomaly(data: Union[Dict, Any]) -> bool:
+        """检查张量是否包含异常值"""
+        if isinstance(data, dict):
+            return has_nan_inf(data)
+        elif isinstance(data, list):
+            return any(AnomalyScene._has_anomaly(x) for x in data)
+        return False
+
+    def get_details(self) -> Dict:
+        """获取异常详情"""
+        return {
+            'api_name': self.api_name,
+            'rank': self.rank.value,
+            'scene_type': self.__class__.__name__,
+            'input_args_anomaly_indices': self._get_anomaly_indices_from_list(self.api_data.input_args),
+            'input_kwargs_anomaly_keys': self._get_anomaly_keys_from_dict(self.api_data.input_kwargs),
+            'output_anomaly_indices': self._get_anomaly_indices_from_list(self.api_data.output_data)
+        }
+
+    def matches(self) -> bool:
+        """
+            待子类实现对应匹配逻辑
+        Returns:
+
+        """
+        raise NotImplementedError
+
+    def _get_anomaly_indices_from_list(self, data_list: List[Dict]) -> List[int]:
+        return [i for i, data in enumerate(data_list) if self._has_anomaly(data)]
+
+    def _get_anomaly_keys_from_dict(self, data_dict: Dict) -> List[str]:
+        return [key for key, data in data_dict.items() if self._has_anomaly(data)]
+
+
+class InputOutputAnomalyScene(AnomalyScene):
+    """输入输出异常检测的基类"""
+    def has_input_anomaly(self) -> bool:
+        """检查输入是否有异常（包括args和kwargs）"""
+        # args
+        args_anomaly = any(self._has_anomaly(x) for x in self.api_data.input_args)
+        # kwargs
+        kwargs_anomaly = any(self._has_anomaly(x) for x in self.api_data.input_kwargs.values())
+        return args_anomaly or kwargs_anomaly
+
+    def has_output_anomaly(self) -> bool:
+        """检查输出是否有异常"""
+        return any(self._has_anomaly(x) for x in self.api_data.output_data)
+
+    def matches(self) -> bool:
+        """判断是否匹配该场景"""
+        raise NotImplementedError
+
+
+class InputAnomalyOutputNormalScene(InputOutputAnomalyScene):
+    """输入异常，输出正常场景"""
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.MEDIUM
+
+    def matches(self) -> bool:
+        return self.has_input_anomaly() and not self.has_output_anomaly()
+
+
+class InputAnomalyOutputAnomalyScene(InputOutputAnomalyScene):
+    """输入异常，输出异常场景"""
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.HIGH
+
+    def matches(self) -> bool:
+        return self.has_input_anomaly() and self.has_output_anomaly()
+
+
+class InputNormalOutputAnomalyScene(InputOutputAnomalyScene):
+    """输入正常，输出异常场景"""
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.CRITICAL
+
+    def matches(self) -> bool:
+        return not self.has_input_anomaly() and self.has_output_anomaly()
+
+
+class NumericalMutationScene(AnomalyScene):
+    """
+        检查数值突变，统计输入args、kwargs中norm值，同时统计输出的norm最大值，计算差异，大于 threshold 则认为是异常情况
+    """
+    def __init__(self, api_info: APIInfo, threshold: float = 100.0):
+        super().__init__(api_info)
+        self.threshold = threshold
+
+    @property
+    def rank(self) -> OverflowLevel:
+        return OverflowLevel.HIGH
+
+    @staticmethod
+    def _get_tensor_norms(data_list: List[Dict]) -> List[float]:
+        norms = []
+        for data in data_list:
+            if isinstance(data, dict) and data.get('type') == 'torch.Tensor':
+                norm = data.get('Norm')
+                if norm is not None and not np.isnan(norm):
+                    norms.append(norm)
+        return norms
+
+    @staticmethod
+    def _get_kwargs_norms(data_dict: Dict) -> List[float]:
+        """
+            获取kwargs中张量的范数列表
+        Args:
+            data_dict:
+        Returns:
+        """
+        norms = []
+        for data in data_dict.values():
+            if isinstance(data, dict) and data.get('type') == 'torch.Tensor':
+                norm = data.get('Norm')
+                if norm is not None and not np.isnan(norm):
+                    norms.append(norm)
+        return norms
+
+    def matches(self) -> bool:
+        """
+            继承父类函数，实现数值突变检查
+        Returns:
+        """
+        # 收集所有输入的范数
+        input_norms = (self._get_tensor_norms(self.api_data.input_args) +
+                       self._get_kwargs_norms(self.api_data.input_kwargs))
+        # 收集所有输出的范数
+        output_norms = self._get_tensor_norms(self.api_data.output_data)
+
+        if not input_norms or not output_norms:
+            return False
+
+        max_input = max(input_norms)
+        max_output = max(output_norms)
+
+        if max_input == 0:
+            return max_output > self.threshold
+        return max_output / max_input > self.threshold
+
+    def get_details(self) -> Dict:
+        details = super().get_details()
+        details.update({
+            'threshold': self.threshold,
+            'scale_change_detected': self.matches()
+        })
+        return details

msprobe/core/overflow_check/api_info.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+
+from typing import Dict, List
+
+from msprobe.core.common.const import Const
+
+
+@dataclass
+class APIInfo:
+    api_name: str
+    torch_api_name: str
+    input_args: List[Dict]
+    input_kwargs: Dict
+    output_data: List[Dict]
+
+    def __init__(self, api_name, input_args=None, input_kwargs=None, output_data=None):
+        self.api_name = api_name
+        self.input_args = input_args
+        self.input_kwargs = input_kwargs
+        self.output_data = output_data
+        self.torch_api_name = self.extract_torch_api(self.api_name)
+
+    @staticmethod
+    def extract_torch_api(api_name) -> str:
+        """
+        Process tensor api name to extract first two fields in lowercase.
+        """
+        # Empty string checking
+        if not api_name.strip():
+            return ""
+
+        parts = api_name.split(Const.SEP)
+
+        # Handle different cases based on number of parts
+        if len(parts) == 0:
+            return ""
+        elif len(parts) == 1:
+            return parts[0].lower()
+        else:
+            return Const.SEP.join(parts[:2]).lower()