mindstudio-probe 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/free_benchmark/compare/grad_saver.py

@@ -102,8 +102,13 @@ class GradSaver:
     def check_grad_input(self, origin_grad, new_grad_index):
         if self.perturbed_grad_input is None:
             raise FreeBenchmarkException(
-                FreeBenchmarkException.InvalidGrad,
-                f"grad not exists : {self.api_name}.",
+                FreeBenchmarkException.InvalidPerturbedOutput,
+                f"perturbed grad not exists for {self.api_name}.",
+            )
+        if len(self.perturbed_grad_input) <= new_grad_index:
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.InvalidPerturbedOutput,
+                f"perturbed grad index {new_grad_index} is out of bounds for {self.api_name}.",
             )
         with torch.no_grad():
             perturbed_grad = self.perturbed_grad_input[new_grad_index].to(
@@ -111,7 +116,7 @@ class GradSaver:
             )
         if origin_grad.shape != perturbed_grad.shape:
             raise FreeBenchmarkException(
-                FreeBenchmarkException.InvalidGrad,
+                FreeBenchmarkException.InvalidPerturbedOutput,
                 f"grad shapes are inconsistent. api:{self.handler_params.api_name}."
                 f"origin:{origin_grad.shape}, perturbation: {perturbed_grad.shape}",
             )
@@ -164,6 +169,18 @@ class GradSaver:
             index_ = 0
             for object_ in inner_args:
                 if object_ is CommonField.HOLD_PLACE:
+                    if index_ >= len(inputs):
+                        err_msg = (
+                            f"[msprobe] Free benchmark: When getting input from vjp, "
+                            f" the input index ({index_}) is out of bounds ({len(inputs)})."
+                        )
+                        logger.error_log_with_exp(
+                            err_msg,
+                            FreeBenchmarkException(
+                                FreeBenchmarkException.InvalidGrad,
+                                error_info=err_msg,
+                            ),
+                        )
                     _real_input.append(inputs[index_])
                     index_ += 1
                 else:

msprobe/pytorch/free_benchmark/compare/single_benchmark.py

@@ -16,6 +16,7 @@
 import math
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.utils import TorchC
@@ -67,6 +68,7 @@ class SingleCompare:
                 return False
         return True
 
+    @recursion_depth_decorator("FreeBenchmark: SingleCompare.compare_seq")
     def compare_seq(self, actual, golden):
         if isinstance(golden, torch.Tensor):
             return self.compare_tensor_seq(actual, golden)

msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -26,6 +27,7 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import
 
 class AddNoiseLayer(NpuBaseLayer):
 
+    @recursion_depth_decorator("FreeBenchmark: AddNoiseLayer.add_noise")
     def add_noise(self, tensor_obj):
         if isinstance(tensor_obj, torch.Tensor):
             self.perturbed_value = ThresholdConfig.PERTURBATION_VALUE_DICT.get(
@@ -99,7 +101,7 @@ class AddNoiseLayer(NpuBaseLayer):
         if max_val < abs_tol:
             logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+                f"Maximun value is less than the minimun threshold. Cancel add noise."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -31,6 +32,7 @@ class BitNoiseLayer(NpuBaseLayer):
         self.bit_tail: int = 1
         self.bit_type = None
 
+    @recursion_depth_decorator("FreeBenchmark: BitNoiseLayer.add_bit_noise")
     def add_bit_noise(self, tensor_obj):
         """
         对输入添加噪声
@@ -79,14 +81,14 @@ class BitNoiseLayer(NpuBaseLayer):
         判断是否需要添加扰动,  bit翻转
         """
         if not self.bit_type:
-            logger.info_on_rank_0(
+            logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
                 f"dtype unsupported. Cancel perturbation."
             )
             return False
         if tensor_obj.numel() == 0:
             logger.warning_on_rank_0(
-                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0"
+                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0."
                 f" Cancel adding noise."
             )
             return False
@@ -102,9 +104,9 @@ class BitNoiseLayer(NpuBaseLayer):
             )
             max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item()
         if max_val < abs_tol:
-            logger.info_on_rank_0(
+            logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+                f"Maximun value is less than the minimun threshold. Cancel add noise."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
 from msprobe.pytorch.free_benchmark.common.params import DataParams
@@ -29,6 +30,7 @@ class ChangeValueLayer(NpuBaseLayer):
         self.head: int = 0
         self.tail: int = -1
 
+    @recursion_depth_decorator("FreeBenchmark: ChangeValueLayer.change_value")
     def change_value(self, tensor_obj):
         """
         交换张量首尾

msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py

@@ -15,6 +15,7 @@
 
 import torch
 from msprobe.core.common.const import Const
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import CommonField
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -26,6 +27,9 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import
 
 class ImprovePrecisionLayer(NpuBaseLayer):
 
+    @recursion_depth_decorator(
+        "FreeBenchmark: ImprovePrecisionLayer.improve_tensor_precision"
+    )
     def improve_tensor_precision(self, tensor_obj):
         if (
             isinstance(tensor_obj, torch.Tensor)

msprobe/pytorch/free_benchmark/result_handlers/base_handler.py

@@ -20,6 +20,7 @@ from typing import Any, Optional, Tuple
 import numpy as np
 import torch
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import FreeBenchmarkException
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import (
@@ -113,6 +114,14 @@ class FuzzHandler(ABC):
         origin_output_chunks, perturbed_output_chunks = (
             self.tensor_split_for_error_calculate(origin_output, perturbed_output)
         )
+        if len(origin_output_chunks) != len(perturbed_output_chunks):
+            err_msg = (
+                f"For {self.params.api_name}, the number of compare tensor chunks is different: "
+                f"{len(origin_output_chunks)} != {len(perturbed_output_chunks)}. please check!"
+            )
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.OutputIndexError, err_msg
+            )
         norm1 = -np.inf
         norm2 = -np.inf
         norm3 = np.inf
@@ -189,6 +198,7 @@ class FuzzHandler(ABC):
                 f"[msprobe] Free Benchmark: For {self.params.api_name} "
                 f"The compare for output type {type(perturbed_output)} is not supported"
             )
+            return True, 1
 
         threshold = self.get_threshold(Tools.get_first_tensor_dtype(origin_output))
         ratio = self.ratio_calculate(

msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py

@@ -15,10 +15,11 @@
 
 from typing import Any
 
+from msprobe.core.common.exceptions import FreeBenchmarkException
+from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.params import DataParams
 from msprobe.pytorch.free_benchmark.common.utils import Tools
 from msprobe.pytorch.free_benchmark.result_handlers.base_handler import FuzzHandler
-from msprobe.pytorch.free_benchmark import logger
 
 
 class FixHandler(FuzzHandler):
@@ -31,9 +32,9 @@ class FixHandler(FuzzHandler):
             return Tools.convert_fuzz_output_to_origin(
                 data_params.original_result, data_params.perturbed_result
             )
-        except Exception as e:
-            logger.warning_on_rank_0(
+        except FreeBenchmarkException as e:
+            logger.warning(
                 f"[msprobe] Free Benchmark: For {self.params.api_name} "
-                f"Fix output failed. "
+                f"Fix output failed because of: \n{e}"
             )
-        return data_params.original_result
+            return data_params.original_result

msprobe/pytorch/grad_probe/grad_monitor.py

@@ -1,15 +1,31 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 from collections import defaultdict
 
 import torch
-if int(torch.__version__.split('.')[0]) >= 2:
-    from torch.optim.optimizer import register_optimizer_step_pre_hook
-from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv
-from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target
+from msprobe.core.common.file_utils import remove_path, save_npy, write_csv, create_directory
 from msprobe.core.grad_probe.constant import level_adp
+from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target
 from msprobe.pytorch.common.log import logger
-from msprobe.core.common.file_utils import remove_path, save_npy, write_csv, create_directory
 from msprobe.pytorch.common.utils import get_rank_id, print_rank_0
+from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv
+
+if int(torch.__version__.split('.')[0]) >= 2:
+    from torch.optim.optimizer import register_optimizer_step_pre_hook
 
 
 class GradientMonitor:
@@ -75,7 +91,7 @@ class GradientMonitor:
                 output_lines.append(grad_info)
                 if self._level_adp["have_grad_direction"]:
                     GradientMonitor.save_grad_direction(param_name, grad,
-                                                    f'{self._output_path}/rank{self._rank}/step{self._step}')
+                                                        f'{self._output_path}/rank{self._rank}/step{self._step}')
             output_dirpath = os.path.join(self._output_path, f"rank{getattr(self, '_rank')}")
             if not os.path.isdir(output_dirpath):
                 create_directory(output_dirpath)
@@ -87,5 +103,6 @@ class GradientMonitor:
             output_lines.insert(0, header_result)
             write_csv(output_lines, output_path)
             logger.info(f"write grad data to {output_path}")
+
         if int(torch.__version__.split('.')[0]) >= 2:
             register_optimizer_step_pre_hook(optimizer_pre_step_hook)

msprobe/pytorch/grad_probe/grad_stat_csv.py

@@ -1,11 +1,27 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import ABC, abstractmethod
 from collections import namedtuple
 import hashlib
+from functools import wraps
 import torch
 from msprobe.core.grad_probe.constant import GradConst
 
-CSV_header_input = namedtuple("CSV_header_input", ["bounds"])
-CSV_content_input = namedtuple("CSV_content_input", ["grad", "bounds"])
+CsvHeaderInput = namedtuple("CsvHeaderInput", ["bounds"])
+CsvContentInput = namedtuple("CsvContentInput", ["grad", "bounds"])
 
 
 class GradStatCsv:
@@ -15,7 +31,7 @@ class GradStatCsv:
     def generate_csv_header(level, bounds):
         header = ["param_name"]
         for key in level["header"]:
-            csv_header_input = CSV_header_input(bounds=bounds)
+            csv_header_input = CsvHeaderInput(bounds=bounds)
             header.extend(GradStatCsv.csv[key].generate_csv_header(csv_header_input))
         return header
 
@@ -23,7 +39,7 @@ class GradStatCsv:
     def generate_csv_line(param_name, level, grad, bounds):
         line = [param_name]
         for key in level["header"]:
-            csv_content_input = CSV_content_input(grad=grad, bounds=bounds)
+            csv_content_input = CsvContentInput(grad=grad, bounds=bounds)
             line.extend(GradStatCsv.csv[key].generate_csv_content(csv_content_input))
         return line
 
@@ -37,20 +53,24 @@ def register_csv_item(key, cls=None):
 
 
 class CsvItem(ABC):
+    @staticmethod
     @abstractmethod
     def generate_csv_header(csv_header_input):
         pass
 
+    @staticmethod
     @abstractmethod
     def generate_csv_content(csv_content_input):
         pass
 
 
 @register_csv_item(GradConst.MD5)
-class CSV_md5(CsvItem):
+class CsvMd5(CsvItem):
+    @staticmethod
     def generate_csv_header(csv_header_input):
         return ["MD5"]
 
+    @staticmethod
     def generate_csv_content(csv_content_input):
         grad = csv_content_input.grad
         tensor_bytes = grad.cpu().detach().float().numpy().tobytes()
@@ -59,7 +79,8 @@ class CSV_md5(CsvItem):
 
 
 @register_csv_item(GradConst.DISTRIBUTION)
-class CSV_distribution(CsvItem):
+class CsvDistribution(CsvItem):
+    @staticmethod
     def generate_csv_header(csv_header_input):
         bounds = csv_header_input.bounds
         intervals = []
@@ -73,6 +94,7 @@ class CSV_distribution(CsvItem):
     
         return intervals
 
+    @staticmethod
     def generate_csv_content(csv_content_input):
         grad = csv_content_input.grad
         bounds = csv_content_input.bounds
@@ -90,40 +112,48 @@ class CSV_distribution(CsvItem):
 
 
 @register_csv_item(GradConst.MAX)
-class CSV_max(CsvItem):
+class CsvMax(CsvItem):
+    @staticmethod
     def generate_csv_header(csv_header_input):
         return ["max"]
 
+    @staticmethod
     def generate_csv_content(csv_content_input):
         grad = csv_content_input.grad
         return [torch.max(grad).cpu().detach().float().numpy().tolist()]
 
 
 @register_csv_item(GradConst.MIN)
-class CSV_min(CsvItem):
+class CsvMin(CsvItem):
+    @staticmethod
     def generate_csv_header(csv_header_input):
         return ["min"]
 
+    @staticmethod
     def generate_csv_content(csv_content_input):
         grad = csv_content_input.grad
         return [torch.min(grad).cpu().detach().float().numpy().tolist()]
 
 
 @register_csv_item(GradConst.NORM)
-class CSV_norm(CsvItem):
+class CsvNorm(CsvItem):
+    @staticmethod
     def generate_csv_header(csv_header_input):
         return ["norm"]
 
+    @staticmethod
     def generate_csv_content(csv_content_input):
         grad = csv_content_input.grad
         return [torch.norm(grad).cpu().detach().float().numpy().tolist()]
     
 
 @register_csv_item(GradConst.SHAPE)
-class CSV_shape(CsvItem):
+class CsvShape(CsvItem):
+    @staticmethod
     def generate_csv_header(csv_header_input):
         return ["shape"]
 
+    @staticmethod
     def generate_csv_content(csv_content_input):
         grad = csv_content_input.grad
         return [list(grad.shape)]

msprobe/pytorch/hook_module/support_wrap_ops.yaml

@@ -1130,6 +1130,7 @@ torch_npu:
   - npu_prompt_flash_attention
   - npu_lstm
   - npu_apply_adam
+  - npu_apply_adam_w
 
 aten:
   - signbit

msprobe/pytorch/hook_module/wrap_functional.py

@@ -30,32 +30,34 @@ def remove_dropout():
         from torch import _VF
         from torch.overrides import has_torch_function_unary, handle_torch_function
 
-        def function_dropout(input: torch.Tensor, p: float = 0.5, training: bool = True,
+        def function_dropout(input_tensor: torch.Tensor, p: float = 0.5, training: bool = True,
                              inplace: bool = False) -> torch.Tensor:
-            if has_torch_function_unary(input):
+            if has_torch_function_unary(input_tensor):
                 return handle_torch_function(
-                    function_dropout, (input,), input, p=0., training=training, inplace=inplace)
+                    function_dropout, (input_tensor,), input_tensor, p=0., training=training, inplace=inplace)
             if p < 0.0 or p > 1.0:
                 raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
-            return _VF.dropout_(input, 0., training) if inplace else _VF.dropout(input, 0., training)
+            return _VF.dropout_(input_tensor, 0., training) if inplace else _VF.dropout(input_tensor, 0., training)
 
-        def function_dropout2d(input: torch.Tensor, p: float = 0.5, training: bool = True,
+        def function_dropout2d(input_tensor: torch.Tensor, p: float = 0.5, training: bool = True,
                                inplace: bool = False) -> torch.Tensor:
-            if has_torch_function_unary(input):
+            if has_torch_function_unary(input_tensor):
                 return handle_torch_function(
-                    function_dropout2d, (input,), input, p=0., training=training, inplace=inplace)
+                    function_dropout2d, (input_tensor,), input_tensor, p=0., training=training, inplace=inplace)
             if p < 0.0 or p > 1.0:
                 raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
-            return _VF.feature_dropout_(input, 0., training) if inplace else _VF.feature_dropout(input, 0., training)
+            return _VF.feature_dropout_(input_tensor, 0., training) if inplace else _VF.feature_dropout(input_tensor,
+                                                                                                        0., training)
 
-        def function_dropout3d(input: torch.Tensor, p: float = 0.5, training: bool = True,
+        def function_dropout3d(input_tensor: torch.Tensor, p: float = 0.5, training: bool = True,
                                inplace: bool = False) -> torch.Tensor:
-            if has_torch_function_unary(input):
+            if has_torch_function_unary(input_tensor):
                 return handle_torch_function(
-                    function_dropout3d, (input,), input, p=0., training=training, inplace=inplace)
+                    function_dropout3d, (input_tensor,), input_tensor, p=0., training=training, inplace=inplace)
             if p < 0.0 or p > 1.0:
                 raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
-            return _VF.feature_dropout_(input, 0., training) if inplace else _VF.feature_dropout(input, 0., training)
+            return _VF.feature_dropout_(input_tensor, 0., training) if inplace else _VF.feature_dropout(input_tensor,
+                                                                                                        0., training)
 
         F.dropout = function_dropout
         F.dropout2d = function_dropout2d

msprobe/pytorch/module_processer.py

@@ -17,7 +17,7 @@ from functools import wraps
 
 import torch
 from msprobe.core.common.const import Const
-from msprobe.core.data_dump.scope import ModuleRangeScope
+from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope
 from torch.utils.hooks import BackwardHook
 
 torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0'
@@ -30,10 +30,7 @@ class ModuleProcesser:
     module_node = {}
 
     def __init__(self, scope):
-        if isinstance(scope, ModuleRangeScope):
-            self.scope = scope
-        else:
-            self.scope = None
+        self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None
         BackwardHook.setup_input_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_input_hook)
         BackwardHook.setup_output_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_output_hook)
         BackwardHook.setup_output_hook = ModuleProcesser.filter_tensor_and_tuple(BackwardHook.setup_output_hook)