mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/pytorch/dump/module_dump/module_dump.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from msprobe.core.common.const import Const
+from msprobe.core.data_dump.scope import BaseScope
+from msprobe.pytorch.common.log import logger
+from msprobe.pytorch.hook_module.api_registry import api_register
+
+torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0'
+
+
+class ModuleDumper:
+    def __init__(self, service):
+        self.service = service
+        self.hook_handle_list = []
+
+    def start_module_dump(self, module, dump_name):
+        api_register.api_originality()
+        self.register_hook(module, dump_name)
+
+    def stop_module_dump(self):
+        api_register.api_modularity()
+        for hook_handle in self.hook_handle_list:
+            if isinstance(hook_handle, torch.utils.hooks.RemovableHandle):
+                hook_handle.remove()
+        self.hook_handle_list.clear()
+
+    def register_hook(self, module, dump_name):
+        prefix_name = (
+                BaseScope.Module_Type_Module + Const.SEP +
+                dump_name + Const.SEP +
+                module.__class__.__name__ + Const.SEP
+        )
+        module_processor = self.service.module_processor
+        _, forward_hook, backward_hook, forward_hook_torch_version_below_2 = self.service.build_hook(
+            BaseScope.Module_Type_Module,
+            prefix_name
+        )
+
+        if module_processor.has_register_backward_hook(module):
+            logger.warning(
+                f"The {dump_name} module has registered deprecated register_backward_hook,"
+                f"which may cause abnormal data dump. The backward data dump for this module will be skipped."
+            )
+        if torch_version_above_or_equal_2:
+            forward_hook_handle = module.register_forward_hook(forward_hook, with_kwargs=True)
+        else:
+            if not module_processor.has_register_backward_hook(module):
+                backward_hook_handle = module.register_full_backward_hook(
+                    module_processor.node_hook(prefix_name + Const.BACKWARD, Const.STOP)
+                )
+                self.hook_handle_list.append(backward_hook_handle)
+            forward_hook_handle = module.register_forward_hook(forward_hook_torch_version_below_2)
+        self.hook_handle_list.append(forward_hook_handle)
+        if not module_processor.has_register_backward_hook(module):
+            backward_hook_handle = module.register_full_backward_hook(backward_hook)
+            self.hook_handle_list.append(backward_hook_handle)
+
+        forward_pre_hook_handle = module.register_forward_pre_hook(
+            module_processor.node_hook(prefix_name + Const.FORWARD, Const.START)
+        )
+        forward_hook_handle = module.register_forward_hook(
+            module_processor.node_hook(prefix_name + Const.FORWARD, Const.STOP)
+        )
+        self.hook_handle_list.extend([forward_pre_hook_handle, forward_hook_handle])
+        if torch_version_above_or_equal_2 and not module_processor.has_register_backward_hook(module):
+            backward_pre_hook_handle = module.register_full_backward_pre_hook(
+                module_processor.node_hook(prefix_name + Const.BACKWARD, Const.START)
+            )
+            backward_hook_handle = module.register_full_backward_hook(
+                module_processor.node_hook(prefix_name + Const.BACKWARD, Const.STOP)
+            )
+            self.hook_handle_list.extend([backward_pre_hook_handle, backward_hook_handle])

msprobe/pytorch/{module_processer.py → dump/module_dump/module_processer.py}

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -17,12 +17,24 @@ from functools import wraps
 
 import torch
 from msprobe.core.common.const import Const
-from msprobe.core.data_dump.scope import ModuleRangeScope
+from msprobe.core.data_dump.scope import BaseScope, ModuleRangeScope, MixRangeScope
+from msprobe.pytorch.common.log import logger
+from torch.utils.checkpoint import checkpoint as origin_checkpoint
+from torch.utils.checkpoint import set_checkpoint_early_stop
 from torch.utils.hooks import BackwardHook
 
 torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0'
 
 
+def checkpoint_without_early_stop(*args, **kwargs):
+    with set_checkpoint_early_stop(False):
+        return origin_checkpoint(*args, **kwargs)
+
+
+def replace_checkpoint():
+    torch.utils.checkpoint.checkpoint = checkpoint_without_early_stop
+
+
 class ModuleProcesser:
     module_count = {}
     module_stack = []
@@ -30,13 +42,11 @@ class ModuleProcesser:
     module_node = {}
 
     def __init__(self, scope):
-        if isinstance(scope, ModuleRangeScope):
-            self.scope = scope
-        else:
-            self.scope = None
+        self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None
         BackwardHook.setup_input_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_input_hook)
         BackwardHook.setup_output_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_output_hook)
         BackwardHook.setup_output_hook = ModuleProcesser.filter_tensor_and_tuple(BackwardHook.setup_output_hook)
+        replace_checkpoint()
 
     @staticmethod
     def filter_tensor_and_tuple(func):
@@ -66,7 +76,7 @@ class ModuleProcesser:
             return ModuleProcesser.clone_if_tensor(result)
 
         return clone_return_value_func
-    
+
     @staticmethod
     def clone_if_tensor(result):
         if isinstance(result, torch.Tensor):
@@ -88,6 +98,22 @@ class ModuleProcesser:
             ModuleProcesser.module_count[module_name] += 1
         return ModuleProcesser.module_count[module_name]
 
+    @staticmethod
+    def has_register_backward_hook(module):
+        return hasattr(module, '_backward_hooks') and \
+            len(module._backward_hooks) > 0 and \
+            module._is_full_backward_hook is False
+    
+    @staticmethod
+    def get_modules_and_names(models):
+        modules_and_names_with_index = {}
+        if isinstance(models, (list, tuple)):
+            for index, model in enumerate(models):
+                modules_and_names_with_index[str(index)] = model.named_modules()
+        else:
+            modules_and_names_with_index["-1"] = models.named_modules()
+        return modules_and_names_with_index
+
     @classmethod
     def reset_module_stats(cls):
         cls.module_count = {}
@@ -95,6 +121,42 @@ class ModuleProcesser:
         cls.api_parent_node = ""
         cls.module_node = {}
 
+    def register_module_hook(self, models, build_hook):
+        logger.info_on_rank_0("The init dump is enabled, and the module dump function will not be available.")
+        modules_and_names_with_index = self.get_modules_and_names(models)
+        for index, modules_and_names in modules_and_names_with_index.items():
+            model = models if index == "-1" else models[int(index)]
+            for name, module in modules_and_names:
+                if module == model:
+                    continue
+                module_index = (index + Const.SEP) if index != "-1" else ""
+                prefix_name = (BaseScope.Module_Type_Module + Const.SEP + module_index + 
+                                name + Const.SEP + module.__class__.__name__ + Const.SEP)
+                pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 = build_hook(
+                    BaseScope.Module_Type_Module,
+                    prefix_name
+                )
+
+                if self.has_register_backward_hook(module):
+                    logger.warning(
+                        f"The {prefix_name[:-1]} has registered deprecated register_backward_hook,"
+                        f"which may cause abnormal data dump. The backward data dump for this module will be skipped."
+                    )
+                if torch_version_above_or_equal_2:
+                    module.register_forward_hook(forward_hook, with_kwargs=True)
+                else:
+                    if not self.has_register_backward_hook(module):
+                        module.register_full_backward_hook(self.node_hook(prefix_name + Const.BACKWARD, Const.STOP))
+                    module.register_forward_hook(forward_hook_torch_version_below_2)
+                if not self.has_register_backward_hook(module):
+                    module.register_full_backward_hook(backward_hook)
+
+                module.register_forward_pre_hook(self.node_hook(prefix_name + Const.FORWARD, Const.START))
+                module.register_forward_hook(self.node_hook(prefix_name + Const.FORWARD, Const.STOP))
+                if torch_version_above_or_equal_2 and not self.has_register_backward_hook(module):
+                    module.register_full_backward_pre_hook(self.node_hook(prefix_name + Const.BACKWARD, Const.START))
+                    module.register_full_backward_hook(self.node_hook(prefix_name + Const.BACKWARD, Const.STOP))
+
     def node_hook(self, name_prefix, start_or_stop, **kwargs):
 
         def pre_hook(module, input, output=None):
@@ -103,7 +165,10 @@ class ModuleProcesser:
             except IndexError as e:
                 index = None
                 pass
-            module.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index)
+            full_name = name_prefix + Const.SEP + str(index)
+            if not hasattr(module, "mindstudio_reserved_name") or not module.mindstudio_reserved_name:
+                module.mindstudio_reserved_name = []
+            module.mindstudio_reserved_name.append(full_name)
             if self.module_stack:
                 ModuleProcesser.module_node[full_name] = self.module_stack[-1]
             else:
@@ -122,8 +187,11 @@ class ModuleProcesser:
                 ModuleProcesser.api_parent_node = self.module_stack[-1]
             else:
                 ModuleProcesser.api_parent_node = None
+            if not hasattr(module, "mindstudio_reserved_name") or not module.mindstudio_reserved_name:
+                raise RuntimeError(f"module reserve name is None when pop")
+            current_name = module.mindstudio_reserved_name.pop()
             if self.scope:
-                self.scope.end_module(module.mindstudio_reserved_name)
+                self.scope.end_module(current_name)
 
         def backward_hook(module, input, output=None):
             try:
@@ -131,7 +199,10 @@ class ModuleProcesser:
             except IndexError as e:
                 index = None
                 pass
-            module.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index)
+            full_name = name_prefix + Const.SEP + str(index)
+            if not hasattr(module, "mindstudio_reserved_name") or not module.mindstudio_reserved_name:
+                module.mindstudio_reserved_name = []
+            module.mindstudio_reserved_name.append(full_name)
             forward_full_name = full_name.replace(Const.BACKWARD, Const.FORWARD)
             ModuleProcesser.module_node[full_name] = ModuleProcesser.module_node[forward_full_name].replace(
                 Const.FORWARD, Const.BACKWARD) if ModuleProcesser.module_node[forward_full_name] else None

msprobe/pytorch/free_benchmark/common/constant.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from typing import Dict
 
 import numpy as np

msprobe/pytorch/free_benchmark/common/counter.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from collections import defaultdict
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 

msprobe/pytorch/free_benchmark/common/enums.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.const import Const
 
 

msprobe/pytorch/free_benchmark/common/params.py

@@ -17,6 +17,7 @@ from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import torch
+from msprobe.core.common.exceptions import FreeBenchmarkException
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.enums import (
     DeviceType,
@@ -38,7 +39,6 @@ class DataParams:
     origin_func: Optional[Callable] = None
     api_type: Optional[str] = None
     fuzz_stage: Optional[str] = None
-    grad_unequal_flag: Optional[bool] = True
 
 
 @dataclass
@@ -126,9 +126,17 @@ def make_unequal_row(
     )
     if isinstance(ratio, float):
         row.max_rel = ratio - 1
+    if isinstance(ratio, str):
+        row.max_rel = ratio
     origin_tensor = data_params.original_result
     perturbed_tensor = data_params.perturbed_result
-    if index:
+    if index is not None:
+        if index >= len(origin_tensor) or index >= len(perturbed_tensor):
+            err_msg = f"When generating unequal results, index {index} of output is out of bounds. please check!"
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.OutputIndexError,
+                error_info=err_msg,
+            )
         origin_tensor = origin_tensor[index]
         perturbed_tensor = perturbed_tensor[index]
         row.output_index = index

msprobe/pytorch/free_benchmark/common/utils.py

@@ -13,7 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 import torch
+from msprobe.core.common.exceptions import FreeBenchmarkException
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark.common.enums import DeviceType
 
 
@@ -51,6 +54,7 @@ class Tools:
         return api_name.rsplit(".", 2)[0]
 
     @staticmethod
+    @recursion_depth_decorator("FreeBenchmark: Tools.convert_device_and_dtype")
     def convert_device_and_dtype(
         tensor_seq, device: str = DeviceType.CPU, change_dtype: bool = False
     ):
@@ -73,23 +77,41 @@ class Tools:
         return tensor_seq
 
     @staticmethod
+    @recursion_depth_decorator("FreeBenchmark: Tools.convert_fuzz_output_to_origin")
     def convert_fuzz_output_to_origin(origin, perturbed):
-        if isinstance(origin, torch.Tensor):
+        if isinstance(origin, torch.Tensor) and isinstance(perturbed, torch.Tensor):
             origin.data = perturbed.to(origin.dtype).to(origin.device)
             return origin
-        if isinstance(origin, dict):
+        if isinstance(origin, dict) and isinstance(perturbed, dict):
             output = dict()
             for key, value in origin.items():
+                if key not in perturbed:
+                    err_msg = f"'{key}' not in perturbed output."
+                    raise FreeBenchmarkException(
+                        FreeBenchmarkException.InvalidPerturbedOutput,
+                        error_info=err_msg,
+                    )
                 output[key] = Tools.convert_fuzz_output_to_origin(value, perturbed[key])
             return output
-        if isinstance(origin, (tuple, list)):
+        if isinstance(origin, (tuple, list)) and isinstance(perturbed, (tuple, list)):
             result = list()
+            if len(perturbed) != len(origin):
+                err_msg = (
+                    f"length of perturbed output ({len(perturbed)}) is different "
+                    f"from the length of original output ({len(origin)})."
+                )
+                raise FreeBenchmarkException(
+                    FreeBenchmarkException.InvalidPerturbedOutput, error_info=err_msg
+                )
             for index_, value in enumerate(origin):
                 result.append(
                     Tools.convert_fuzz_output_to_origin(value, perturbed[index_])
                 )
             return type(origin)(result)
-        return origin
+        err_msg = f"conversion of two outputs with types ({type(origin)}, {type(perturbed)}) is not supported."
+        raise FreeBenchmarkException(
+            FreeBenchmarkException.UnsupportedType, error_info=err_msg
+        )
 
 
 class TorchC:
@@ -102,6 +124,7 @@ class TorchC:
     abs = torch._C._VariableFunctionsClass.abs
     where = torch._C._VariableFunctionsClass.where
     div = torch._C._VariableFunctionsClass.div
+    mul = torch._C._VariableFunctionsClass.mul
     max = torch._C._VariableFunctionsClass.max
     min = torch._C._VariableFunctionsClass.min
     gt = torch._C._VariableFunctionsClass.gt
@@ -116,3 +139,5 @@ class TorchC:
     tensor_split = torch._C._VariableFunctionsClass.tensor_split
     stack = torch._C._VariableFunctionsClass.stack
     reshape = torch._C._VariableFunctionsClass.reshape
+    nan_to_num = torch._C._VariableFunctionsClass.nan_to_num
+    aminmax = torch._C._VariableFunctionsClass.aminmax

msprobe/pytorch/free_benchmark/compare/grad_saver.py

@@ -82,13 +82,11 @@ class GradSaver:
         data_params = DataParams()
         data_params.original_result = origin_grad
         data_params.perturbed_result = perturbed_grad
-        data_params.grad_unequal_flag = False
         data_params.valid_input_index = index
         try:
             handler.handle(data_params)
             if not data_params.is_consistent:
                 self.is_compare = False
-                data_params.grad_unequal_flag = True
                 data_params.is_consistent = True
                 data_params.perturbed_result = self.perturbed_grad_input
                 data_params.original_result = self.origin_grad_input
@@ -102,8 +100,13 @@ class GradSaver:
     def check_grad_input(self, origin_grad, new_grad_index):
         if self.perturbed_grad_input is None:
             raise FreeBenchmarkException(
-                FreeBenchmarkException.InvalidGrad,
-                f"grad not exists : {self.api_name}.",
+                FreeBenchmarkException.InvalidPerturbedOutput,
+                f"perturbed grad not exists for {self.api_name}.",
+            )
+        if len(self.perturbed_grad_input) <= new_grad_index:
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.InvalidPerturbedOutput,
+                f"perturbed grad index {new_grad_index} is out of bounds for {self.api_name}.",
             )
         with torch.no_grad():
             perturbed_grad = self.perturbed_grad_input[new_grad_index].to(
@@ -111,7 +114,7 @@ class GradSaver:
             )
         if origin_grad.shape != perturbed_grad.shape:
             raise FreeBenchmarkException(
-                FreeBenchmarkException.InvalidGrad,
+                FreeBenchmarkException.InvalidPerturbedOutput,
                 f"grad shapes are inconsistent. api:{self.handler_params.api_name}."
                 f"origin:{origin_grad.shape}, perturbation: {perturbed_grad.shape}",
             )
@@ -164,6 +167,18 @@ class GradSaver:
             index_ = 0
             for object_ in inner_args:
                 if object_ is CommonField.HOLD_PLACE:
+                    if index_ >= len(inputs):
+                        err_msg = (
+                            f"[msprobe] Free benchmark: When getting input from vjp, "
+                            f" the input index ({index_}) is out of bounds ({len(inputs)})."
+                        )
+                        logger.error_log_with_exp(
+                            err_msg,
+                            FreeBenchmarkException(
+                                FreeBenchmarkException.InvalidGrad,
+                                error_info=err_msg,
+                            ),
+                        )
                     _real_input.append(inputs[index_])
                     index_ += 1
                 else:

msprobe/pytorch/free_benchmark/compare/single_benchmark.py

@@ -16,6 +16,7 @@
 import math
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.utils import TorchC
@@ -67,6 +68,7 @@ class SingleCompare:
                 return False
         return True
 
+    @recursion_depth_decorator("FreeBenchmark: SingleCompare.compare_seq")
     def compare_seq(self, actual, golden):
         if isinstance(golden, torch.Tensor):
             return self.compare_tensor_seq(actual, golden)

msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -26,6 +27,7 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import
 
 class AddNoiseLayer(NpuBaseLayer):
 
+    @recursion_depth_decorator("FreeBenchmark: AddNoiseLayer.add_noise")
     def add_noise(self, tensor_obj):
         if isinstance(tensor_obj, torch.Tensor):
             self.perturbed_value = ThresholdConfig.PERTURBATION_VALUE_DICT.get(
@@ -99,7 +101,7 @@ class AddNoiseLayer(NpuBaseLayer):
         if max_val < abs_tol:
             logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+                f"Maximun value is less than the minimun threshold. Cancel add noise."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -31,6 +32,7 @@ class BitNoiseLayer(NpuBaseLayer):
         self.bit_tail: int = 1
         self.bit_type = None
 
+    @recursion_depth_decorator("FreeBenchmark: BitNoiseLayer.add_bit_noise")
     def add_bit_noise(self, tensor_obj):
         """
         对输入添加噪声
@@ -79,14 +81,14 @@ class BitNoiseLayer(NpuBaseLayer):
         判断是否需要添加扰动,  bit翻转
         """
         if not self.bit_type:
-            logger.info_on_rank_0(
+            logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
                 f"dtype unsupported. Cancel perturbation."
             )
             return False
         if tensor_obj.numel() == 0:
             logger.warning_on_rank_0(
-                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0"
+                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0."
                 f" Cancel adding noise."
             )
             return False
@@ -102,9 +104,9 @@ class BitNoiseLayer(NpuBaseLayer):
             )
             max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item()
         if max_val < abs_tol:
-            logger.info_on_rank_0(
+            logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+                f"Maximun value is less than the minimun threshold. Cancel add noise."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
 from msprobe.pytorch.free_benchmark.common.params import DataParams
@@ -29,6 +30,7 @@ class ChangeValueLayer(NpuBaseLayer):
         self.head: int = 0
         self.tail: int = -1
 
+    @recursion_depth_decorator("FreeBenchmark: ChangeValueLayer.change_value")
     def change_value(self, tensor_obj):
         """
         交换张量首尾

msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py

@@ -15,6 +15,7 @@
 
 import torch
 from msprobe.core.common.const import Const
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import CommonField
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -26,6 +27,9 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import
 
 class ImprovePrecisionLayer(NpuBaseLayer):
 
+    @recursion_depth_decorator(
+        "FreeBenchmark: ImprovePrecisionLayer.improve_tensor_precision"
+    )
     def improve_tensor_precision(self, tensor_obj):
         if (
             isinstance(tensor_obj, torch.Tensor)