mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/free_benchmark/compare/grad_saver.py

@@ -1,8 +1,27 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
 from msprobe.core.common.exceptions import FreeBenchmarkException
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import CommonField
-from msprobe.pytorch.free_benchmark.common.params import DataParams, HandlerParams, data_pre_deal
+from msprobe.pytorch.free_benchmark.common.params import (
+    DataParams,
+    HandlerParams,
+    data_pre_deal,
+)
 from msprobe.pytorch.free_benchmark.perturbed_layers.layer_factory import LayerFactory
 from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import (
     FuzzHandlerFactory,
@@ -83,8 +102,13 @@ class GradSaver:
     def check_grad_input(self, origin_grad, new_grad_index):
         if self.perturbed_grad_input is None:
             raise FreeBenchmarkException(
-                FreeBenchmarkException.InvalidGrad,
-                f"grad not exists : {self.api_name}."
+                FreeBenchmarkException.InvalidPerturbedOutput,
+                f"perturbed grad not exists for {self.api_name}.",
+            )
+        if len(self.perturbed_grad_input) <= new_grad_index:
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.InvalidPerturbedOutput,
+                f"perturbed grad index {new_grad_index} is out of bounds for {self.api_name}.",
             )
         with torch.no_grad():
             perturbed_grad = self.perturbed_grad_input[new_grad_index].to(
@@ -92,9 +116,9 @@ class GradSaver:
             )
         if origin_grad.shape != perturbed_grad.shape:
             raise FreeBenchmarkException(
-                FreeBenchmarkException.InvalidGrad,
+                FreeBenchmarkException.InvalidPerturbedOutput,
                 f"grad shapes are inconsistent. api:{self.handler_params.api_name}."
-                f"origin:{origin_grad.shape}, perturbation: {perturbed_grad.shape}"
+                f"origin:{origin_grad.shape}, perturbation: {perturbed_grad.shape}",
             )
         return perturbed_grad
 
@@ -145,13 +169,25 @@ class GradSaver:
             index_ = 0
             for object_ in inner_args:
                 if object_ is CommonField.HOLD_PLACE:
+                    if index_ >= len(inputs):
+                        err_msg = (
+                            f"[msprobe] Free benchmark: When getting input from vjp, "
+                            f" the input index ({index_}) is out of bounds ({len(inputs)})."
+                        )
+                        logger.error_log_with_exp(
+                            err_msg,
+                            FreeBenchmarkException(
+                                FreeBenchmarkException.InvalidGrad,
+                                error_info=err_msg,
+                            ),
+                        )
                     _real_input.append(inputs[index_])
                     index_ += 1
                 else:
                     _real_input.append(object_)
             kwargs = self.kwargs.copy()
-            if 'inplace' in kwargs:
-                kwargs['inplace'] = False
+            if "inplace" in kwargs:
+                kwargs["inplace"] = False
             return self.origin_func(*_real_input, **kwargs)
 
         _, grad_input = torch.autograd.functional.vjp(
@@ -159,12 +195,14 @@ class GradSaver:
         )
         return grad_input
 
-    def calculate_perturbed_grad_input(self, grad_output, need_grad_tensors, inner_args):
+    def calculate_perturbed_grad_input(
+        self, grad_output, need_grad_tensors, inner_args
+    ):
         data_params = data_pre_deal(
             self.handler_params.api_name,
             self.get_grad_input_from_vjp,
             [need_grad_tensors, grad_output, inner_args],
-            {}
+            {},
         )
         layer = LayerFactory.create(
             self.handler_params.api_name,

msprobe/pytorch/free_benchmark/compare/single_benchmark.py

@@ -1,6 +1,22 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import math
 
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.utils import TorchC
@@ -52,6 +68,7 @@ class SingleCompare:
                 return False
         return True
 
+    @recursion_depth_decorator("FreeBenchmark: SingleCompare.compare_seq")
     def compare_seq(self, actual, golden):
         if isinstance(golden, torch.Tensor):
             return self.compare_tensor_seq(actual, golden)

msprobe/pytorch/free_benchmark/main.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import ABC
 
 import torch
@@ -36,9 +51,9 @@ class FreeBenchmarkCheck(ABC):
 
     def update_iter(self, update_iter):
         self.current_iter = update_iter
-    
+
     def if_fix(self):
-        if self.config.handler_type==HandlerType.FIX:
+        if self.config.handler_type == HandlerType.FIX:
             return True
         return False
 
@@ -73,9 +88,9 @@ class FreeBenchmarkCheck(ABC):
         layer.handle(data_params)
         handler_params = make_handler_params(name, self.config, self.current_iter)
         handler = FuzzHandlerFactory.create(handler_params)
-        perturbed_output = handler.handle(data_params) 
+        perturbed_output = handler.handle(data_params)
         return perturbed_output, handler.get_unequal_rows()
-    
+
     def backward(self, name, module, grad_output):
 
         if not self.config.fuzz_stage == Const.BACKWARD:

msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import ABC, abstractmethod
 from typing import Any
 

msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py

@@ -1,14 +1,29 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.pytorch.free_benchmark import FreeBenchmarkException
 from msprobe.pytorch.free_benchmark.common.enums import DeviceType, PerturbationMode
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.improve_precision import (
-    ImprovePrecisionLayer,
-)
 from msprobe.pytorch.free_benchmark.perturbed_layers.npu.add_noise import AddNoiseLayer
 from msprobe.pytorch.free_benchmark.perturbed_layers.npu.bit_noise import BitNoiseLayer
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.no_change import NoChangeLayer
 from msprobe.pytorch.free_benchmark.perturbed_layers.npu.change_value import (
     ChangeValueLayer,
 )
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.improve_precision import (
+    ImprovePrecisionLayer,
+)
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.no_change import NoChangeLayer
 from msprobe.pytorch.free_benchmark.perturbed_layers.run_cpu import CpuLayer
 
 

msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py

@@ -1,4 +1,20 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -11,6 +27,7 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import
 
 class AddNoiseLayer(NpuBaseLayer):
 
+    @recursion_depth_decorator("FreeBenchmark: AddNoiseLayer.add_noise")
     def add_noise(self, tensor_obj):
         if isinstance(tensor_obj, torch.Tensor):
             self.perturbed_value = ThresholdConfig.PERTURBATION_VALUE_DICT.get(
@@ -84,7 +101,7 @@ class AddNoiseLayer(NpuBaseLayer):
         if max_val < abs_tol:
             logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+                f"Maximun value is less than the minimun threshold. Cancel add noise."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py

@@ -1,4 +1,20 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -16,6 +32,7 @@ class BitNoiseLayer(NpuBaseLayer):
         self.bit_tail: int = 1
         self.bit_type = None
 
+    @recursion_depth_decorator("FreeBenchmark: BitNoiseLayer.add_bit_noise")
     def add_bit_noise(self, tensor_obj):
         """
         对输入添加噪声
@@ -64,14 +81,14 @@ class BitNoiseLayer(NpuBaseLayer):
         判断是否需要添加扰动,  bit翻转
         """
         if not self.bit_type:
-            logger.info_on_rank_0(
+            logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
                 f"dtype unsupported. Cancel perturbation."
             )
             return False
         if tensor_obj.numel() == 0:
             logger.warning_on_rank_0(
-                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0"
+                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0."
                 f" Cancel adding noise."
             )
             return False
@@ -87,9 +104,9 @@ class BitNoiseLayer(NpuBaseLayer):
             )
             max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item()
         if max_val < abs_tol:
-            logger.info_on_rank_0(
+            logger.warning_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+                f"Maximun value is less than the minimun threshold. Cancel add noise."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py

@@ -1,4 +1,20 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
 from msprobe.pytorch.free_benchmark.common.params import DataParams
@@ -14,6 +30,7 @@ class ChangeValueLayer(NpuBaseLayer):
         self.head: int = 0
         self.tail: int = -1
 
+    @recursion_depth_decorator("FreeBenchmark: ChangeValueLayer.change_value")
     def change_value(self, tensor_obj):
         """
         交换张量首尾
@@ -54,10 +71,19 @@ class ChangeValueLayer(NpuBaseLayer):
         """
         判断是否需要添加扰动,  首尾值交换
         """
-        if tensor_obj.size(0) < 2:
+        # 对于维度大于1的张量、要求1维至少大于1且0维和1维至少一个长度大于2
+        if tensor_obj.ndim > 1:
+            if tensor_obj.size(1) == 0 or (tensor_obj.size(1) < 2 and tensor_obj.size(0) < 2):
+                logger.info_on_rank_0(
+                    f"[msprobe] Free Benchmark: For {self.api_name} with ndim {tensor_obj.ndim}, "
+                    f"at least one of 0-dimension or 1-dimension greater than 1. Cancel change value."
+                )
+                return False
+        # 不支持维度等于0的张量、对于维度等于1的张量、要求0维长度大于2
+        elif tensor_obj.dim() == 0 or tensor_obj.size(0) < 2:
             logger.info_on_rank_0(
                 f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"size 0 must greater than 1. Cancel change value."
+                f"0-dimension must greater than 1. Cancel change value."
             )
             return False
         return True

msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py

@@ -1,5 +1,21 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
 from msprobe.core.common.const import Const
+from msprobe.core.common.utils import recursion_depth_decorator
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import CommonField
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
@@ -11,6 +27,9 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import
 
 class ImprovePrecisionLayer(NpuBaseLayer):
 
+    @recursion_depth_decorator(
+        "FreeBenchmark: ImprovePrecisionLayer.improve_tensor_precision"
+    )
     def improve_tensor_precision(self, tensor_obj):
         if (
             isinstance(tensor_obj, torch.Tensor)

msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode

msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import abstractmethod
 from typing import Any
 

msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import torch
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.params import DataParams

msprobe/pytorch/free_benchmark/result_handlers/base_handler.py

@@ -1,10 +1,26 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import math
 from abc import ABC, abstractmethod
 from typing import Any, Optional, Tuple
-import numpy as np
 
+import numpy as np
 import torch
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import FreeBenchmarkException
 from msprobe.pytorch.free_benchmark import logger
 from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
 from msprobe.pytorch.free_benchmark.common.enums import (
@@ -35,7 +51,9 @@ class FuzzHandler(ABC):
             origin_ouput = origin_ouput.values
             perturbed_output = perturbed_output.values
         if hasattr(perturbed_output, "dtype"):
-            abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(perturbed_output.dtype, FuzzThreshold.F32_THD)
+            abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(
+                perturbed_output.dtype, FuzzThreshold.F32_THD
+            )
         else:
             abs_tol = FuzzThreshold.F32_THD
         return (
@@ -53,16 +71,22 @@ class FuzzHandler(ABC):
         :return origin_output_chunks: 切块后原始输出列表
         :return perturbed_output_chunks: 切块后扰动后输出列表
         """
-        single_output_mem = origin_output.element_size() * origin_output.nelement() / Const.ONE_MB
+        single_output_mem = (
+            origin_output.element_size() * origin_output.nelement() / Const.ONE_MB
+        )
         if single_output_mem == 0 or origin_output.ndim == 0:
             return [origin_output], [perturbed_output]
         # 张量大小和批数之间的关系：chunks_exp=math.log(M,2)-4, chunks=2**chunks_exp (M为对比张量数据大小[Mb])
         chunks_exp = int(math.log(single_output_mem, 2)) - 4
-        chunks = 2 ** chunks_exp
+        chunks = 2**chunks_exp
         chunks = max(chunks, 1)
         chunks = min(chunks, ThresholdConfig.TENSOR_SPLIT_MAX_CHUNK)
-        origin_output_chunks = TorchC.tensor_split(TorchC.reshape(origin_output, (-1,)), chunks)
-        perturbed_output_chunks = TorchC.tensor_split(TorchC.reshape(perturbed_output, (-1,)), chunks)
+        origin_output_chunks = TorchC.tensor_split(
+            TorchC.reshape(origin_output, (-1,)), chunks
+        )
+        perturbed_output_chunks = TorchC.tensor_split(
+            TorchC.reshape(perturbed_output, (-1,)), chunks
+        )
         return origin_output_chunks, perturbed_output_chunks
 
     @staticmethod
@@ -80,14 +104,24 @@ class FuzzHandler(ABC):
         pass
 
     def get_ratio_from_specific_norm(
-            self, origin_output, perturbed_output, norm_type, abs_tol
+        self, origin_output, perturbed_output, norm_type, abs_tol
     ):
         if norm_type == NormType.ENDLESS_NORM:
             return self.calculate_error(origin_output, perturbed_output, abs_tol)
         return ThresholdConfig.COMP_CONSISTENT
 
     def calculate_error(self, origin_output, perturbed_output, abs_tol):
-        origin_output_chunks, perturbed_output_chunks = self.tensor_split_for_error_calculate(origin_output, perturbed_output)
+        origin_output_chunks, perturbed_output_chunks = (
+            self.tensor_split_for_error_calculate(origin_output, perturbed_output)
+        )
+        if len(origin_output_chunks) != len(perturbed_output_chunks):
+            err_msg = (
+                f"For {self.params.api_name}, the number of compare tensor chunks is different: "
+                f"{len(origin_output_chunks)} != {len(perturbed_output_chunks)}. please check!"
+            )
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.OutputIndexError, err_msg
+            )
         norm1 = -np.inf
         norm2 = -np.inf
         norm3 = np.inf
@@ -95,11 +129,25 @@ class FuzzHandler(ABC):
             if chunk_origin.nelement() == 0:
                 break
             chunk_perturbed = perturbed_output_chunks[i]
-            ratio_tensor1 = TorchC.where(TorchC.abs(chunk_perturbed) > abs_tol,
-                                         TorchC.div(TorchC.clamp(chunk_origin, min=abs_tol), TorchC.clamp(chunk_perturbed, min=abs_tol)), 1)
-            ratio_tensor2 = TorchC.where(TorchC.abs(chunk_origin) > abs_tol,
-                                         TorchC.div(TorchC.clamp(chunk_perturbed, min=abs_tol), TorchC.clamp(chunk_origin, min=abs_tol)), 1)
-            norm_values = TorchC.stack([TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)])
+            ratio_tensor1 = TorchC.where(
+                TorchC.abs(chunk_perturbed) > abs_tol,
+                TorchC.div(
+                    TorchC.clamp(chunk_origin, min=abs_tol),
+                    TorchC.clamp(chunk_perturbed, min=abs_tol),
+                ),
+                1,
+            )
+            ratio_tensor2 = TorchC.where(
+                TorchC.abs(chunk_origin) > abs_tol,
+                TorchC.div(
+                    TorchC.clamp(chunk_perturbed, min=abs_tol),
+                    TorchC.clamp(chunk_origin, min=abs_tol),
+                ),
+                1,
+            )
+            norm_values = TorchC.stack(
+                [TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)]
+            )
             max_ratio1, max_ratio2 = norm_values.tolist()
             norm1 = max(norm1, self.convert_overflow_ratio_to_consistent(max_ratio1))
             norm2 = max(norm2, self.convert_overflow_ratio_to_consistent(max_ratio2))
@@ -126,13 +174,13 @@ class FuzzHandler(ABC):
         if self.params.fuzz_stage == Const.BACKWARD:
             abs_tol = ThresholdConfig.BACKWARD_OUTPUT_LOWER_BOUND
         else:
-            abs_tol = abs_tol ** 0.5
+            abs_tol = abs_tol**0.5
         return self.get_ratio_from_specific_norm(
             origin_output, perturbed_output, norm_type, abs_tol
         )
 
     def npu_compare(
-            self, origin_output, perturbed_output
+        self, origin_output, perturbed_output
     ) -> Tuple[bool, Optional[float]]:
 
         if isinstance(perturbed_output, int):
@@ -150,6 +198,7 @@ class FuzzHandler(ABC):
                 f"[msprobe] Free Benchmark: For {self.params.api_name} "
                 f"The compare for output type {type(perturbed_output)} is not supported"
             )
+            return True, 1
 
         threshold = self.get_threshold(Tools.get_first_tensor_dtype(origin_output))
         ratio = self.ratio_calculate(
@@ -189,7 +238,7 @@ class FuzzHandler(ABC):
                         max_fuzz_ratio if ratio is None else max(max_fuzz_ratio, ratio)
                     )
                     data_params.is_consistent = (
-                            is_consistent and data_params.is_consistent
+                        is_consistent and data_params.is_consistent
                     )
                     if not is_consistent and data_params.grad_unequal_flag:
                         self.unequal_rows.append(

msprobe/pytorch/free_benchmark/result_handlers/check_handler.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from typing import Any
 
 from msprobe.pytorch.free_benchmark import logger