mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.4__py3-none-any.whl

msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py

@@ -1,28 +1,28 @@
-import torch
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
-from msprobe.pytorch.free_benchmark.common.params import DataParams
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import (
-    NpuBaseLayer,
-)
-
-
-class NoChangeLayer(NpuBaseLayer):
-
-    def no_change(self, tensor_obj):
-        """
-        不对输入做任何改变、直接二次执行
-        """
-        self.is_added = True
-        return tensor_obj
-
-    def handle(self, params: DataParams) -> torch.Any:
-        """
-        对输入添加扰动并返回
-        """
-        logger.info_on_rank_0(
-            f"[msprobe] Free benchmark: Perturbation is "
-            f"{PerturbationMode.NO_CHANGE} of {self.api_name}."
-        )
-        params.perturbed_value = self.no_change(params.args[params.valid_input_index])
-        return self.perturbed_result(params)
+import torch
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
+from msprobe.pytorch.free_benchmark.common.params import DataParams
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import (
+    NpuBaseLayer,
+)
+
+
+class NoChangeLayer(NpuBaseLayer):
+
+    def no_change(self, tensor_obj):
+        """
+        不对输入做任何改变、直接二次执行
+        """
+        self.is_added = True
+        return tensor_obj
+
+    def handle(self, params: DataParams):
+        """
+        对输入添加扰动并返回
+        """
+        logger.info_on_rank_0(
+            f"[msprobe] Free benchmark: Perturbation is "
+            f"{PerturbationMode.NO_CHANGE} of {self.api_name}."
+        )
+        params.perturbed_value = self.no_change(params.args[params.valid_input_index])
+        return self.perturbed_result(params)

msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py

@@ -1,45 +1,45 @@
-from abc import abstractmethod
-from typing import Any
-
-import torch
-from msprobe.pytorch.free_benchmark.common.params import DataParams
-from msprobe.pytorch.free_benchmark.perturbed_layers.base_layer import BaseLayer
-
-
-class NpuBaseLayer(BaseLayer):
-    def __init__(self, api_name: str) -> None:
-        super().__init__(api_name)
-        self.perturbed_value = None  # 扰动的元素
-        self.is_added = False  # 标记当前算子输入是否调整
-
-    @staticmethod
-    def perturbed_result(params: DataParams) -> Any:
-        args_front = params.args[: params.valid_input_index]
-        args_rear = params.args[params.valid_input_index + 1:]
-        # 此处会将有inplace属性的算子换为非inplace
-        if "inplace" in params.kwargs:
-            params.kwargs["inplace"] = False
-        params.perturbed_result = params.origin_func(
-            *args_front, params.perturbed_value, *args_rear, **params.kwargs
-        )
-        return params.perturbed_result
-
-    @abstractmethod
-    def handle(self, params: DataParams) -> Any:
-        pass
-
-    def pre_check(self, tensor_obj):
-        """
-        检查张量是否符合标准(float类型且最大值大于对应精度最小值)
-        """
-        # 只针对第一个满足要求的添加扰动
-        if self.is_added:
-            return False
-        if not torch.is_floating_point(tensor_obj):
-            return False
-        if not self._check_details(tensor_obj):
-            return False
-        return True
-
-    def _check_details(self, tensor_obj):
-        return True
+from abc import abstractmethod
+from typing import Any
+
+import torch
+from msprobe.pytorch.free_benchmark.common.params import DataParams
+from msprobe.pytorch.free_benchmark.perturbed_layers.base_layer import BaseLayer
+
+
+class NpuBaseLayer(BaseLayer):
+    def __init__(self, api_name: str) -> None:
+        super().__init__(api_name)
+        self.perturbed_value = None  # 扰动的元素
+        self.is_added = False  # 标记当前算子输入是否调整
+
+    @staticmethod
+    def perturbed_result(params: DataParams) -> Any:
+        args_front = params.args[: params.valid_input_index]
+        args_rear = params.args[params.valid_input_index + 1:]
+        # 此处会将有inplace属性的算子换为非inplace
+        if "inplace" in params.kwargs:
+            params.kwargs["inplace"] = False
+        params.perturbed_result = params.origin_func(
+            *args_front, params.perturbed_value, *args_rear, **params.kwargs
+        )
+        return params.perturbed_result
+
+    @abstractmethod
+    def handle(self, params: DataParams) -> Any:
+        pass
+
+    def pre_check(self, tensor_obj):
+        """
+        检查张量是否符合标准(float类型且最大值大于对应精度最小值)
+        """
+        # 只针对第一个满足要求的添加扰动
+        if self.is_added:
+            return False
+        if not torch.is_floating_point(tensor_obj):
+            return False
+        if not self._check_details(tensor_obj):
+            return False
+        return True
+
+    def _check_details(self, tensor_obj):
+        return True

msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py

@@ -1,19 +1,19 @@
-import torch
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.params import DataParams
-from msprobe.pytorch.free_benchmark.common.utils import Tools
-from msprobe.pytorch.free_benchmark.common.enums import DeviceType
-from msprobe.pytorch.free_benchmark.perturbed_layers.base_layer import BaseLayer
-
-
-class CpuLayer(BaseLayer):
-
-    def handle(self, params: DataParams) -> torch.Any:
-
-        logger.info_on_rank_0(
-            f"[msprobe] Free benchmark: Perturbation is to_cpu of {self.api_name}."
-        )
-        new_args = Tools.convert_device_and_dtype(params.args, DeviceType.CPU, change_dtype=True)
-        new_kwargs = Tools.convert_device_and_dtype(params.kwargs, DeviceType.CPU, change_dtype=True)
-        params.perturbed_result = params.origin_func(*new_args, **new_kwargs)
-        return params.perturbed_result
+import torch
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.params import DataParams
+from msprobe.pytorch.free_benchmark.common.utils import Tools
+from msprobe.pytorch.free_benchmark.common.enums import DeviceType
+from msprobe.pytorch.free_benchmark.perturbed_layers.base_layer import BaseLayer
+
+
+class CpuLayer(BaseLayer):
+
+    def handle(self, params: DataParams):
+
+        logger.info_on_rank_0(
+            f"[msprobe] Free benchmark: Perturbation is to_cpu of {self.api_name}."
+        )
+        new_args = Tools.convert_device_and_dtype(params.args, DeviceType.CPU, change_dtype=True)
+        new_kwargs = Tools.convert_device_and_dtype(params.kwargs, DeviceType.CPU, change_dtype=True)
+        params.perturbed_result = params.origin_func(*new_args, **new_kwargs)
+        return params.perturbed_result

msprobe/pytorch/free_benchmark/result_handlers/base_handler.py

@@ -1,203 +1,217 @@
-import math
-from abc import ABC, abstractmethod
-from typing import Any, Optional, Tuple
-
-import torch
-from msprobe.core.common.const import Const
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
-from msprobe.pytorch.free_benchmark.common.enums import (
-    FuzzThreshold,
-    NormType,
-    PerturbationMode,
-)
-from msprobe.pytorch.free_benchmark.common.params import (
-    DataParams,
-    HandlerParams,
-    make_unequal_row,
-)
-from msprobe.pytorch.free_benchmark.common.utils import Tools, TorchC
-
-
-class FuzzHandler(ABC):
-    def __init__(self, params: HandlerParams) -> None:
-        self.params = params
-        self.unequal_rows = []
-
-    @staticmethod
-    def pre_process(origin_ouput, perturbed_output):
-        if (
-            isinstance(origin_ouput, tuple)
-            and hasattr(origin_ouput, "values")
-            and hasattr(origin_ouput, "indices")
-        ):
-            origin_ouput = origin_ouput.values
-            perturbed_output = perturbed_output.values
-        if hasattr(perturbed_output, "dtype"):
-            abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(perturbed_output.dtype)
-        else:
-            abs_tol = FuzzThreshold.F32_THD.value
-        return (
-            origin_ouput.to(perturbed_output.dtype).to(perturbed_output.device),
-            perturbed_output,
-            abs_tol,
-        )
-
-    @staticmethod
-    def convert_overflow_ratio_to_consistent(ratio):
-        if math.isnan(ratio) or math.isinf(ratio):
-            return ThresholdConfig.COMP_CONSISTENT
-        return ratio
-
-    @abstractmethod
-    def get_threshold(self, dtype):
-        pass
-
-    @abstractmethod
-    def handle(self, data_params: DataParams) -> Any:
-        pass
-
-    def get_ratio_from_specific_norm(
-            self, origin_output, perturbed_output, norm_type, abs_tol
-    ):
-        if norm_type == NormType.ENDLESS_NORM:
-            return self.get_endless_norm(origin_output, perturbed_output, abs_tol)
-        return ThresholdConfig.COMP_CONSISTENT
-
-    def get_endless_norm(self, origin_output, perturbed_output, abs_tol):
-        ratio_tensor1 = TorchC.where(
-            TorchC.gt(TorchC.abs(perturbed_output), abs_tol),
-            TorchC.div(
-                TorchC.abs(origin_output),
-                TorchC.add(TorchC.abs(perturbed_output), abs_tol),
-            ),
-            1,
-        )
-        ratio_tensor2 = TorchC.where(
-            TorchC.gt(TorchC.abs(origin_output), abs_tol),
-            TorchC.div(
-                TorchC.abs(perturbed_output),
-                TorchC.add(TorchC.abs(origin_output), abs_tol),
-            ),
-            1,
-        )
-
-        norm1 = self.convert_overflow_ratio_to_consistent(
-            TorchC.max(ratio_tensor1).item()
-        )
-        norm2 = self.convert_overflow_ratio_to_consistent(
-            TorchC.max(ratio_tensor2).item()
-        )
-        norm3 = self.convert_overflow_ratio_to_consistent(
-            TorchC.min(ratio_tensor1).item()
-        )
-        if norm3 < 0:
-            ratio = ThresholdConfig.SYMBOL_FLIPPING
-        else:
-            ratio = max(norm1, norm2)
-        return ratio
-
-    def ratio_calculate(self, origin_output, perturbed_output, norm_type) -> float:
-        try:
-            origin_output, perturbed_output, abs_tol = self.pre_process(
-                origin_output, perturbed_output
-            )
-        except Exception as e:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free Benchmark: For {self.params.api_name}, "
-                f"when computing ratio,"
-                f" y1 or y2 dtype is not supported {e}"
-            )
-            return ThresholdConfig.COMP_NAN
-        if self.params.fuzz_stage == Const.BACKWARD:
-            abs_tol = ThresholdConfig.BACKWARD_OUTPUT_LOWER_BOUND
-        else:
-            abs_tol = abs_tol ** 0.5
-        return self.get_ratio_from_specific_norm(
-            origin_output, perturbed_output, norm_type, abs_tol
-        )
-
-    def npu_compare(
-            self, origin_output, perturbed_output
-    ) -> Tuple[bool, Optional[float]]:
-
-        if isinstance(perturbed_output, int):
-            return origin_output == perturbed_output, None
-        elif isinstance(perturbed_output, float):
-            if perturbed_output == 0:
-                origin_output += FuzzThreshold.F32_THD
-                perturbed_output += FuzzThreshold.F32_THD
-            return (
-                math.isclose(origin_output, perturbed_output),
-                origin_output / perturbed_output,
-            )
-        elif not isinstance(perturbed_output, torch.Tensor):
-            logger.warning_on_rank_0(
-                f"[msprobe] Free Benchmark: For {self.params.api_name} "
-                f"The compare for output type {type(perturbed_output)} is not supported"
-            )
-
-        threshold = self.get_threshold(Tools.get_first_tensor_dtype(origin_output))
-        ratio = self.ratio_calculate(
-            origin_output, perturbed_output, norm_type=NormType.ENDLESS_NORM
-        )
-        if ratio == ThresholdConfig.SYMBOL_FLIPPING:
-            is_consistent = False
-        else:
-            is_consistent = threshold >= ratio >= 1 / threshold
-        return is_consistent, ratio
-
-    def cmp_output_npu(self, data_params: DataParams):
-        npu_consistent = True
-        max_fuzz_ratio = 0
-        try:
-            if isinstance(data_params.original_result, torch.Tensor):
-                is_consistent, ratio = self.npu_compare(
-                    data_params.original_result, data_params.perturbed_result
-                )
-                npu_consistent = is_consistent
-                max_fuzz_ratio = (
-                    max_fuzz_ratio if ratio is None else max(max_fuzz_ratio, ratio)
-                )
-                data_params.is_consistent = is_consistent and data_params.is_consistent
-                if not is_consistent and data_params.grad_unequal_flag:
-                    self.unequal_rows.append(
-                        make_unequal_row(data_params, self.params, ratio=ratio)
-                    )
-
-            elif isinstance(data_params.original_result, (list, tuple)):
-                for index_, origin_item in enumerate(data_params.original_result):
-                    is_consistent, ratio = self.npu_compare(
-                        origin_item, data_params.perturbed_result[index_]
-                    )
-                    npu_consistent = npu_consistent and is_consistent
-                    max_fuzz_ratio = (
-                        max_fuzz_ratio if ratio is None else max(max_fuzz_ratio, ratio)
-                    )
-                    data_params.is_consistent = (
-                            is_consistent and data_params.is_consistent
-                    )
-                    if not is_consistent and data_params.grad_unequal_flag:
-                        self.unequal_rows.append(
-                            make_unequal_row(
-                                data_params, self.params, ratio=ratio, index=index_
-                            )
-                        )
-        except Exception as e:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free Benchmark: For {self.params.api_name}, "
-                f"when campare the result exception raise {e}"
-            )
-        return npu_consistent, max_fuzz_ratio
-
-    def get_unequal_rows(self):
-        return self.unequal_rows
-
-    def _get_default_threshold(self, dtype):
-        if self.params.pert_mode == PerturbationMode.NO_CHANGE:
-            threshold = ThresholdConfig.COMP_CONSISTENT
-        else:
-            threshold = ThresholdConfig.DTYPE_PER_THD.get(
-                dtype, ThresholdConfig.DTYPE_PER_THD.get(torch.float32)
-            )
-        return threshold
+import math
+from abc import ABC, abstractmethod
+from typing import Any, Optional, Tuple
+import numpy as np
+
+import torch
+from msprobe.core.common.const import Const
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
+from msprobe.pytorch.free_benchmark.common.enums import (
+    FuzzThreshold,
+    NormType,
+    PerturbationMode,
+)
+from msprobe.pytorch.free_benchmark.common.params import (
+    DataParams,
+    HandlerParams,
+    make_unequal_row,
+)
+from msprobe.pytorch.free_benchmark.common.utils import Tools, TorchC
+
+
+class FuzzHandler(ABC):
+    def __init__(self, params: HandlerParams) -> None:
+        self.params = params
+        self.unequal_rows = []
+
+    @staticmethod
+    def pre_process(origin_ouput, perturbed_output):
+        if (
+            isinstance(origin_ouput, tuple)
+            and hasattr(origin_ouput, "values")
+            and hasattr(origin_ouput, "indices")
+        ):
+            origin_ouput = origin_ouput.values
+            perturbed_output = perturbed_output.values
+        if hasattr(perturbed_output, "dtype"):
+            abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(perturbed_output.dtype, FuzzThreshold.F32_THD)
+        else:
+            abs_tol = FuzzThreshold.F32_THD
+        return (
+            origin_ouput.to(perturbed_output.dtype).to(perturbed_output.device),
+            perturbed_output,
+            abs_tol,
+        )
+
+    @staticmethod
+    def tensor_split_for_error_calculate(origin_output, perturbed_output):
+        """
+        对将投入误差值计算的扰动前后输出张量进行分块
+        :param origin_output: 原始输出
+        :param perturbed_output: 扰动后输出
+        :return origin_output_chunks: 切块后原始输出列表
+        :return perturbed_output_chunks: 切块后扰动后输出列表
+        """
+        single_output_mem = origin_output.element_size() * origin_output.nelement() / Const.ONE_MB
+        if single_output_mem == 0 or origin_output.ndim == 0:
+            return [origin_output], [perturbed_output]
+        # 张量大小和批数之间的关系：chunks_exp=math.log(M,2)-4, chunks=2**chunks_exp (M为对比张量数据大小[Mb])
+        chunks_exp = int(math.log(single_output_mem, 2)) - 4
+        chunks = 2 ** chunks_exp
+        chunks = max(chunks, 1)
+        chunks = min(chunks, ThresholdConfig.TENSOR_SPLIT_MAX_CHUNK)
+        origin_output_chunks = TorchC.tensor_split(TorchC.reshape(origin_output, (-1,)), chunks)
+        perturbed_output_chunks = TorchC.tensor_split(TorchC.reshape(perturbed_output, (-1,)), chunks)
+        return origin_output_chunks, perturbed_output_chunks
+
+    @staticmethod
+    def convert_overflow_ratio_to_consistent(ratio):
+        if math.isnan(ratio) or math.isinf(ratio):
+            return ThresholdConfig.COMP_CONSISTENT
+        return ratio
+
+    @abstractmethod
+    def get_threshold(self, dtype):
+        pass
+
+    @abstractmethod
+    def handle(self, data_params: DataParams) -> Any:
+        pass
+
+    def get_ratio_from_specific_norm(
+            self, origin_output, perturbed_output, norm_type, abs_tol
+    ):
+        if norm_type == NormType.ENDLESS_NORM:
+            return self.calculate_error(origin_output, perturbed_output, abs_tol)
+        return ThresholdConfig.COMP_CONSISTENT
+
+    def calculate_error(self, origin_output, perturbed_output, abs_tol):
+        origin_output_chunks, perturbed_output_chunks = self.tensor_split_for_error_calculate(origin_output, perturbed_output)
+        norm1 = -np.inf
+        norm2 = -np.inf
+        norm3 = np.inf
+        for i, chunk_origin in enumerate(origin_output_chunks):
+            if chunk_origin.nelement() == 0:
+                break
+            chunk_perturbed = perturbed_output_chunks[i]
+            ratio_tensor1 = TorchC.where(TorchC.abs(chunk_perturbed) > abs_tol,
+                                         TorchC.div(TorchC.clamp(chunk_origin, min=abs_tol), TorchC.clamp(chunk_perturbed, min=abs_tol)), 1)
+            ratio_tensor2 = TorchC.where(TorchC.abs(chunk_origin) > abs_tol,
+                                         TorchC.div(TorchC.clamp(chunk_perturbed, min=abs_tol), TorchC.clamp(chunk_origin, min=abs_tol)), 1)
+            norm_values = TorchC.stack([TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)])
+            max_ratio1, max_ratio2 = norm_values.tolist()
+            norm1 = max(norm1, self.convert_overflow_ratio_to_consistent(max_ratio1))
+            norm2 = max(norm2, self.convert_overflow_ratio_to_consistent(max_ratio2))
+            norm3 = min(norm3, self.convert_overflow_ratio_to_consistent(max_ratio1))
+
+        if norm3 < 0:
+            ratio = ThresholdConfig.SYMBOL_FLIPPING
+        else:
+            ratio = max(norm1, norm2)
+        return ratio
+
+    def ratio_calculate(self, origin_output, perturbed_output, norm_type) -> float:
+        try:
+            origin_output, perturbed_output, abs_tol = self.pre_process(
+                origin_output, perturbed_output
+            )
+        except Exception as e:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free Benchmark: For {self.params.api_name}, "
+                f"when computing ratio,"
+                f" y1 or y2 dtype is not supported {e}"
+            )
+            return ThresholdConfig.COMP_NAN
+        if self.params.fuzz_stage == Const.BACKWARD:
+            abs_tol = ThresholdConfig.BACKWARD_OUTPUT_LOWER_BOUND
+        else:
+            abs_tol = abs_tol ** 0.5
+        return self.get_ratio_from_specific_norm(
+            origin_output, perturbed_output, norm_type, abs_tol
+        )
+
+    def npu_compare(
+            self, origin_output, perturbed_output
+    ) -> Tuple[bool, Optional[float]]:
+
+        if isinstance(perturbed_output, int):
+            return origin_output == perturbed_output, None
+        elif isinstance(perturbed_output, float):
+            if perturbed_output == 0:
+                origin_output += FuzzThreshold.F32_THD
+                perturbed_output += FuzzThreshold.F32_THD
+            return (
+                math.isclose(origin_output, perturbed_output),
+                origin_output / perturbed_output,
+            )
+        elif not isinstance(perturbed_output, torch.Tensor):
+            logger.warning_on_rank_0(
+                f"[msprobe] Free Benchmark: For {self.params.api_name} "
+                f"The compare for output type {type(perturbed_output)} is not supported"
+            )
+
+        threshold = self.get_threshold(Tools.get_first_tensor_dtype(origin_output))
+        ratio = self.ratio_calculate(
+            origin_output, perturbed_output, norm_type=NormType.ENDLESS_NORM
+        )
+        if ratio == ThresholdConfig.SYMBOL_FLIPPING:
+            is_consistent = False
+        else:
+            is_consistent = threshold >= ratio >= 1 / threshold
+        return is_consistent, ratio
+
+    def cmp_output_npu(self, data_params: DataParams):
+        npu_consistent = True
+        max_fuzz_ratio = 0
+        try:
+            if isinstance(data_params.original_result, torch.Tensor):
+                is_consistent, ratio = self.npu_compare(
+                    data_params.original_result, data_params.perturbed_result
+                )
+                npu_consistent = is_consistent
+                max_fuzz_ratio = (
+                    max_fuzz_ratio if ratio is None else max(max_fuzz_ratio, ratio)
+                )
+                data_params.is_consistent = is_consistent and data_params.is_consistent
+                if not is_consistent and data_params.grad_unequal_flag:
+                    self.unequal_rows.append(
+                        make_unequal_row(data_params, self.params, ratio=ratio)
+                    )
+
+            elif isinstance(data_params.original_result, (list, tuple)):
+                for index_, origin_item in enumerate(data_params.original_result):
+                    is_consistent, ratio = self.npu_compare(
+                        origin_item, data_params.perturbed_result[index_]
+                    )
+                    npu_consistent = npu_consistent and is_consistent
+                    max_fuzz_ratio = (
+                        max_fuzz_ratio if ratio is None else max(max_fuzz_ratio, ratio)
+                    )
+                    data_params.is_consistent = (
+                            is_consistent and data_params.is_consistent
+                    )
+                    if not is_consistent and data_params.grad_unequal_flag:
+                        self.unequal_rows.append(
+                            make_unequal_row(
+                                data_params, self.params, ratio=ratio, index=index_
+                            )
+                        )
+        except Exception as e:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free Benchmark: For {self.params.api_name}, "
+                f"when campare the result exception raise {e}"
+            )
+        return npu_consistent, max_fuzz_ratio
+
+    def get_unequal_rows(self):
+        return self.unequal_rows
+
+    def _get_default_threshold(self, dtype):
+        if self.params.pert_mode == PerturbationMode.NO_CHANGE:
+            threshold = ThresholdConfig.COMP_CONSISTENT
+        else:
+            threshold = ThresholdConfig.DTYPE_PER_THD.get(
+                dtype, ThresholdConfig.DTYPE_PER_THD.get(torch.float32)
+            )
+        return threshold