mindstudio-probe 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/pytorch/free_benchmark/main.py

@@ -1,105 +1,120 @@
-from abc import ABC
-
-import torch
-from msprobe.core.common.const import Const
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.constant import CommonField
-from msprobe.pytorch.free_benchmark.common.enums import (
-    DeviceType,
-    FuzzLevel,
-    HandlerType,
-    PerturbationMode,
-)
-from msprobe.pytorch.free_benchmark.common.params import (
-    data_pre_deal,
-    make_handler_params,
-)
-from msprobe.pytorch.free_benchmark.compare.grad_saver import GradSaver
-from msprobe.pytorch.free_benchmark.perturbed_layers.layer_factory import LayerFactory
-from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import (
-    FuzzHandlerFactory,
-)
-
-
-class FreeBenchmarkCheck(ABC):
-
-    def __init__(self, config) -> None:
-        super().__init__()
-        self.config = config
-        if self.config.pert_mode is None:
-            self.config.pert_mode = PerturbationMode.IMPROVE_PRECISION
-        if self.config.fuzz_level is None:
-            self.config.fuzz_level = FuzzLevel.BASE_LEVEL
-        if self.config.fuzz_device is None:
-            self.config.fuzz_device = DeviceType.NPU
-        self.current_iter = 0
-
-    def update_iter(self, update_iter):
-        self.current_iter = update_iter
-    
-    def if_fix(self):
-        if self.config.handler_type==HandlerType.FIX:
-            return True
-        return False
-
-    def pre_forward(self, name, module, data_processor, args, kwargs):
-        if not self.config.fuzz_stage == Const.BACKWARD:
-            return
-        origin_func = (
-            module._slow_forward if torch._C._get_tracing_state() else module.forward
-        )
-        handler_params = make_handler_params(name, self.config, self.current_iter)
-        grad_saver = GradSaver(origin_func, handler_params)
-        grad_saver.kwargs = kwargs
-        grad_saver.register_compare_func_for_inputs(args, data_processor)
-        grad_saver.cache_backward_input(args)
-        setattr(module, CommonField.GRADSAVER, grad_saver)
-
-    def forward(self, name, module, args, kwargs, output):
-        if not self.config.fuzz_stage == Const.FORWARD:
-            return output, []
-        origin_func = (
-            module._slow_forward if torch._C._get_tracing_state() else module.forward
-        )
-        data_params = data_pre_deal(name, origin_func, args, kwargs)
-        if data_params.valid_input_index == -1:
-            return output, []
-        data_params.original_result = output
-        data_params.fuzz_stage = self.config.fuzz_stage
-
-        layer = LayerFactory.create(
-            name, self.config.fuzz_device, self.config.pert_mode
-        )
-        layer.handle(data_params)
-        handler_params = make_handler_params(name, self.config, self.current_iter)
-        handler = FuzzHandlerFactory.create(handler_params)
-        perturbed_output = handler.handle(data_params) 
-        return perturbed_output, handler.get_unequal_rows()
-    
-    def backward(self, name, module, grad_output):
-
-        if not self.config.fuzz_stage == Const.BACKWARD:
-            return
-        try:
-            grad_saver = getattr(module, CommonField.GRADSAVER)
-        except AttributeError:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free benchmark:  get grad saver failed. api_name:{name}"
-            )
-            return
-
-        _new_grad_output = grad_output
-        try:
-            need_grad_tensors, _inner_args = grad_saver.get_vjp_input()
-            origin_grad_input = grad_saver.get_grad_input_from_vjp(
-                tuple(need_grad_tensors), _new_grad_output, _inner_args
-            )
-            grad_saver.origin_grad_input = tuple([x.cpu() for x in origin_grad_input])
-            grad_saver.calculate_perturbed_grad_input(
-                _new_grad_output, need_grad_tensors, _inner_args
-            )
-        except Exception as e:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free benchmark: grad vjp calculate failed. api_name:{name} error: {e}"
-            )
-            return
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC
+
+import torch
+from msprobe.core.common.const import Const
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.constant import CommonField
+from msprobe.pytorch.free_benchmark.common.enums import (
+    DeviceType,
+    FuzzLevel,
+    HandlerType,
+    PerturbationMode,
+)
+from msprobe.pytorch.free_benchmark.common.params import (
+    data_pre_deal,
+    make_handler_params,
+)
+from msprobe.pytorch.free_benchmark.compare.grad_saver import GradSaver
+from msprobe.pytorch.free_benchmark.perturbed_layers.layer_factory import LayerFactory
+from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import (
+    FuzzHandlerFactory,
+)
+
+
+class FreeBenchmarkCheck(ABC):
+
+    def __init__(self, config) -> None:
+        super().__init__()
+        self.config = config
+        if self.config.pert_mode is None:
+            self.config.pert_mode = PerturbationMode.IMPROVE_PRECISION
+        if self.config.fuzz_level is None:
+            self.config.fuzz_level = FuzzLevel.BASE_LEVEL
+        if self.config.fuzz_device is None:
+            self.config.fuzz_device = DeviceType.NPU
+        self.current_iter = 0
+
+    def update_iter(self, update_iter):
+        self.current_iter = update_iter
+
+    def if_fix(self):
+        if self.config.handler_type == HandlerType.FIX:
+            return True
+        return False
+
+    def pre_forward(self, name, module, data_processor, args, kwargs):
+        if not self.config.fuzz_stage == Const.BACKWARD:
+            return
+        origin_func = (
+            module._slow_forward if torch._C._get_tracing_state() else module.forward
+        )
+        handler_params = make_handler_params(name, self.config, self.current_iter)
+        grad_saver = GradSaver(origin_func, handler_params)
+        grad_saver.kwargs = kwargs
+        grad_saver.register_compare_func_for_inputs(args, data_processor)
+        grad_saver.cache_backward_input(args)
+        setattr(module, CommonField.GRADSAVER, grad_saver)
+
+    def forward(self, name, module, args, kwargs, output):
+        if not self.config.fuzz_stage == Const.FORWARD:
+            return output, []
+        origin_func = (
+            module._slow_forward if torch._C._get_tracing_state() else module.forward
+        )
+        data_params = data_pre_deal(name, origin_func, args, kwargs)
+        if data_params.valid_input_index == -1:
+            return output, []
+        data_params.original_result = output
+        data_params.fuzz_stage = self.config.fuzz_stage
+
+        layer = LayerFactory.create(
+            name, self.config.fuzz_device, self.config.pert_mode
+        )
+        layer.handle(data_params)
+        handler_params = make_handler_params(name, self.config, self.current_iter)
+        handler = FuzzHandlerFactory.create(handler_params)
+        perturbed_output = handler.handle(data_params)
+        return perturbed_output, handler.get_unequal_rows()
+
+    def backward(self, name, module, grad_output):
+
+        if not self.config.fuzz_stage == Const.BACKWARD:
+            return
+        try:
+            grad_saver = getattr(module, CommonField.GRADSAVER)
+        except AttributeError:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free benchmark:  get grad saver failed. api_name:{name}"
+            )
+            return
+
+        _new_grad_output = grad_output
+        try:
+            need_grad_tensors, _inner_args = grad_saver.get_vjp_input()
+            origin_grad_input = grad_saver.get_grad_input_from_vjp(
+                tuple(need_grad_tensors), _new_grad_output, _inner_args
+            )
+            grad_saver.origin_grad_input = tuple([x.cpu() for x in origin_grad_input])
+            grad_saver.calculate_perturbed_grad_input(
+                _new_grad_output, need_grad_tensors, _inner_args
+            )
+        except Exception as e:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free benchmark: grad vjp calculate failed. api_name:{name} error: {e}"
+            )
+            return

msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py

@@ -1,13 +1,28 @@
-from abc import ABC, abstractmethod
-from typing import Any
-
-from msprobe.pytorch.free_benchmark.common.params import DataParams
-
-
-class BaseLayer(ABC):
-    def __init__(self, api_name: str) -> None:
-        self.api_name = api_name
-
-    @abstractmethod
-    def handle(self, params: DataParams) -> Any:
-        pass
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from msprobe.pytorch.free_benchmark.common.params import DataParams
+
+
+class BaseLayer(ABC):
+    def __init__(self, api_name: str) -> None:
+        self.api_name = api_name
+
+    @abstractmethod
+    def handle(self, params: DataParams) -> Any:
+        pass

msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py

@@ -1,41 +1,56 @@
-from msprobe.pytorch.free_benchmark import FreeBenchmarkException
-from msprobe.pytorch.free_benchmark.common.enums import DeviceType, PerturbationMode
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.improve_precision import (
-    ImprovePrecisionLayer,
-)
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.add_noise import AddNoiseLayer
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.bit_noise import BitNoiseLayer
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.no_change import NoChangeLayer
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.change_value import (
-    ChangeValueLayer,
-)
-from msprobe.pytorch.free_benchmark.perturbed_layers.run_cpu import CpuLayer
-
-
-class LayerFactory:
-    layers = {
-        DeviceType.NPU: {
-            PerturbationMode.ADD_NOISE: AddNoiseLayer,
-            PerturbationMode.CHANGE_VALUE: ChangeValueLayer,
-            PerturbationMode.NO_CHANGE: NoChangeLayer,
-            PerturbationMode.BIT_NOISE: BitNoiseLayer,
-            PerturbationMode.IMPROVE_PRECISION: ImprovePrecisionLayer,
-        },
-        DeviceType.CPU: {PerturbationMode.TO_CPU: CpuLayer},
-    }
-
-    @staticmethod
-    def create(api_name: str, device_type: str, mode: str):
-        layer = LayerFactory.layers.get(device_type)
-        if not layer:
-            raise FreeBenchmarkException(
-                FreeBenchmarkException.UnsupportedType,
-                f"无标杆工具不支持当前设备 {device_type}",
-            )
-        layer = layer.get(mode)
-        if not layer:
-            raise FreeBenchmarkException(
-                FreeBenchmarkException.UnsupportedType,
-                f"无标杆工具无法识别该扰动因子 {mode} on {device_type}",
-            )
-        return layer(api_name)
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.pytorch.free_benchmark import FreeBenchmarkException
+from msprobe.pytorch.free_benchmark.common.enums import DeviceType, PerturbationMode
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.add_noise import AddNoiseLayer
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.bit_noise import BitNoiseLayer
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.change_value import (
+    ChangeValueLayer,
+)
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.improve_precision import (
+    ImprovePrecisionLayer,
+)
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.no_change import NoChangeLayer
+from msprobe.pytorch.free_benchmark.perturbed_layers.run_cpu import CpuLayer
+
+
+class LayerFactory:
+    layers = {
+        DeviceType.NPU: {
+            PerturbationMode.ADD_NOISE: AddNoiseLayer,
+            PerturbationMode.CHANGE_VALUE: ChangeValueLayer,
+            PerturbationMode.NO_CHANGE: NoChangeLayer,
+            PerturbationMode.BIT_NOISE: BitNoiseLayer,
+            PerturbationMode.IMPROVE_PRECISION: ImprovePrecisionLayer,
+        },
+        DeviceType.CPU: {PerturbationMode.TO_CPU: CpuLayer},
+    }
+
+    @staticmethod
+    def create(api_name: str, device_type: str, mode: str):
+        layer = LayerFactory.layers.get(device_type)
+        if not layer:
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.UnsupportedType,
+                f"无标杆工具不支持当前设备 {device_type}",
+            )
+        layer = layer.get(mode)
+        if not layer:
+            raise FreeBenchmarkException(
+                FreeBenchmarkException.UnsupportedType,
+                f"无标杆工具无法识别该扰动因子 {mode} on {device_type}",
+            )
+        return layer(api_name)

msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py

@@ -1,90 +1,105 @@
-import torch
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
-from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
-from msprobe.pytorch.free_benchmark.common.params import DataParams
-from msprobe.pytorch.free_benchmark.common.utils import TorchC
-from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import (
-    NpuBaseLayer,
-)
-
-
-class AddNoiseLayer(NpuBaseLayer):
-
-    def add_noise(self, tensor_obj):
-        if isinstance(tensor_obj, torch.Tensor):
-            self.perturbed_value = ThresholdConfig.PERTURBATION_VALUE_DICT.get(
-                tensor_obj.dtype
-            )
-            if not self.pre_check(tensor_obj):
-                return tensor_obj
-            noise = self._get_noise(tensor_obj)
-            result = TorchC.where(
-                TorchC.gt(TorchC.abs(tensor_obj), self.perturbed_value ** 0.5),
-                TorchC.add(noise, tensor_obj),
-                tensor_obj,
-            ).to(tensor_obj.dtype)
-            self.is_added = True
-            return result
-        if isinstance(tensor_obj, dict):
-            return {key: self.add_noise(value) for key, value in tensor_obj.items()}
-        if isinstance(tensor_obj, (tuple, list)):
-            return type(tensor_obj)([self.add_noise(value) for value in tensor_obj])
-        return tensor_obj
-
-    def handle(self, params: DataParams):
-        """
-        对输入添加扰动并返回
-        """
-        logger.info_on_rank_0(
-            f"[msprobe] Free benchmark: Perturbation is "
-            f"{PerturbationMode.ADD_NOISE} of {self.api_name}."
-        )
-        params.perturbed_value = self.add_noise(params.args[params.valid_input_index])
-        return self.perturbed_result(params)
-
-    def _get_noise(self, tensor_obj):
-        dtype = tensor_obj.dtype
-        device = str(tensor_obj.device)
-        noise = TorchC.full(
-            tensor_obj.shape,
-            self.perturbed_value,
-            device=device,
-            dtype=dtype,
-        )
-        return noise
-
-    def _check_details(self, tensor_obj):
-        """
-        判断是否需要添加扰动
-        """
-        if not self.perturbed_value:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"dtype unsupported. Cancel perturbation."
-            )
-            return False
-        if tensor_obj.numel() == 0:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0."
-                f" Cancel adding noise."
-            )
-            return False
-        abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(
-            tensor_obj.dtype, ThresholdConfig.NOISE_INPUT_LOWER_BOUND
-        )
-        try:
-            max_val = TorchC.max(TorchC.abs(tensor_obj)).item()
-        except Exception:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"when calculate maximun value, tensor is changed to float32."
-            )
-            max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item()
-        if max_val < abs_tol:
-            logger.warning_on_rank_0(
-                f"[msprobe] Free Benchmark: For {self.api_name}, "
-                f"Maximun value is less than the  minimun threshold. Cancel add noise."
-            )
-            return False
-        return True
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
+from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode
+from msprobe.pytorch.free_benchmark.common.params import DataParams
+from msprobe.pytorch.free_benchmark.common.utils import TorchC
+from msprobe.pytorch.free_benchmark.perturbed_layers.npu.npu_base_layser import (
+    NpuBaseLayer,
+)
+
+
+class AddNoiseLayer(NpuBaseLayer):
+
+    def add_noise(self, tensor_obj):
+        if isinstance(tensor_obj, torch.Tensor):
+            self.perturbed_value = ThresholdConfig.PERTURBATION_VALUE_DICT.get(
+                tensor_obj.dtype
+            )
+            if not self.pre_check(tensor_obj):
+                return tensor_obj
+            noise = self._get_noise(tensor_obj)
+            result = TorchC.where(
+                TorchC.gt(TorchC.abs(tensor_obj), self.perturbed_value ** 0.5),
+                TorchC.add(noise, tensor_obj),
+                tensor_obj,
+            ).to(tensor_obj.dtype)
+            self.is_added = True
+            return result
+        if isinstance(tensor_obj, dict):
+            return {key: self.add_noise(value) for key, value in tensor_obj.items()}
+        if isinstance(tensor_obj, (tuple, list)):
+            return type(tensor_obj)([self.add_noise(value) for value in tensor_obj])
+        return tensor_obj
+
+    def handle(self, params: DataParams):
+        """
+        对输入添加扰动并返回
+        """
+        logger.info_on_rank_0(
+            f"[msprobe] Free benchmark: Perturbation is "
+            f"{PerturbationMode.ADD_NOISE} of {self.api_name}."
+        )
+        params.perturbed_value = self.add_noise(params.args[params.valid_input_index])
+        return self.perturbed_result(params)
+
+    def _get_noise(self, tensor_obj):
+        dtype = tensor_obj.dtype
+        device = str(tensor_obj.device)
+        noise = TorchC.full(
+            tensor_obj.shape,
+            self.perturbed_value,
+            device=device,
+            dtype=dtype,
+        )
+        return noise
+
+    def _check_details(self, tensor_obj):
+        """
+        判断是否需要添加扰动
+        """
+        if not self.perturbed_value:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free Benchmark: For {self.api_name}, "
+                f"dtype unsupported. Cancel perturbation."
+            )
+            return False
+        if tensor_obj.numel() == 0:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free benchmark: For {self.api_name}, tensor shape must > 0."
+                f" Cancel adding noise."
+            )
+            return False
+        abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(
+            tensor_obj.dtype, ThresholdConfig.NOISE_INPUT_LOWER_BOUND
+        )
+        try:
+            max_val = TorchC.max(TorchC.abs(tensor_obj)).item()
+        except Exception:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free Benchmark: For {self.api_name}, "
+                f"when calculate maximun value, tensor is changed to float32."
+            )
+            max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item()
+        if max_val < abs_tol:
+            logger.warning_on_rank_0(
+                f"[msprobe] Free Benchmark: For {self.api_name}, "
+                f"Maximun value is less than the  minimun threshold. Cancel add noise."
+            )
+            return False
+        return True