mindstudio-probe 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/pytorch/free_benchmark/common/enums.py

@@ -1,37 +1,65 @@
-class PerturbationMode:
-    ADD_NOISE = "add_noise"
-    CHANGE_VALUE = "change_value"
-    IMPROVE_PRECISION = "improve_precision"
-    NO_CHANGE = "no_change"
-    BIT_NOISE = "bit_noise"
-    TO_CPU = "to_cpu"
-
-
-class DeviceType:
-    NPU = "npu"
-    CPU = "cpu"
-
-
-class FuzzThreshold:
-    BF16_THD = 1e-4
-    F16_THD = 1e-6
-    F32_THD = 1e-8
-    F64_THD = 1e-16
-
-
-class NormType:
-    ONE_NORM = (1, "one_norm")
-    TWO_NORM = (2, "two_norm")
-    ENDLESS_NORM = (3, "endless_norm")
-
-
-class HandlerType:
-    CHECK = "check"
-    PREHEAT = "preheat"
-    FIX = "fix"
-
-
-class FuzzLevel:
-    BASE_LEVEL = "L1"
-    ADV_LEVEL = "L2"
-    REAL_LEVEL = "L3"
+from msprobe.core.common.const import Const
+
+
+class PerturbationMode:
+    ADD_NOISE = "add_noise"
+    CHANGE_VALUE = "change_value"
+    IMPROVE_PRECISION = "improve_precision"
+    NO_CHANGE = "no_change"
+    BIT_NOISE = "bit_noise"
+    TO_CPU = "to_cpu"
+
+
+class DeviceType:
+    NPU = "npu"
+    CPU = "cpu"
+
+
+class FuzzThreshold:
+    BF16_THD = 1e-4
+    F16_THD = 1e-6
+    F32_THD = 1e-8
+    F64_THD = 1e-16
+
+
+class NormType:
+    ONE_NORM = (1, "one_norm")
+    TWO_NORM = (2, "two_norm")
+    ENDLESS_NORM = (3, "endless_norm")
+
+
+class HandlerType:
+    CHECK = "check"
+    PREHEAT = "preheat"
+    FIX = "fix"
+
+
+class FuzzLevel:
+    BASE_LEVEL = "L1"
+    ADV_LEVEL = "L2"
+    REAL_LEVEL = "L3"
+
+
+class PytorchFreeBenchmarkConst:
+    PERTURBATION_MODE_LIST = [
+        PerturbationMode.ADD_NOISE,
+        PerturbationMode.CHANGE_VALUE,
+        PerturbationMode.IMPROVE_PRECISION,
+        PerturbationMode.NO_CHANGE,
+        PerturbationMode.BIT_NOISE,
+        PerturbationMode.TO_CPU,
+    ]
+    DEFAULT_MODE = PerturbationMode.IMPROVE_PRECISION
+    DEVICE_LIST = [DeviceType.NPU, DeviceType.CPU]
+    DEFAULT_DEVICE = DeviceType.NPU
+    HANDLER_LIST = [HandlerType.CHECK, HandlerType.FIX]
+    DEFAULT_HANDLER = HandlerType.CHECK
+    FUZZ_LEVEL_LIST = [FuzzLevel.BASE_LEVEL]
+    DEFAULT_FUZZ_LEVEL = FuzzLevel.BASE_LEVEL
+    FUZZ_STAGE_LIST = [Const.FORWARD, Const.BACKWARD]
+    FIX_MODE_LIST = [PerturbationMode.IMPROVE_PRECISION, PerturbationMode.TO_CPU]
+    DEFAULT_FUZZ_STAGE = Const.FORWARD
+    DEFAULT_PREHEAT_STEP = 15
+    DEFAULT_MAX_SAMPLE = 20
+    CPU_MODE_LIST = [PerturbationMode.TO_CPU]
+    FIX_STAGE_LIST = [Const.FORWARD]

msprobe/pytorch/free_benchmark/common/params.py

@@ -1,129 +1,144 @@
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Optional, Tuple
-
-import torch
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.enums import (
-    DeviceType,
-    FuzzLevel,
-    PerturbationMode,
-)
-from msprobe.pytorch.free_benchmark.common.utils import Tools
-
-
-@dataclass
-class DataParams:
-    args: Optional[Tuple] = None
-    kwargs: Optional[Dict] = None
-    valid_input_index: Optional[int] = None
-    original_result: Optional[Any] = None
-    perturbed_result: Optional[Any] = None
-    is_consistent: Optional[bool] = True
-    perturbed_value: Optional[Any] = None
-    origin_func: Optional[Callable] = None
-    api_type: Optional[str] = None
-    fuzz_stage: Optional[str] = None
-    grad_unequal_flag: Optional[bool] = True
-
-
-@dataclass
-class HandlerParams:
-    handler_type: Optional[str] = None
-    api_name: Optional[str] = None
-    pert_mode: Optional[PerturbationMode] = None
-    step: Optional[int] = None
-    fuzz_stage: Optional[str] = None
-    fuzz_device: Optional[DeviceType] = None
-    preheat_config: Optional[Dict] = None
-    fuzz_level: Optional[str] = None
-
-
-@dataclass
-class UnequalRow:
-    rank: Optional[int] = None
-    pert_mode: Optional[PerturbationMode] = None
-    stage: Optional[str] = None
-    step: Optional[int] = None
-    api_name: Optional[str] = None
-    max_rel: Optional[float] = None
-    dtype: Optional[str] = None
-    shape: Optional[str] = None
-    output_index: Optional[int] = None
-
-
-@dataclass
-class BenchmarkThd:
-    rtol: Optional[float] = None  # 相对误差阈值
-    small_value: Optional[float] = None  # 小值域
-    small_value_atol: Optional[float] = None  # 小值域绝对阈值
-    err_balance: Optional[float] = None  # 误差均衡性
-
-
-def check_args_type(args: Tuple) -> int:
-    for i, arg in enumerate(args):
-        if torch.is_tensor(arg):
-            if arg.is_meta:
-                continue
-            if not torch.is_floating_point(arg):
-                continue
-            return i
-        if isinstance(arg, (List, Tuple, Dict)):
-            return i
-    return -1
-
-
-def data_pre_deal(name, func, args, kwargs):
-    data_params = DataParams(args=args, kwargs=kwargs, origin_func=func)
-    index = check_args_type(args)
-    data_params.valid_input_index = index
-    if index == -1:
-        logger.warning_on_rank_0(
-            f"[msprobe] Free benchmark: 无标杆工具不支持当前算子的输入类型 {name}."
-        )
-    return data_params
-
-
-def make_handler_params(name, config, step):
-    handler_params = HandlerParams()
-    handler_params.api_name = name
-    handler_params.step = step
-    handler_params.handler_type = config.handler_type
-    handler_params.fuzz_stage = config.fuzz_stage
-    handler_params.fuzz_device = config.fuzz_device
-    handler_params.preheat_config = config.preheat_config
-    handler_params.fuzz_level = config.fuzz_level
-    handler_params.pert_mode = config.pert_mode
-    return handler_params
-
-
-def make_unequal_row(
-    data_params: DataParams,
-    handle_params: HandlerParams,
-    ratio: float = None,
-    index: int = None,
-):
-    row = UnequalRow(
-        api_name=handle_params.api_name,
-        pert_mode=handle_params.pert_mode,
-        output_index=index,
-        stage=handle_params.fuzz_stage,
-        step=handle_params.step,
-    )
-    if isinstance(ratio, float):
-        row.max_rel = ratio - 1
-    origin_tensor = data_params.original_result
-    perturbed_tensor = data_params.perturbed_result
-    if index:
-        origin_tensor = origin_tensor[index]
-        perturbed_tensor = perturbed_tensor[index]
-        row.output_index = index
-    if isinstance(origin_tensor, torch.Tensor):
-        row.dtype = origin_tensor.dtype
-        row.shape = origin_tensor.shape
-    row.rank = Tools.get_dist_rank()
-    # 以下暂不支持
-    if handle_params.fuzz_level == FuzzLevel.ADV_LEVEL:
-        pass
-    if handle_params.fuzz_level == FuzzLevel.REAL_LEVEL:
-        pass
-    return row
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+import torch
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.enums import (
+    DeviceType,
+    FuzzLevel,
+    PerturbationMode,
+)
+from msprobe.pytorch.free_benchmark.common.utils import Tools
+
+
+@dataclass
+class DataParams:
+    args: Optional[Tuple] = None
+    kwargs: Optional[Dict] = None
+    valid_input_index: Optional[int] = None
+    original_result: Optional[Any] = None
+    perturbed_result: Optional[Any] = None
+    is_consistent: Optional[bool] = True
+    perturbed_value: Optional[Any] = None
+    origin_func: Optional[Callable] = None
+    api_type: Optional[str] = None
+    fuzz_stage: Optional[str] = None
+    grad_unequal_flag: Optional[bool] = True
+
+
+@dataclass
+class HandlerParams:
+    handler_type: Optional[str] = None
+    api_name: Optional[str] = None
+    pert_mode: Optional[PerturbationMode] = None
+    step: Optional[int] = None
+    fuzz_stage: Optional[str] = None
+    fuzz_device: Optional[DeviceType] = None
+    preheat_config: Optional[Dict] = None
+    fuzz_level: Optional[str] = None
+
+
+@dataclass
+class UnequalRow:
+    rank: Optional[int] = None
+    pert_mode: Optional[PerturbationMode] = None
+    stage: Optional[str] = None
+    step: Optional[int] = None
+    api_name: Optional[str] = None
+    max_rel: Optional[float] = None
+    dtype: Optional[str] = None
+    shape: Optional[str] = None
+    output_index: Optional[int] = None
+
+
+@dataclass
+class BenchmarkThd:
+    rtol: Optional[float] = None  # 相对误差阈值
+    small_value: Optional[float] = None  # 小值域
+    small_value_atol: Optional[float] = None  # 小值域绝对阈值
+    err_balance: Optional[float] = None  # 误差均衡性
+
+
+def check_args_type(args: Tuple) -> int:
+    for i, arg in enumerate(args):
+        if torch.is_tensor(arg):
+            if arg.is_meta:
+                continue
+            if not torch.is_floating_point(arg):
+                continue
+            return i
+        if isinstance(arg, (List, Tuple, Dict)):
+            return i
+    return -1
+
+
+def data_pre_deal(name, func, args, kwargs):
+    data_params = DataParams(args=args, kwargs=kwargs, origin_func=func)
+    index = check_args_type(args)
+    data_params.valid_input_index = index
+    if index == -1:
+        logger.warning_on_rank_0(
+            f"[msprobe] Free benchmark: 无标杆工具不支持当前算子的输入类型 {name}."
+        )
+    return data_params
+
+
+def make_handler_params(name, config, step):
+    handler_params = HandlerParams()
+    handler_params.api_name = name
+    handler_params.step = step
+    handler_params.handler_type = config.handler_type
+    handler_params.fuzz_stage = config.fuzz_stage
+    handler_params.fuzz_device = config.fuzz_device
+    handler_params.preheat_config = config.preheat_config
+    handler_params.fuzz_level = config.fuzz_level
+    handler_params.pert_mode = config.pert_mode
+    return handler_params
+
+
+def make_unequal_row(
+    data_params: DataParams,
+    handle_params: HandlerParams,
+    ratio: float = None,
+    index: int = None,
+):
+    row = UnequalRow(
+        api_name=handle_params.api_name,
+        pert_mode=handle_params.pert_mode,
+        output_index=index,
+        stage=handle_params.fuzz_stage,
+        step=handle_params.step,
+    )
+    if isinstance(ratio, float):
+        row.max_rel = ratio - 1
+    origin_tensor = data_params.original_result
+    perturbed_tensor = data_params.perturbed_result
+    if index:
+        origin_tensor = origin_tensor[index]
+        perturbed_tensor = perturbed_tensor[index]
+        row.output_index = index
+    if isinstance(origin_tensor, torch.Tensor):
+        row.dtype = origin_tensor.dtype
+        row.shape = origin_tensor.shape
+    row.rank = Tools.get_dist_rank()
+    # 以下暂不支持
+    if handle_params.fuzz_level == FuzzLevel.ADV_LEVEL:
+        pass
+    if handle_params.fuzz_level == FuzzLevel.REAL_LEVEL:
+        pass
+    return row

msprobe/pytorch/free_benchmark/common/utils.py

@@ -1,102 +1,118 @@
-import torch
-from msprobe.pytorch.free_benchmark.common.enums import DeviceType
-
-
-class Tools:
-
-    @staticmethod
-    def is_float_tensor(tensor) -> bool:
-        if isinstance(tensor, torch.Tensor) and torch.is_floating_point(tensor):
-            return True
-        if isinstance(tensor, (list, tuple)):
-            for value in tensor:
-                if isinstance(value, torch.Tensor) and torch.is_floating_point(value):
-                    return True
-        return False
-
-    @staticmethod
-    def get_dist_rank():
-        try:
-            return torch.distributed.get_rank()
-        except RuntimeError:
-            return 0
-
-    @staticmethod
-    def get_first_tensor_dtype(tensor_seq):
-        if isinstance(tensor_seq, torch.Tensor):
-            return tensor_seq.dtype
-        if isinstance(tensor_seq, (list, tuple)):
-            for object_ in tensor_seq:
-                if isinstance(object_, torch.Tensor):
-                    return object_.dtype
-        raise RuntimeError("The sequence does not contain tensors.")
-
-    @staticmethod
-    def get_pure_api_name(api_name: str):
-        return api_name.rsplit(".", 2)[0]
-
-    @staticmethod
-    def convert_device_and_dtype(
-        tensor_seq, device: str = DeviceType.CPU, change_dtype: bool = False
-    ):
-        if isinstance(tensor_seq, torch.Tensor):
-            if change_dtype and tensor_seq.dtype in [torch.float16, torch.bfloat16]:
-                return tensor_seq.detach().to(device).to(torch.float32)
-            return tensor_seq.detach().to(device)
-        if isinstance(tensor_seq, dict):
-            return {
-                key: Tools.convert_device_and_dtype(value, device, change_dtype)
-                for key, value in tensor_seq.items()
-            }
-        if isinstance(tensor_seq, (tuple, list)):
-            return type(tensor_seq)(
-                [
-                    Tools.convert_device_and_dtype(value, device, change_dtype)
-                    for value in tensor_seq
-                ]
-            )
-        return tensor_seq
-
-    @staticmethod
-    def convert_fuzz_output_to_origin(origin, perturbed):
-        if isinstance(origin, torch.Tensor):
-            origin.data = perturbed.to(origin.dtype).to(origin.device)
-            return origin
-        if isinstance(origin, dict):
-            output = dict()
-            for key, value in origin.items():
-                output[key] = Tools.convert_fuzz_output_to_origin(value, perturbed[key])
-            return output
-        if isinstance(origin, (tuple, list)):
-            result = list()
-            for index_, value in enumerate(origin):
-                result.append(
-                    Tools.convert_fuzz_output_to_origin(value, perturbed[index_])
-                )
-            return type(origin)(result)
-        return origin
-    
-class TorchC:
-    sum = torch._C._VariableFunctionsClass.sum
-    isinf = torch._C._VariableFunctionsClass.isinf
-    isfinite = torch._C._VariableFunctionsClass.isfinite
-    isnan = torch._C._VariableFunctionsClass.isnan
-    logical_not = torch._C._VariableFunctionsClass.logical_not
-    subtract = torch._C._VariableFunctionsClass.subtract
-    abs = torch._C._VariableFunctionsClass.abs
-    where = torch._C._VariableFunctionsClass.where
-    div = torch._C._VariableFunctionsClass.div
-    max = torch._C._VariableFunctionsClass.max
-    min = torch._C._VariableFunctionsClass.min
-    gt = torch._C._VariableFunctionsClass.gt
-    ge = torch._C._VariableFunctionsClass.ge
-    lt = torch._C._VariableFunctionsClass.lt
-    mean = torch._C._VariableFunctionsClass.mean
-    full = torch._C._VariableFunctionsClass.full
-    add = torch._C._VariableFunctionsClass.add
-    bitwise_xor = torch._C._VariableFunctionsClass.bitwise_xor
-    clone = torch._C._VariableFunctionsClass.clone
-    clamp = torch._C._VariableFunctionsClass.clamp
-    tensor_split = torch._C._VariableFunctionsClass.tensor_split
-    stack = torch._C._VariableFunctionsClass.stack
-    reshape = torch._C._VariableFunctionsClass.reshape
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from msprobe.pytorch.free_benchmark.common.enums import DeviceType
+
+
+class Tools:
+
+    @staticmethod
+    def is_float_tensor(tensor) -> bool:
+        if isinstance(tensor, torch.Tensor) and torch.is_floating_point(tensor):
+            return True
+        if isinstance(tensor, (list, tuple)):
+            for value in tensor:
+                if isinstance(value, torch.Tensor) and torch.is_floating_point(value):
+                    return True
+        return False
+
+    @staticmethod
+    def get_dist_rank():
+        try:
+            return torch.distributed.get_rank()
+        except RuntimeError:
+            return 0
+
+    @staticmethod
+    def get_first_tensor_dtype(tensor_seq):
+        if isinstance(tensor_seq, torch.Tensor):
+            return tensor_seq.dtype
+        if isinstance(tensor_seq, (list, tuple)):
+            for object_ in tensor_seq:
+                if isinstance(object_, torch.Tensor):
+                    return object_.dtype
+        raise RuntimeError("The sequence does not contain tensors.")
+
+    @staticmethod
+    def get_pure_api_name(api_name: str):
+        return api_name.rsplit(".", 2)[0]
+
+    @staticmethod
+    def convert_device_and_dtype(
+        tensor_seq, device: str = DeviceType.CPU, change_dtype: bool = False
+    ):
+        if isinstance(tensor_seq, torch.Tensor):
+            if change_dtype and tensor_seq.dtype in [torch.float16, torch.bfloat16]:
+                return tensor_seq.detach().to(device).to(torch.float32)
+            return tensor_seq.detach().to(device)
+        if isinstance(tensor_seq, dict):
+            return {
+                key: Tools.convert_device_and_dtype(value, device, change_dtype)
+                for key, value in tensor_seq.items()
+            }
+        if isinstance(tensor_seq, (tuple, list)):
+            return type(tensor_seq)(
+                [
+                    Tools.convert_device_and_dtype(value, device, change_dtype)
+                    for value in tensor_seq
+                ]
+            )
+        return tensor_seq
+
+    @staticmethod
+    def convert_fuzz_output_to_origin(origin, perturbed):
+        if isinstance(origin, torch.Tensor):
+            origin.data = perturbed.to(origin.dtype).to(origin.device)
+            return origin
+        if isinstance(origin, dict):
+            output = dict()
+            for key, value in origin.items():
+                output[key] = Tools.convert_fuzz_output_to_origin(value, perturbed[key])
+            return output
+        if isinstance(origin, (tuple, list)):
+            result = list()
+            for index_, value in enumerate(origin):
+                result.append(
+                    Tools.convert_fuzz_output_to_origin(value, perturbed[index_])
+                )
+            return type(origin)(result)
+        return origin
+
+
+class TorchC:
+    sum = torch._C._VariableFunctionsClass.sum
+    isinf = torch._C._VariableFunctionsClass.isinf
+    isfinite = torch._C._VariableFunctionsClass.isfinite
+    isnan = torch._C._VariableFunctionsClass.isnan
+    logical_not = torch._C._VariableFunctionsClass.logical_not
+    subtract = torch._C._VariableFunctionsClass.subtract
+    abs = torch._C._VariableFunctionsClass.abs
+    where = torch._C._VariableFunctionsClass.where
+    div = torch._C._VariableFunctionsClass.div
+    max = torch._C._VariableFunctionsClass.max
+    min = torch._C._VariableFunctionsClass.min
+    gt = torch._C._VariableFunctionsClass.gt
+    ge = torch._C._VariableFunctionsClass.ge
+    lt = torch._C._VariableFunctionsClass.lt
+    mean = torch._C._VariableFunctionsClass.mean
+    full = torch._C._VariableFunctionsClass.full
+    add = torch._C._VariableFunctionsClass.add
+    bitwise_xor = torch._C._VariableFunctionsClass.bitwise_xor
+    clone = torch._C._VariableFunctionsClass.clone
+    clamp = torch._C._VariableFunctionsClass.clamp
+    tensor_split = torch._C._VariableFunctionsClass.tensor_split
+    stack = torch._C._VariableFunctionsClass.stack
+    reshape = torch._C._VariableFunctionsClass.reshape