mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.4__py3-none-any.whl

msprobe/pytorch/free_benchmark/common/constant.py

@@ -1,67 +1,70 @@
-from typing import Dict
-
-import numpy as np
-import torch
-from msprobe.pytorch.free_benchmark.common.enums import FuzzThreshold
-from msprobe.pytorch.free_benchmark.common.params import BenchmarkThd
-
-
-class CommonField:
-    DEVICE = "device"
-    META = "meta"
-    FUZZ_TENSOR = "fuzz_tensor"
-    REQUIRES_GRAD = "requires_grad"
-    HOLD_PLACE = "hold_place"
-    DISTRIBUTED_OP = "torch.distributed"
-    GRADSAVER = "grad_saver"
-
-
-class ThresholdConfig:
-    PERTURBATION_VALUE_DICT: Dict = {
-        torch.bfloat16: FuzzThreshold.BF16_THD,
-        torch.float16: FuzzThreshold.F16_THD,
-        torch.float32: FuzzThreshold.F32_THD,
-        torch.float64: FuzzThreshold.F64_THD,
-    }
-
-    ABS_TOL_VALUE_DICT: Dict = {
-        torch.bfloat16: FuzzThreshold.BF16_THD,
-        torch.float16: FuzzThreshold.F16_THD,
-        torch.float32: FuzzThreshold.F32_THD,
-        torch.float64: FuzzThreshold.F64_THD,
-    }
-
-    # bit翻转需要匹配到等长或更长的整型
-    PERTURBATION_BIT_DICT = {
-        torch.bfloat16: torch.int16,
-        torch.float16: torch.int16,
-        torch.float32: torch.int32,
-        torch.float64: torch.int64,
-    }
-
-    # 输入噪声下界
-    NOISE_INPUT_LOWER_BOUND = 1e-8
-    COMP_CONSISTENT = 1.0
-    COMP_NAN = np.nan
-    SYMBOL_FLIPPING = "symbol_flipping"
-    BACKWARD_OUTPUT_LOWER_BOUND = 1e-3
-    SMALL_VALUE = 1.0
-    # 预热初始阈值
-    PREHEAT_INITIAL_THD = 2.05
-    API_THD_STEP = 2.0
-
-    DTYPE_PER_THD = {
-        torch.float16: 1.002,
-        torch.float32: 1.0002,
-    }
-    BENCHMARK_THD_DICT = {
-        torch.float32: BenchmarkThd(2**-14, 1.0, 2**-14, 1e-4),
-        torch.float16: BenchmarkThd(2**-11, 1.0, 2**-11, 1e-4),
-        torch.bfloat16: BenchmarkThd(2**-8, 1.0, 2**-8, 1e-4),
-    }
-
-
-class PreheatConfig:
-    IF_PREHEAT = "if_preheat"
-    PREHEAT_STEP = "preheat_step"
-    MAX_SAMPLE = "max_sample"
+from typing import Dict
+
+import numpy as np
+import torch
+from msprobe.pytorch.free_benchmark.common.enums import FuzzThreshold
+from msprobe.pytorch.free_benchmark.common.params import BenchmarkThd
+
+
+class CommonField:
+    DEVICE = "device"
+    META = "meta"
+    FUZZ_TENSOR = "fuzz_tensor"
+    REQUIRES_GRAD = "requires_grad"
+    HOLD_PLACE = "hold_place"
+    DISTRIBUTED_OP = "torch.distributed"
+    GRADSAVER = "grad_saver"
+
+
+class ThresholdConfig:
+    PERTURBATION_VALUE_DICT: Dict = {
+        torch.bfloat16: FuzzThreshold.BF16_THD,
+        torch.float16: FuzzThreshold.F16_THD,
+        torch.float32: FuzzThreshold.F32_THD,
+        torch.float64: FuzzThreshold.F64_THD,
+    }
+
+    ABS_TOL_VALUE_DICT: Dict = {
+        torch.bfloat16: FuzzThreshold.BF16_THD,
+        torch.float16: FuzzThreshold.F16_THD,
+        torch.float32: FuzzThreshold.F32_THD,
+        torch.float64: FuzzThreshold.F64_THD,
+    }
+
+    # bit翻转需要匹配到等长或更长的整型
+    PERTURBATION_BIT_DICT = {
+        torch.bfloat16: torch.int16,
+        torch.float16: torch.int16,
+        torch.float32: torch.int32,
+        torch.float64: torch.int64,
+    }
+
+    # 输入噪声下界
+    NOISE_INPUT_LOWER_BOUND = 1e-8
+    COMP_CONSISTENT = 1.0
+    COMP_NAN = np.nan
+    SYMBOL_FLIPPING = "symbol_flipping"
+    BACKWARD_OUTPUT_LOWER_BOUND = 1e-3
+    SMALL_VALUE = 1.0
+    # 预热初始阈值
+    PREHEAT_INITIAL_THD = 2.05
+    API_THD_STEP = 2.0
+
+    DTYPE_PER_THD = {
+        torch.float16: 1.002,
+        torch.bfloat16: 1.004,
+        torch.float32: 1.0002,
+    }
+    BENCHMARK_THD_DICT = {
+        torch.float32: BenchmarkThd(2**-14, 1.0, 2**-14, 1e-4),
+        torch.float16: BenchmarkThd(2**-11, 1.0, 2**-11, 1e-4),
+        torch.bfloat16: BenchmarkThd(2**-8, 1.0, 2**-8, 1e-4),
+    }
+
+    TENSOR_SPLIT_MAX_CHUNK = 128
+
+
+class PreheatConfig:
+    IF_PREHEAT = "if_preheat"
+    PREHEAT_STEP = "preheat_step"
+    MAX_SAMPLE = "max_sample"

msprobe/pytorch/free_benchmark/common/counter.py

@@ -1,72 +1,72 @@
-from collections import defaultdict
-from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
-
-
-class PreheatCounter:
-    def __init__(self) -> None:
-        self.api_called_time: dict = defaultdict(int)
-        self.api_sample_time: dict = defaultdict(int)
-        self.one_step_used_api: dict = defaultdict(int)
-        self.api_thd: dict = defaultdict(dict)
-        self.preheat_record: dict = defaultdict(dict)
-        self.dtype_map: dict = {}
-        self.if_preheat: dict = defaultdict(dict)
-        self.step = 0
-
-    def clear_step(self):
-        self.preheat_record.clear()
-        self.api_called_time.clear()
-        self.api_sample_time.clear()
-    
-    def check_step(self, current_step):
-        if current_step != self.step:
-            self.clear_step()
-            self.step = current_step
-
-    def add_api_called_time(self, api_name: str):
-        self.api_called_time[api_name] += 1
-
-    def get_api_called_time(self, api_name: str) -> int:
-        return self.api_called_time[api_name]
-    
-    def add_api_sample_time(self, api_name: str):
-        self.api_sample_time[api_name] += 1
-
-    def get_api_sample_time(self, api_name: str) -> int:
-        return self.api_sample_time[api_name]
-
-    def add_one_step_used_api(self, api_name: str):
-        self.one_step_used_api[api_name] += 1
-
-    def get_one_step_used_api(self, api_name: str):
-        return self.one_step_used_api[api_name]
-
-    def update_preheat_record(self, api_name, dtype, cmp_result):
-        # 记录预热阶段CPU标杆比对的结果
-        if str(dtype) not in self.preheat_record[api_name].keys():
-            self.preheat_record[api_name][str(dtype)] = list()
-        self.preheat_record[api_name][str(dtype)].append(cmp_result)
-        self.dtype_map[str(dtype)] = dtype
-
-    def update_api_thd(self, api_name, dtype, threshold, dthreshold):
-        self.api_thd[api_name][str(dtype)] = (
-            threshold if threshold > dthreshold else dthreshold
-        )
-
-    def get_api_thd(self, api_name, dtype):
-        if not str(dtype) in self.api_thd[api_name]:
-            self.api_thd[api_name][str(dtype)] = ThresholdConfig.PREHEAT_INITIAL_THD
-            self.dtype_map[str(dtype)] = dtype
-        return self.api_thd[api_name][str(dtype)]
-
-    def set_api_preheat(self, api_name, dtype_str, is_preheat=True):
-        # 标记cpu不一致的dtype 不再进行预热
-        self.if_preheat[api_name][dtype_str] = is_preheat
-
-    def get_api_preheat(self, api_name, dtype):
-        # 标记cpu不一致的dtype 不再进行预热
-        if str(dtype) not in self.if_preheat[api_name]:
-            return True
-        return self.if_preheat[api_name][str(dtype)]
-
+from collections import defaultdict
+from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig
+
+
+class PreheatCounter:
+    def __init__(self) -> None:
+        self.api_called_time: dict = defaultdict(int)
+        self.api_sample_time: dict = defaultdict(int)
+        self.one_step_used_api: dict = defaultdict(int)
+        self.api_thd: dict = defaultdict(dict)
+        self.preheat_record: dict = defaultdict(dict)
+        self.dtype_map: dict = {}
+        self.if_preheat: dict = defaultdict(dict)
+        self.step = 0
+
+    def clear_step(self):
+        self.preheat_record.clear()
+        self.api_called_time.clear()
+        self.api_sample_time.clear()
+    
+    def check_step(self, current_step):
+        if current_step != self.step:
+            self.clear_step()
+            self.step = current_step
+
+    def add_api_called_time(self, api_name: str):
+        self.api_called_time[api_name] += 1
+
+    def get_api_called_time(self, api_name: str) -> int:
+        return self.api_called_time[api_name]
+    
+    def add_api_sample_time(self, api_name: str):
+        self.api_sample_time[api_name] += 1
+
+    def get_api_sample_time(self, api_name: str) -> int:
+        return self.api_sample_time[api_name]
+
+    def add_one_step_used_api(self, api_name: str):
+        self.one_step_used_api[api_name] += 1
+
+    def get_one_step_used_api(self, api_name: str):
+        return self.one_step_used_api[api_name]
+
+    def update_preheat_record(self, api_name, dtype, cmp_result):
+        # 记录预热阶段CPU标杆比对的结果
+        if str(dtype) not in self.preheat_record[api_name].keys():
+            self.preheat_record[api_name][str(dtype)] = list()
+        self.preheat_record[api_name][str(dtype)].append(cmp_result)
+        self.dtype_map[str(dtype)] = dtype
+
+    def update_api_thd(self, api_name, dtype, threshold, dthreshold):
+        self.api_thd[api_name][str(dtype)] = (
+            threshold if threshold > dthreshold else dthreshold
+        )
+
+    def get_api_thd(self, api_name, dtype):
+        if not str(dtype) in self.api_thd[api_name]:
+            self.api_thd[api_name][str(dtype)] = ThresholdConfig.PREHEAT_INITIAL_THD
+            self.dtype_map[str(dtype)] = dtype
+        return self.api_thd[api_name][str(dtype)]
+
+    def set_api_preheat(self, api_name, dtype_str, is_preheat=True):
+        # 标记cpu不一致的dtype 不再进行预热
+        self.if_preheat[api_name][dtype_str] = is_preheat
+
+    def get_api_preheat(self, api_name, dtype):
+        # 标记cpu不一致的dtype 不再进行预热
+        if str(dtype) not in self.if_preheat[api_name]:
+            return True
+        return self.if_preheat[api_name][str(dtype)]
+
 preheat_counter = PreheatCounter()

msprobe/pytorch/free_benchmark/common/enums.py

@@ -1,37 +1,37 @@
-class PerturbationMode:
-    ADD_NOISE = "add_noise"
-    CHANGE_VALUE = "change_value"
-    IMPROVE_PRECISION = "improve_precision"
-    NO_CHANGE = "no_change"
-    BIT_NOISE = "bit_noise"
-    TO_CPU = "to_cpu"
-
-
-class DeviceType:
-    NPU = "npu"
-    CPU = "cpu"
-
-
-class FuzzThreshold:
-    BF16_THD = 1e-4
-    F16_THD = 1e-6
-    F32_THD = 1e-8
-    F64_THD = 1e-16
-
-
-class NormType:
-    ONE_NORM = (1, "one_norm")
-    TWO_NORM = (2, "two_norm")
-    ENDLESS_NORM = (3, "endless_norm")
-
-
-class HandlerType:
-    CHECK = "check"
-    PREHEAT = "preheat"
-    FIX = "fix"
-
-
-class FuzzLevel:
-    BASE_LEVEL = "L1"
-    ADV_LEVEL = "L2"
-    REAL_LEVEL = "L3"
+class PerturbationMode:
+    ADD_NOISE = "add_noise"
+    CHANGE_VALUE = "change_value"
+    IMPROVE_PRECISION = "improve_precision"
+    NO_CHANGE = "no_change"
+    BIT_NOISE = "bit_noise"
+    TO_CPU = "to_cpu"
+
+
+class DeviceType:
+    NPU = "npu"
+    CPU = "cpu"
+
+
+class FuzzThreshold:
+    BF16_THD = 1e-4
+    F16_THD = 1e-6
+    F32_THD = 1e-8
+    F64_THD = 1e-16
+
+
+class NormType:
+    ONE_NORM = (1, "one_norm")
+    TWO_NORM = (2, "two_norm")
+    ENDLESS_NORM = (3, "endless_norm")
+
+
+class HandlerType:
+    CHECK = "check"
+    PREHEAT = "preheat"
+    FIX = "fix"
+
+
+class FuzzLevel:
+    BASE_LEVEL = "L1"
+    ADV_LEVEL = "L2"
+    REAL_LEVEL = "L3"

msprobe/pytorch/free_benchmark/common/params.py

@@ -1,129 +1,129 @@
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Optional, Tuple
-
-import torch
-from msprobe.pytorch.free_benchmark import logger
-from msprobe.pytorch.free_benchmark.common.enums import (
-    DeviceType,
-    FuzzLevel,
-    PerturbationMode,
-)
-from msprobe.pytorch.free_benchmark.common.utils import Tools
-
-
-@dataclass
-class DataParams:
-    args: Optional[Tuple] = None
-    kwargs: Optional[Dict] = None
-    valid_input_index: Optional[int] = None
-    original_result: Optional[Any] = None
-    perturbed_result: Optional[Any] = None
-    is_consistent: Optional[bool] = True
-    perturbed_value: Optional[Any] = None
-    origin_func: Optional[Callable] = None
-    api_type: Optional[str] = None
-    fuzz_stage: Optional[str] = None
-    grad_unequal_flag: Optional[bool] = True
-
-
-@dataclass
-class HandlerParams:
-    handler_type: Optional[str] = None
-    api_name: Optional[str] = None
-    pert_mode: Optional[PerturbationMode] = None
-    step: Optional[int] = None
-    fuzz_stage: Optional[str] = None
-    fuzz_device: Optional[DeviceType] = None
-    preheat_config: Optional[Dict] = None
-    fuzz_level: Optional[str] = None
-
-
-@dataclass
-class UnequalRow:
-    rank: Optional[int] = None
-    pert_mode: Optional[PerturbationMode] = None
-    stage: Optional[str] = None
-    step: Optional[int] = None
-    api_name: Optional[str] = None
-    max_rel: Optional[float] = None
-    dtype: Optional[str] = None
-    shape: Optional[str] = None
-    output_index: Optional[int] = None
-
-
-@dataclass
-class BenchmarkThd:
-    rtol: Optional[float] = None  # 相对误差阈值
-    small_value: Optional[float] = None  # 小值域
-    small_value_atol: Optional[float] = None  # 小值域绝对阈值
-    err_balance: Optional[float] = None  # 误差均衡性
-
-
-def check_args_type(args: Tuple) -> int:
-    for i, arg in enumerate(args):
-        if torch.is_tensor(arg):
-            if arg.is_meta:
-                continue
-            if not torch.is_floating_point(arg):
-                continue
-            return i
-        if isinstance(arg, (List, Tuple, Dict)):
-            return i
-    return -1
-
-
-def data_pre_deal(name, func, args, kwargs):
-    data_params = DataParams(args=args, kwargs=kwargs, origin_func=func)
-    index = check_args_type(args)
-    data_params.valid_input_index = index
-    if index == -1:
-        logger.warning_on_rank_0(
-            f"[msprobe] Free benchmark: 无标杆工具不支持当前算子的输入类型 {name}."
-        )
-    return data_params
-
-
-def make_handler_params(name, config, step):
-    handler_params = HandlerParams()
-    handler_params.api_name = name
-    handler_params.step = step
-    handler_params.handler_type = config.handler_type
-    handler_params.fuzz_stage = config.fuzz_stage
-    handler_params.fuzz_device = config.fuzz_device
-    handler_params.preheat_config = config.preheat_config
-    handler_params.fuzz_level = config.fuzz_level
-    handler_params.pert_mode = config.pert_mode
-    return handler_params
-
-
-def make_unequal_row(
-    data_params: DataParams,
-    handle_params: HandlerParams,
-    ratio: float = None,
-    index: int = None,
-):
-    row = UnequalRow(
-        api_name=handle_params.api_name,
-        pert_mode=handle_params.pert_mode,
-        output_index=index,
-        stage=handle_params.fuzz_stage,
-        step=handle_params.step,
-    )
-    if isinstance(ratio, float):
-        row.max_rel = ratio - 1
-    origin_tensor = data_params.original_result
-    perturbed_tensor = data_params.perturbed_result
-    if index:
-        origin_tensor = origin_tensor[index]
-        perturbed_tensor = perturbed_tensor[index]
-        row.output_index = index
-    if isinstance(origin_tensor, torch.Tensor):
-        row.dtype = origin_tensor.dtype
-        row.shape = origin_tensor.shape
-    row.rank = Tools.get_dist_rank()
-    # 以下暂不支持
-    if handle_params.fuzz_level == FuzzLevel.ADV_LEVEL:
-        pass
-    if handle_params.fuzz_level == FuzzLevel.REAL_LEVEL:
-        pass
-    return row
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+import torch
+from msprobe.pytorch.free_benchmark import logger
+from msprobe.pytorch.free_benchmark.common.enums import (
+    DeviceType,
+    FuzzLevel,
+    PerturbationMode,
+)
+from msprobe.pytorch.free_benchmark.common.utils import Tools
+
+
+@dataclass
+class DataParams:
+    args: Optional[Tuple] = None
+    kwargs: Optional[Dict] = None
+    valid_input_index: Optional[int] = None
+    original_result: Optional[Any] = None
+    perturbed_result: Optional[Any] = None
+    is_consistent: Optional[bool] = True
+    perturbed_value: Optional[Any] = None
+    origin_func: Optional[Callable] = None
+    api_type: Optional[str] = None
+    fuzz_stage: Optional[str] = None
+    grad_unequal_flag: Optional[bool] = True
+
+
+@dataclass
+class HandlerParams:
+    handler_type: Optional[str] = None
+    api_name: Optional[str] = None
+    pert_mode: Optional[PerturbationMode] = None
+    step: Optional[int] = None
+    fuzz_stage: Optional[str] = None
+    fuzz_device: Optional[DeviceType] = None
+    preheat_config: Optional[Dict] = None
+    fuzz_level: Optional[str] = None
+
+
+@dataclass
+class UnequalRow:
+    rank: Optional[int] = None
+    pert_mode: Optional[PerturbationMode] = None
+    stage: Optional[str] = None
+    step: Optional[int] = None
+    api_name: Optional[str] = None
+    max_rel: Optional[float] = None
+    dtype: Optional[str] = None
+    shape: Optional[str] = None
+    output_index: Optional[int] = None
+
+
+@dataclass
+class BenchmarkThd:
+    rtol: Optional[float] = None  # 相对误差阈值
+    small_value: Optional[float] = None  # 小值域
+    small_value_atol: Optional[float] = None  # 小值域绝对阈值
+    err_balance: Optional[float] = None  # 误差均衡性
+
+
+def check_args_type(args: Tuple) -> int:
+    for i, arg in enumerate(args):
+        if torch.is_tensor(arg):
+            if arg.is_meta:
+                continue
+            if not torch.is_floating_point(arg):
+                continue
+            return i
+        if isinstance(arg, (List, Tuple, Dict)):
+            return i
+    return -1
+
+
+def data_pre_deal(name, func, args, kwargs):
+    data_params = DataParams(args=args, kwargs=kwargs, origin_func=func)
+    index = check_args_type(args)
+    data_params.valid_input_index = index
+    if index == -1:
+        logger.warning_on_rank_0(
+            f"[msprobe] Free benchmark: 无标杆工具不支持当前算子的输入类型 {name}."
+        )
+    return data_params
+
+
+def make_handler_params(name, config, step):
+    handler_params = HandlerParams()
+    handler_params.api_name = name
+    handler_params.step = step
+    handler_params.handler_type = config.handler_type
+    handler_params.fuzz_stage = config.fuzz_stage
+    handler_params.fuzz_device = config.fuzz_device
+    handler_params.preheat_config = config.preheat_config
+    handler_params.fuzz_level = config.fuzz_level
+    handler_params.pert_mode = config.pert_mode
+    return handler_params
+
+
+def make_unequal_row(
+    data_params: DataParams,
+    handle_params: HandlerParams,
+    ratio: float = None,
+    index: int = None,
+):
+    row = UnequalRow(
+        api_name=handle_params.api_name,
+        pert_mode=handle_params.pert_mode,
+        output_index=index,
+        stage=handle_params.fuzz_stage,
+        step=handle_params.step,
+    )
+    if isinstance(ratio, float):
+        row.max_rel = ratio - 1
+    origin_tensor = data_params.original_result
+    perturbed_tensor = data_params.perturbed_result
+    if index:
+        origin_tensor = origin_tensor[index]
+        perturbed_tensor = perturbed_tensor[index]
+        row.output_index = index
+    if isinstance(origin_tensor, torch.Tensor):
+        row.dtype = origin_tensor.dtype
+        row.shape = origin_tensor.shape
+    row.rank = Tools.get_dist_rank()
+    # 以下暂不支持
+    if handle_params.fuzz_level == FuzzLevel.ADV_LEVEL:
+        pass
+    if handle_params.fuzz_level == FuzzLevel.REAL_LEVEL:
+        pass
+    return row