mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

msprobe/mindspore/free_benchmark/decorator/dec_forward.py

@@ -0,0 +1,42 @@
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory
+from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory
+
+
+class ForwardSelfChecker:
+
+    def __init__(self, api_name: str):
+        self.api_name = api_name
+
+    def handle(self, params: HandlerParams):
+        """
+        装饰器实际执行逻辑
+
+        """
+        perturbation = PerturbationFactory.create(self.api_name)
+        params.fuzzed_result = perturbation.handle(params)
+        params.original_result = params.original_func(*params.args, **params.kwargs)
+        if params.fuzzed_result is not False:
+            return self.deal_fuzzed_and_original_result(params)
+        return params.original_result
+
+    def get_compare_data(self, params: HandlerParams):
+        if self.api_name not in FreeBenchmarkConst.COMMUNICATION_API_LIST:
+            return
+        # 以下为通讯类api处理逻辑
+        params.fuzzed_result = params.fuzzed_value
+        if Config.pert_type == FreeBenchmarkConst.IMPROVE_PRECISION:
+            params.original_result = params.args
+        else:
+            params.original_result = params.args[params.index]
+
+    def deal_fuzzed_and_original_result(self, params: HandlerParams):
+        original_result = params.original_result
+        self.get_compare_data(params)
+        handler = HandlerFactory.create(self.api_name)
+        result = handler.handle(params)
+        if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST:
+            result = original_result
+        return result

msprobe/mindspore/free_benchmark/decorator/decorator_factory.py

@@ -0,0 +1,107 @@
+import os
+import sys
+import traceback
+from functools import wraps
+from typing import Tuple, Dict, List
+
+from mindspore import ops
+
+from msprobe.mindspore.runtime import Runtime
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from .dec_forward import ForwardSelfChecker
+
+
+def decorate(original_func, decorate_func, api_name=None):
+    """
+    总装饰器
+    """
+    @wraps(original_func)
+    def fuzz_wrapper(*args, **kwargs):
+
+        def __exec_decorate_func():
+            params = data_pre_deal(api_name, original_func, *args, **kwargs)
+            result = decorate_func(params)
+            return result
+
+        try:
+            if Runtime.rank_id == -1:
+                Runtime.rank_id = os.environ.get("RANK_ID", -1)
+            if need_wrapper_func():
+                logger.info(f"[{api_name}] is checking.")
+                return __exec_decorate_func()
+        except Exception as e:
+            logger.error(f"[{api_name}] Error: {str(e)}")
+            logger.error(f"[{api_name}] Error detail: {traceback.format_exc()}")
+
+        return original_func(*args, **kwargs)
+
+    return fuzz_wrapper
+
+
+def decorate_forward_function(func, api_name=None):
+    """
+    前向装饰器
+    """
+
+    if not api_name:
+        api_name = func.__name__
+
+    def forward_func(params: HandlerParams):
+        forward = ForwardSelfChecker(api_name)
+        result = forward.handle(params)
+        return result
+
+    return decorate(func, forward_func, api_name)
+
+
+def stack_depth_check() -> bool:
+    nested_depth = 1
+    frame = sys._getframe(1)
+    while frame:
+        if frame.f_code.co_name == "fuzz_wrapper":
+            nested_depth -= 1
+            if nested_depth < 0:
+                return False
+        frame = frame.f_back
+    return True
+
+
+def get_target_arg_index(args: Tuple) -> int:
+    """
+    类型校验
+
+    """
+    for i, arg in enumerate(args):
+        if ops.is_tensor(arg):
+            if not ops.is_floating_point(arg):
+                continue
+            return i
+        if isinstance(arg, (List, Tuple, Dict)):
+            return i
+    return -1
+
+
+def data_pre_deal(api_name, func, *args, **kwargs):
+    params = HandlerParams()
+    params.args = args
+    params.kwargs = kwargs
+    params.original_func = func
+    index = get_target_arg_index(args)
+    if index == -1:
+        raise Exception(f"{api_name} has no supported input type")
+    params.index = index
+    return params
+
+
+def need_wrapper_func():
+    if not (Runtime.is_running and Config.is_enable):
+        return False
+    if not stack_depth_check():
+        return False
+    if Config.steps and Runtime.step_count not in Config.steps:
+        return False
+    if Config.ranks and Runtime.rank_id != -1 and Runtime.rank_id not in Config.ranks:
+        return False
+    return True

msprobe/mindspore/free_benchmark/handler/base_handler.py

@@ -0,0 +1,90 @@
+import math
+from abc import ABC, abstractmethod
+from typing import Any, Tuple, Optional
+
+import mindspore as ms
+from mindspore import Tensor, ops
+
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.utils import Tools
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+
+
+class BaseHandler(ABC):
+
+    def __init__(self, api_name: str):
+        self.api_name = api_name
+
+    @staticmethod
+    def pre_calculate(original_output, fuzzed_output):
+        abs_tol = FreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype,
+                                                         FreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32))
+
+        return original_output.to(fuzzed_output.dtype), fuzzed_output, abs_tol
+
+    @staticmethod
+    def get_threshold(dtype):
+        err = Tools.get_default_error_threshold(dtype)
+        return err
+
+    @staticmethod
+    def convert_overflow_ratio_to_consistent(ratio):
+        if math.isnan(ratio) or math.isinf(ratio):
+            return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD
+        return ratio
+
+    @staticmethod
+    def get_endless_norm(first_tensor, second_tensor, abs_tol):
+        if first_tensor.dtype != ms.bfloat16 and second_tensor.dtype != ms.bfloat16:
+            ratio_tensor1 = ops.where(ops.abs(second_tensor) > abs_tol, ops.div(first_tensor, second_tensor), 1)
+            ratio_tensor2 = ops.where(ops.abs(first_tensor) > abs_tol, ops.div(second_tensor, first_tensor), 1)
+        else:
+            ratio_tensor1 = ops.where(ops.abs(second_tensor).to(ms.float32) > abs_tol,
+                                      ops.div(first_tensor.to(ms.float32), second_tensor.to(ms.float32)), 1)
+            ratio_tensor2 = ops.where(ops.abs(first_tensor).to(ms.float32) > abs_tol,
+                                      ops.div(second_tensor.to(ms.float32), first_tensor.to(ms.float32)), 1)
+        norm1 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor1)[0].to(ms.float32).item())
+        norm2 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor2)[0].to(ms.float32).item())
+        norm3 = BaseHandler.convert_overflow_ratio_to_consistent(ops.min(ratio_tensor1)[0].to(ms.float32).item())
+        ratio = FreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2)
+
+        return ratio
+
+    @staticmethod
+    def ratio_calculate(original_output, fuzzed_output) -> float:
+        try:
+            original_output, fuzzed_output, abs_tol = BaseHandler.pre_calculate(original_output, fuzzed_output)
+        except Exception as e:
+            logger.error(f"When computing ratio, y1 or y2 dtype is not supported {str(e)}")
+            return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD
+
+        abs_tol = abs_tol ** 0.5
+
+        return BaseHandler.get_endless_norm(original_output, fuzzed_output, abs_tol)
+
+    @staticmethod
+    def npu_compare(original_output, fuzzed_output) -> Tuple[bool, Optional[float]]:
+        if not isinstance(fuzzed_output, Tensor):
+            logger.error(f"The compare for output type `{type(fuzzed_output)}` is not supported")
+            return True, 1.0
+
+        # 范数计算等
+        err_thd = BaseHandler.get_threshold(original_output.dtype)
+        ratio = BaseHandler.ratio_calculate(original_output, fuzzed_output)
+        is_consistent = err_thd >= ratio >= 1.0 / err_thd
+        return is_consistent, ratio
+
+    @staticmethod
+    def is_float_tensor(output) -> bool:
+        if isinstance(output, Tensor) and ops.is_floating_point(output):
+            return True
+        if isinstance(output, (list, tuple)):
+            for i in output:
+                if isinstance(i, Tensor) and ops.is_floating_point(i):
+                    return True
+        return False
+
+    @abstractmethod
+    def handle(self, params: HandlerParams) -> Any:
+        pass

msprobe/mindspore/free_benchmark/handler/check_handler.py

@@ -0,0 +1,41 @@
+from typing import Any
+from dataclasses import asdict
+
+from mindspore import Tensor, ops
+
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.common.utils import make_unequal_row
+from msprobe.core.data_dump.json_writer import DataWriter
+
+
+class CheckHandler(BaseHandler):
+
+    def npu_compare_and_save(self, original_output, fuzzed_output, params: HandlerParams, output_index=None):
+        is_consistent, ratio = self.npu_compare(original_output, fuzzed_output)
+        params.is_consistent = params.is_consistent and is_consistent
+        if not is_consistent:
+            row = make_unequal_row(self.api_name, params, ratio, output_index)
+            data_dict = asdict(row)
+            DataWriter.write_data_to_csv(
+                data_dict.values(),
+                data_dict.keys(),
+                Config.dump_path
+            )
+            logger.error(f"{self.api_name} is not consistent")
+
+    def handle(self, params: HandlerParams) -> Any:
+        try:
+            if not self.is_float_tensor(params.fuzzed_result):
+                return params.original_result
+            if isinstance(params.fuzzed_result, Tensor):
+                self.npu_compare_and_save(params.original_result, params.fuzzed_result, params)
+            elif isinstance(params.fuzzed_result, (list, tuple)):
+                for i, item in enumerate(params.original_result):
+                    if ops.is_tensor(item) and ops.is_floating_point(item):
+                        self.npu_compare_and_save(item, params.fuzzed_result[i], params, output_index=i)
+        except Exception as e:
+            logger.error(str(e))
+        return params.original_result

msprobe/mindspore/free_benchmark/handler/fix_handler.py

@@ -0,0 +1,36 @@
+from typing import Any
+
+from mindspore import Tensor
+
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+
+
+class FixHandler:
+
+    def __init__(self, api_name: str):
+        self.api_name = api_name
+
+    @staticmethod
+    def use_fuzzed_result(original_result, fuzzed_result):
+        if isinstance(original_result, Tensor):
+            return fuzzed_result.to(original_result.dtype)
+        if isinstance(original_result, dict):
+            dict_fixed_result = dict()
+            for k, v in original_result.items():
+                dict_fixed_result[k] = FixHandler.use_fuzzed_result(v, fuzzed_result[k])
+            return dict_fixed_result
+        if isinstance(original_result, (tuple, list)):
+            list_fixed_result = list()
+            for i, v in enumerate(original_result):
+                list_fixed_result.append(FixHandler.use_fuzzed_result(v, fuzzed_result[i]))
+            return type(original_result)(list_fixed_result)
+        return original_result
+
+    def handle(self, params: HandlerParams) -> Any:
+        try:
+            return FixHandler.use_fuzzed_result(params.original_result, params.fuzzed_result)
+        except Exception as e:
+            logger.error(f"{self.api_name} failed to fix.")
+            logger.error(str(e))
+            return params.original_result

msprobe/mindspore/free_benchmark/handler/handler_factory.py

@@ -0,0 +1,21 @@
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from .check_handler import CheckHandler
+from .fix_handler import FixHandler
+
+
+class HandlerFactory:
+    result_handlers = {
+        FreeBenchmarkConst.CHECK: CheckHandler,
+        FreeBenchmarkConst.FIX: FixHandler,
+    }
+
+    @staticmethod
+    def create(api_name: str):
+        handler = HandlerFactory.result_handlers.get(Config.handler_type)
+        if handler:
+            return handler(api_name)
+        else:
+            logger.error(f"{Config.handler_type} is not supported.")
+            raise Exception

msprobe/mindspore/free_benchmark/perturbation/add_noise.py

@@ -0,0 +1,67 @@
+from typing import Any
+
+from mindspore import Tensor, ops
+
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+
+
+class AddNoisePerturbation(BasePerturbation):
+
+    def handle(self, params: HandlerParams) -> Any:
+        """
+        返回增加扰动后的api输出
+
+        """
+        params.fuzzed_value = self.add_noise(params.args[params.index])
+        if not self.is_fuzzed:
+            logger.warning(f"{self.api_name} can not add noise.")
+            return False
+        return self.get_fuzzed_result(params)
+
+    def add_noise(self, inputs) -> Any:
+        """
+        返回增加扰动后的api输入
+
+        """
+        if isinstance(inputs, Tensor):
+            noise = self._get_noise(inputs)
+            if noise is not False:
+                result = ops.where(ops.abs(inputs) > self.perturbation_value ** 0.5,
+                                   ops.add(noise, inputs), inputs)
+                result = result.type(dtype=inputs.dtype)
+                self.is_fuzzed = True
+                return result
+
+        if isinstance(inputs, dict):
+            return {k: self.add_noise(v) for k, v in inputs.items()}
+
+        if isinstance(inputs, (list, tuple)):
+            return [self.add_noise(v) for v in inputs]
+
+        return inputs
+
+    def _get_noise(self, input):
+        """
+        得到要添加的噪声值
+
+        """
+        if self.is_fuzzed:
+            return False
+        if not ops.is_floating_point(input) or ops.numel(input) == 0:
+            return False
+
+        pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype)
+        if not pert_value:
+            return False
+        else:
+            self.perturbation_value = pert_value
+
+        max_val = ops.max(ops.abs(input))[0].item()
+        if max_val < pert_value:
+            return False
+
+        noise = ops.full(input.shape, self.perturbation_value, dtype=input.dtype)
+        return noise

msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py

@@ -0,0 +1,21 @@
+from typing import Any
+
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+
+
+class BasePerturbation:
+
+    def __init__(self, api_name: str):
+        self.api_name = api_name
+        self.is_fuzzed = False
+        self.perturbation_value = None
+
+    @staticmethod
+    def get_fuzzed_result(params: HandlerParams):
+        args_front = params.args[:params.index]
+        args_rear = params.args[params.index + 1:]
+        fuzzed_result = params.original_func(*args_front, params.fuzzed_value, *args_rear, **params.kwargs)
+        return fuzzed_result
+
+    def handler(self, params: HandlerParams) -> Any:
+        pass

msprobe/mindspore/free_benchmark/perturbation/bit_noise.py

@@ -0,0 +1,63 @@
+from typing import Any
+
+import numpy as np
+from mindspore import Tensor, ops
+
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
+
+
+class BitNoisePerturbation(BasePerturbation):
+
+    def add_bit_noise(self, inputs) -> Any:
+        if isinstance(inputs, Tensor):
+            bit_len_type = self._get_bit_len_type(inputs)
+            if bit_len_type is not False:
+                sub_normal_np = np.finfo(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal
+                sub_normal = Tensor(sub_normal_np)
+                noise_type = list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[
+                             list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.values()).index(bit_len_type)]
+                noise = ops.full(inputs.shape, 1, dtype=noise_type)
+                input_np = inputs.asnumpy()
+                input_np_int = input_np.view(bit_len_type)
+                result = Tensor(input_np_int)
+                result = ops.where(ops.abs(inputs) > sub_normal,
+                                   ops.bitwise_xor(result, noise), result)
+                result_np = result.asnumpy()
+                result_np_float = result_np.view(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype))
+                self.is_fuzzed = True
+                return Tensor(result_np_float)
+
+        if isinstance(inputs, dict):
+            return {k: self.add_bit_noise(v) for k, v in inputs.items()}
+        if isinstance(inputs, (tuple, list)):
+            return type(inputs)([self.add_bit_noise(v) for v in inputs])
+        return inputs
+
+    def handle(self, params: HandlerParams) -> any:
+        args = params.args
+        params.fuzzed_value = self.add_bit_noise(params.args[params.index])
+        if not self.is_fuzzed:
+            logger.warning(f"{self.api_name} can not add bit noise.")
+            return False
+        params.args = args
+        return self.get_fuzzed_result(params)
+
+    def _get_bit_len_type(self, input):
+        if self.is_fuzzed:
+            return False
+        if not isinstance(input, Tensor) or not ops.is_floating_point(input) or \
+           input.numel() == 0:
+            return False
+        bit_len_type = FreeBenchmarkConst.PERT_BIT_DICT.get(input.dtype)
+        if not bit_len_type:
+            return False
+        pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype)
+        if not pert_value:
+            return False
+        max_val = ops.max(ops.abs(input))[0].item()
+        if max_val < pert_value:
+            return False
+        return bit_len_type

msprobe/mindspore/free_benchmark/perturbation/improve_precision.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+import mindspore as ms
+from mindspore import Tensor, ops
+
+from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.common.log import logger
+
+
+class ImprovePrecisionPerturbation(BasePerturbation):
+
+    def improve_tensor_precision(self, target_tensor):
+        if isinstance(target_tensor, Tensor) and ops.is_floating_point(target_tensor) and \
+           target_tensor.dtype not in [ms.float64, ms.float32]:
+            self.is_fuzzed = True
+            return target_tensor.to(ms.float32)
+        if isinstance(target_tensor, dict):
+            return {k: self.improve_tensor_precision(v) for k, v in target_tensor.items()}
+        if isinstance(target_tensor, (tuple, list)):
+            return type(target_tensor)([self.improve_tensor_precision(v) for v in target_tensor])
+        return target_tensor
+
+    def handle(self, params: HandlerParams) -> Any:
+        args = self.improve_tensor_precision(params.args)
+        kwargs = self.improve_tensor_precision(params.kwargs)
+        fuzzed_value = args
+        if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST:
+            params.fuzzed_value = fuzzed_value
+        if not self.is_fuzzed:
+            logger.warning(f"{self.api_name} can not improve precision.")
+            return False
+        return params.original_func(*args, **kwargs)

msprobe/mindspore/free_benchmark/perturbation/no_change.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+
+
+class NoChangePerturbation(BasePerturbation):
+
+    def handle(self, params: HandlerParams) -> Any:
+        params.fuzzed_value = params.args[params.index]
+        self.is_fuzzed = True
+        return self.get_fuzzed_result(params)

msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py

@@ -0,0 +1,27 @@
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.free_benchmark.common.config import Config
+from .add_noise import AddNoisePerturbation
+from .bit_noise import BitNoisePerturbation
+from .no_change import NoChangePerturbation
+from .improve_precision import ImprovePrecisionPerturbation
+
+
+class PerturbationFactory:
+    """
+    扰动工厂类
+
+    """
+    perturbations = {
+        FreeBenchmarkConst.IMPROVE_PRECISION: ImprovePrecisionPerturbation,
+        FreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation,
+        FreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation,
+        FreeBenchmarkConst.NO_CHANGE: NoChangePerturbation,
+    }
+
+    @staticmethod
+    def create(api_name: str):
+        perturbation = PerturbationFactory.perturbations.get(Config.pert_type)
+        if perturbation:
+            return perturbation(api_name)
+        else:
+            raise Exception(f'{Config.pert_type} is a invalid perturbation type')

msprobe/mindspore/free_benchmark/self_check_tool_factory.py

@@ -0,0 +1,33 @@
+from msprobe.mindspore.common.const import Const
+from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
+from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelFCheck
+
+
+class SelfCheckToolFactory:
+    tools = {
+        Const.CELL: {
+            Const.GRAPH_KBYK_MODE: None,
+            Const.GRAPH_GE_MODE: None,
+            Const.PYNATIVE_MODE: None
+        },
+        Const.API: {
+            Const.GRAPH_KBYK_MODE: None,
+            Const.GRAPH_GE_MODE: None,
+            Const.PYNATIVE_MODE: ApiPyNativeSelFCheck
+        },
+        Const.KERNEL: {
+            Const.GRAPH_KBYK_MODE: None,
+            Const.GRAPH_GE_MODE: None,
+            Const.PYNATIVE_MODE: None
+        }
+    }
+
+    @staticmethod
+    def create(config: DebuggerConfig):
+        tool = SelfCheckToolFactory.tools.get(config.level)
+        if not tool:
+            raise Exception(f"{config.level} is not supported.")
+        tool = tool.get(config.execution_mode)
+        if not tool:
+            raise Exception(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.")
+        return tool(config)