mindstudio-probe 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/mindspore/free_benchmark/decorator/dec_forward.py

@@ -1,42 +1,57 @@
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.common.const import FreeBenchmarkConst
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory
-from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory
-
-
-class ForwardSelfChecker:
-
-    def __init__(self, api_name: str):
-        self.api_name = api_name
-
-    def handle(self, params: HandlerParams):
-        """
-        装饰器实际执行逻辑
-
-        """
-        perturbation = PerturbationFactory.create(self.api_name)
-        params.fuzzed_result = perturbation.handle(params)
-        params.original_result = params.original_func(*params.args, **params.kwargs)
-        if params.fuzzed_result is not False:
-            return self.deal_fuzzed_and_original_result(params)
-        return params.original_result
-
-    def get_compare_data(self, params: HandlerParams):
-        if self.api_name not in FreeBenchmarkConst.COMMUNICATION_API_LIST:
-            return
-        # 以下为通讯类api处理逻辑
-        params.fuzzed_result = params.fuzzed_value
-        if Config.pert_type == FreeBenchmarkConst.IMPROVE_PRECISION:
-            params.original_result = params.args
-        else:
-            params.original_result = params.args[params.index]
-
-    def deal_fuzzed_and_original_result(self, params: HandlerParams):
-        original_result = params.original_result
-        self.get_compare_data(params)
-        handler = HandlerFactory.create(self.api_name)
-        result = handler.handle(params)
-        if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST:
-            result = original_result
-        return result
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.mindspore.common.const import Const, FreeBenchmarkConst
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory
+from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory
+
+
+class ForwardSelfChecker:
+
+    def __init__(self, api_name: str):
+        self.api_name = api_name
+
+    def handle(self, params: HandlerParams):
+        """
+        装饰器实际执行逻辑
+
+        """
+        perturbation = PerturbationFactory.create(self.api_name)
+        params.fuzzed_result = perturbation.handle(params)
+        params.original_result = params.original_func(*params.args, **params.kwargs)
+        if params.fuzzed_result is not False:
+            return self.deal_fuzzed_and_original_result(params)
+        return params.original_result
+
+    def get_compare_data(self, params: HandlerParams):
+        if self.api_name not in Const.COMMUNICATION_API_LIST:
+            return
+        # 以下为通讯类api处理逻辑
+        params.fuzzed_result = params.fuzzed_value
+        if Config.pert_type == FreeBenchmarkConst.IMPROVE_PRECISION:
+            params.original_result = params.args
+        else:
+            params.original_result = params.args[params.index]
+
+    def deal_fuzzed_and_original_result(self, params: HandlerParams):
+        original_result = params.original_result
+        self.get_compare_data(params)
+        handler = HandlerFactory.create(self.api_name)
+        result = handler.handle(params)
+        if self.api_name in Const.COMMUNICATION_API_LIST:
+            result = original_result
+        return result

msprobe/mindspore/free_benchmark/decorator/decorator_factory.py

@@ -1,107 +1,122 @@
-import os
-import sys
-import traceback
-from functools import wraps
-from typing import Tuple, Dict, List
-
-from mindspore import ops
-
-from msprobe.mindspore.runtime import Runtime
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from .dec_forward import ForwardSelfChecker
-
-
-def decorate(original_func, decorate_func, api_name=None):
-    """
-    总装饰器
-    """
-    @wraps(original_func)
-    def fuzz_wrapper(*args, **kwargs):
-
-        def __exec_decorate_func():
-            params = data_pre_deal(api_name, original_func, *args, **kwargs)
-            result = decorate_func(params)
-            return result
-
-        try:
-            if Runtime.rank_id == -1:
-                Runtime.rank_id = os.environ.get("RANK_ID", -1)
-            if need_wrapper_func():
-                logger.info(f"[{api_name}] is checking.")
-                return __exec_decorate_func()
-        except Exception as e:
-            logger.error(f"[{api_name}] Error: {str(e)}")
-            logger.error(f"[{api_name}] Error detail: {traceback.format_exc()}")
-
-        return original_func(*args, **kwargs)
-
-    return fuzz_wrapper
-
-
-def decorate_forward_function(func, api_name=None):
-    """
-    前向装饰器
-    """
-
-    if not api_name:
-        api_name = func.__name__
-
-    def forward_func(params: HandlerParams):
-        forward = ForwardSelfChecker(api_name)
-        result = forward.handle(params)
-        return result
-
-    return decorate(func, forward_func, api_name)
-
-
-def stack_depth_check() -> bool:
-    nested_depth = 1
-    frame = sys._getframe(1)
-    while frame:
-        if frame.f_code.co_name == "fuzz_wrapper":
-            nested_depth -= 1
-            if nested_depth < 0:
-                return False
-        frame = frame.f_back
-    return True
-
-
-def get_target_arg_index(args: Tuple) -> int:
-    """
-    类型校验
-
-    """
-    for i, arg in enumerate(args):
-        if ops.is_tensor(arg):
-            if not ops.is_floating_point(arg):
-                continue
-            return i
-        if isinstance(arg, (List, Tuple, Dict)):
-            return i
-    return -1
-
-
-def data_pre_deal(api_name, func, *args, **kwargs):
-    params = HandlerParams()
-    params.args = args
-    params.kwargs = kwargs
-    params.original_func = func
-    index = get_target_arg_index(args)
-    if index == -1:
-        raise Exception(f"{api_name} has no supported input type")
-    params.index = index
-    return params
-
-
-def need_wrapper_func():
-    if not (Runtime.is_running and Config.is_enable):
-        return False
-    if not stack_depth_check():
-        return False
-    if Config.steps and Runtime.step_count not in Config.steps:
-        return False
-    if Config.ranks and Runtime.rank_id != -1 and Runtime.rank_id not in Config.ranks:
-        return False
-    return True
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import traceback
+from functools import wraps
+from typing import Dict, List, Tuple
+
+from mindspore import ops
+
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.decorator.dec_forward import ForwardSelfChecker
+from msprobe.mindspore.runtime import Runtime
+
+
+def decorate(original_func, decorate_func, api_name=None):
+    """
+    总装饰器
+    """
+    @wraps(original_func)
+    def fuzz_wrapper(*args, **kwargs):
+
+        def __exec_decorate_func():
+            params = data_pre_deal(api_name, original_func, *args, **kwargs)
+            result = decorate_func(params)
+            return result
+
+        try:
+            if Runtime.rank_id == -1:
+                Runtime.rank_id = os.environ.get("RANK_ID", -1)
+            if need_wrapper_func():
+                logger.info(f"[{api_name}] is checking.")
+                return __exec_decorate_func()
+        except Exception as e:
+            logger.error(f"[{api_name}] Error: {str(e)}")
+            logger.error(f"[{api_name}] Error detail: {traceback.format_exc()}")
+
+        return original_func(*args, **kwargs)
+
+    return fuzz_wrapper
+
+
+def decorate_forward_function(func, api_name=None):
+    """
+    前向装饰器
+    """
+
+    if not api_name:
+        api_name = func.__name__
+
+    def forward_func(params: HandlerParams):
+        forward = ForwardSelfChecker(api_name)
+        result = forward.handle(params)
+        return result
+
+    return decorate(func, forward_func, api_name)
+
+
+def stack_depth_check() -> bool:
+    nested_depth = 1
+    frame = sys._getframe(1)
+    while frame:
+        if frame.f_code.co_name == "fuzz_wrapper":
+            nested_depth -= 1
+            if nested_depth < 0:
+                return False
+        frame = frame.f_back
+    return True
+
+
+def get_target_arg_index(args: Tuple) -> int:
+    """
+    类型校验
+
+    """
+    for i, arg in enumerate(args):
+        if ops.is_tensor(arg):
+            if not ops.is_floating_point(arg):
+                continue
+            return i
+        if isinstance(arg, (List, Tuple, Dict)):
+            return i
+    return -1
+
+
+def data_pre_deal(api_name, func, *args, **kwargs):
+    params = HandlerParams()
+    params.args = args
+    params.kwargs = kwargs
+    params.original_func = func
+    index = get_target_arg_index(args)
+    if index == -1:
+        raise Exception(f"{api_name} has no supported input type")
+    params.index = index
+    return params
+
+
+def need_wrapper_func():
+    if not (Runtime.is_running and Config.is_enable):
+        return False
+    if not stack_depth_check():
+        return False
+    if Config.steps and Runtime.step_count not in Config.steps:
+        return False
+    if Config.ranks and Runtime.rank_id != -1 and Runtime.rank_id not in Config.ranks:
+        return False
+    return True

msprobe/mindspore/free_benchmark/handler/base_handler.py

@@ -1,90 +1,105 @@
-import math
-from abc import ABC, abstractmethod
-from typing import Any, Tuple, Optional
-
-import mindspore as ms
-from mindspore import Tensor, ops
-
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.utils import Tools
-from msprobe.mindspore.common.const import FreeBenchmarkConst
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-
-
-class BaseHandler(ABC):
-
-    def __init__(self, api_name: str):
-        self.api_name = api_name
-
-    @staticmethod
-    def pre_calculate(original_output, fuzzed_output):
-        abs_tol = FreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype,
-                                                         FreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32))
-
-        return original_output.to(fuzzed_output.dtype), fuzzed_output, abs_tol
-
-    @staticmethod
-    def get_threshold(dtype):
-        err = Tools.get_default_error_threshold(dtype)
-        return err
-
-    @staticmethod
-    def convert_overflow_ratio_to_consistent(ratio):
-        if math.isnan(ratio) or math.isinf(ratio):
-            return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD
-        return ratio
-
-    @staticmethod
-    def get_endless_norm(first_tensor, second_tensor, abs_tol):
-        if first_tensor.dtype != ms.bfloat16 and second_tensor.dtype != ms.bfloat16:
-            ratio_tensor1 = ops.where(ops.abs(second_tensor) > abs_tol, ops.div(first_tensor, second_tensor), 1)
-            ratio_tensor2 = ops.where(ops.abs(first_tensor) > abs_tol, ops.div(second_tensor, first_tensor), 1)
-        else:
-            ratio_tensor1 = ops.where(ops.abs(second_tensor).to(ms.float32) > abs_tol,
-                                      ops.div(first_tensor.to(ms.float32), second_tensor.to(ms.float32)), 1)
-            ratio_tensor2 = ops.where(ops.abs(first_tensor).to(ms.float32) > abs_tol,
-                                      ops.div(second_tensor.to(ms.float32), first_tensor.to(ms.float32)), 1)
-        norm1 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor1)[0].to(ms.float32).item())
-        norm2 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor2)[0].to(ms.float32).item())
-        norm3 = BaseHandler.convert_overflow_ratio_to_consistent(ops.min(ratio_tensor1)[0].to(ms.float32).item())
-        ratio = FreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2)
-
-        return ratio
-
-    @staticmethod
-    def ratio_calculate(original_output, fuzzed_output) -> float:
-        try:
-            original_output, fuzzed_output, abs_tol = BaseHandler.pre_calculate(original_output, fuzzed_output)
-        except Exception as e:
-            logger.error(f"When computing ratio, y1 or y2 dtype is not supported {str(e)}")
-            return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD
-
-        abs_tol = abs_tol ** 0.5
-
-        return BaseHandler.get_endless_norm(original_output, fuzzed_output, abs_tol)
-
-    @staticmethod
-    def npu_compare(original_output, fuzzed_output) -> Tuple[bool, Optional[float]]:
-        if not isinstance(fuzzed_output, Tensor):
-            logger.error(f"The compare for output type `{type(fuzzed_output)}` is not supported")
-            return True, 1.0
-
-        # 范数计算等
-        err_thd = BaseHandler.get_threshold(original_output.dtype)
-        ratio = BaseHandler.ratio_calculate(original_output, fuzzed_output)
-        is_consistent = err_thd >= ratio >= 1.0 / err_thd
-        return is_consistent, ratio
-
-    @staticmethod
-    def is_float_tensor(output) -> bool:
-        if isinstance(output, Tensor) and ops.is_floating_point(output):
-            return True
-        if isinstance(output, (list, tuple)):
-            for i in output:
-                if isinstance(i, Tensor) and ops.is_floating_point(i):
-                    return True
-        return False
-
-    @abstractmethod
-    def handle(self, params: HandlerParams) -> Any:
-        pass
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from abc import ABC, abstractmethod
+from typing import Any, Optional, Tuple
+
+import mindspore as ms
+from mindspore import Tensor, ops
+
+from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.common.utils import Tools
+
+
+class BaseHandler(ABC):
+
+    def __init__(self, api_name: str):
+        self.api_name = api_name
+
+    @staticmethod
+    def pre_calculate(original_output, fuzzed_output):
+        abs_tol = FreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype,
+                                                         FreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32))
+
+        return original_output.to(fuzzed_output.dtype), fuzzed_output, abs_tol
+
+    @staticmethod
+    def get_threshold(dtype):
+        err = Tools.get_default_error_threshold(dtype)
+        return err
+
+    @staticmethod
+    def convert_overflow_ratio_to_consistent(ratio):
+        if math.isnan(ratio) or math.isinf(ratio):
+            return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD
+        return ratio
+
+    @staticmethod
+    def get_endless_norm(first_tensor, second_tensor, abs_tol):
+        if first_tensor.dtype != ms.bfloat16 and second_tensor.dtype != ms.bfloat16:
+            ratio_tensor1 = ops.where(ops.abs(second_tensor) > abs_tol, ops.div(first_tensor, second_tensor), 1)
+            ratio_tensor2 = ops.where(ops.abs(first_tensor) > abs_tol, ops.div(second_tensor, first_tensor), 1)
+        else:
+            ratio_tensor1 = ops.where(ops.abs(second_tensor).to(ms.float32) > abs_tol,
+                                      ops.div(first_tensor.to(ms.float32), second_tensor.to(ms.float32)), 1)
+            ratio_tensor2 = ops.where(ops.abs(first_tensor).to(ms.float32) > abs_tol,
+                                      ops.div(second_tensor.to(ms.float32), first_tensor.to(ms.float32)), 1)
+        norm1 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor1)[0].to(ms.float32).item())
+        norm2 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor2)[0].to(ms.float32).item())
+        norm3 = BaseHandler.convert_overflow_ratio_to_consistent(ops.min(ratio_tensor1)[0].to(ms.float32).item())
+        ratio = FreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2)
+
+        return ratio
+
+    @staticmethod
+    def ratio_calculate(original_output, fuzzed_output) -> float:
+        try:
+            original_output, fuzzed_output, abs_tol = BaseHandler.pre_calculate(original_output, fuzzed_output)
+        except Exception as e:
+            logger.error(f"When computing ratio, y1 or y2 dtype is not supported {str(e)}")
+            return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD
+
+        abs_tol = abs_tol ** 0.5
+
+        return BaseHandler.get_endless_norm(original_output, fuzzed_output, abs_tol)
+
+    @staticmethod
+    def npu_compare(original_output, fuzzed_output) -> Tuple[bool, Optional[float]]:
+        if not isinstance(fuzzed_output, Tensor):
+            logger.error(f"The compare for output type `{type(fuzzed_output)}` is not supported")
+            return True, 1.0
+
+        # 范数计算等
+        err_thd = BaseHandler.get_threshold(original_output.dtype)
+        ratio = BaseHandler.ratio_calculate(original_output, fuzzed_output)
+        is_consistent = err_thd >= ratio >= 1.0 / err_thd
+        return is_consistent, ratio
+
+    @staticmethod
+    def is_float_tensor(output) -> bool:
+        if isinstance(output, Tensor) and ops.is_floating_point(output):
+            return True
+        if isinstance(output, (list, tuple)):
+            for i in output:
+                if isinstance(i, Tensor) and ops.is_floating_point(i):
+                    return True
+        return False
+
+    @abstractmethod
+    def handle(self, params: HandlerParams) -> Any:
+        pass

msprobe/mindspore/free_benchmark/handler/check_handler.py

@@ -1,41 +1,56 @@
-from typing import Any
-from dataclasses import asdict
-
-from mindspore import Tensor, ops
-
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.free_benchmark.common.config import Config
-from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler
-from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
-from msprobe.mindspore.free_benchmark.common.utils import make_unequal_row
-from msprobe.core.data_dump.json_writer import DataWriter
-
-
-class CheckHandler(BaseHandler):
-
-    def npu_compare_and_save(self, original_output, fuzzed_output, params: HandlerParams, output_index=None):
-        is_consistent, ratio = self.npu_compare(original_output, fuzzed_output)
-        params.is_consistent = params.is_consistent and is_consistent
-        if not is_consistent:
-            row = make_unequal_row(self.api_name, params, ratio, output_index)
-            data_dict = asdict(row)
-            DataWriter.write_data_to_csv(
-                data_dict.values(),
-                data_dict.keys(),
-                Config.dump_path
-            )
-            logger.error(f"{self.api_name} is not consistent")
-
-    def handle(self, params: HandlerParams) -> Any:
-        try:
-            if not self.is_float_tensor(params.fuzzed_result):
-                return params.original_result
-            if isinstance(params.fuzzed_result, Tensor):
-                self.npu_compare_and_save(params.original_result, params.fuzzed_result, params)
-            elif isinstance(params.fuzzed_result, (list, tuple)):
-                for i, item in enumerate(params.original_result):
-                    if ops.is_tensor(item) and ops.is_floating_point(item):
-                        self.npu_compare_and_save(item, params.fuzzed_result[i], params, output_index=i)
-        except Exception as e:
-            logger.error(str(e))
-        return params.original_result
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import asdict
+from typing import Any
+
+from mindspore import Tensor, ops
+
+from msprobe.core.data_dump.json_writer import DataWriter
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.free_benchmark.common.config import Config
+from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
+from msprobe.mindspore.free_benchmark.common.utils import make_unequal_row
+from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler
+
+
+class CheckHandler(BaseHandler):
+
+    def npu_compare_and_save(self, original_output, fuzzed_output, params: HandlerParams, output_index=None):
+        is_consistent, ratio = self.npu_compare(original_output, fuzzed_output)
+        params.is_consistent = params.is_consistent and is_consistent
+        if not is_consistent:
+            row = make_unequal_row(self.api_name, params, ratio, output_index)
+            data_dict = asdict(row)
+            DataWriter.write_data_to_csv(
+                data_dict.values(),
+                data_dict.keys(),
+                Config.dump_path
+            )
+            logger.error(f"{self.api_name} is not consistent")
+
+    def handle(self, params: HandlerParams) -> Any:
+        try:
+            if not self.is_float_tensor(params.fuzzed_result):
+                return params.original_result
+            if isinstance(params.fuzzed_result, Tensor):
+                self.npu_compare_and_save(params.original_result, params.fuzzed_result, params)
+            elif isinstance(params.fuzzed_result, (list, tuple)):
+                for i, item in enumerate(params.original_result):
+                    if ops.is_tensor(item) and ops.is_floating_point(item):
+                        self.npu_compare_and_save(item, params.fuzzed_result[i], params, output_index=i)
+        except Exception as e:
+            logger.error(str(e))
+        return params.original_result