mindstudio-probe 1.1.1__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/precision_standard/standard_config.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from msprobe.core.common.const import CompareConst
+
+
+class StandardConfig:
+    """
+    Standard configuration class for managing precision and comparison thresholds.
+
+    This class provides a centralized way to manage the small value thresholds, absolute tolerances,
+    and relative tolerances (rtol) used in precision comparisons. It allows for different thresholds
+    based on the data type, with default values provided for common data types.
+
+    Attributes:
+        _small_value (dict): A dictionary mapping data types to their corresponding small value thresholds.
+        _small_value_atol (dict): A dictionary mapping data types to their corresponding absolute tolerances.
+        _rtol (dict): A dictionary mapping data types to their corresponding relative tolerances.
+
+    Methods:
+        get_small_value(dtype): Retrieves the small value threshold for the given data type.
+        get_small_value_atol(dtype): Retrieves the absolute tolerance for the given data type.
+        get_rtol(dtype): Retrieves the relative tolerance for the given data type.
+
+    Example:
+        >>> small_value = StandardConfig.get_small_value(torch.float32)
+        >>> atol = StandardConfig.get_small_value_atol(torch.float32)
+        >>> rtol = StandardConfig.get_rtol(torch.float32)
+        >>> print(small_value, atol, rtol)
+        1e-6 1e-9 1e-6
+
+    Note:
+        The data type is expected to be a PyTorch data type. If the data type is not found in the dictionary,
+        the default value is returned.
+
+    See Also:
+        torch.dtype: PyTorch data types.
+    """
+    _small_value = {
+        torch.float16: 2**-10,
+        torch.bfloat16: 2**-10,
+        torch.float32: 2**-20,
+        "default": 2**-20
+    }
+    _threshold_small_value_atol = {
+        torch.float16: 2**-16,
+        torch.bfloat16: 1e-16,
+        torch.float32: 2**-30,
+        "default": 2**-30
+    }
+    _benchmark_small_value_atol = {
+        torch.float16: 1e-16,
+        torch.bfloat16: 1e-16,
+        torch.float32: 2**-30,
+        "default": 2**-30
+    }
+    _rtol = {
+        torch.float16: 2**-10,
+        torch.bfloat16: 2**-8,
+        torch.float32: 2**-20,
+        "default": 2**-20
+    }
+    _accumulative_error_bound = {
+        torch.float16: 2**-8,
+        torch.bfloat16: 2**-7,
+        torch.float32: 2**-11,
+        "default": 2**-11
+    }
+    _small_value_threshold = {
+        'error_threshold': 2,
+        'warning_threshold': 1,
+        "default": 1
+    }
+    _rmse_threshold = {
+        'error_threshold': 2,
+        'warning_threshold': 1,
+        "default": 1
+    }
+    _max_rel_err_threshold = {
+        'error_threshold': 10,
+        'warning_threshold': 1,
+        "default": 1
+    }
+    _mean_rel_err_threshold = {
+        'error_threshold': 2,
+        'warning_threshold': 1,
+        "default": 1
+    }
+    _eb_threshold = {
+        'error_threshold': 2,
+        'warning_threshold': 1,
+        "default": 1
+    }
+    _minmum_err = {
+        'torch.float16': 2**-11,
+        'torch.bfloat16': 2**-8,
+        'torch.float32': 2**-14,
+        'default': 2**-14
+    }
+    _accumulative_error_eb_threshold = {
+        'torch.float16': 2**-20,
+        'torch.bfloat16': 2**-7,
+        'torch.float32': 2**-14,
+        'default': 2**-14
+    }
+    
+    _fp32_mean_ulp_err_threshold = 64
+    ulp_err_proportion_ratio = 1
+    _fp32_ulp_err_proportion = 0.05
+    _fp16_ulp_err_proportion = 0.001
+    _special_samll_value = 1
+    
+    @classmethod
+    def get_small_value(cls, dtype, standard):
+        if standard == CompareConst.ACCUMULATIVE_ERROR_COMPARE:
+            return cls._special_samll_value
+        return cls._small_value.get(dtype, cls._small_value["default"])
+    
+    @classmethod
+    def get_small_value_atol(cls, dtype, standard):
+        standard_dict = {
+            CompareConst.ABSOLUTE_THRESHOLD: cls._threshold_small_value_atol,
+            CompareConst.BENCHMARK: cls._benchmark_small_value_atol
+        }
+        small_value_atol_standard = standard_dict.get(standard, cls._benchmark_small_value_atol)
+        return small_value_atol_standard.get(dtype, small_value_atol_standard["default"])
+    
+    @classmethod
+    def get_rtol(cls, dtype):
+        return cls._rtol.get(dtype, cls._rtol["default"])
+    
+    @classmethod
+    def get_small_value_threshold(cls, threshold_type):
+        return cls._small_value_threshold.get(threshold_type, cls._small_value_threshold["default"])
+    
+    @classmethod
+    def get_rmse_threshold(cls, threshold_type):
+        return cls._rmse_threshold.get(threshold_type, cls._rmse_threshold["default"])
+    
+    @classmethod
+    def get_max_rel_err_threshold(cls, threshold_type):
+        return cls._max_rel_err_threshold.get(threshold_type, cls._max_rel_err_threshold["default"])
+    
+    @classmethod
+    def get_mean_rel_err_threshold(cls, threshold_type):
+        return cls._mean_rel_err_threshold.get(threshold_type, cls._mean_rel_err_threshold["default"])
+    
+    @classmethod
+    def get_eb_threshold(cls, threshold_type):
+        return cls._eb_threshold.get(threshold_type, cls._eb_threshold["default"])
+    
+    @classmethod
+    def get_benchmark_threshold(cls, metric):
+        metric_threshold_functions = {
+            'small_value': StandardConfig.get_small_value_threshold,
+            'rmse': StandardConfig.get_rmse_threshold,
+            'max_rel_err': StandardConfig.get_max_rel_err_threshold,
+            'mean_rel_err': StandardConfig.get_mean_rel_err_threshold,
+            'eb': StandardConfig.get_eb_threshold
+        }
+        
+        threshold_func = metric_threshold_functions.get(metric)
+        return threshold_func('error_threshold')
+
+    @classmethod
+    def get_fp32_mean_ulp_err_threshold(cls):
+        return cls._fp32_mean_ulp_err_threshold
+
+    @classmethod
+    def get_ulp_err_proportion_ratio_threshold(cls):
+        return cls.ulp_err_proportion_ratio
+
+    @classmethod
+    def get_fp32_ulp_err_proportion_threshold(cls):
+        return cls._fp32_ulp_err_proportion
+
+    @classmethod
+    def get_fp16_ulp_err_proportion_threshold(cls):
+        return cls._fp16_ulp_err_proportion
+    
+    @classmethod
+    def get_ulp_threshold(cls, dtype):
+        ulp_err_proportion_ratio_threshold = StandardConfig.get_ulp_err_proportion_ratio_threshold()
+        if dtype == torch.float32:
+            mean_ulp_err_threshold = StandardConfig.get_fp32_mean_ulp_err_threshold()
+            ulp_err_proportion_threshold = StandardConfig.get_fp32_ulp_err_proportion_threshold()
+            return mean_ulp_err_threshold, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold
+        else:
+            ulp_err_proportion_threshold = StandardConfig.get_fp16_ulp_err_proportion_threshold()
+            return None, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold
+
+    @classmethod
+    def get_minmum_err(cls, dtype):
+        return cls._minmum_err.get(dtype, cls._minmum_err["default"])
+    
+    @classmethod
+    def get_accumulative_error_bound(cls, dtype):
+        return cls._accumulative_error_bound.get(dtype, cls._accumulative_error_bound["default"])
+    
+    @classmethod
+    def get_accumulative_error_eb_threshold(cls, dtype):
+        return cls._accumulative_error_eb_threshold.get(dtype, cls._accumulative_error_eb_threshold["default"])

msprobe/pytorch/api_accuracy_checker/precision_standard/standard_register.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import absolute_standard_api, binary_standard_api, \
+    ulp_standard_api, thousandth_standard_api, accumulative_error_standard_api, BINARY_COMPARE_UNSUPPORT_LIST
+from msprobe.core.common.const import CompareConst
+
+
+class StandardRegistry:
+    """
+    Registry class for managing comparison standards and functions.
+
+    This class provides a centralized registry for different comparison standards and their corresponding functions.
+    It allows for dynamic registration of comparison functions based on the standard category.
+
+    Attributes:
+        comparison_functions (dict): A dictionary mapping standard categories to their corresponding comparison 
+        functions.
+        standard_categories (dict): A dictionary mapping standard names to their corresponding API categories.
+
+    Methods:
+        _get_standard_category(api_name, dtype): Determines the standard category for a given API name and data type.
+        register(standard, func): Registers a comparison function for a given standard category.
+        get_comparison_function(api_name, dtype): Retrieves the comparison function for a given API name and data type.
+
+    Note:
+        The data type is used to determine the standard category if it is not supported by binary comparison.
+        If the API name is not found in any standard category, it defaults to the 'benchmark' category.
+
+    See Also:
+        BaseCompare: The base class for comparison classes.
+    """
+    def __init__(self):
+        self.comparison_functions = {}
+        self.api_standard_function_map = {
+            CompareConst.ABSOLUTE_THRESHOLD: absolute_standard_api,
+            CompareConst.BINARY_CONSISTENCY: binary_standard_api,
+            CompareConst.ULP_COMPARE: ulp_standard_api,
+            CompareConst.THOUSANDTH_STANDARD: thousandth_standard_api,
+            CompareConst.ACCUMULATIVE_ERROR_COMPARE: accumulative_error_standard_api
+        }
+
+    def register(self, standard: str, func: Callable) -> None:
+        """
+        Registers a comparison function for a given standard category.
+
+        Args:
+            standard (str): The name of the standard category.
+            func (Callable): The comparison function to be registered.
+
+        Raises:
+            ValueError: If the standard category is not supported.
+        """
+        if not callable(func):
+            raise ValueError("The function to be registered must be callable.")
+        self.comparison_functions[standard] = func
+
+    def get_comparison_function(self, api_name, dtype=None):
+        standard = self._get_standard_category(api_name, dtype)
+        return self.comparison_functions.get(standard)
+
+    def _get_standard_category(self, api_name, dtype=None):
+        """
+        Determines the standard category for a given API name and data type.
+
+        This method checks if the provided data type is supported for binary comparison.
+        If it is, the method returns 'binary_consistency'. Otherwise, it iterates over the
+        api_standard_function_map to find a matching category for the API name.
+
+        Args:
+            api_name (str): The name of the API for which to determine the standard category.
+            dtype (type, optional): The data type to check against the BINARY_COMPARE_UNSUPPORT_LIST. Defaults to None.
+
+        Returns:
+            str: The name of the standard category that matches the API name and data type, or 'benchmark' if no match 
+            is found.
+
+        Note:
+        This method assumes that the api_standard_function_map is properly populated with standard categories and 
+        their corresponding API functions.
+        The BINARY_COMPARE_UNSUPPORT_LIST should be defined and contain all data types that are not supported for 
+        binary comparison.
+        """
+        if dtype and dtype not in BINARY_COMPARE_UNSUPPORT_LIST:
+            return CompareConst.BINARY_CONSISTENCY
+        for name, category in self.api_standard_function_map.items():
+            if api_name in category:
+                return name
+        return CompareConst.BENCHMARK

msprobe/pytorch/api_accuracy_checker/precision_standard/thousandth_standard.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rel_err_ratio
+from msprobe.core.common.const import CompareConst
+from msprobe.pytorch.api_accuracy_checker.precision_standard.base_standard import BaseCompare
+
+
+class ThousandthStdCompare(BaseCompare):
+    """
+    Thousandth standard comparison class for calculating accuracy metrics.
+    
+    A subclass of BaseCompare, specifically designed to compare the relative error 
+    between benchmark and device outputs, focusing on errors within a thousandth (0.001) threshold.
+
+    Attributes:
+        rel_err_orign (float or array-like): The original relative error values to be compared.
+        compare_column (object): An object to store and update comparison metrics.
+
+    Methods:
+        _compute_metrics(): Computes the relative error metrics, specifically the thousandth error ratio.
+    """
+    def __init__(self, input_data):
+        self.rel_err_orign = input_data.rel_err_orign
+        self.compare_column = input_data.compare_column
+
+    def _pre_compare(self):
+        pass
+
+    def _compute_metrics(self):
+        """
+        Computes the relative error metrics for the comparison, specifically focusing on errors within a thousandth 
+        (0.001) threshold.
+
+        This method calculates the proportion of relative errors that are within the thousandth threshold.
+        It uses the `get_rel_err_ratio` function to determine the ratio of relative errors that are less than or 
+        equal to the
+        specified threshold defined in `CompareConst.THOUSAND_RATIO_THRESHOLD`.
+
+        Returns:
+        dict: A dictionary containing the computed relative error metric.
+            The dictionary has the following key:
+            - 'rel_err_thousandth': The proportion of relative errors within the thousandth threshold.
+        """
+        rel_err_thousandth, _ = get_rel_err_ratio(self.rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD)
+
+        return {
+            'rel_err_thousandth': rel_err_thousandth
+        }

msprobe/pytorch/api_accuracy_checker/precision_standard/ulp_compare.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+import numpy as np
+import torch
+
+from msprobe.pytorch.api_accuracy_checker.precision_standard.standard_config import StandardConfig
+from msprobe.pytorch.api_accuracy_checker.precision_standard.base_standard import BaseCompare, BasePrecisionCompare
+from msprobe.core.common.const import Const, CompareConst
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import calc_ratio, get_ulp_err
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ApiPrecisionCompareColumn, check_inf_or_nan, \
+    is_inf_or_nan
+
+
+UlpInfNanConsistency = namedtuple('UlpInfNanConsistency', ['mean_ulp_err_inf_nan_consistency', 
+                                             'ulp_err_proportion_ratio_inf_nan_consistency'])
+
+
+class UlpCompare(BaseCompare):
+    """
+    Ulp compare comparison class for calculating accuracy metrics.
+
+    Attributes:
+        bench_output (array-like): The benchmark output values.
+        device_output (array-like): The device output values.
+        dtype (torch.dtype): The data type of the outputs (e.g., torch.float32 or torch.float16).
+        ulp_err (array-like): The ULP errors calculated from the benchmark and device outputs.
+
+    Methods:
+        _stat_max_ulp_err(ulp_err): Calculates the maximum ULP error.
+        _stat_mean_ulp_err(ulp_err): Calculates the mean ULP error.
+        _stat_ulp_error_proportion(ulp_err): Calculates the proportion of ULP errors exceeding a threshold.
+        _pre_compare(): Prepares for comparison by calculating ULP errors.
+        _compute_metrics(): Computes the ULP error metrics.
+    """
+    def __init__(self, input_data):
+        super(UlpCompare, self).__init__(input_data)
+    
+    @staticmethod
+    def _stat_max_ulp_err(ulp_err):
+        return np.max(ulp_err)
+    
+    @staticmethod
+    def _stat_mean_ulp_err(ulp_err):
+        return np.mean(ulp_err)
+    
+    def _stat_ulp_error_proportion(self, ulp_err):
+        if self.dtype == torch.float32:
+            return np.sum(ulp_err > CompareConst.ULP_FLOAT32_THRESHOLD) / self.bench_output.size
+        else:
+            return np.sum(ulp_err > CompareConst.ULP_FLOAT16_THRESHOLD) / self.bench_output.size
+    
+    def _pre_compare(self):
+        self.ulp_err = get_ulp_err(self.bench_output, self.device_output, self.dtype)
+    
+    def _compute_metrics(self):
+        """
+        Computes the ULP error metrics for the comparison.
+
+        This method calculates three key metrics:
+        1. Maximum ULP error: The maximum difference in ULPs between the benchmark and device outputs.
+        2. Mean ULP error: The average difference in ULPs between the benchmark and device outputs.
+        3. ULP error proportion: The proportion of ULP errors that exceed a certain threshold.
+
+        Args:
+        None (this method uses instance variables)
+
+        Returns:
+        dict: A dictionary containing the computed ULP error metrics.
+            The dictionary has the following keys:
+            - "max_ulp_error": The maximum ULP error.
+            - "mean_ulp_error": The mean ULP error.
+            - "ulp_error_proportion": The proportion of ULP errors exceeding the threshold.
+        """
+        max_ulp_error = self._stat_max_ulp_err(self.ulp_err)
+        mean_ulp_error = self._stat_mean_ulp_err(self.ulp_err)
+        
+        ulp_error_proportion = self._stat_ulp_error_proportion(self.ulp_err)
+        
+        return {
+            "max_ulp_error": max_ulp_error,
+            "mean_ulp_error": mean_ulp_error,
+            "ulp_error_proportion": ulp_error_proportion
+        }
+
+
+class UlpPrecisionCompare(BasePrecisionCompare):
+    def __init__(self, input_data):
+        super().__init__(input_data)
+        self.compare_algorithm = CompareConst.ULP_COMPARE_ALGORITHM_NAME
+
+    @staticmethod
+    def _compute_ulp_err_proportion_ratio(npu_value, gpu_value, dtype):
+        column_name = ApiPrecisionCompareColumn.ULP_ERR_PROPORTION
+        if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value):
+            return check_inf_or_nan(npu_value, gpu_value, column_name)
+        else:
+            return calc_ratio(npu_value, gpu_value, dtype), True, ""
+
+    def _compute_mean_ulp_err(self):
+        column_name = ApiPrecisionCompareColumn.MEAN_ULP_ERR
+        npu_value, gpu_value = self._get_and_convert_values(column_name)
+        if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value):
+            _, mean_ulp_err_inf_nan_consistency, message = check_inf_or_nan(npu_value, gpu_value, column_name)
+            return npu_value, mean_ulp_err_inf_nan_consistency, message
+        else:
+            return npu_value, True, ""
+    
+    def _compute_ulp_err_proportion(self):
+        column_name = ApiPrecisionCompareColumn.ULP_ERR_PROPORTION
+        npu_value, gpu_value = self._get_and_convert_values(column_name)
+        return npu_value, gpu_value
+        
+    def _get_status(self, metrics, inf_nan_consistency):
+        ulp_inf_nan_consistency = inf_nan_consistency.mean_ulp_err_inf_nan_consistency and \
+                                  inf_nan_consistency.ulp_err_proportion_ratio_inf_nan_consistency  
+
+        if not ulp_inf_nan_consistency:
+            status_dict = {
+                CompareConst.ULP_ERR_STATUS: CompareConst.ERROR
+            }
+            compare_result = CompareConst.ERROR
+            metrics[CompareConst.COMPARE_MESSAGE] = metrics.get(CompareConst.COMPARE_MESSAGE, "") + \
+                "ERROR: ULP误差不满足标准\n"
+            metrics.update({CompareConst.COMPARE_RESULT: compare_result})
+            return metrics
+        
+        dtype = self.row_npu.get(ApiPrecisionCompareColumn.DEVICE_DTYPE)
+        mean_ulp_err = metrics.get(CompareConst.MEAN_ULP_ERR)
+        ulp_err_proportion = metrics.get(CompareConst.ULP_ERR_PROPORTION)
+        ulp_err_proportion_ratio = metrics.get(CompareConst.ULP_ERR_PROPORTION_RATIO)
+        if dtype == Const.TORCH_FLOAT32:
+            status, final_message = \
+                self._get_fp32_ulp_err_status(mean_ulp_err, ulp_err_proportion, ulp_err_proportion_ratio)
+        else:
+            status, final_message = \
+                self._get_fp16_ulp_err_status(ulp_err_proportion, ulp_err_proportion_ratio)
+        metrics[CompareConst.COMPARE_MESSAGE] = metrics.get(CompareConst.COMPARE_MESSAGE, "") + final_message
+
+        status_dict = {
+            CompareConst.ULP_ERR_STATUS: status
+        }
+        compare_result = status
+        metrics.update(status_dict)
+        metrics.update({CompareConst.COMPARE_RESULT: compare_result})
+        return metrics
+
+    def _get_fp32_ulp_err_status(self, mean_ulp_err, ulp_err_proportion, ulp_err_proportion_ratio):
+        mean_ulp_err_threshold, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold = \
+                                                        StandardConfig.get_ulp_threshold(torch.float32)
+        if mean_ulp_err < mean_ulp_err_threshold:
+            return CompareConst.PASS, ""
+        elif ulp_err_proportion < ulp_err_proportion_threshold:
+            return CompareConst.PASS, ""
+        elif ulp_err_proportion_ratio < ulp_err_proportion_ratio_threshold:
+            return CompareConst.PASS, ""
+        compare_message = "ERROR: ULP误差不满足标准\n"
+        return CompareConst.ERROR, compare_message
+        
+    def _get_fp16_ulp_err_status(self, ulp_err_proportion, ulp_err_proportion_ratio):
+        _, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold = \
+                                                        StandardConfig.get_ulp_threshold(torch.float16)
+        if ulp_err_proportion < ulp_err_proportion_threshold:
+            return CompareConst.PASS, ""
+        elif ulp_err_proportion_ratio < ulp_err_proportion_ratio_threshold:
+            return CompareConst.PASS, ""
+        compare_message = "ERROR: ULP误差不满足标准\n"
+        return CompareConst.ERROR, compare_message
+
+    def _compute_ratio(self):
+        compare_message = ""
+        mean_ulp_err, mean_ulp_err_inf_nan_consistency, mean_ulp_err_message = self._compute_mean_ulp_err()
+        compare_message += mean_ulp_err_message
+        npu_ulp_err_proportion, gpu_ulp_err_proportion = self._compute_ulp_err_proportion()
+        ulp_err_proportion_ratio, ulp_err_proportion_ratio_inf_nan_consistency, ulp_err_proportion_ratio_message = \
+            self._compute_ulp_err_proportion_ratio(npu_ulp_err_proportion, gpu_ulp_err_proportion, str(self.dtype))
+        compare_message += ulp_err_proportion_ratio_message
+        metrics = {
+            CompareConst.MEAN_ULP_ERR: mean_ulp_err,
+            CompareConst.ULP_ERR_PROPORTION: npu_ulp_err_proportion,
+            CompareConst.ULP_ERR_PROPORTION_RATIO: ulp_err_proportion_ratio,
+            CompareConst.COMPARE_MESSAGE: compare_message
+        }
+        return metrics, UlpInfNanConsistency(mean_ulp_err_inf_nan_consistency, 
+                                             ulp_err_proportion_ratio_inf_nan_consistency)

msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py

@@ -28,6 +28,7 @@ from msprobe.pytorch.common.log import logger
 from msprobe.pytorch.common.utils import load_pt
 from msprobe.core.common.const import Const, FileCheckConst, CompareConst
 
+
 TORCH_TYPE = ["torch.device", "torch.dtype"]
 TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"]
 FLOAT_TYPE = [
@@ -310,6 +311,19 @@ def gen_kwargs(api_info, api_name, convert_type=None, real_data_path=None):
             kwargs_params[key] = gen_list_kwargs(value, api_name, convert_type, real_data_path)
         elif value is None:
             kwargs_params[key] = None
+        elif key == 'atten_mask' and api_name == 'npu_fusion_attention':
+            sparse_mode = kwargs_params.get('sparse_mode', {})
+            if isinstance(sparse_mode, dict):
+                sparse_mode_value = sparse_mode.get('value', 0)
+            elif isinstance(sparse_mode, int):
+                sparse_mode_value = sparse_mode
+            else:
+                msg = f'The sparse_mode value is not int or dict, but {type(sparse_mode)}'
+                raise CompareException(CompareException.INVALID_PARAM_ERROR, msg)
+            if sparse_mode_value in Const.FA_SPECIAL_SPARSE_MODE:
+                kwargs_params[key] = gen_atten_mask(value, convert_type, real_data_path)
+            else:
+                kwargs_params[key] = gen_data(value, api_name, True, convert_type, real_data_path)
         elif value.get('type') in TENSOR_DATA_LIST or value.get('type').startswith("numpy"):
             kwargs_params[key] = gen_data(value, api_name, True, convert_type, real_data_path)
         elif value.get('type') in TORCH_TYPE:
@@ -319,6 +333,30 @@ def gen_kwargs(api_info, api_name, convert_type=None, real_data_path=None):
     return kwargs_params
 
 
+def gen_atten_mask(info, convert_type, real_data_path):
+    """
+    Function Description:
+        Based on API basic information, generate input parameters: atten_mask, for API forward running
+    Parameter:
+        info: API basic information. Dict
+        convert_type: convert ori_type to dist_type flag.
+        real_data_path: the root directory for storing real data.
+    """
+    check_object_type(info, dict)
+    data_type = info.get('type')
+    data_path = info.get('datapath', info.get('data_name'))
+    data_path = get_full_data_path(data_path, real_data_path)
+    data = None
+    if data_type in TENSOR_DATA_LIST:
+        if data_path:
+            data = gen_real_tensor(data_path, convert_type)
+        else:
+            # 生成一个2048x2048的三角矩阵，对角线为1，其余为0
+            # 这是npu_fusion_attention的sparse_mode为[2, 3, 4]时，atten_mask的shape
+            data = torch.triu(torch.ones([2048, 2048]), diagonal=1).to(torch.bool)
+    return data
+
+
 def gen_torch_kwargs(kwargs_params, key, value):
     if value.get('type') != "torch.device":
         module_name, attribute_name = get_module_and_atttribute_name(value.get('value'))
@@ -346,6 +384,23 @@ def gen_list_kwargs(kwargs_item_value, api_name, convert_type, real_data_path=No
     return kwargs_item_result
 
 
+def get_output_dtype(api_info):
+    """
+    Function Description:
+        Based on API basic information, get the output data dtype
+    Parameter:
+        api_info: API basic information. Dict
+    """
+    output_dtype = None
+    output_info = api_info.get(Const.OUTPUT)
+    if output_info and isinstance(output_info[0], dict):
+        output_str_dtype = output_info[0].get(Const.DTYPE)
+        if output_str_dtype in Const.TORCH_FLOAT_DTYPE:
+            module_name, attribute_name = get_module_and_atttribute_name(output_str_dtype)
+            output_dtype = get_attribute(module_name, attribute_name)
+    return output_dtype
+
+
 def gen_api_params(api_info, api_name, need_grad=True, convert_type=None, real_data_path=None):
     """
     Function Description:
@@ -372,4 +427,5 @@ def gen_api_params(api_info, api_name, need_grad=True, convert_type=None, real_d
     else:
         logger.warning(f'Warning: No args in {api_info} ')
         args_params = []
-    return args_params, kwargs_params
+    output_dtype = get_output_dtype(api_info)
+    return args_params, kwargs_params, output_dtype