mindstudio-probe 1.1.1__py3-none-any.whl → 1.2.2__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/precision_standard/absolute_threshold.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import check_inf_nan_value, check_norm_value, \
+    check_small_value
+from msprobe.pytorch.api_accuracy_checker.precision_standard.base_standard import BaseCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.standard_config import StandardConfig
+from msprobe.core.common.const import CompareConst
+
+
+
+class AbsolutethdCompare(BaseCompare):
+    """
+    Absolute threshold compare class.
+
+    This class is used to compare the absolute threshold of benchmark outputs and device outputs.
+    It calculates various metrics such as inf_nan_error_ratio, rel_err_ratio, and abs_err_ratio
+    to determine the accuracy of the device output compared to the benchmark output.
+
+    Attributes:
+        bench_output (np.ndarray): The output from the benchmark.
+        device_output (np.ndarray): The output from the device.
+        dtype (torch.dtype): The data type of the outputs.
+        abs_bench (np.ndarray): The absolute value of the benchmark output.
+        abs_bench_with_eps (np.ndarray): The absolute value of the benchmark output with epsilon.
+        both_finite_mask (np.ndarray): A mask indicating where both outputs are finite.
+        inf_nan_mask (np.ndarray): A mask indicating where either output is infinite or NaN.
+        rtol (float): The relative tolerance for comparison.
+        rel_err (np.ndarray): The relative error between the benchmark and device outputs.
+        small_value (float): The small value threshold for comparison.
+        small_value_atol (float): The absolute tolerance for small values.
+        small_value_mask (np.ndarray): A mask indicating where values are small.
+        normal_value_mask (np.ndarray): A mask indicating where values are normal.
+
+    Methods:
+        _get_rtol(): Gets the relative tolerance based on the data type.
+        _get_rel_err(abs_bench_with_eps): Calculates the relative error.
+        _get_normal_value_mask(small_value_mask): Gets the mask for normal values.
+        _pre_compare(): Prepares the comparison by calculating various metrics.
+        _compute_metrics(): Computes the comparison metrics.
+
+    Note:
+        This class assumes that the input data is a dictionary containing 'bench_output', 'device_output', 
+        'compare_column' and 'dtype'.
+        The 'dtype' should be a PyTorch data type.
+
+    See Also:
+        BaseCompare: The base class for comparison classes.
+        StandardConfig: The class containing standard configuration values.
+    """
+    def __init__(self, input_data):
+        super(AbsolutethdCompare, self).__init__(input_data)
+        self.compare_algorithm = CompareConst.ABSOLUTE_THRESHOLD
+
+    def _get_rtol(self):
+        return StandardConfig.get_rtol(self.dtype)
+
+    def _pre_compare(self):
+        """
+        Prepares the comparison by calculating various metrics.
+
+        This method performs the following steps:
+            1. Calculates the absolute benchmark values and their epsilon-adjusted versions.
+            2. Determines masks for finite and infinite/NaN values in the outputs.
+            3. Computes the absolute error between benchmark and device outputs.
+            4. Retrieves the relative tolerance based on the data type.
+            5. Calculates the relative error using the absolute error and epsilon-adjusted benchmark values.
+            6. Determines the small value threshold and its absolute tolerance.
+            7. Creates a mask for small values based on the benchmark values and finite mask.
+            8. Creates a mask for normal values by excluding small values from the finite mask.
+        """
+        self.abs_bench, self.abs_bench_with_eps = self.stat_abs_bench_with_eps()
+        self.both_finite_mask, self.inf_nan_mask = self.stat_finite_and_infinite_mask()
+        self.abs_err = self.stat_abs_error()
+        self.rtol = self._get_rtol()
+        self.rel_err = self._get_rel_err(self.abs_err, self.abs_bench_with_eps)
+        self.small_value, self.small_value_atol = self.get_small_value_threshold()
+        self.small_value_mask = self.stat_small_value_mask(self.abs_bench, self.both_finite_mask, self.small_value)
+        self.normal_value_mask = self._get_normal_value_mask(self.both_finite_mask, self.small_value_mask)
+
+    def _compute_metrics(self):
+        inf_nan_error_ratio = check_inf_nan_value(self.inf_nan_mask, self.bench_output, self.device_output, self.dtype,
+                                                  self.rtol)
+        rel_err_ratio = check_norm_value(self.normal_value_mask, self.rel_err, self.rtol)
+        abs_err_ratio = check_small_value(self.abs_err, self.small_value_mask, self.small_value_atol)
+        return {
+            "inf_nan_error_ratio": inf_nan_error_ratio,
+            "rel_err_ratio": rel_err_ratio,
+            "abs_err_ratio": abs_err_ratio
+        }

msprobe/pytorch/api_accuracy_checker/precision_standard/accumulative_error_compare.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import check_inf_nan_value, check_norm_value, \
+    check_small_value, get_error_balance
+from msprobe.pytorch.api_accuracy_checker.precision_standard.base_standard import BaseCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.standard_config import StandardConfig
+from msprobe.core.common.const import CompareConst
+
+
+class AccumulativeErrorCompare(BaseCompare):
+    """
+    Absolute threshold compare class.
+
+    This class is used to compare the absolute threshold of benchmark outputs and device outputs.
+    It calculates various metrics such as inf_nan_error_ratio, rel_err_ratio, and abs_err_ratio
+    to determine the accuracy of the device output compared to the benchmark output.
+
+    Attributes:
+        bench_output (np.ndarray): The output from the benchmark.
+        device_output (np.ndarray): The output from the device.
+        dtype (torch.dtype): The data type of the outputs.
+        abs_bench (np.ndarray): The absolute value of the benchmark output.
+        abs_bench_with_eps (np.ndarray): The absolute value of the benchmark output with epsilon.
+        both_finite_mask (np.ndarray): A mask indicating where both outputs are finite.
+        inf_nan_mask (np.ndarray): A mask indicating where either output is infinite or NaN.
+        bound (float): The tolerance for comparison.
+        rel_err (np.ndarray): The relative error between the benchmark and device outputs.
+        small_value (float): The small value threshold for comparison.
+        small_value_atol (float): The absolute tolerance for small values.
+        small_value_mask (np.ndarray): A mask indicating where values are small.
+        normal_value_mask (np.ndarray): A mask indicating where values are normal.
+
+    Methods:
+        _get_rtol(): Gets the relative tolerance based on the data type.
+        _get_rel_err(abs_bench_with_eps): Calculates the relative error.
+        _get_normal_value_mask(small_value_mask): Gets the mask for normal values.
+        _pre_compare(): Prepares the comparison by calculating various metrics.
+        _compute_metrics(): Computes the comparison metrics.
+
+    Note:
+        This class assumes that the input data is a dictionary containing 'bench_output', 'device_output', 
+        'compare_column' and 'dtype'.
+        The 'dtype' should be a PyTorch data type.
+
+    See Also:
+        BaseCompare: The base class for comparison classes.
+        StandardConfig: The class containing standard configuration values.
+    """
+    def __init__(self, input_data):
+        super(AccumulativeErrorCompare, self).__init__(input_data)
+        self.compare_algorithm = CompareConst.ACCUMULATIVE_ERROR_COMPARE
+
+    def _get_bound(self):
+        return StandardConfig.get_accumulative_error_bound(self.dtype)
+
+    def _pre_compare(self):
+        """
+        Prepares the comparison by calculating various metrics.
+
+        This method performs the following steps:
+            1. Calculates the absolute benchmark values and their epsilon-adjusted versions.
+            2. Determines masks for finite and infinite/NaN values in the outputs.
+            3. Computes the absolute error between benchmark and device outputs.
+            4. Retrieves the tolerance based on the data type.
+            5. Calculates the relative error using the absolute error and epsilon-adjusted benchmark values.
+            6. Determines the small value threshold and its absolute tolerance.
+            7. Creates a mask for small values based on the benchmark values and finite mask.
+            8. Creates a mask for normal values by excluding small values from the finite mask.
+        """
+        self.abs_bench, self.abs_bench_with_eps = self.stat_abs_bench_with_eps()
+        self.both_finite_mask, self.inf_nan_mask = self.stat_finite_and_infinite_mask()
+        self.abs_err = self.stat_abs_error()
+        self.bound = self._get_bound()
+        self.rel_err = self._get_rel_err(self.abs_err, self.abs_bench_with_eps)
+        self.small_value, self.small_value_atol = self.get_small_value_threshold()
+        self.small_value_mask = self.stat_small_value_mask(self.abs_bench, self.both_finite_mask, self.small_value)
+        self.normal_value_mask = self._get_normal_value_mask(self.both_finite_mask, self.small_value_mask)
+
+    def _compute_metrics(self):
+        inf_nan_error_ratio = check_inf_nan_value(self.inf_nan_mask, self.bench_output, self.device_output, self.dtype,
+                                                  self.bound)
+        rel_err_ratio = check_norm_value(self.normal_value_mask, self.rel_err, self.bound)
+        abs_err_ratio = check_small_value(self.abs_err, self.small_value_mask, self.bound)
+        eb = get_error_balance(self.bench_output, self.device_output)
+        return {
+            "inf_nan_error_ratio": inf_nan_error_ratio,
+            "rel_err_ratio": rel_err_ratio,
+            "abs_err_ratio": abs_err_ratio,
+            "eb": eb
+        }

msprobe/pytorch/api_accuracy_checker/precision_standard/base_standard.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+import numpy as np
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import convert_str_to_float
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_abs_bench_with_eps, get_abs_err, \
+    get_finite_and_infinite_mask, get_small_value_mask
+from msprobe.pytorch.api_accuracy_checker.precision_standard.standard_config import StandardConfig
+
+
+class BaseCompare(ABC):
+    """
+    Base comparison class for benchmarking and device output.
+
+    This class provides a foundation for comparing benchmark outputs with device outputs.
+    It encapsulates the common logic for calculating accuracy metrics and
+    provides a framework for subclasses to implement specific comparison logic.
+
+    Attributes:
+        bench_output (np.ndarray): The output from the benchmark.
+        device_output (np.ndarray): The output from the device.
+        compare_column (object): The column object to store comparison results.
+        dtype (torch.dtype): The data type of the outputs.
+
+    Methods:
+        get_small_value_threshold(): Retrieves the small value threshold for the given data type.
+        stat_abs_bench_with_eps(): Calculates the absolute benchmark output with epsilon.
+        stat_abs_error(): Calculates the absolute error between the benchmark and device outputs.
+        stat_finite_and_infinite_mask(): Generates masks for finite and infinite/NaN values.
+        stat_small_value_mask(abs_bench, both_finite_mask, small_value): Creates a mask for small values.
+        compare(): Performs the comparison and computes metrics.
+        _pre_compare(): Pre-comparison hook for subclass-specific initialization.
+        _compute_metrics(): Computes the comparison metrics.
+        _post_compare(metrics): Post-comparison hook to update comparison results.
+
+    Note:
+        This class assumes that the input data is an instance of InputData containing the benchmark output,
+        device output, comparison column, and data type. Subclasses should implement the _pre_compare,
+        _compute_metrics, and _post_compare methods to provide specific comparison logic.
+
+    See Also:
+        InputData: The class containing input data for comparison.
+        StandardConfig: The class containing standard configuration values.
+    """
+    def __init__(self, input_data):
+        self.bench_output = input_data.bench_output
+        self.device_output = input_data.device_output
+        self.compare_column = input_data.compare_column
+        self.dtype = input_data.dtype
+        self.compare_algorithm = None
+
+    @staticmethod
+    def stat_small_value_mask(abs_bench, both_finite_mask, small_value):
+        small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, small_value)
+        return small_value_mask
+    
+    @staticmethod
+    def _get_rel_err(abs_err, abs_bench_with_eps):
+        rel_err = abs_err / abs_bench_with_eps
+        return rel_err
+    
+    @staticmethod
+    def _get_normal_value_mask(both_finite_mask, small_value_mask):
+        return np.logical_and(both_finite_mask, np.logical_not(small_value_mask))
+
+    @abstractmethod
+    def _pre_compare(self):
+        raise NotImplementedError
+
+    def get_small_value_threshold(self):
+        small_value = StandardConfig.get_small_value(self.dtype, self.compare_algorithm)
+        small_value_atol = StandardConfig.get_small_value_atol(self.dtype, self.compare_algorithm)
+        return small_value, small_value_atol
+    
+    def stat_abs_bench_with_eps(self):
+        abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(self.bench_output, self.dtype)
+        return abs_bench, abs_bench_with_eps
+    
+    def stat_abs_error(self):
+        abs_err = get_abs_err(self.bench_output, self.device_output)
+        return abs_err
+    
+    def stat_finite_and_infinite_mask(self):
+        both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(self.bench_output, self.device_output)
+        return both_finite_mask, inf_nan_mask
+
+    def compare(self):
+        self._pre_compare()
+        metrics = self._compute_metrics()
+        self._post_compare(metrics)
+
+    def _compute_metrics(self):
+        return {}
+    
+    def _post_compare(self, metrics):
+        self.compare_column.update(metrics)
+
+
+class BasePrecisionCompare:
+    def __init__(self, input_data):
+        self.row_npu = input_data.row_npu
+        self.row_gpu = input_data.row_gpu
+        self.dtype = input_data.dtype
+        self.compare_column = input_data.compare_column
+        self.compare_algorithm = None
+    
+    @abstractmethod
+    def _get_status(self, metrics, inf_nan_consistency):
+        pass
+
+    @abstractmethod
+    def _compute_ratio(self):
+        pass
+
+    def compare(self):
+        metrics, inf_nan_consistency = self._compute_ratio()
+        compare_result = self._post_compare(metrics, inf_nan_consistency)
+        return compare_result
+    
+    def _get_and_convert_values(self, column_name):
+        npu_value = self.row_npu.get(column_name)
+        gpu_value = self.row_gpu.get(column_name)
+        if npu_value is None:
+            raise ValueError(f"NPU value for column '{column_name}' is None.")
+        if gpu_value is None:
+            raise ValueError(f"GPU value for column '{column_name}' is None.")
+        npu_value = convert_str_to_float(npu_value)
+        gpu_value = convert_str_to_float(gpu_value)
+        return npu_value, gpu_value
+
+    def _post_compare(self, metrics, inf_nan_consistency):
+        metrics = self._get_status(metrics, inf_nan_consistency)
+        metrics.update({'compare_algorithm': self.compare_algorithm})
+        self.compare_column.update(metrics)
+        compare_result = metrics.get('compare_result')
+        return compare_result

msprobe/pytorch/api_accuracy_checker/precision_standard/benchmark_compare.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from collections import namedtuple
+import numpy as np
+
+from msprobe.pytorch.api_accuracy_checker.precision_standard.standard_config import StandardConfig
+from msprobe.pytorch.api_accuracy_checker.precision_standard.base_standard import BaseCompare, BasePrecisionCompare
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import calc_ratio, get_small_value_err_ratio, get_rel_err, \
+    get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ApiPrecisionCompareColumn, check_inf_or_nan, \
+    is_inf_or_nan
+from msprobe.core.common.const import CompareConst
+
+
+BenchmarkInfNanConsistency = namedtuple('BenchmarkInfNanConsistency', ['small_value_inf_nan_consistency', 
+                                                                           'rmse_inf_nan_consistency', 
+                                                                           'max_rel_inf_nan_consistency', 
+                                                                           'mean_rel_inf_nan_consistency', 
+                                                                           'eb_inf_nan_consistency'])
+
+
+class BenchmarkCompare(BaseCompare):
+    """
+    Benchmark comparison class for calculating accuracy metrics.
+
+    This class is designed to compare the output of a benchmark test with the output of a device.
+    It calculates various metrics such as small value error ratio, RMSE, error balance, max relative error,
+    and mean relative error to assess the accuracy of the device output against the benchmark output.
+
+    Attributes:
+        bench_output (np.ndarray): The output from the benchmark.
+        device_output (np.ndarray): The output from the device.
+        dtype (torch.dtype): The data type of the outputs.
+        abs_bench (np.ndarray): The absolute value of the benchmark output.
+        abs_bench_with_eps (np.ndarray): The absolute value of the benchmark output with epsilon.
+        both_finite_mask (np.ndarray): A mask indicating where both outputs are finite.
+        inf_nan_mask (np.ndarray): A mask indicating where either output is infinite or NaN.
+        abs_err (np.ndarray): The absolute error between the benchmark and device outputs.
+        small_value (float): The small value threshold for comparison.
+        small_value_atol (float): The absolute tolerance for small values.
+        small_value_mask (np.ndarray): A mask indicating where values are small.
+        rel_err (np.ndarray): The relative error between the benchmark and device outputs.
+        abs_err_greater_mask (np.ndarray): A mask indicating where absolute error is greater than the small value 
+        tolerance.
+
+    Methods:
+        _get_abs_err_greater_mask(small_value_atol): Calculates a mask where absolute error is greater than the small 
+        value tolerance.
+        _compute_rel_err(): Computes the relative error between the benchmark and device outputs.
+        _pre_compare(): Prepares the comparison by calculating various metrics.
+        _compute_metrics(): Computes the accuracy metrics.
+
+    Note:
+        This class assumes that the input data is a dictionary containing 'bench_output', 'device_output', 
+        'compare_column' and 'dtype'. 
+        The data type should be a PyTorch data type.
+
+    See Also:
+        BaseCompare: The base class for comparison classes.
+        InputData: The class containing input data for comparison.
+    """
+
+    def __init__(self, input_data):
+        super(BenchmarkCompare, self).__init__(input_data)
+        self.compare_algorithm = CompareConst.BENCHMARK
+
+    def _get_abs_err_greater_mask(self, small_value_atol):
+        abs_err_greater_mask = np.greater(self.abs_err, small_value_atol)
+        return abs_err_greater_mask
+    
+    def _compute_rel_err(self):
+        rel_err = get_rel_err(self.abs_err, self.abs_bench_with_eps, self.small_value_mask, self.inf_nan_mask)
+        return rel_err
+    
+    def _pre_compare(self):
+        self.abs_bench, self.abs_bench_with_eps = self.stat_abs_bench_with_eps()
+        self.both_finite_mask, self.inf_nan_mask = self.stat_finite_and_infinite_mask()
+        self.abs_err = self.stat_abs_error()
+        self.small_value, self.small_value_atol = self.get_small_value_threshold()
+        self.small_value_mask = self.stat_small_value_mask(self.abs_bench, self.both_finite_mask, self.small_value)
+        self.rel_err = self._compute_rel_err()
+        self.abs_err_greater_mask = self._get_abs_err_greater_mask(self.small_value_atol)
+
+    def _compute_metrics(self):
+        """
+        Computes a comprehensive set of error metrics for the comparison between benchmark and device outputs.
+
+        This method calculates five key metrics:
+        1. Small Value Error Ratio: The proportion of errors associated with small values.
+        2. Root Mean Square Error (RMSE): The square root of the mean of the squared errors.
+        3. Error Balance (EB): A measure of the balance between the errors in the benchmark and device outputs.
+        4. Maximum Relative Error: The maximum relative error between the benchmark and device outputs.
+        5. Mean Relative Error: The mean relative error between the benchmark and device outputs.
+        
+        Returns:
+        dict: A dictionary containing the computed error metrics.
+            The dictionary has the following keys:
+            - "small_value_err_ratio": The proportion of errors associated with small values.
+            - "max_rel_error": The maximum relative error.
+            - "mean_rel_error": The mean relative error.
+            - "rmse": The root mean square error.
+            - "eb": The error balance.
+        """
+        small_value_err_ratio = get_small_value_err_ratio(self.small_value_mask, self.abs_err_greater_mask)
+        rmse = get_rmse(self.abs_err, np.logical_or(self.inf_nan_mask, self.small_value_mask))
+        eb = get_error_balance(self.bench_output, self.device_output)
+        max_rel_error = get_max_rel_err(self.rel_err)
+        mean_rel_error = get_mean_rel_err(self.rel_err)
+
+        return {
+            "small_value_err_ratio": small_value_err_ratio,
+            "max_rel_error": max_rel_error,
+            "mean_rel_error": mean_rel_error,
+            "rmse": rmse,
+            "eb": eb
+        }
+
+
+class BenchmarkPrecisionCompare(BasePrecisionCompare):
+    def __init__(self, input_data):
+        super().__init__(input_data)
+        self.compare_algorithm = CompareConst.BENCHMARK_COMPARE_ALGORITHM_NAME
+
+    @staticmethod
+    def get_final_status(status_list):
+        compare_result = CompareConst.PASS
+        if CompareConst.ERROR in status_list:
+            compare_result = CompareConst.ERROR
+        elif CompareConst.WARNING in status_list:
+            compare_result = CompareConst.WARNING
+        return compare_result
+    
+    def _calc_ratio(self, column_name):
+        npu_value, gpu_value = self._get_and_convert_values(column_name)
+        if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value):
+            return check_inf_or_nan(npu_value, gpu_value, column_name)
+        else:
+            return calc_ratio(npu_value, gpu_value, str(self.dtype)), True, ""
+
+    def _compute_ratio(self):
+        compare_message = ""
+        small_value_err_ratio, small_value_inf_nan_consistency, small_value_message = \
+            self._calc_ratio(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE)
+        compare_message += small_value_message
+        rmse_ratio, rmse_inf_nan_consistency, rmse_message = self._calc_ratio(ApiPrecisionCompareColumn.RMSE)
+        compare_message += rmse_message
+        max_rel_err_ratio, max_rel_inf_nan_consistency, max_rel_message = \
+            self._calc_ratio(ApiPrecisionCompareColumn.MAX_REL_ERR)
+        compare_message += max_rel_message
+        mean_rel_err_ratio, mean_rel_inf_nan_consistency, mean_rel_message = \
+            self._calc_ratio(ApiPrecisionCompareColumn.MEAN_REL_ERR)
+        compare_message += mean_rel_message
+        eb_ratio, eb_inf_nan_consistency, eb_message = self._calc_ratio(ApiPrecisionCompareColumn.EB)
+        compare_message += eb_message
+        
+        metrics = {
+            CompareConst.SMALL_VALUE_ERR_RATIO: small_value_err_ratio,
+            CompareConst.RMSE_RATIO: rmse_ratio,
+            CompareConst.MAX_REL_ERR_RATIO: max_rel_err_ratio,
+            CompareConst.MEAN_REL_ERR_RATIO: mean_rel_err_ratio,
+            CompareConst.EB_RATIO: eb_ratio,
+            CompareConst.COMPARE_MESSAGE: compare_message
+        }
+        
+        return metrics, \
+               BenchmarkInfNanConsistency(small_value_inf_nan_consistency, rmse_inf_nan_consistency, 
+                                          max_rel_inf_nan_consistency, mean_rel_inf_nan_consistency,
+                                          eb_inf_nan_consistency)
+
+    def _get_threshold(self, metric):
+        error_threshold = StandardConfig.get_benchmark_threshold(metric)
+        return error_threshold
+
+    def _get_single_metric_status(self, ratio, metric):
+        if is_inf_or_nan(ratio):
+            return CompareConst.PASS
+        error_threshold = self._get_threshold(metric)
+        if ratio > error_threshold:
+            return CompareConst.ERROR
+        return CompareConst.PASS
+
+    def _get_status(self, metrics, inf_nan_consistency):
+        small_value_err_ratio = metrics.get(CompareConst.SMALL_VALUE_ERR_RATIO)
+        rmse_ratio = metrics.get(CompareConst.RMSE_RATIO)
+        max_rel_err_ratio = metrics.get(CompareConst.MAX_REL_ERR_RATIO)
+        mean_rel_err_ratio = metrics.get(CompareConst.MEAN_REL_ERR_RATIO)
+        eb_ratio = metrics.get(CompareConst.EB_RATIO)
+        
+        small_value_err_status = self._get_single_metric_status(small_value_err_ratio, CompareConst.SMALL_VALUE) \
+                                        if inf_nan_consistency.small_value_inf_nan_consistency else CompareConst.ERROR
+        rmse_status = self._get_single_metric_status(rmse_ratio, CompareConst.RMSE)  \
+                                        if inf_nan_consistency.rmse_inf_nan_consistency else CompareConst.ERROR
+        max_rel_err_status = self._get_single_metric_status(max_rel_err_ratio, CompareConst.MAX_REL_ERR) \
+                                        if inf_nan_consistency.max_rel_inf_nan_consistency else CompareConst.ERROR
+        mean_rel_err_status = self._get_single_metric_status(mean_rel_err_ratio, CompareConst.MEAN_REL_ERR) \
+                                        if inf_nan_consistency.mean_rel_inf_nan_consistency else CompareConst.ERROR
+        eb_status = self._get_single_metric_status(eb_ratio, CompareConst.EB) \
+                                        if inf_nan_consistency.eb_inf_nan_consistency else CompareConst.ERROR
+        status_list = [small_value_err_status, rmse_status, max_rel_err_status, mean_rel_err_status]
+        compare_result = self.get_final_status(status_list)
+        status_dict = {
+            CompareConst.SMALL_VALUE_ERR_STATUS: small_value_err_status,
+            CompareConst.RMSE_STATUS: rmse_status,
+            CompareConst.MAX_REL_ERR_STATUS: max_rel_err_status,
+            CompareConst.MEAN_REL_ERR_STATUS: mean_rel_err_status,
+            CompareConst.EB_STATUS: eb_status
+        }
+        metrics.update(status_dict)
+        metrics.update({CompareConst.COMPARE_RESULT: compare_result})
+        return metrics

msprobe/pytorch/api_accuracy_checker/precision_standard/binary_consistency.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import compare_bool_tensor
+from msprobe.pytorch.api_accuracy_checker.precision_standard.base_standard import BaseCompare
+
+
+class BinaryCompare(BaseCompare):
+    """
+    Binary comparison class for comparing boolean tensors.
+
+    This class is designed to compare the output of a binary operation between a benchmark and a device.
+    It calculates the error rate of the comparison and provides a simple metric for assessing the accuracy.
+
+    Attributes:
+        bench_output (np.ndarray): The output from the benchmark.
+        device_output (np.ndarray): The output from the device.
+        compare_column (object): The column object to store comparison results.
+        dtype (torch.dtype): The data type of the outputs.
+
+    Methods:
+        _compute_metrics(): Computes the comparison metrics, specifically the error rate.
+
+    Note:
+        This class assumes that the input data is an instance of InputData containing the benchmark output,
+        device output, comparison column, and data type. The outputs are expected to be boolean tensors.
+
+    See Also:
+        BaseCompare: The base class for comparison classes.
+        compare_bool_tensor: The function used to compare boolean tensors.
+    """
+    def __init__(self, input_data):
+        super(BinaryCompare, self).__init__(input_data)
+
+    def _pre_compare(self):
+        pass
+
+    def _compute_metrics(self):
+        """
+        Computes the error rate metric for the comparison between benchmark and device outputs.
+
+        This method calculates the proportion of mismatches between the benchmark output and the device output.
+        It uses the `compare_bool_tensor` function to compare the two tensors and extract the error rate.
+
+        Returns:
+        dict: A dictionary containing the computed error rate metric.
+            The dictionary has the following key:
+            - "error_rate": The proportion of mismatches between the benchmark and device outputs.
+        """
+        error_rate, _, _ = compare_bool_tensor(self.bench_output, self.device_output)
+
+        return {
+            "error_rate": error_rate
+        }