mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/compare/compare.py

@@ -24,15 +24,20 @@ from msprobe.core.common.utils import CompareException
 from msprobe.core.common.file_utils import get_json_contents, write_csv
 import torch
 from msprobe.core.common.const import CompareConst
-from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \
-    get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \
-    get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \
-    check_small_value, check_norm_value, get_abs_bench_with_eps, get_ulp_err
+from msprobe.pytorch.api_accuracy_checker.precision_standard.standard_register import StandardRegistry
+from msprobe.pytorch.api_accuracy_checker.precision_standard.absolute_threshold import AbsolutethdCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.benchmark_compare import BenchmarkCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.ulp_compare import UlpCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.binary_consistency import BinaryCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.thousandth_standard import ThousandthStdCompare
+from msprobe.pytorch.api_accuracy_checker.precision_standard.accumulative_error_compare import AccumulativeErrorCompare
+from msprobe.pytorch.api_accuracy_checker.compare.compare_input import CompareInput
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_abs_err, get_max_abs_err, get_rel_err_ratio, \
+    cosine_sim, get_rel_err_origin, get_abs_bench_with_eps, compare_bool_tensor
 from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig
 from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn
 from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \
-    DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \
-    ulp_standard_api, thousandth_standard_api, apis_threshold
+    DETAIL_TEST_ROWS, BENCHMARK_COMPARE_SUPPORT_LIST
 from msprobe.pytorch.api_accuracy_checker.common.utils import extract_basic_api_segments
 from msprobe.pytorch.common.log import logger
 
@@ -42,6 +47,7 @@ ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'b
 
 
 INDEX_TEST_RESULT_GROUP = 3
+BACKWARD_RESULT_GROUP = 4
 INDEX_FIRST_GROUP = 0
 INDEX_MESSAGE = -1
 
@@ -66,6 +72,8 @@ class Comparator:
             self.detail_save_path_list = \
                 [self.detail_save_path_str.format(rank) for rank in config.online_config.rank_list]
 
+        self.registry = self._register_compare_func()
+
         if not is_continue_run_ut:
             self.write_csv_title()
         if stack_info_json_path:
@@ -101,22 +109,6 @@ class Comparator:
         compare_column.error_rate = 0
         return CompareConst.PASS, compare_column, ""
 
-    @staticmethod
-    def _compare_bool_tensor(bench_output, device_output):
-        error_nums = (bench_output != device_output).sum()
-        if bench_output.size == 0:
-            return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result."
-        error_rate = float(error_nums / bench_output.size)
-        result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR
-        return error_rate, result, ""
-
-    @staticmethod
-    def _get_absolute_threshold_attribute(api_name, dtype):
-        small_value_threshold = apis_threshold.get(api_name).get(dtype).get('small_value')
-        small_value_atol = apis_threshold.get(api_name).get(dtype).get('small_value_atol')
-        rtol = apis_threshold.get(api_name).get(dtype).get('rtol')
-        return small_value_threshold, small_value_atol, rtol
-
     @staticmethod
     def _get_run_ut_detail(test_result):
         """get run_ut detail before write to csv, called by online run_ut"""
@@ -143,6 +135,36 @@ class Comparator:
                 test_rows.append([subject] + list(test_subject))
         return test_rows
 
+    @staticmethod
+    def _binary_standard_compare(input_data):
+        binary_compare = BinaryCompare(input_data)
+        binary_compare.compare()
+    
+    @staticmethod
+    def _thousandth_standard_compare(input_data):
+        thousandth_compare = ThousandthStdCompare(input_data)
+        thousandth_compare.compare()
+    
+    @staticmethod
+    def _absolute_standard_compare(input_data):
+        absolute_compare = AbsolutethdCompare(input_data)
+        absolute_compare.compare()
+
+    @staticmethod
+    def _ulp_compare(input_data):
+        ulp_compare = UlpCompare(input_data)
+        ulp_compare.compare()
+    
+    @staticmethod
+    def _benchmark_compare(input_data):
+        benchmark_compare = BenchmarkCompare(input_data)
+        benchmark_compare.compare()
+    
+    @staticmethod
+    def _accumulative_error_compare(input_data):
+        accumulative_error_compare = AccumulativeErrorCompare(input_data)
+        accumulative_error_compare.compare()
+
     def write_csv_title(self):
         summary_test_rows = [
             [self.COLUMN_API_NAME, 
@@ -163,6 +185,8 @@ class Comparator:
             df_row = list(test_result[:INDEX_TEST_RESULT_GROUP])
             if test_result[1] == CompareConst.SKIP:
                 df_row.append(test_result[INDEX_TEST_RESULT_GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE])
+            elif test_result[2] == CompareConst.SKIP:
+                df_row.append(test_result[BACKWARD_RESULT_GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE])
             if self.stack_info:
                 stack_info = "\n".join(self.stack_info[name])
                 df_row.append(stack_info)
@@ -211,6 +235,7 @@ class Comparator:
         if backward_message:
             backward_column = CompareColumn()
             bwd_compare_alg_results = [backward_column.to_column_value(CompareConst.SKIP, backward_message)]
+            bwd_success_status = CompareConst.SKIP
         else:
             bwd_success_status = bwd_success_status if bwd_compare_alg_results is not None else CompareConst.SPACE
         result_info = ResultInfo(full_api_name,
@@ -226,6 +251,16 @@ class Comparator:
         return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS \
                or bwd_success_status == CompareConst.SPACE
 
+    def _register_compare_func(self):
+        registry = StandardRegistry()
+        registry.register(CompareConst.ABSOLUTE_THRESHOLD, self._absolute_standard_compare)
+        registry.register(CompareConst.BINARY_CONSISTENCY, self._binary_standard_compare)
+        registry.register(CompareConst.ULP_COMPARE, self._ulp_compare)
+        registry.register(CompareConst.THOUSANDTH_STANDARD, self._thousandth_standard_compare)
+        registry.register(CompareConst.BENCHMARK, self._benchmark_compare)
+        registry.register(CompareConst.ACCUMULATIVE_ERROR_COMPARE, self._accumulative_error_compare)
+        return registry
+
     def _compare_core_wrapper(self, api_name, bench_output, device_output):
         detailed_result_total = []
         test_final_success = CompareConst.PASS
@@ -308,11 +343,13 @@ class Comparator:
             return CompareConst.ERROR, compare_column, f"Bench out dtype is {bench_output.dtype} but " \
                                                        f"npu output dtype is {device_output.dtype}, cannot compare."
         message = ""
+        if bench_output.size == 0:
+            return CompareConst.ERROR, compare_column, "There is not bench calculation result."
         if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32,
                                   np.int64, np.uint64]:
             message += f"Compare algorithm is not supported for {bench_output.dtype} data. " \
                        f"Only judged by Error Rate."
-            err_rate, status, msg = self._compare_bool_tensor(bench_output, device_output)
+            err_rate, status, msg = compare_bool_tensor(bench_output, device_output)
             message += msg + "\n"
             compare_column.error_rate = err_rate
             return status, compare_column, message
@@ -321,56 +358,20 @@ class Comparator:
                                                                          compare_column, npu_dtype)
             return status, compare_column, message
 
+    def _perform_comparison(self, api_name, input_data):
+        comparison_func = self.registry.get_comparison_function(api_name, None)
+        comparison_func(input_data)
+            
     def _compare_float_tensor(self, api_name, bench_output, device_output, compare_column, dtype):
         message = ""
-        abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype)
+        _, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype)
         abs_err = get_abs_err(bench_output, device_output)
         rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps)
-        if api_name in thousandth_standard_api:
-            thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD)
-            compare_column.rel_err_thousandth = thousand_res
+        input_data = CompareInput(bench_output, device_output, compare_column, dtype, rel_err_orign)
         if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST:
-            both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output)
-            if api_name in binary_standard_api:
-                err_rate, _, _ = self._compare_bool_tensor(bench_output, device_output)
-                compare_column.error_rate = err_rate
-            elif api_name in absolute_standard_api:
-                small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute(
-                    api_name, str(dtype))
-                rel_err = abs_err / abs_bench_with_eps
-                small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold)
-                normal_value_mask = np.logical_and(both_finite_mask, np.logical_not(small_value_mask))
-                compare_column.inf_nan_error_ratio = check_inf_nan_value(inf_nan_mask, bench_output, device_output,
-                                                                         dtype, rtol)
-                compare_column.rel_err_ratio = check_norm_value(normal_value_mask, rel_err, rtol)
-                compare_column.abs_err_ratio = check_small_value(abs_err, small_value_mask, small_value_atol)
-            elif api_name in ulp_standard_api:
-                if bench_output.size == 0:
-                    compare_column.max_ulp_error = 0
-                    compare_column.mean_ulp_error = 0
-                    compare_column.ulp_error_proportion = 0
-                else:
-                    ulp_err = get_ulp_err(bench_output, device_output, dtype)
-                    compare_column.max_ulp_error = np.max(ulp_err)
-                    compare_column.mean_ulp_error = np.mean(ulp_err)
-                    if dtype == torch.float32:
-                        compare_column.ulp_error_proportion = \
-                        np.sum(ulp_err > CompareConst.ULP_FLOAT32_THRESHOLD) / bench_output.size
-                    else:
-                        compare_column.ulp_error_proportion = \
-                            np.sum(ulp_err > CompareConst.ULP_FLOAT16_THRESHOLD) / bench_output.size
-            else:
-                dtype_config = precision_configs.get(dtype)
-                small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0])
-                abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0])
-                compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask)
-                rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask)
-                compare_column.rmse = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask))
-                compare_column.eb = get_error_balance(bench_output, device_output)
-                if rel_err.size == 0:
-                    return CompareConst.ERROR, compare_column, "Relative error result list is empty."
-                compare_column.max_rel_error = get_max_rel_err(rel_err)
-                compare_column.mean_rel_error = get_mean_rel_err(rel_err)
+            self._perform_comparison(api_name, input_data)
+        else:
+            message += f"The data type {dtype} is not supported for new precision standard."
 
         cos_res, cos_status, msg = cosine_sim(bench_output, device_output)
         compare_column.cosine_sim = cos_res

msprobe/pytorch/api_accuracy_checker/compare/compare_column.py

@@ -16,9 +16,17 @@
 # limitations under the License.
 
 from msprobe.core.common.const import CompareConst
+from msprobe.pytorch.common.log import logger
 
 
 class CompareColumn:
+    __slots__ = [
+        'bench_type', 'npu_type', 'shape', 'cosine_sim', 'max_abs_err', 'rel_err_hundredth',
+        'rel_err_ten_thousandth', 'inf_nan_error_ratio', 'rel_err_ratio', 'abs_err_ratio', 
+        'small_value_err_ratio', 'max_rel_error', 'mean_rel_error', 'rmse', 'eb', 'max_ulp_error', 
+        'mean_ulp_error', 'ulp_error_proportion', 'error_rate', 'rel_err_thousandth'
+    ]
+    
     def __init__(self):
         self.bench_type = CompareConst.SPACE
         self.npu_type = CompareConst.SPACE
@@ -41,6 +49,24 @@ class CompareColumn:
         self.mean_ulp_error = CompareConst.SPACE
         self.ulp_error_proportion = CompareConst.SPACE
 
+    def update(self, metrics):
+        """
+        Updates the object's attributes with the provided metrics.
+
+        Args:
+            metrics (dict): A dictionary containing attribute names and their corresponding values.
+
+        Raises:
+            AttributeError: If the metric key is not a valid attribute of CompareColumn.
+        """
+        for key, value in metrics.items():
+            if value is None:
+                continue
+            if key not in self.__slots__:
+                logger.error(f"The key '{key}' is not a valid attribute of CompareColumn.")
+                continue
+            setattr(self, key, value)
+
     def to_column_value(self, is_pass, message):
         return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth,
                 self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.eb, self.rmse, 
@@ -50,6 +76,16 @@ class CompareColumn:
 
 
 class ApiPrecisionOutputColumn:
+    __slots__ = [
+                'api_name', 'small_value_err_ratio', 'small_value_err_status', 'rmse_ratio', 'rmse_status', 
+                'max_rel_err_ratio', 'max_rel_err_status', 'mean_rel_err_ratio', 'mean_rel_err_status', 'eb_ratio', 
+                'eb_status', 'inf_nan_error_ratio', 'inf_nan_error_ratio_status', 'rel_err_ratio', 
+                'rel_err_ratio_status', 'abs_err_ratio', 'abs_err_ratio_status', 'error_rate', 'error_rate_status', 
+                'mean_ulp_err', 'ulp_err_proportion', 'ulp_err_proportion_ratio', 'ulp_err_status', 
+                'rel_err_thousandth', 'rel_err_thousandth_status', 'compare_result', 'compare_algorithm', 
+                'compare_message'
+                ]
+    
     def __init__(self):
         self.api_name = CompareConst.SPACE
         self.small_value_err_ratio = CompareConst.SPACE
@@ -80,6 +116,24 @@ class ApiPrecisionOutputColumn:
         self.compare_algorithm = CompareConst.SPACE
         self.compare_message = CompareConst.SPACE
 
+    def update(self, metrics):
+        """
+        Updates the object's attributes with the provided metrics.
+
+        Args:
+            metrics (dict): A dictionary containing attribute names and their corresponding values.
+
+        Raises:
+            AttributeError: If the metric key is not a valid attribute of CompareColumn.
+        """
+        for key, value in metrics.items():
+            if value is None:
+                continue
+            if key not in self.__slots__:
+                logger.error("The key '%s' is not a valid attribute of CompareColumn.", key)
+                continue
+            setattr(self, key, value)
+
     def to_column_value(self):
         return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, 
                 self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, 

msprobe/pytorch/api_accuracy_checker/compare/compare_input.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+
+class CompareInput:
+    """
+    A class to encapsulate the input data required for comparison operations.
+
+    Attributes:
+        bench_output (np.ndarray): The benchmark output values.
+        device_output (np.ndarray): The device output values.
+        compare_column (class): A clasee to store and update comparison metrics.
+        dtype (type, optional): The data type of the outputs. Defaults to None.
+        rel_err_orign (float or array-like, optional): The original relative error values. Defaults to None.
+
+    Methods:
+        __init__(bench_output, device_output, compare_column, dtype, rel_err_orign): 
+        Initializes an instance of CompareInput.
+    """
+    def __init__(self, bench_output, device_output, compare_column, dtype=None, rel_err_orign=None):
+        self.bench_output = bench_output
+        self.device_output = device_output
+        if not isinstance(bench_output, np.ndarray) or not isinstance(device_output, np.ndarray):
+            raise TypeError("The input should be numpy array")
+        self.compare_column = compare_column
+        self.dtype = dtype
+        self.rel_err_orign = rel_err_orign
+
+
+class PrecisionCompareInput:
+    def __init__(self, row_npu, row_gpu, dtype, compare_column):
+        self.row_npu = row_npu
+        self.row_gpu = row_gpu
+        self.dtype = dtype
+        self.compare_column = compare_column

msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py

@@ -43,10 +43,7 @@ absolute_standard_api = apis.get('AbsoluteThreshStandard')
 binary_standard_api = apis.get('BinaryCompareStandard')
 ulp_standard_api = apis.get('ULPStandard')
 thousandth_standard_api = apis.get('ThousandthStandard')
-
-
-threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml")
-apis_threshold = load_yaml(threshold_yaml_path)
+accumulative_error_standard_api = apis.get('AccumulativeErrorStandard')
 
 
 DETAIL_TEST_ROWS = [
@@ -134,6 +131,7 @@ ULP_PARAMETERS = {
 class ApiPrecisionCompareColumn:
     API_NAME = 'API Name'
     DEVICE_DTYPE = 'DEVICE Dtype'
+    SHAPE = 'Shape'
     SMALL_VALUE_ERROR_RATE = '小值域错误占比'
     RMSE = '均方根误差'
     MAX_REL_ERR = '相对误差最大值'

msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json

@@ -0,0 +1,9 @@
+{
+    "dump_json_path": "./dump.json",
+    "api_name": "",
+    "extract_api_path": "",
+    "propagation": "forward",
+    "data_mode": "random_data",
+    "random_seed": 1234,
+    "iter_times": 1
+}