mindstudio-probe 1.0.1__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/compare/compare.py

@@ -0,0 +1,345 @@
+# 进行比对及结果展示
+import os
+from collections import namedtuple
+import torch
+import numpy as np
+from msprobe.pytorch.common.log import logger
+from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents, write_csv
+from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \
+    DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, \
+    ULPStandardApi, ThousandthStandardApi, apis_threshold
+from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn
+from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \
+    get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \
+    get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \
+    check_small_value, check_norm_value, get_abs_bench_with_eps, get_ulp_err
+from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig
+from msprobe.core.common.const import Const, CompareConst
+
+
+ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'bwd_success_status',
+                                       'fwd_compare_alg_results', 'bwd_compare_alg_results', 'rank'])
+
+
+INDEX_TEST_RESULT__GROUP = 3
+INDEX_FIRST_GROUP = 0
+INDEX_MESSAGE = -1
+
+
+class Comparator:
+    # consts for result csv
+    COLUMN_API_NAME = "API name"
+    COLUMN_FORWARD_SUCCESS = "Forward Test Success"
+    COLUMN_BACKWARD_SUCCESS = "Backward Test Success"
+    COLUMN_STACK_INFO = "Traceback callstack info"
+
+    def __init__(self, result_csv_path, details_csv_path, is_continue_run_ut, stack_info_json_path=None):
+        self.save_path = result_csv_path
+        self.detail_save_path = details_csv_path
+        if not is_continue_run_ut and not os.path.exists(self.save_path) and not os.path.exists(self.detail_save_path):
+            self.write_csv_title()
+        if stack_info_json_path:
+            self.stack_info = get_json_contents(stack_info_json_path)
+        else:
+            self.stack_info = None
+
+    @staticmethod
+    def print_pretest_result():
+        logger.info("Successfully completed run_ut/multi_run_ut.")
+    
+    @staticmethod
+    def _compare_dropout(bench_output, device_output):
+        tensor_num = bench_output.numel()
+        if tensor_num >= 100:
+            if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1:
+                return CompareConst.PASS, 1
+            else:
+                return CompareConst.ERROR, 0
+        else:
+            return CompareConst.PASS, 1
+
+    @staticmethod
+    def _compare_builtin_type(bench_output, device_output, compare_column):
+        if not isinstance(bench_output, (bool, int, float, str)):
+            return CompareConst.PASS, compare_column, ""
+        if bench_output != device_output:
+            return CompareConst.ERROR, compare_column, ""
+        compare_column.error_rate = 0
+        return CompareConst.PASS, compare_column, ""
+
+    @staticmethod
+    def _compare_bool_tensor(bench_output, device_output):
+        error_nums = (bench_output != device_output).sum()
+        if bench_output.size == 0:
+            return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result."
+        error_rate = float(error_nums / bench_output.size)
+        result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR
+        return error_rate, result, ""
+    
+    @staticmethod
+    def _get_absolute_threshold_attribute(api_name, dtype):
+        small_value_threshold = apis_threshold.get(api_name).get(dtype).get('small_value')
+        small_value_atol = apis_threshold.get(api_name).get(dtype).get('small_value_atol')
+        rtol = apis_threshold.get(api_name).get(dtype).get('rtol')
+        return small_value_threshold, small_value_atol, rtol
+
+    def write_csv_title(self):
+        summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS,
+                              self.COLUMN_BACKWARD_SUCCESS, "Message"]]
+        if not os.path.exists(self.save_path):
+            write_csv(summary_test_rows, self.save_path)
+        if not os.path.exists(self.detail_save_path):
+            write_csv(DETAIL_TEST_ROWS, self.detail_save_path)
+
+    def write_summary_csv(self, test_result):
+        test_rows = []
+        if self.stack_info:
+            test_rows[0].append(self.COLUMN_STACK_INFO)
+
+        name = test_result[0]
+        df_row = list(test_result[:INDEX_TEST_RESULT__GROUP])
+        if test_result[1] == "SKIP":
+            df_row.append(test_result[INDEX_TEST_RESULT__GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE])
+        if self.stack_info:
+            stack_info = "\n".join(self.stack_info[name])
+            df_row.append(stack_info)
+        test_rows.append(df_row)
+        write_csv(test_rows, self.save_path)
+
+    def write_detail_csv(self, test_result):
+        test_rows = []
+
+        subject_prefix = test_result[0]
+        fwd_result = test_result[3]
+        bwd_result = test_result[4]
+        if isinstance(fwd_result, list):
+            for i, test_subject in enumerate(fwd_result):
+                subject = subject_prefix + ".forward.output." + str(i)
+                test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision)
+                                if isinstance(item, float) else item for item in test_subject]
+                test_rows.append([subject] + list(test_subject))
+        if isinstance(bwd_result, list):
+            for i, test_subject in enumerate(bwd_result):
+                subject = subject_prefix + ".backward.output." + str(i)
+                test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision)
+                                if isinstance(item, float) else item for item in test_subject]
+                test_rows.append([subject] + list(test_subject))
+
+        write_csv(test_rows, self.detail_save_path)
+
+    def record_results(self, args):
+        self.write_summary_csv(args)
+        self.write_detail_csv(args)
+
+    def compare_output(self, full_api_name, data_info):
+        _, api_name, _ = full_api_name.split(Const.SEP)
+        bench_output, device_output = data_info.bench_output, data_info.device_output
+        bench_grad, device_grad = data_info.bench_grad, data_info.device_grad
+        backward_message = data_info.backward_message
+        if "dropout" in full_api_name:
+            fwd_success_status, fwd_compare_alg_results = self._compare_dropout(bench_output, device_output)
+        else:
+            fwd_success_status, fwd_compare_alg_results = self._compare_core_wrapper(api_name, bench_output,
+                                                                                     device_output)
+        if not (bench_grad and device_grad):
+            bwd_success_status, bwd_compare_alg_results = (CompareConst.SPACE, [])
+        else:
+            if "dropout" in full_api_name:
+                bwd_success_status, bwd_compare_alg_results = self._compare_dropout(bench_grad[0], device_grad[0])
+            else:
+                bwd_success_status, bwd_compare_alg_results = self._compare_core_wrapper(api_name, bench_grad,
+                                                                                         device_grad)
+        if backward_message:
+            backward_column = CompareColumn()
+            bwd_compare_alg_results = [backward_column.to_column_value(CompareConst.SKIP, backward_message)]
+        else:
+            bwd_success_status = bwd_success_status if bwd_compare_alg_results is not None else CompareConst.SPACE
+        result_info = ResultInfo(full_api_name,
+                                 fwd_success_status,
+                                 bwd_success_status,
+                                 fwd_compare_alg_results,
+                                 bwd_compare_alg_results,
+                                 data_info.rank)
+        self.record_results(result_info)
+        return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS \
+               or bwd_success_status == CompareConst.SPACE
+
+    def _compare_core_wrapper(self, api_name, bench_output, device_output):
+        detailed_result_total = []
+        test_final_success = CompareConst.PASS
+        if isinstance(bench_output, (list, tuple)):
+            status, compare_result, message = [], [], []
+            if len(bench_output) > len(device_output):
+                status = [CompareConst.ERROR]
+                message = ["bench and npu output structure is different."]
+            else:
+                device_output = device_output[:len(bench_output)]
+                for b_out_i, n_out_i in zip(bench_output, device_output):
+                    status_i, compare_result_i, message_i = self._compare_core(api_name, b_out_i, n_out_i)
+                    status.append(status_i)
+                    compare_result.append(compare_result_i)
+                    message.append(message_i)
+        else:
+            status, compare_result, message = self._compare_core(api_name, bench_output, device_output)
+        if not isinstance(status, list):
+            detailed_result_total.append(compare_result.to_column_value(status, message))
+            if status == CompareConst.ERROR:
+                test_final_success = CompareConst.ERROR
+            elif status == CompareConst.WARNING:
+                test_final_success = CompareConst.WARNING
+        else:
+            for item, item_status in enumerate(status):
+                detailed_result_total.append(compare_result[item].to_column_value(item_status, message[item]))
+                if item_status == CompareConst.ERROR:
+                    test_final_success = CompareConst.ERROR
+                elif item_status == CompareConst.WARNING:
+                    test_final_success = CompareConst.WARNING
+        return test_final_success, detailed_result_total
+
+    def _compare_core(self, api_name, bench_output, device_output):
+        compare_column = CompareColumn()
+        if not isinstance(bench_output, type(device_output)):
+            return CompareConst.ERROR, compare_column, "bench and npu output type is different."
+        elif isinstance(bench_output, dict):
+            b_keys, n_keys = set(bench_output.keys()), set(device_output.keys())
+            if b_keys != n_keys:
+                return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different."
+            else:
+                status, compare_result, message = self._compare_core(api_name, list(bench_output.values()),
+                                                                     list(device_output.values()))
+        elif isinstance(bench_output, torch.Tensor):
+            copy_bench_out = bench_output.detach().clone()
+            copy_device_output = device_output.detach().clone()
+            compare_column.bench_type = str(copy_bench_out.dtype)
+            compare_column.npu_type = str(copy_device_output.dtype)
+            compare_column.shape = tuple(device_output.shape)
+            status, compare_result, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output,
+                                                                         compare_column)
+        elif isinstance(bench_output, (bool, int, float, str)):
+            compare_column.bench_type = str(type(bench_output))
+            compare_column.npu_type = str(type(device_output))
+            status, compare_result, message = self._compare_builtin_type(bench_output, device_output, compare_column)
+        elif bench_output is None:
+            return CompareConst.SKIP, compare_column, "Bench output is None, skip this test."
+        else:
+            return CompareConst.PASS, compare_column,
+        "Unexpected output type in compare_core: {}".format(type(bench_output))
+
+        return status, compare_result, message
+
+    def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column):
+        cpu_shape = bench_output.shape
+        npu_shape = device_output.shape
+        npu_dtype = device_output.dtype
+        if npu_dtype == torch.bfloat16:
+            bench_output = bench_output.to(torch.float32)
+            device_output = device_output.to(torch.float32)
+        bench_output = bench_output.numpy()
+        device_output = device_output.cpu().numpy()
+        if cpu_shape != npu_shape:
+            return CompareConst.ERROR, compare_column, f"The shape of bench{str(cpu_shape)} " \
+                                                       f"and npu{str(npu_shape)} not equal."
+        if not check_dtype_comparable(bench_output, device_output):
+            return CompareConst.ERROR, compare_column, f"Bench out dtype is {bench_output.dtype} but " \
+                                                       f"npu output dtype is {device_output.dtype}, cannot compare."
+        message = ""
+        if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32,
+                                  np.int64, np.uint64]:
+            message += f"Compare algorithm is not supported for {bench_output.dtype} data. " \
+                       f"Only judged by Error Rate."
+            err_rate, status, msg = self._compare_bool_tensor(bench_output, device_output)
+            message += msg + "\n"
+            compare_column.error_rate = err_rate
+            return status, compare_column, message
+        else:
+            status, compare_column, message = self._compare_float_tensor(api_name, bench_output, device_output,
+                                                                         compare_column, npu_dtype)
+            return status, compare_column, message
+
+    def _compare_float_tensor(self, api_name, bench_output, device_output, compare_column, dtype):
+        message = ""
+        abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype)
+        abs_err = get_abs_err(bench_output, device_output)
+        rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps)
+        if api_name in ThousandthStandardApi:
+            thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD)
+            compare_column.rel_err_thousandth = thousand_res
+        if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST:
+            both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output)
+            if api_name in BinaryStandardApi:
+                err_rate, _, _ = self._compare_bool_tensor(bench_output, device_output)
+                compare_column.error_rate = err_rate
+            elif api_name in AbsoluteStandardApi:
+                small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute(
+                    api_name, str(dtype))
+                rel_err = abs_err / abs_bench_with_eps
+                small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold)
+                normal_value_mask = np.logical_and(both_finite_mask, np.logical_not(small_value_mask))
+                compare_column.inf_nan_error_ratio = check_inf_nan_value(inf_nan_mask, bench_output, device_output,
+                                                                         dtype, rtol)
+                compare_column.rel_err_ratio = check_norm_value(normal_value_mask, rel_err, rtol)
+                compare_column.abs_err_ratio = check_small_value(abs_err, small_value_mask, small_value_atol)
+            elif api_name in ULPStandardApi:
+                if bench_output.size == 0:
+                    compare_column.max_ulp_error = 0
+                    compare_column.mean_ulp_error = 0
+                    compare_column.ulp_error_proportion = 0
+                else:
+                    ulp_err = get_ulp_err(bench_output, device_output, dtype)
+                    compare_column.max_ulp_error = np.max(ulp_err)
+                    compare_column.mean_ulp_error = np.mean(ulp_err)
+                    if dtype == torch.float32:
+                        compare_column.ulp_error_proportion = np.sum(ulp_err > CompareConst.ULP_FLOAT32_THRESHOLD) / bench_output.size
+                    else:
+                        compare_column.ulp_error_proportion = np.sum(ulp_err > CompareConst.ULP_FLOAT16_THRESHOLD) / bench_output.size
+            else:
+                dtype_config = precision_configs.get(dtype)
+                small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0])
+                abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0])
+                compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask)
+                rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask)
+                compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask))
+                compare_column.EB = get_error_balance(bench_output, device_output)
+                if rel_err.size == 0:
+                    return CompareConst.ERROR, compare_column, "Relative error result list is empty."
+                compare_column.Max_rel_error = get_max_rel_err(rel_err)
+                compare_column.Mean_rel_error = get_mean_rel_err(rel_err)
+
+        cos_res, cos_status, msg = cosine_sim(bench_output, device_output)
+        compare_column.cosine_sim = cos_res
+        message += msg + "\n"
+        if not cos_status:
+            message += "Cosine similarity is less than 0.99, consider as error, skip other check and set to SPACE.\n"
+            return CompareConst.ERROR, compare_column, message
+
+        max_abs_res, max_abs_status = get_max_abs_err(abs_err)
+        compare_column.max_abs_err = max_abs_res
+        if max_abs_status:
+            message += "Max abs error is less than 0.001, consider as pass, skip other check and set to SPACE.\n"
+            return CompareConst.PASS, compare_column, message
+
+        if dtype in [torch.float16, torch.bfloat16]:
+            hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, CompareConst.HUNDRED_RATIO_THRESHOLD)
+            compare_column.rel_err_hundredth = hundred_res
+            if not hundred_status:
+                message += "Relative error is greater than 0.01, consider as error, skip other check and set to SPACE.\n"
+                return CompareConst.ERROR, compare_column, message
+        thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD)
+        compare_column.rel_err_thousandth = thousand_res
+        if dtype in [torch.float16, torch.bfloat16]:
+            if thousand_status:
+                message += "Relative error is less than 0.001, consider as pass, skip other check and set to SPACE.\n"
+                return CompareConst.PASS, compare_column, message
+            message += "Relative error is greater than 0.001, consider as warning, skip other check and set to SPACE.\n"
+            return CompareConst.WARNING, compare_column, message
+        ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.TEN_THOUSAND_RATIO_THRESHOLD)
+        compare_column.rel_err_ten_thousandth = ten_thousand_res
+        if dtype in [torch.float32, torch.float64]:
+            if not thousand_status:
+                message += "Relative error is greater than 0.001, consider as error, skip other check and set to SPACE.\n"
+                return CompareConst.ERROR, compare_column, message
+            if not ten_thousand_status:
+                message += "Relative error is greater than 0.0001, consider as warning, skip other check and set to SPACE.\n"
+                return CompareConst.WARNING, compare_column, message
+            message += "Relative error is less than 0.0001, consider as pass.\n"
+        return CompareConst.PASS, compare_column, message

msprobe/pytorch/api_accuracy_checker/compare/compare_column.py

@@ -0,0 +1,74 @@
+from msprobe.core.common.const import CompareConst
+
+
+class CompareColumn:
+    def __init__(self):
+        self.bench_type = CompareConst.SPACE
+        self.npu_type = CompareConst.SPACE
+        self.shape = CompareConst.SPACE
+        self.cosine_sim = CompareConst.SPACE
+        self.max_abs_err = CompareConst.SPACE
+        self.rel_err_hundredth = CompareConst.SPACE
+        self.rel_err_thousandth = CompareConst.SPACE
+        self.rel_err_ten_thousandth = CompareConst.SPACE
+        self.error_rate = CompareConst.SPACE
+        self.EB = CompareConst.SPACE
+        self.RMSE = CompareConst.SPACE
+        self.small_value_err_ratio = CompareConst.SPACE
+        self.Max_rel_error = CompareConst.SPACE
+        self.Mean_rel_error = CompareConst.SPACE
+        self.inf_nan_error_ratio = CompareConst.SPACE
+        self.rel_err_ratio = CompareConst.SPACE
+        self.abs_err_ratio = CompareConst.SPACE
+        self.max_ulp_error = CompareConst.SPACE
+        self.mean_ulp_error = CompareConst.SPACE
+        self.ulp_error_proportion = CompareConst.SPACE
+
+    def to_column_value(self, is_pass, message):
+        return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth,
+                self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, 
+                self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, self.inf_nan_error_ratio, 
+                self.rel_err_ratio, self.abs_err_ratio, self.max_ulp_error, self.mean_ulp_error, 
+                self.ulp_error_proportion, is_pass, message]
+
+
+class ApiPrecisionOutputColumn:
+    def __init__(self):
+        self.api_name = CompareConst.SPACE
+        self.small_value_err_ratio = CompareConst.SPACE
+        self.small_value_err_status = CompareConst.SPACE
+        self.rmse_ratio = CompareConst.SPACE
+        self.rmse_status = CompareConst.SPACE
+        self.max_rel_err_ratio = CompareConst.SPACE
+        self.max_rel_err_status = CompareConst.SPACE
+        self.mean_rel_err_ratio = CompareConst.SPACE
+        self.mean_rel_err_status = CompareConst.SPACE
+        self.eb_ratio = CompareConst.SPACE
+        self.eb_status = CompareConst.SPACE
+        self.inf_nan_error_ratio = CompareConst.SPACE
+        self.inf_nan_error_ratio_status = CompareConst.SPACE
+        self.rel_err_ratio = CompareConst.SPACE
+        self.rel_err_ratio_status = CompareConst.SPACE
+        self.abs_err_ratio = CompareConst.SPACE
+        self.abs_err_ratio_status = CompareConst.SPACE
+        self.error_rate = CompareConst.SPACE
+        self.error_rate_status = CompareConst.SPACE
+        self.mean_ulp_err = CompareConst.SPACE
+        self.ulp_err_proportion = CompareConst.SPACE
+        self.ulp_err_proportion_ratio = CompareConst.SPACE
+        self.ulp_err_status = CompareConst.SPACE
+        self.rel_err_thousandth = CompareConst.SPACE
+        self.rel_err_thousandth_status = CompareConst.SPACE
+        self.compare_result = CompareConst.SPACE
+        self.compare_algorithm = CompareConst.SPACE
+        self.compare_message = CompareConst.SPACE
+
+    def to_column_value(self):
+        return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, 
+                self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, 
+                self.mean_rel_err_status, self.eb_ratio, self.eb_status, self.inf_nan_error_ratio, 
+                self.inf_nan_error_ratio_status, self.rel_err_ratio, self.rel_err_ratio_status, self.abs_err_ratio, 
+                self.abs_err_ratio_status, self.error_rate, self.error_rate_status, self.mean_ulp_err, 
+                self.ulp_err_proportion, self.ulp_err_proportion_ratio, self.ulp_err_status, self.rel_err_thousandth, 
+                self.rel_err_thousandth_status, self.compare_result, self.compare_algorithm, self.compare_message]
+        

msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py

@@ -0,0 +1,249 @@
+import time
+import os
+import math
+
+import numpy as np
+import torch
+import yaml
+from msprobe.core.common.utils import CompareException
+from msprobe.core.common.const import Const
+from msprobe.pytorch.common.log import logger
+from msprobe.core.common.file_check import FileOpen
+
+
+current_time = time.strftime("%Y%m%d%H%M%S")
+API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv"
+API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv"
+BENCHMARK_COMPARE_SUPPORT_LIST = ['torch.float16', 'torch.bfloat16', 'torch.float32']
+API_PRECISION_COMPARE_UNSUPPORT_LIST = ['torch.float64', 'torch.complex64', 'torch.complex128']
+ULP_COMPARE_SUPPORT_LIST = ['torch.float16', 'torch.bfloat16', 'torch.float32']
+BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_COMPARE_UNSUPPORT_LIST
+
+
+cur_path = os.path.dirname(os.path.realpath(__file__))
+standard_yaml_path = os.path.join(cur_path, "api_precision_standard.yaml")
+with FileOpen(standard_yaml_path, 'r') as f:
+    Apis = yaml.safe_load(f)
+    AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard')
+    BinaryStandardApi = Apis.get('BinaryCompareStandard')
+    ULPStandardApi = Apis.get('ULPStandard')
+    ThousandthStandardApi = Apis.get('ThousandthStandard')
+
+
+threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml")
+with FileOpen(threshold_yaml_path, 'r') as f:
+    apis_threshold = yaml.safe_load(f)
+
+
+DETAIL_TEST_ROWS = [[
+            "API Name", "Bench Dtype", "DEVICE Dtype", "Shape",
+            "余弦相似度",
+            "最大绝对误差",
+            "双百指标",
+            "双千指标",
+            "双万指标",
+            "二进制一致错误率",
+            "误差均衡性",
+            "均方根误差",
+            "小值域错误占比",
+            "相对误差最大值",
+            "相对误差平均值",
+            "inf/nan错误率",
+            "相对误差错误率",
+            "绝对误差错误率",
+            "ULP误差最大值",
+            "ULP误差平均值",
+            "ULP误差大于阈值占比",
+            "Status",
+            "Message"
+        ]]
+
+
+precision_configs = {
+    torch.float16 : {
+        'small_value' : [
+            1e-3
+        ],
+        'small_value_atol' : [
+            1e-5
+        ]
+    },
+    torch.bfloat16: {
+        'small_value' : [
+            1e-3
+        ],
+        'small_value_atol' : [
+            1e-5
+        ]
+    },
+    torch.float32:{
+        'small_value' : [
+            1e-6
+        ],
+        'small_value_atol' : [
+            1e-9
+        ]
+    }
+}
+
+
+ULP_PARAMETERS = {
+    torch.float16 : {
+        'min_eb' : [
+            -14
+        ],
+        'exponent_num' : [
+            10
+        ]
+    },
+    torch.bfloat16 : {
+        'min_eb' : [
+            -126
+        ],
+        'exponent_num' : [
+            7
+        ]
+    },
+    torch.float32 : {
+        'min_eb' : [
+            -126
+        ],
+        'exponent_num' : [
+            23
+        ]
+    }
+}
+  
+    
+class ApiPrecisionCompareColumn:
+    API_NAME = 'API Name'
+    DEVICE_DTYPE = 'DEVICE Dtype'
+    SMALL_VALUE_ERROR_RATE = '小值域错误占比'
+    RMSE = '均方根误差'
+    MAX_REL_ERR = '相对误差最大值'
+    MEAN_REL_ERR = '相对误差平均值'
+    EB = '误差均衡性'
+    SMALL_VALUE_ERROR_RATIO = '小值域错误比值'
+    SMALL_VALUE_ERROR_STATUS = '小值域判定结果'
+    RMSE_RATIO = '均方根误差比值'
+    RMSE_STATUS = '均方根误差判定结果'
+    MAX_REL_ERR_RATIO = '相对误差最大值比值'
+    MAX_REL_ERR_STATUS = '相对误差最大值判定结果'
+    MEAN_REL_ERR_RATIO = '相对误差平均值比值'
+    MEAN_REL_ERR_STATUS = '相对误差平均值判定结果'
+    EB_RATIO = '误差均衡性比值'
+    EB_STATUS = '误差均衡性判定结果'
+    ERROR_RATE = '二进制一致错误率'
+    ERROR_RATE_STATUS = '二进制一致错误率判定结果'
+    INF_NAN_ERROR_RATIO = 'inf/nan错误率'
+    INF_NAN_ERROR_RATIO_STATUS = 'inf/nan判定结果'
+    REL_ERR_RATIO = '相对误差错误率'
+    REL_ERR_RATIO_STATUS = '相对误差判定结果'
+    ABS_ERR_RATIO = '绝对误差错误率'
+    ABS_ERR_RATIO_STATUS = '绝对误差判定结果'
+    MEAN_ULP_ERR = 'ULP误差平均值'
+    ULP_ERR_PROPORTION = 'ULP误差大于阈值占比'
+    ULP_ERR_PROPORTION_RATIO = 'ULP误差大于阈值占比比值'
+    ULP_ERR_STATUS = 'ULP误差判定结果'
+    REL_ERR_THOUSANDTH = '双千指标'
+    REL_ERR_THOUSANDTH_STATUS = '双千指标判定结果'
+    FINAL_RESULT = '比对结果'
+    ALGORITHM = '比对算法'
+    FORWWARD_STATUS = 'Forward Test Success'
+    BACKWARD_STATUS = 'Backward Test Success'
+    MESSAGE = 'Message'
+    
+    @staticmethod
+    def to_required_columns():
+        return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.DEVICE_DTYPE, 
+                ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE, ApiPrecisionCompareColumn.RMSE, 
+                ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR, ApiPrecisionCompareColumn.EB,
+                ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, 
+                ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO, 
+                ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION,
+                ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH]
+
+    @staticmethod
+    def get_detail_csv_title():
+        return [ApiPrecisionCompareColumn.API_NAME, 
+                ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATIO, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_STATUS, 
+                ApiPrecisionCompareColumn.RMSE_RATIO, ApiPrecisionCompareColumn.RMSE_STATUS, 
+                ApiPrecisionCompareColumn.MAX_REL_ERR_RATIO, ApiPrecisionCompareColumn.MAX_REL_ERR_STATUS, 
+                ApiPrecisionCompareColumn.MEAN_REL_ERR_RATIO, ApiPrecisionCompareColumn.MEAN_REL_ERR_STATUS, 
+                ApiPrecisionCompareColumn.EB_RATIO, ApiPrecisionCompareColumn.EB_STATUS, 
+                ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO_STATUS, 
+                ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO_STATUS, 
+                ApiPrecisionCompareColumn.ABS_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO_STATUS, 
+                ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.ERROR_RATE_STATUS, 
+                ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION, 
+                ApiPrecisionCompareColumn.ULP_ERR_PROPORTION_RATIO, ApiPrecisionCompareColumn.ULP_ERR_STATUS,
+                ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH, ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH_STATUS,
+                ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, ApiPrecisionCompareColumn.MESSAGE]
+    
+    @staticmethod
+    def get_result_csv_title():
+        return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.FORWWARD_STATUS, 
+                ApiPrecisionCompareColumn.BACKWARD_STATUS, ApiPrecisionCompareColumn.MESSAGE]
+
+
+CompareMessage = {
+    "topk" : "在npu上，topk的入参sorted=False时不生效，会返回有序tensor，而cpu上会返回无序tensor。 如果topk精度不达标，请检查是否是该原因导致的。"
+}
+
+
+def check_dtype_comparable(x, y):
+    if x.dtype in Const.FLOAT_TYPE:
+        if y.dtype in Const.FLOAT_TYPE:
+            return True 
+        return False 
+    if x.dtype in Const.BOOL_TYPE:
+        if y.dtype in Const.BOOL_TYPE:
+            return True 
+        return False 
+    if x.dtype in Const.INT_TYPE:
+        if y.dtype in Const.INT_TYPE:
+            return True 
+        return False
+    logger.warning(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}")
+    return False
+
+
+def convert_str_to_float(input_data):
+    if isinstance(input_data, str) and input_data.strip() == "":
+        msg = 'ERROR: Input data is an empty string'
+        raise CompareException(CompareException.INVALID_DATA_ERROR, msg)
+    try:
+        float_data = float(input_data)
+        return float_data
+    except ValueError as e:
+        msg = 'ERROR: Input data cannot be converted to float'
+        raise CompareException(CompareException.INVALID_DATA_ERROR, msg) from e
+
+
+def is_inf_or_nan(x):
+    return math.isnan(x) or math.isinf(x)
+
+
+def handle_infinity(x, y, column_name):
+    if math.isinf(x) and math.isinf(y):
+        if x == y:
+            return float("nan"), True, f"{column_name}同为同号inf或nan\n"
+        else:
+            return float("nan"), False, f"{column_name}inf或nan不一致\n"
+    else:
+        return float("nan"), False, f"{column_name}inf或nan不一致\n"
+
+
+def handle_nan(x, y, column_name):
+    if math.isnan(x) and math.isnan(y):
+        return float("nan"), True, f"{column_name}同为同号inf或nan\n"
+    else:
+        return float("nan"), False, f"{column_name}inf或nan不一致\n"
+
+
+def check_inf_or_nan(x, y, column_name):
+    if math.isinf(x) or math.isinf(y):
+        return handle_infinity(x, y, column_name)
+    else:
+        return handle_nan(x, y, column_name)
+    

msprobe/pytorch/api_accuracy_checker/config.yaml

@@ -0,0 +1,4 @@
+white_list: []
+error_data_path: './'
+precision: 14
+