mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/common/utils.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2023-2023. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,10 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import os
 import re
 from collections import namedtuple
+import importlib
 
 import torch
 
@@ -96,7 +98,8 @@ def cross_entropy_process(api_info_dict):
     Return api_info_dict:
         api_info_dict: Processed argument of the API.
     """
-    if 'input_args' in api_info_dict and len(api_info_dict['input_args']) > 1 and 'Min' in api_info_dict['input_args'][1]:
+    if 'input_args' in api_info_dict and len(api_info_dict['input_args']) > 1 \
+        and 'Min' in api_info_dict['input_args'][1]:
         if api_info_dict['input_args'][1]['Min'] <= 0:
             # The second argument in cross_entropy should be -100 or not less than 0
             api_info_dict['input_args'][1]['Min'] = 0
@@ -109,18 +112,6 @@ def initialize_save_path(save_path, dir_name):
     return data_path
 
 
-def get_real_data_path(file_path):
-    targets = ['forward_real_data', 'backward_real_data', 'ut_error_data\d+']
-    pattern = re.compile(r'({})'.format('|'.join(targets)))
-    match = pattern.search(file_path)
-    if match:
-        target_index = match.start()
-        target_path = file_path[target_index:]
-        return target_path
-    else:
-        raise DumpException(DumpException.INVALID_PATH_ERROR)
-
-
 def get_full_data_path(data_path, real_data_path):
     if not data_path:
         return data_path
@@ -137,7 +128,10 @@ class UtDataProcessor:
         self.index = 0
         self._save_recursive(api_name, element)
 
-    def _save_recursive(self, api_name, element):
+    def _save_recursive(self, api_name, element, depth=0):
+        if depth > Const.MAX_DEPTH:
+            logger.error(f"Maximum depth of {Const.MAX_DEPTH} exceeded for {api_name}")
+            raise DumpException(DumpException.RECURSION_LIMIT_ERROR)
         if isinstance(element, torch.Tensor):
             api_args = api_name + Const.SEP + str(self.index)
             create_directory(self.save_path)
@@ -153,10 +147,10 @@ class UtDataProcessor:
             self.index += 1
         elif isinstance(element, (list, tuple)):
             for item in element:
-                self._save_recursive(api_name, item)
+                self._save_recursive(api_name, item, depth=depth+1)
         elif isinstance(element, dict):
             for value in element.values():
-                self._save_recursive(api_name, value)
+                self._save_recursive(api_name, value, depth=depth+1)
         else:
             self.index += 1
 
@@ -211,4 +205,42 @@ def extract_detailed_api_segments(full_api_name_with_direction_status):
     else:
         full_api_name = None
     return api_name, full_api_name, direction_status
-    
+
+
+def get_module_and_atttribute_name(attribute):
+    '''
+    Function Description:
+        Get the module and attribute name.
+    Parameter:
+        name: Attribute of a module. Example: torch.float16
+    Return:
+        module_name: Name of the module. Example: torch.
+        attribute_name: Name of the attribute. Example: float16.
+    '''
+    try:
+        module_name, attribute_name = attribute.split(Const.SEP)
+    except ValueError as e:
+        logger.error(f"Failed to get module and attribute name from {attribute}")
+        raise CompareException(CompareException.INVALID_DATA_ERROR) from e
+    return module_name, attribute_name
+
+
+def get_attribute(module_name, attribute_name):
+    '''
+    Function Description:
+        Get the attribute of the module.
+    Parameter:
+        module_name: Name of the module.
+        attribute_name: Name of the attribute.
+    '''
+    attribute = None
+    if module_name not in Const.MODULE_WHITE_LIST:
+        logger.error(f"Module {module_name} is not in white list")
+        raise CompareException(CompareException.INVALID_DATA_ERROR)
+    try:
+        module = importlib.import_module(module_name)
+        attribute = getattr(module, attribute_name)
+    except (ImportError, AttributeError) as e:
+        logger.error(f"Failed to get attribute {attribute_name} from module {module_name}: {e}")
+        raise CompareException(CompareException.INVALID_ATTRIBUTE_ERROR) from e
+    return attribute

msprobe/pytorch/api_accuracy_checker/compare/algorithm.py

@@ -1,3 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # 定义比对算法及比对标准
 import torch
 import numpy as np
@@ -142,7 +159,7 @@ def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol):
     输出： 
         inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例
     '''
-    abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype)
+    _, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype)
     golden_same_dtype = bench_output.astype(device_output.dtype)
     a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MIN
     a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MAX
@@ -209,5 +226,5 @@ def get_ulp_err(bench_output, device_output, dtype):
 
 
 def calc_ulp_err(bench_output, device_output, eb, exponent_num, data_type):
-    return  (device_output.astype(data_type) - bench_output).astype(data_type) * \
+    return (device_output.astype(data_type) - bench_output).astype(data_type) * \
             np.exp2(-eb + exponent_num).astype(data_type)

msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py

@@ -1,3 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import argparse
 import math
 import os
@@ -7,7 +24,7 @@ from collections import namedtuple
 import torch
 import pandas as pd
 
-from msprobe.core.common.file_utils import write_csv
+from msprobe.core.common.file_utils import write_csv, read_csv
 from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig
 from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECISION_COMPARE_RESULT_FILE_NAME, \
     API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \
@@ -17,18 +34,18 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECI
 from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn
 from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path
 from msprobe.pytorch.api_accuracy_checker.common.utils import extract_detailed_api_segments
-from msprobe.core.common.file_utils import FileChecker, change_mode, check_path_before_create, create_directory
+from msprobe.core.common.file_utils import FileChecker, change_mode, create_directory
 from msprobe.pytorch.common.log import logger
 from msprobe.core.common.utils import CompareException
 from msprobe.core.common.const import Const, CompareConst, FileCheckConst
 
 CompareConfig = namedtuple('CompareConfig', ['npu_csv_path', 'gpu_csv_path', 'result_csv_path', 'details_csv_path'])
-BenchmarkInf_Nan_Consistency = namedtuple('BenchmarkInf_Nan_Consistency', ['small_value_inf_nan_consistency', 
+BenchmarkInfNanConsistency = namedtuple('BenchmarkInfNanConsistency', ['small_value_inf_nan_consistency', 
                                                                            'rmse_inf_nan_consistency', 
                                                                            'max_rel_inf_nan_consistency', 
                                                                            'mean_rel_inf_nan_consistency', 
                                                                            'eb_inf_nan_consistency'])
-unsupported_message = 'This data type does not support benchmark compare.'
+UNSUPPORTED_MESSAGE = 'This data type does not support benchmark compare.'
 
 DEFAULT_THRESHOLD = 1
 
@@ -154,11 +171,11 @@ class BenchmarkStandard(Standard):
         self.rmse_status = self._get_status(self.rmse_ratio, 'rmse') if rmse_inf_nan_consistency \
             else CompareConst.ERROR
         self.check_result_list.append(self.rmse_status)
-        self.max_rel_err_status = self._get_status(self.max_rel_err_ratio, 'max_rel_err') if max_rel_inf_nan_consistency \
-            else CompareConst.ERROR
+        self.max_rel_err_status = self._get_status(
+             self.max_rel_err_ratio, 'max_rel_err') if max_rel_inf_nan_consistency else CompareConst.ERROR
         self.check_result_list.append(self.max_rel_err_status)
-        self.mean_rel_err_status = self._get_status(self.mean_rel_err_ratio, 'mean_rel_err') if mean_rel_inf_nan_consistency \
-            else CompareConst.ERROR
+        self.mean_rel_err_status = self._get_status(
+            self.mean_rel_err_ratio, 'mean_rel_err') if mean_rel_inf_nan_consistency else CompareConst.ERROR
         self.check_result_list.append(self.mean_rel_err_status)
         self.eb_status = self._get_status(self.eb_ratio, 'eb')
         if CompareConst.ERROR in self.check_result_list:
@@ -187,7 +204,8 @@ class BenchmarkStandard(Standard):
                                         self.npu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR),
                                         self.gpu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), 10000.0)
         self.compare_message += max_rel_message
-        self.mean_rel_err_ratio, mean_rel_inf_nan_consistency, mean_rel_message = self._calc_ratio(ApiPrecisionCompareColumn.MEAN_REL_ERR,
+        self.mean_rel_err_ratio, mean_rel_inf_nan_consistency, mean_rel_message = self._calc_ratio(
+                                        ApiPrecisionCompareColumn.MEAN_REL_ERR,
                                         self.npu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR),
                                         self.gpu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR), 10000.0)
         self.compare_message += mean_rel_message
@@ -196,8 +214,9 @@ class BenchmarkStandard(Standard):
                                         self.gpu_precision.get(ApiPrecisionCompareColumn.EB), 10000.0)
         self.compare_message += eb_message
         
-        return BenchmarkInf_Nan_Consistency(small_value_inf_nan_consistency, rmse_inf_nan_consistency, 
-                                            max_rel_inf_nan_consistency, mean_rel_inf_nan_consistency, eb_inf_nan_consistency)
+        return BenchmarkInfNanConsistency(small_value_inf_nan_consistency, rmse_inf_nan_consistency, 
+                                          max_rel_inf_nan_consistency, mean_rel_inf_nan_consistency,
+                                          eb_inf_nan_consistency)
 
 
 class ULPStandard(Standard):
@@ -269,12 +288,12 @@ def api_precision_compare(config):
     logger.info(f"Compare task result will be saved in {config.result_csv_path}")
     logger.info(f"Compare task detail will be saved in {config.details_csv_path}")
     try:
-        npu_data = pd.read_csv(config.npu_csv_path)
+        npu_data = read_csv(config.npu_csv_path)
     except Exception as err:
         logger.error(f"Open npu csv Error: %s" % str(err))
     check_csv_columns(npu_data.columns, "npu_csv")
     try:
-        gpu_data = pd.read_csv(config.gpu_csv_path)
+        gpu_data = read_csv(config.gpu_csv_path)
     except Exception as err:
         logger.error(f"Open gpu csv Error: %s" % str(err))
     check_csv_columns(gpu_data.columns, "gpu_csv")
@@ -292,8 +311,10 @@ def api_precision_compare(config):
 
 def online_api_precision_compare(online_config):
     rank = online_config.rank
-    result_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.result_csv_path).replace("_rank*.csv", f"_rank{rank}.csv")
-    details_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.details_csv_path).replace("_rank*.csv", f"_rank{rank}.csv")
+    result_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.result_csv_path).replace(
+                    "_rank*.csv", f"_rank{rank}.csv")
+    details_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.details_csv_path).replace(
+                    "_rank*.csv", f"_rank{rank}.csv")
     detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()]
     result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()]
     if not os.path.exists(result_csv_path):
@@ -315,6 +336,7 @@ def online_api_precision_compare(online_config):
 def analyse_csv(npu_data, gpu_data, config):
     forward_status, backward_status = [], []
     last_api_name, last_api_dtype, last_api_full_name = None, None, None
+    last_api_skip_message = ''
     for _, row_npu in npu_data.iterrows():
         message = ''
         compare_column = ApiPrecisionOutputColumn()
@@ -328,7 +350,7 @@ def analyse_csv(npu_data, gpu_data, config):
             compare_column.compare_result = CompareConst.SKIP
             compare_column.compare_message = err_message
             write_detail_csv(compare_column.to_column_value(), config.details_csv_path)
-            write_csv([[full_api_name_with_direction_status, CompareConst.SKIP, CompareConst.SKIP,  err_message]], 
+            write_csv([[full_api_name_with_direction_status, CompareConst.SKIP, CompareConst.SKIP, err_message]], 
                       config.result_csv_path)
             continue
         if row_gpu.empty:
@@ -355,19 +377,19 @@ def analyse_csv(npu_data, gpu_data, config):
 
         if last_api_name is not None and api_full_name != last_api_name:
             if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST:
-                message = unsupported_message
+                message = UNSUPPORTED_MESSAGE
                 write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path)
                 print_test_success(last_api_name, CompareConst.SKIP, CompareConst.SKIP)
-                forward_status, backward_status = [], []
-                message = ''
             else:
                 forward_result = get_api_checker_result(forward_status)
                 backward_result = get_api_checker_result(backward_status)
                 message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else ""
+                message += last_api_skip_message if forward_result == CompareConst.SKIP else ""
                 write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path)
                 print_test_success(last_api_name, forward_result, backward_result)
-                forward_status, backward_status = [], []
-                message = ''
+                last_api_skip_message = ''
+            forward_status, backward_status = [], []
+            message = ''
 
         is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST
         last_api_name = api_full_name
@@ -378,6 +400,8 @@ def analyse_csv(npu_data, gpu_data, config):
 
         if direction_status == 'forward':
             forward_status.append(new_status)
+            last_api_skip_message = str(row_npu[ApiPrecisionCompareColumn.MESSAGE]) if new_status == CompareConst.SKIP \
+                                    else ''
         elif direction_status == 'backward':
             backward_status.append(new_status)
         else:
@@ -385,15 +409,17 @@ def analyse_csv(npu_data, gpu_data, config):
 
     if last_api_name is not None:
         if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST:
-            message = unsupported_message
+            message = UNSUPPORTED_MESSAGE
             write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path)
             print_test_success(last_api_name, CompareConst.SKIP, CompareConst.SKIP)
         else:
             forward_result = get_api_checker_result(forward_status)
             backward_result = get_api_checker_result(backward_status)
             message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else ""
+            message += last_api_skip_message if forward_result == CompareConst.SKIP else ""
             write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path)
             print_test_success(last_api_name, forward_result, backward_result)
+            last_api_skip_message = ''
 
 
 def get_api_status(row_npu, row_gpu, api_name, compare_column):
@@ -576,8 +602,7 @@ def _api_precision_compare(parser=None):
 def _api_precision_compare_command(args):
     npu_csv_path = get_validated_result_csv_path(args.npu_csv_path, 'detail')
     gpu_csv_path = get_validated_result_csv_path(args.gpu_csv_path, 'detail')
-    out_path = os.path.realpath(args.out_path) if args.out_path else "./"
-    check_path_before_create(out_path)
+    out_path = args.out_path if args.out_path else Const.DEFAULT_PATH
     create_directory(out_path)
     out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE)
     out_path = out_path_checker.common_check()
@@ -595,7 +620,7 @@ def _api_precision_compare_parser(parser):
     parser.add_argument("-gpu", "--gpu_csv_path", dest="gpu_csv_path", default="", type=str,
                         help="<Required> Accuracy_checking_details.csv generated on the GPU by using the "
                              "api_accuracy_checker tool.",
-                        required=False)
+                        required=True)
     parser.add_argument("-o", "--out_path", dest="out_path", default="", type=str,
                         help="<optional> The api precision compare task result out path.",
                         required=False)

msprobe/pytorch/api_accuracy_checker/compare/compare.py

@@ -1,3 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # 进行比对及结果展示
 import os
 from collections import namedtuple
@@ -127,8 +144,12 @@ class Comparator:
         return test_rows
 
     def write_csv_title(self):
-        summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS,
-                              self.COLUMN_BACKWARD_SUCCESS, "Message"]]
+        summary_test_rows = [
+            [self.COLUMN_API_NAME, 
+             self.COLUMN_FORWARD_SUCCESS,
+             self.COLUMN_BACKWARD_SUCCESS,
+             "Message"]
+            ]
         for save_path, detail_save_path in zip(self.save_path_list, self.detail_save_path_list):
             if not os.path.exists(save_path):
                 write_csv(summary_test_rows, save_path)
@@ -240,13 +261,15 @@ class Comparator:
     def _compare_core(self, api_name, bench_output, device_output):
         compare_column = CompareColumn()
         if not isinstance(bench_output, type(device_output)):
-            return CompareConst.ERROR, compare_column, "bench and npu output type is different."
+            status = CompareConst.ERROR
+            message = "bench and npu output type is different."
         elif isinstance(bench_output, dict):
             b_keys, n_keys = set(bench_output.keys()), set(device_output.keys())
             if b_keys != n_keys:
-                return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different."
+                status = CompareConst.ERROR
+                message = "bench and npu output dict keys are different."
             else:
-                status, compare_result, message = self._compare_core(api_name, list(bench_output.values()),
+                status, compare_column, message = self._compare_core(api_name, list(bench_output.values()),
                                                                      list(device_output.values()))
         elif isinstance(bench_output, torch.Tensor):
             copy_bench_out = bench_output.detach().clone()
@@ -254,19 +277,20 @@ class Comparator:
             compare_column.bench_type = str(copy_bench_out.dtype)
             compare_column.npu_type = str(copy_device_output.dtype)
             compare_column.shape = tuple(device_output.shape)
-            status, compare_result, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output,
+            status, compare_column, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output,
                                                                          compare_column)
         elif isinstance(bench_output, (bool, int, float, str)):
             compare_column.bench_type = str(type(bench_output))
             compare_column.npu_type = str(type(device_output))
-            status, compare_result, message = self._compare_builtin_type(bench_output, device_output, compare_column)
+            status, compare_column, message = self._compare_builtin_type(bench_output, device_output, compare_column)
         elif bench_output is None:
-            return CompareConst.SKIP, compare_column, "Bench output is None, skip this test."
+            status = CompareConst.SKIP
+            message = "Bench output is None, skip this test."
         else:
-            return CompareConst.PASS, compare_column,
-        "Unexpected output type in compare_core: {}".format(type(bench_output))
+            status = CompareConst.ERROR
+            message = "Unexpected output type in compare_core: {}".format(type(bench_output))
 
-        return status, compare_result, message
+        return status, compare_column, message
 
     def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column):
         cpu_shape = bench_output.shape
@@ -330,21 +354,23 @@ class Comparator:
                     compare_column.max_ulp_error = np.max(ulp_err)
                     compare_column.mean_ulp_error = np.mean(ulp_err)
                     if dtype == torch.float32:
-                        compare_column.ulp_error_proportion = np.sum(ulp_err > CompareConst.ULP_FLOAT32_THRESHOLD) / bench_output.size
+                        compare_column.ulp_error_proportion = \
+                        np.sum(ulp_err > CompareConst.ULP_FLOAT32_THRESHOLD) / bench_output.size
                     else:
-                        compare_column.ulp_error_proportion = np.sum(ulp_err > CompareConst.ULP_FLOAT16_THRESHOLD) / bench_output.size
+                        compare_column.ulp_error_proportion = \
+                            np.sum(ulp_err > CompareConst.ULP_FLOAT16_THRESHOLD) / bench_output.size
             else:
                 dtype_config = precision_configs.get(dtype)
                 small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0])
                 abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0])
                 compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask)
                 rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask)
-                compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask))
-                compare_column.EB = get_error_balance(bench_output, device_output)
+                compare_column.rmse = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask))
+                compare_column.eb = get_error_balance(bench_output, device_output)
                 if rel_err.size == 0:
                     return CompareConst.ERROR, compare_column, "Relative error result list is empty."
-                compare_column.Max_rel_error = get_max_rel_err(rel_err)
-                compare_column.Mean_rel_error = get_mean_rel_err(rel_err)
+                compare_column.max_rel_error = get_max_rel_err(rel_err)
+                compare_column.mean_rel_error = get_mean_rel_err(rel_err)
 
         cos_res, cos_status, msg = cosine_sim(bench_output, device_output)
         compare_column.cosine_sim = cos_res
@@ -363,7 +389,8 @@ class Comparator:
             hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, CompareConst.HUNDRED_RATIO_THRESHOLD)
             compare_column.rel_err_hundredth = hundred_res
             if not hundred_status:
-                message += "Relative error is greater than 0.01, consider as error, skip other check and set to SPACE.\n"
+                message += "Relative error is greater than 0.01, consider as error, " \
+                           "skip other check and set to SPACE.\n"
                 return CompareConst.ERROR, compare_column, message
         thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD)
         compare_column.rel_err_thousandth = thousand_res
@@ -373,14 +400,17 @@ class Comparator:
                 return CompareConst.PASS, compare_column, message
             message += "Relative error is greater than 0.001, consider as warning, skip other check and set to SPACE.\n"
             return CompareConst.WARNING, compare_column, message
-        ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.TEN_THOUSAND_RATIO_THRESHOLD)
+        ten_thousand_res, ten_thousand_status = get_rel_err_ratio(
+                                                rel_err_orign, CompareConst.TEN_THOUSAND_RATIO_THRESHOLD)
         compare_column.rel_err_ten_thousandth = ten_thousand_res
         if dtype in [torch.float32, torch.float64]:
             if not thousand_status:
-                message += "Relative error is greater than 0.001, consider as error, skip other check and set to SPACE.\n"
+                message += "Relative error is greater than 0.001, consider as error, " \
+                           "skip other check and set to SPACE.\n"
                 return CompareConst.ERROR, compare_column, message
             if not ten_thousand_status:
-                message += "Relative error is greater than 0.0001, consider as warning, skip other check and set to SPACE.\n"
+                message += "Relative error is greater than 0.0001, consider as warning, " \
+                           "skip other check and set to SPACE.\n"
                 return CompareConst.WARNING, compare_column, message
             message += "Relative error is less than 0.0001, consider as pass.\n"
         return CompareConst.PASS, compare_column, message

msprobe/pytorch/api_accuracy_checker/compare/compare_column.py

@@ -1,3 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.const import CompareConst
 
 
@@ -12,11 +29,11 @@ class CompareColumn:
         self.rel_err_thousandth = CompareConst.SPACE
         self.rel_err_ten_thousandth = CompareConst.SPACE
         self.error_rate = CompareConst.SPACE
-        self.EB = CompareConst.SPACE
-        self.RMSE = CompareConst.SPACE
+        self.eb = CompareConst.SPACE
+        self.rmse = CompareConst.SPACE
         self.small_value_err_ratio = CompareConst.SPACE
-        self.Max_rel_error = CompareConst.SPACE
-        self.Mean_rel_error = CompareConst.SPACE
+        self.max_rel_error = CompareConst.SPACE
+        self.mean_rel_error = CompareConst.SPACE
         self.inf_nan_error_ratio = CompareConst.SPACE
         self.rel_err_ratio = CompareConst.SPACE
         self.abs_err_ratio = CompareConst.SPACE
@@ -26,8 +43,8 @@ class CompareColumn:
 
     def to_column_value(self, is_pass, message):
         return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth,
-                self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, 
-                self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, self.inf_nan_error_ratio, 
+                self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.eb, self.rmse, 
+                self.small_value_err_ratio, self.max_rel_error, self.mean_rel_error, self.inf_nan_error_ratio, 
                 self.rel_err_ratio, self.abs_err_ratio, self.max_ulp_error, self.mean_ulp_error, 
                 self.ulp_error_proportion, is_pass, message]
 

msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py

@@ -1,3 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import time
 import os
 import math
@@ -32,7 +49,8 @@ threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml")
 apis_threshold = load_yaml(threshold_yaml_path)
 
 
-DETAIL_TEST_ROWS = [[
+DETAIL_TEST_ROWS = [
+            [
             "API Name", "Bench Dtype", "DEVICE Dtype", "Shape",
             "余弦相似度",
             "最大绝对误差",
@@ -53,7 +71,8 @@ DETAIL_TEST_ROWS = [[
             "ULP误差大于阈值占比",
             "Status",
             "Message"
-        ]]
+            ]
+        ]
 
 
 precision_configs = {
@@ -154,11 +173,11 @@ class ApiPrecisionCompareColumn:
     def to_required_columns():
         return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.DEVICE_DTYPE, 
                 ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE, ApiPrecisionCompareColumn.RMSE, 
-                ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR, ApiPrecisionCompareColumn.EB,
-                ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, 
-                ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO, 
-                ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION,
-                ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH]
+                ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR,
+                ApiPrecisionCompareColumn.EB, ApiPrecisionCompareColumn.ERROR_RATE, 
+                ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO, 
+                ApiPrecisionCompareColumn.ABS_ERR_RATIO, ApiPrecisionCompareColumn.MEAN_ULP_ERR, 
+                ApiPrecisionCompareColumn.ULP_ERR_PROPORTION, ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH]
 
     @staticmethod
     def get_detail_csv_title():
@@ -175,7 +194,8 @@ class ApiPrecisionCompareColumn:
                 ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION, 
                 ApiPrecisionCompareColumn.ULP_ERR_PROPORTION_RATIO, ApiPrecisionCompareColumn.ULP_ERR_STATUS,
                 ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH, ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH_STATUS,
-                ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, ApiPrecisionCompareColumn.MESSAGE]
+                ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, 
+                ApiPrecisionCompareColumn.MESSAGE]
     
     @staticmethod
     def get_result_csv_title():

msprobe/pytorch/api_accuracy_checker/config.yaml

@@ -7,4 +7,4 @@ nfs_path: ""
 host: ""
 port: -1
 rank_list: [0]
-tls_path: ""
+tls_path: "./"

msprobe/pytorch/api_accuracy_checker/generate_op_script/config_op.json

@@ -0,0 +1,9 @@
+{
+    "dump_json_path": "./dump.json",
+    "api_name": "",
+    "extract_api_path": "",
+    "propagation": "forward",
+    "data_mode": "random_data",
+    "random_seed": 1234,
+    "iter_times": 1
+}