mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py

@@ -1,15 +1,34 @@
-import json
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
+from tqdm import tqdm
 
-from msprobe.core.common.file_utils import FileOpen, create_directory, write_csv
-from msprobe.core.common.utils import add_time_as_suffix
 from msprobe.core.common.const import Const, CompareConst, MsCompareConst
-from msprobe.mindspore.common.log import logger
+from msprobe.core.common.file_utils import FileOpen, create_directory, write_csv, load_json, load_yaml
+from msprobe.core.common.utils import add_time_as_suffix
 from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo
 from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation
 from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms
+from msprobe.mindspore.api_accuracy_checker.data_manager import DataManager
 from msprobe.mindspore.api_accuracy_checker.utils import (check_and_get_from_json_dict, global_context,
                                                           trim_output_compute_element_list)
+from msprobe.mindspore.common.log import logger
+
+cur_path = os.path.dirname(os.path.realpath(__file__))
+yaml_path = os.path.join(cur_path, MsCompareConst.SUPPORTED_API_LIST_FILE)
 
 
 class BasicInfoAndStatus:
@@ -21,6 +40,7 @@ class BasicInfoAndStatus:
         self.status = status
         self.err_msg = err_msg
 
+
 class ResultCsvEntry:
     def __init__(self) -> None:
         self.forward_pass_status = None
@@ -30,14 +50,21 @@ class ResultCsvEntry:
         self.overall_err_msg = None
 
 
+class ProcessResultPacket:
+    def __init__(self, process_status, result, err_msg) -> None:
+        self.process_status = process_status
+        self.result = result
+        self.err_msg = err_msg
+
+
 class ApiAccuracyChecker:
-    def __init__(self):
+    def __init__(self, args):
         self.api_infos = dict()
-        self.results = dict()
+        self.data_manager = DataManager(args.out_path, args.result_csv_path)  # 在初始化时实例化 DataManager
 
     @staticmethod
     def run_and_compare_helper(api_info, api_name_str, api_input_aggregation, forward_or_backward):
-        '''
+        """
         Args:
             api_info: ApiInfo
             api_name_str: str
@@ -51,7 +78,7 @@ class ApiAccuracyChecker:
             get mindspore api output, run torch api and get output.
             compare output.
             record compare result.
-        '''
+        """
         # get output
         if global_context.get_is_constructed():
             # constructed situation, need use constructed input to run mindspore api getting tested_output
@@ -80,13 +107,13 @@ class ApiAccuracyChecker:
                 compare_result_dict[compare_algorithm_name] = compare_result
 
             if compare_result_dict.get(CompareConst.COSINE).pass_status == CompareConst.PASS and \
-                compare_result_dict.get(CompareConst.MAX_ABS_ERR).pass_status == CompareConst.PASS:
+                    compare_result_dict.get(CompareConst.MAX_ABS_ERR).pass_status == CompareConst.PASS:
                 status = CompareConst.PASS
                 err_msg = ""
             else:
                 status = CompareConst.ERROR
-                err_msg = compare_result_dict.get(CompareConst.COSINE).err_msg + \
-                    compare_result_dict.get(CompareConst.MAX_ABS_ERR).err_msg
+                err_msg = (compare_result_dict.get(CompareConst.COSINE).err_msg +
+                           compare_result_dict.get(CompareConst.MAX_ABS_ERR).err_msg)
             basic_info_status = \
                 BasicInfoAndStatus(api_name_with_slot, bench_dtype, tested_dtype, shape, status, err_msg)
             output_list.append(tuple([api_name_str, forward_or_backward, basic_info_status, compare_result_dict]))
@@ -94,13 +121,13 @@ class ApiAccuracyChecker:
 
     @staticmethod
     def prepare_api_input_aggregation(api_info, forward_or_backward=Const.FORWARD):
-        '''
+        """
         Args:
             api_info: ApiInfo
             forward_or_backward: str
         Returns:
             ApiInputAggregation
-        '''
+        """
         forward_inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT)
         kwargs = api_info.get_kwargs()
         if forward_or_backward == Const.FORWARD:
@@ -109,19 +136,42 @@ class ApiAccuracyChecker:
             gradient_inputs = api_info.get_compute_element_list(Const.BACKWARD, Const.INPUT)
         return ApiInputAggregation(forward_inputs, kwargs, gradient_inputs)
 
+    @staticmethod
+    def is_api_checkable(api_name_str):
+        '''
+        Args:
+            api_name_str: str, e.g. "MintFunctional.relu.0.forward", key in data field of api_info.json
+        Returns:
+            is_checkable: bool
+        Description:
+            tell whether this api is checkable based on the key in "data" dict in api_info.json
+        '''
+        api_name_str_list = api_name_str.split(Const.SEP)
+        if len(api_name_str_list) < MsCompareConst.API_NAME_STR_LENGTH:
+            return False
+        api_type_str = api_name_str_list[0]
+        real_api_str = Const.SEP.join(api_name_str_list[1:-2])
+        api_list = load_yaml(yaml_path)
+        supported_tensor_api_list = api_list.get(MsCompareConst.SUPPORTED_TENSOR_LIST_KEY)
+        if api_type_str in (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL):
+            return True
+        if api_type_str == MsCompareConst.TENSOR_API and real_api_str in supported_tensor_api_list:
+            return True
+        return False
+
     def parse(self, api_info_path):
-        with FileOpen(api_info_path, "r") as f:
-            api_info_dict = json.load(f)
+        api_info_dict = load_json(api_info_path)
 
         # init global context
         task = check_and_get_from_json_dict(api_info_dict, MsCompareConst.TASK_FIELD,
-                                            "task field in api_info.json",accepted_type=str,
+                                            "task field in api_info.json", accepted_type=str,
                                             accepted_value=(MsCompareConst.STATISTICS_TASK,
                                                             MsCompareConst.TENSOR_TASK))
         is_constructed = task == MsCompareConst.STATISTICS_TASK
         if not is_constructed:
             dump_data_dir = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DUMP_DATA_DIR_FIELD,
-                                                         "dump_data_dir field in api_info.json", accepted_type=str)
+                                                         "dump_data_dir field in api_info.json",
+                                                         accepted_type=str)
         else:
             dump_data_dir = ""
         global_context.init(is_constructed, dump_data_dir)
@@ -129,14 +179,12 @@ class ApiAccuracyChecker:
         api_info_data = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DATA_FIELD,
                                                      "data field in api_info.json", accepted_type=dict)
         for api_name, api_info in api_info_data.items():
-            is_mint = api_name.split(Const.SEP)[0] in \
-                (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL)
-            if not is_mint:
+            if not self.is_api_checkable(api_name):
                 continue
             forbackward_str = api_name.split(Const.SEP)[-1]
             if forbackward_str not in (Const.FORWARD, Const.BACKWARD):
                 logger.warning(f"api: {api_name} is not recognized as forward api or backward api, skip this.")
-            api_name = Const.SEP.join(api_name.split(Const.SEP)[:-1]) # www.xxx.yyy.zzz --> www.xxx.yyy
+            api_name = Const.SEP.join(api_name.split(Const.SEP)[:-1])  # www.xxx.yyy.zzz --> www.xxx.yyy
             if api_name not in self.api_infos:
                 self.api_infos[api_name] = ApiInfo(api_name)
 
@@ -145,135 +193,87 @@ class ApiAccuracyChecker:
             else:
                 self.api_infos[api_name].load_backward_info(api_info)
 
+    def process_forward(self, api_name_str, api_info):
+        """处理前向检查"""
+        if not api_info.check_forward_info():
+            logger.debug(f"api: {api_name_str} is lack of forward information, skip forward check.")
+            process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.API_NOT_FOUND,
+                                                        result=None,
+                                                        err_msg=f"forward info of {api_name_str} is not found")
+            return process_result_packet
+
+        try:
+            forward_inputs_aggregation = self.prepare_api_input_aggregation(api_info, Const.FORWARD)
+        except Exception as e:
+            logger.warning(f"Exception occurs when getting inputs for {api_name_str} forward api. "
+                           f"Skipping forward check. Detailed exception information: {e}.")
+            process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.EXCEPTION_SKIP,
+                                                        result=None, err_msg=f"{e}")
+            return process_result_packet
+
+        try:
+            forward_output_list = self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation,
+                                                              Const.FORWARD)
+        except Exception as e:
+            logger.warning(f"Exception occurs when running and comparing {api_name_str} forward api. "
+                           f"Detailed exception information: {e}.")
+            process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.EXCEPTION_SKIP,
+                                                        result=None, err_msg=f"{e}")
+            return process_result_packet
+
+        process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.SUCCESS,
+                                                    result=forward_output_list, err_msg="")
+        return process_result_packet
+
+    def process_backward(self, api_name_str, api_info):
+        """处理反向检查"""
+        if not api_info.check_backward_info():
+            logger.debug(f"api: {api_name_str} is lack of backward information, skipping backward check.")
+            process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.API_NOT_FOUND,
+                                                        result=None,
+                                                        err_msg=f"backward info of {api_name_str} is not found")
+            return process_result_packet
+
+        try:
+            backward_inputs_aggregation = self.prepare_api_input_aggregation(api_info, Const.BACKWARD)
+        except Exception as e:
+            logger.warning(f"Exception occurs when getting inputs for {api_name_str} backward api. "
+                           f"Skipping backward check. Detailed exception information: {e}.")
+            process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.EXCEPTION_SKIP,
+                                                        result=None, err_msg=f"{e}")
+            return process_result_packet
+
+        try:
+            backward_output_list = self.run_and_compare_helper(api_info, api_name_str, backward_inputs_aggregation,
+                                                               Const.BACKWARD)
+        except Exception as e:
+            logger.warning(f"Exception occurs when running and comparing {api_name_str} backward api. "
+                           f"Detailed exception information: {e}.")
+            process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.EXCEPTION_SKIP,
+                                                        result=None, err_msg=f"{e}")
+            return process_result_packet
+
+        process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.SUCCESS,
+                                                    result=backward_output_list, err_msg="")
+        return process_result_packet
+
     def run_and_compare(self):
-        for api_name_str, api_info in self.api_infos.items():
-            if not api_info.check_forward_info():
-                logger.warning(f"api: {api_name_str} is lack of forward infomation, skip forward and backward check.")
-                continue
-            try:
-                forward_inputs_aggregation = self.prepare_api_input_aggregation(api_info, Const.FORWARD)
-            except Exception as e:
-                logger.warning(f"exception occurs when getting inputs for {api_name_str} forward api. "
-                               f"skip forward and backward check. detailed exception information: {e}.")
-                continue
-            forward_output_list = None
-            try:
-                forward_output_list = \
-                    self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation, Const.FORWARD)
-            except Exception as e:
-                logger.warning(f"exception occurs when running and comparing {api_name_str} forward api. "
-                               f"detailed exception information: {e}.")
-            self.record(forward_output_list)
-
-            if not api_info.check_backward_info():
-                logger.warning(f"api: {api_name_str} is lack of backward infomation, skip backward check.")
+        for api_name_str, api_info in tqdm(self.api_infos.items()):
+            if not self.data_manager.is_unique_api(api_name_str):
                 continue
-            try:
-                backward_inputs_aggregation = self.prepare_api_input_aggregation(api_info, Const.BACKWARD)
-            except Exception as e:
-                logger.warning(f"exception occurs when getting inputs for {api_name_str} backward api. "
-                               f"skip backward check. detailed exception information: {e}.")
-                continue
-            backward_output_list = None
-            try:
-                backward_output_list = \
-                    self.run_and_compare_helper(api_info, api_name_str, backward_inputs_aggregation, Const.BACKWARD)
-            except Exception as e:
-                logger.warning(f"exception occurs when running and comparing {api_name_str} backward api. "
-                               f"detailed exception information: {e}.")
-            self.record(backward_output_list)
-
-    def record(self, output_list):
-        if output_list is None:
-            return
-        for output in output_list:
-            api_real_name, forward_or_backward, basic_info, compare_result_dict = output
-            key = tuple([api_real_name, forward_or_backward])
-            if key not in self.results:
-                self.results[key] = []
-            self.results[key].append(tuple([basic_info, compare_result_dict]))
-
-
-    def to_detail_csv(self, csv_dir):
-        # detail_csv
-        detail_csv = []
-        detail_csv_header_basic_info = [
-            MsCompareConst.DETAIL_CSV_API_NAME,
-            MsCompareConst.DETAIL_CSV_BENCH_DTYPE,
-            MsCompareConst.DETAIL_CSV_TESTED_DTYPE,
-            MsCompareConst.DETAIL_CSV_SHAPE,
-        ]
-        detail_csv_header_compare_result = list(compare_algorithms.keys())
-        detail_csv_header_status = [
-            MsCompareConst.DETAIL_CSV_PASS_STATUS,
-            MsCompareConst.DETAIL_CSV_MESSAGE,
-        ]
-
-        detail_csv_header = detail_csv_header_basic_info + detail_csv_header_compare_result + detail_csv_header_status
-        detail_csv.append(detail_csv_header)
-
-        for _, results in self.results.items():
-            # detail csv
-            for res in results:
-                basic_info, compare_result_dict = res
-                csv_row_basic_info = \
-                    [basic_info.api_name, basic_info.bench_dtype, basic_info.tested_dtype, basic_info.shape]
-                csv_row_compare_result = list(compare_result_dict.get(algorithm_name).compare_value \
-                                            for algorithm_name in detail_csv_header_compare_result)
-                csv_row_status = [basic_info.status, basic_info.err_msg]
-                csv_row = csv_row_basic_info  + csv_row_compare_result + csv_row_status
-                detail_csv.append(csv_row)
-
-        file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.DETAIL_CSV_FILE_NAME))
-        create_directory(csv_dir)
-        write_csv(detail_csv, file_name, mode="w")
-
-
-    def to_result_csv(self, csv_dir):
-        result_csv_dict = dict()
-        for key, results in self.results.items():
-            api_real_name, forward_or_backward = key
-            forward_or_backward_pass_status = CompareConst.PASS
-            forward_or_backward_overall_err_msg = ""
-            # detail csv
-            for res in results:
-                basic_info, _ = res
-                if basic_info.status != CompareConst.PASS:
-                    forward_or_backward_pass_status = CompareConst.ERROR
-                forward_or_backward_overall_err_msg += basic_info.err_msg
-            forward_or_backward_overall_err_msg = \
-                "" if forward_or_backward_pass_status == CompareConst.PASS else forward_or_backward_overall_err_msg
-
-            #result_csv_dict
-            if api_real_name not in result_csv_dict:
-                result_csv_dict[api_real_name] = ResultCsvEntry()
-            if forward_or_backward == Const.FORWARD:
-                result_csv_dict[api_real_name].forward_pass_status = forward_or_backward_pass_status
-                result_csv_dict[api_real_name].forward_err_msg = forward_or_backward_overall_err_msg
-            else:
-                result_csv_dict[api_real_name].backward_pass_status = forward_or_backward_pass_status
-                result_csv_dict[api_real_name].backward_err_msg = forward_or_backward_overall_err_msg
-
-        #result_csv
-        result_csv = []
-        result_csv_header = [
-            MsCompareConst.DETAIL_CSV_API_NAME,
-            MsCompareConst.RESULT_CSV_FORWARD_TEST_SUCCESS,
-            MsCompareConst.RESULT_CSV_BACKWARD_TEST_SUCCESS,
-            MsCompareConst.DETAIL_CSV_MESSAGE,
-        ]
-        result_csv.append(result_csv_header)
-
-        for api_name, result_csv_entry in result_csv_dict.items():
-            if result_csv_entry.forward_pass_status == CompareConst.PASS and \
-                result_csv_entry.backward_pass_status == CompareConst.PASS:
-                overall_err_msg = ""
-            else:
-                overall_err_msg = result_csv_entry.forward_err_msg + result_csv_entry.backward_err_msg
-            row = [api_name, result_csv_entry.forward_pass_status,
-                   result_csv_entry.backward_pass_status, overall_err_msg]
-            result_csv.append(row)
-
-        file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.RESULT_CSV_FILE_NAME))
-        create_directory(csv_dir)
-        write_csv(result_csv, file_name, mode="w")
+
+            # 处理前向
+            process_result_packet = self.process_forward(api_name_str, api_info)
+            if process_result_packet.process_status is MsCompareConst.ProcessStatus.SUCCESS:
+                self.data_manager.record(process_result_packet.result)
+            elif process_result_packet.process_status == MsCompareConst.ProcessStatus.EXCEPTION_SKIP:
+                self.data_manager.record_exception_skip(api_name_str, Const.FORWARD, process_result_packet.err_msg)
+
+            # 处理反向
+            process_result_packet = self.process_backward(api_name_str, api_info)
+            if process_result_packet.process_status is MsCompareConst.ProcessStatus.SUCCESS:
+                self.data_manager.record(process_result_packet.result)
+            elif process_result_packet.process_status == MsCompareConst.ProcessStatus.EXCEPTION_SKIP:
+                self.data_manager.record_exception_skip(api_name_str, Const.BACKWARD, process_result_packet.err_msg)
+
+            self.data_manager.save_results(api_name_str)

msprobe/mindspore/api_accuracy_checker/api_info.py

@@ -1,9 +1,25 @@
-from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.const import Const
-from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict
 from msprobe.core.common.exceptions import ApiAccuracyCheckerException
-from msprobe.mindspore.common.log import logger
 from msprobe.core.common.utils import is_invalid_pattern
+from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement
+from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict
+from msprobe.mindspore.common.log import logger
+
 
 class ApiInfo:
     def __init__(self, api_name):
@@ -66,11 +82,10 @@ class ApiInfo:
                 err_msg = "ApiInfo.get_kwargs failed: compute_element_dict key is not a string"
                 logger.error_log_with_exp(err_msg,
                                           ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed))
-            if not isinstance(compute_element_info, (list, dict)):
-                err_msg = "ApiInfo.get_kwargs failed: compute_element_dict value is not a list or dict"
+            if not (isinstance(compute_element_info, (list, dict)) or compute_element_info is None):
+                err_msg = "ApiInfo.get_kwargs failed: compute_element_dict value is not a list, dict or null"
                 logger.error_log_with_exp(err_msg,
                                           ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed))
         kwargs_compute_element_dict = {key_str: ComputeElement(compute_element_info=compute_element_info)
                                        for key_str, compute_element_info in kwargs_dict.items()}
         return kwargs_compute_element_dict
-

msprobe/mindspore/api_accuracy_checker/api_runner.py

@@ -1,15 +1,27 @@
-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import mindspore
 import torch
 from mindspore import ops
-
-from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement
 from msprobe.core.common.const import Const, MsCompareConst
 from msprobe.core.common.exceptions import ApiAccuracyCheckerException
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple
+from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement
 from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list, torch_dtype_to_dtype_str
+from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple
+from msprobe.mindspore.common.log import logger
 
 
 class ApiInputAggregation:
@@ -24,11 +36,23 @@ class ApiInputAggregation:
         self.kwargs = kwargs
         self.gradient_inputs = gradient_inputs
 
+
 api_parent_module_mapping = {
     (MsCompareConst.MINT, Const.MS_FRAMEWORK): mindspore.mint,
     (MsCompareConst.MINT, Const.PT_FRAMEWORK): torch,
     (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional,
-    (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional
+    (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional,
+    (MsCompareConst.TENSOR_API, Const.MS_FRAMEWORK): mindspore.Tensor,
+    (MsCompareConst.TENSOR_API, Const.PT_FRAMEWORK): torch.Tensor
+}
+
+api_parent_module_str_mapping = {
+    (MsCompareConst.MINT, Const.MS_FRAMEWORK): "mindspore.mint",
+    (MsCompareConst.MINT, Const.PT_FRAMEWORK): "torch",
+    (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): "mindspore.mint.nn.functional",
+    (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): "torch.nn.functional",
+    (MsCompareConst.TENSOR_API, Const.MS_FRAMEWORK): "mindspore.Tensor",
+    (MsCompareConst.TENSOR_API, Const.PT_FRAMEWORK): "torch.Tensor"
 }
 
 
@@ -60,7 +84,7 @@ class ApiRunner:
             api_name_str: str, the trimmed key of data dict in api_info.json. e.g. "MintFunctional.relu.0"
 
         Return:
-            api_type_str: str, Union["MintFunctional", "Mint"]
+            api_type_str: str, Union["MintFunctional", "Mint", "Tensor"]
             api_sub_name: str, e.g. "relu"
         '''
         api_name_list = api_name_str.split(Const.SEP)
@@ -68,8 +92,8 @@ class ApiRunner:
             err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue))
         api_type_str, api_sub_name = api_name_list[0], api_name_list[1]
-        if api_type_str not in [MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL]:
-            err_msg = f"ApiRunner.get_info_from_name failed: not mint or mint.nn.functional api"
+        if api_type_str not in [MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL, MsCompareConst.TENSOR_API]:
+            err_msg = f"ApiRunner.get_info_from_name failed: not mint, mint.nn.functional or Tensor api"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue))
 
         return api_type_str, api_sub_name
@@ -78,7 +102,7 @@ class ApiRunner:
     def get_api_instance(api_type_str, api_sub_name, api_platform):
         '''
         Args:
-            api_type_str: str, Union["MintFunctional", "Mint"]
+            api_type_str: str, Union["MintFunctional", "Mint", "Tensor"]
             api_sub_name: str, e.g. "relu"
             api_platform: str: Union["mindpore", "torch"]
 
@@ -92,9 +116,8 @@ class ApiRunner:
         '''
 
         api_parent_module = api_parent_module_mapping.get((api_type_str, api_platform))
-        module_str = "mindspore.mint." if api_platform == Const.MS_FRAMEWORK else "torch."
-        submodule_str = "nn.functional." if api_type_str == MsCompareConst.MINT_FUNCTIONAL else ""
-        full_api_name = module_str + submodule_str + api_sub_name
+        api_parent_module_str = api_parent_module_str_mapping.get((api_type_str, api_platform))
+        full_api_name = api_parent_module_str + Const.SEP + api_sub_name
         if not hasattr(api_parent_module, api_sub_name):
             err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not found"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong))
@@ -115,7 +138,7 @@ class ApiRunner:
         gradient_inputs = api_input_aggregation.gradient_inputs
 
         if forward_or_backward == Const.FORWARD:
-            forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple
+            forward_result = api_instance(*inputs, **kwargs)  # can be single tensor or tuple
             forward_result_tuple = convert_to_tuple(forward_result)
             res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple]
         else:
@@ -127,18 +150,20 @@ class ApiRunner:
             if api_platform == Const.MS_FRAMEWORK:
                 if len(gradient_inputs) == 1:
                     gradient_inputs = gradient_inputs[0]
+
                 def api_with_kwargs(*forward_inputs):
                     return api_instance(*forward_inputs, **kwargs)
+
                 grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_with_kwargs)
-                backward_result = grad_func(*inputs, gradient_inputs) # can be single tensor or tuple
+                backward_result = grad_func(*inputs, gradient_inputs)  # can be single tensor or tuple
                 backward_result_tuple = convert_to_tuple(backward_result)
                 res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple]
             else:
-                #set requires_grad
+                # set requires_grad
                 requires_grad_index = []
                 for index, tensor in enumerate(inputs):
                     if isinstance(tensor, torch.Tensor) and \
-                        torch_dtype_to_dtype_str.get(tensor.dtype) in float_dtype_str_list:
+                            torch_dtype_to_dtype_str.get(tensor.dtype) in float_dtype_str_list:
                         setattr(tensor, "requires_grad", True)
                         requires_grad_index.append(index)
                 forward_results = api_instance(*inputs, **kwargs)
@@ -153,4 +178,4 @@ class ApiRunner:
         return res_compute_element_list
 
 
-api_runner = ApiRunner()
+api_runner = ApiRunner()

msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py

@@ -1,12 +1,27 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import ABC, abstractmethod
 
 import mindspore
-import torch
 import numpy as np
-
+import torch
+from msprobe.core.common.const import CompareConst, MsCompareConst
 from msprobe.core.common.exceptions import ApiAccuracyCheckerException
 from msprobe.mindspore.common.log import logger
-from msprobe.core.common.const import CompareConst, MsCompareConst
+
 
 class CompareResult:
     def __init__(self, compare_value, pass_status, err_msg):
@@ -28,7 +43,7 @@ class BaseCompareAlgorithm(ABC):
             CompareConst.MAX_ABS_ERR: {
                 CompareConst.PASS: "",
                 CompareConst.ERROR: "max absolute difference is greater than " \
-                    f"threshold: {CompareConst.MAX_ABS_ERR_THRESHOLD} ",
+                                    f"threshold: {CompareConst.MAX_ABS_ERR_THRESHOLD} ",
                 CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing ",
             },
             CompareConst.MAX_RELATIVE_ERR: {
@@ -68,7 +83,7 @@ class BaseCompareAlgorithm(ABC):
             ndarray = tensor.to(torch.float64, copy=True).numpy()
         else:
             err_msg = "BaseCompareAlgorithm.convert_to_np_float64_ndarray failed: " \
-                "input is not mindspore.Tensor or torch.Tensor"
+                      "input is not mindspore.Tensor or torch.Tensor"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType))
         return ndarray
 
@@ -189,9 +204,8 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm):
             return CompareConst.ERROR
 
 
-
 compare_algorithms = {
     CompareConst.COSINE: CosineSimilarityCompareAlgorithm(),
     CompareConst.MAX_ABS_ERR: MaxAbsoluteDiffCompareAlgorithm(),
     CompareConst.MAX_RELATIVE_ERR: MaxRelativeDiffCompareAlgorithm(),
-}
+}