mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/mindspore/api_accuracy_checker/api_runner.py

@@ -1,15 +1,27 @@
-
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import mindspore
 import torch
 from mindspore import ops
-
-from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement
 from msprobe.core.common.const import Const, MsCompareConst
 from msprobe.core.common.exceptions import ApiAccuracyCheckerException
-from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple
+from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement
 from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list, torch_dtype_to_dtype_str
+from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple
+from msprobe.mindspore.common.log import logger
 
 
 class ApiInputAggregation:
@@ -24,11 +36,23 @@ class ApiInputAggregation:
         self.kwargs = kwargs
         self.gradient_inputs = gradient_inputs
 
+
 api_parent_module_mapping = {
     (MsCompareConst.MINT, Const.MS_FRAMEWORK): mindspore.mint,
     (MsCompareConst.MINT, Const.PT_FRAMEWORK): torch,
     (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional,
-    (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional
+    (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional,
+    (MsCompareConst.TENSOR_API, Const.MS_FRAMEWORK): mindspore.Tensor,
+    (MsCompareConst.TENSOR_API, Const.PT_FRAMEWORK): torch.Tensor
+}
+
+api_parent_module_str_mapping = {
+    (MsCompareConst.MINT, Const.MS_FRAMEWORK): "mindspore.mint",
+    (MsCompareConst.MINT, Const.PT_FRAMEWORK): "torch",
+    (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): "mindspore.mint.nn.functional",
+    (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): "torch.nn.functional",
+    (MsCompareConst.TENSOR_API, Const.MS_FRAMEWORK): "mindspore.Tensor",
+    (MsCompareConst.TENSOR_API, Const.PT_FRAMEWORK): "torch.Tensor"
 }
 
 
@@ -60,7 +84,7 @@ class ApiRunner:
             api_name_str: str, the trimmed key of data dict in api_info.json. e.g. "MintFunctional.relu.0"
 
         Return:
-            api_type_str: str, Union["MintFunctional", "Mint"]
+            api_type_str: str, Union["MintFunctional", "Mint", "Tensor"]
             api_sub_name: str, e.g. "relu"
         '''
         api_name_list = api_name_str.split(Const.SEP)
@@ -68,8 +92,8 @@ class ApiRunner:
             err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue))
         api_type_str, api_sub_name = api_name_list[0], api_name_list[1]
-        if api_type_str not in [MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL]:
-            err_msg = f"ApiRunner.get_info_from_name failed: not mint or mint.nn.functional api"
+        if api_type_str not in [MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL, MsCompareConst.TENSOR_API]:
+            err_msg = f"ApiRunner.get_info_from_name failed: not mint, mint.nn.functional or Tensor api"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue))
 
         return api_type_str, api_sub_name
@@ -78,7 +102,7 @@ class ApiRunner:
     def get_api_instance(api_type_str, api_sub_name, api_platform):
         '''
         Args:
-            api_type_str: str, Union["MintFunctional", "Mint"]
+            api_type_str: str, Union["MintFunctional", "Mint", "Tensor"]
             api_sub_name: str, e.g. "relu"
             api_platform: str: Union["mindpore", "torch"]
 
@@ -92,9 +116,8 @@ class ApiRunner:
         '''
 
         api_parent_module = api_parent_module_mapping.get((api_type_str, api_platform))
-        module_str = "mindspore.mint." if api_platform == Const.MS_FRAMEWORK else "torch."
-        submodule_str = "nn.functional." if api_type_str == MsCompareConst.MINT_FUNCTIONAL else ""
-        full_api_name = module_str + submodule_str + api_sub_name
+        api_parent_module_str = api_parent_module_str_mapping.get((api_type_str, api_platform))
+        full_api_name = api_parent_module_str + Const.SEP + api_sub_name
         if not hasattr(api_parent_module, api_sub_name):
             err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not found"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong))
@@ -115,7 +138,7 @@ class ApiRunner:
         gradient_inputs = api_input_aggregation.gradient_inputs
 
         if forward_or_backward == Const.FORWARD:
-            forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple
+            forward_result = api_instance(*inputs, **kwargs)  # can be single tensor or tuple
             forward_result_tuple = convert_to_tuple(forward_result)
             res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple]
         else:
@@ -127,18 +150,20 @@ class ApiRunner:
             if api_platform == Const.MS_FRAMEWORK:
                 if len(gradient_inputs) == 1:
                     gradient_inputs = gradient_inputs[0]
+
                 def api_with_kwargs(*forward_inputs):
                     return api_instance(*forward_inputs, **kwargs)
+
                 grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_with_kwargs)
-                backward_result = grad_func(*inputs, gradient_inputs) # can be single tensor or tuple
+                backward_result = grad_func(*inputs, gradient_inputs)  # can be single tensor or tuple
                 backward_result_tuple = convert_to_tuple(backward_result)
                 res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple]
             else:
-                #set requires_grad
+                # set requires_grad
                 requires_grad_index = []
                 for index, tensor in enumerate(inputs):
                     if isinstance(tensor, torch.Tensor) and \
-                        torch_dtype_to_dtype_str.get(tensor.dtype) in float_dtype_str_list:
+                            torch_dtype_to_dtype_str.get(tensor.dtype) in float_dtype_str_list:
                         setattr(tensor, "requires_grad", True)
                         requires_grad_index.append(index)
                 forward_results = api_instance(*inputs, **kwargs)
@@ -153,4 +178,4 @@ class ApiRunner:
         return res_compute_element_list
 
 
-api_runner = ApiRunner()
+api_runner = ApiRunner()

msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py

@@ -1,12 +1,27 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import ABC, abstractmethod
 
 import mindspore
-import torch
 import numpy as np
-
+import torch
+from msprobe.core.common.const import CompareConst, MsCompareConst
 from msprobe.core.common.exceptions import ApiAccuracyCheckerException
 from msprobe.mindspore.common.log import logger
-from msprobe.core.common.const import CompareConst, MsCompareConst
+
 
 class CompareResult:
     def __init__(self, compare_value, pass_status, err_msg):
@@ -28,7 +43,7 @@ class BaseCompareAlgorithm(ABC):
             CompareConst.MAX_ABS_ERR: {
                 CompareConst.PASS: "",
                 CompareConst.ERROR: "max absolute difference is greater than " \
-                    f"threshold: {CompareConst.MAX_ABS_ERR_THRESHOLD} ",
+                                    f"threshold: {CompareConst.MAX_ABS_ERR_THRESHOLD} ",
                 CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing ",
             },
             CompareConst.MAX_RELATIVE_ERR: {
@@ -68,7 +83,7 @@ class BaseCompareAlgorithm(ABC):
             ndarray = tensor.to(torch.float64, copy=True).numpy()
         else:
             err_msg = "BaseCompareAlgorithm.convert_to_np_float64_ndarray failed: " \
-                "input is not mindspore.Tensor or torch.Tensor"
+                      "input is not mindspore.Tensor or torch.Tensor"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType))
         return ndarray
 
@@ -189,9 +204,8 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm):
             return CompareConst.ERROR
 
 
-
 compare_algorithms = {
     CompareConst.COSINE: CosineSimilarityCompareAlgorithm(),
     CompareConst.MAX_ABS_ERR: MaxAbsoluteDiffCompareAlgorithm(),
     CompareConst.MAX_RELATIVE_ERR: MaxRelativeDiffCompareAlgorithm(),
-}
+}

msprobe/mindspore/api_accuracy_checker/checker_support_api.yaml

@@ -0,0 +1,77 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+# list of api that can be checked
+
+tensor:
+  - add_
+  - add
+  - addmm_
+  - all
+  - allclose
+  - any
+  - bool
+  - byte
+  - ceil
+  - clamp
+  - contiguous
+  - copy_
+  - cos
+  - clone
+  - cumprod
+  - expand_as
+  - flatten
+  - float
+  - half
+  - int
+  - is_contiguous
+  - isnan
+  - item
+  - log
+  - log2
+  - long
+  - masked_fill
+  - max
+  - mean
+  - min
+  - numel
+  - numpy
+  - repeat
+  - repeat_interleave
+  - reshape
+  - round
+  - select
+  - sin
+  - size
+  - split
+  - sqrt
+  - square
+  - sub
+  - swapaxes
+  - to
+  - t
+  - tolist
+  - topk
+  - transpose
+  - trunc
+  - type
+  - unsqueeze
+  - view
+  - view_as
+  - fill_
+  - floor_
+  - clamp_
+  - type_as
+  - zero_

msprobe/mindspore/api_accuracy_checker/cmd_parser.py

@@ -1,6 +1,68 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+
+
+from msprobe.core.common.file_utils import check_file_or_directory_path, create_directory
+from msprobe.core.common.utils import Const, MsprobeBaseException
+
+class UniqueDeviceAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        unique_values = set(values)
+        if len(values) != len(unique_values):
+            parser.error("device id must be unique")
+        for device_id in values:
+            if not 0 <= device_id <= 4095:
+                parser.error(f"the argument 'device_id' must be in range [0, 4095], but got {device_id}")
+        setattr(namespace, self.dest, values)
+
+
 def add_api_accuracy_checker_argument(parser):
     parser.add_argument("-api_info", "--api_info_file", dest="api_info_file", type=str, required=True,
                         help="<Required> The api param tool result file: generate from api param tool, "
                              "a json file.")
     parser.add_argument("-o", "--out_path", dest="out_path", default="./", type=str, required=False,
-                        help="<optional> The ut task result out path.")
+                        help="<optional> The ut task result out path.")
+    parser.add_argument("-csv_path", "--result_csv_path", dest="result_csv_path", default="", type=str, required=False,
+                        help="<optional> the exit csv for continue")
+
+def multi_add_api_accuracy_checker_argument(parser):
+    parser.add_argument("-api_info", "--api_info_file", dest="api_info_file", type=str, required=True,
+                        help="<Required> The api param tool result file: generate from api param tool, "
+                             "a json file.")
+    parser.add_argument("-o", "--out_path", dest="out_path", default="./", type=str, required=False,
+                        help="<optional> The ut task result out path.")
+    parser.add_argument("-csv_path", "--result_csv_path", dest="result_csv_path", default="", type=str, required=False,
+                        help="<optional> the exit csv for continue")
+    #以下属于多线程参数
+    parser.add_argument("-d", "--device", dest="device_id", nargs='+', type=int,
+                        help="<optional> set device id to run ut, must be unique and in range 0-7",
+                        default=[0], required=False, action=UniqueDeviceAction)
+
+
+def check_args(args):
+    args.api_info_file = os.path.abspath(args.api_info_file)
+    check_file_or_directory_path(args.api_info_file)
+
+    if args.out_path == "":
+        args.out_path = "./"
+    args.out_path = os.path.abspath(args.out_path)
+    create_directory(args.out_path)
+
+    if args.result_csv_path:
+        args.result_csv_path = os.path.abspath(args.result_csv_path)
+        check_file_or_directory_path(args.result_csv_path)

msprobe/mindspore/api_accuracy_checker/compute_element.py

@@ -1,21 +1,37 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 
 import mindspore
-import torch
 import numpy as np
-
-from msprobe.mindspore.common.log import logger
+import torch
+from mindspore._c_expression import typing
+from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import ApiAccuracyCheckerException
 from msprobe.core.common.file_utils import load_npy
-from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_dtype, api_info_type_str_to_type,
+from msprobe.mindspore.api_accuracy_checker.type_mapping import (api_info_type_str_to_type,
                                                                  ms_dtype_to_dtype_str, torch_dtype_to_dtype_str,
                                                                  dtype_str_to_ms_dtype, dtype_str_to_np_dtype,
                                                                  dtype_str_to_torch_dtype, type_to_api_info_type_str,
                                                                  DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE, TUPLE_TYPE_STR,
-                                                                 MINDSPORE_TENSOR_TYPE_STR, float_dtype_str_list,
-                                                                 int_dtype_str_list)
-from msprobe.core.common.const import Const
+                                                                 MINDSPORE_TENSOR_TYPE_STR, MINDSPORE_DTYPE_TYPE_STR,
+                                                                 SLICE_TYPE_STR, TORCH_DTYPE_TYPE_STR,
+                                                                 float_dtype_str_list, int_dtype_str_list)
 from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context
+from msprobe.mindspore.common.log import logger
 
 
 class MstensorMetaData:
@@ -26,6 +42,12 @@ class MstensorMetaData:
         self.minimum = minimum
         self.shape = shape
 
+
+class DtypeMetaData:
+    def __init__(self, dtype_str) -> None:
+        self.dtype_str = dtype_str
+
+
 class ComputeElement:
     def __init__(self, compute_element_info=None, parameter=None):
         self.supported_parameter_type = tuple(type_to_api_info_type_str.keys()) + tuple([torch.Tensor, tuple])
@@ -118,6 +140,11 @@ class ComputeElement:
                           for compute_element in self.parameter])
         elif isinstance(self.parameter, self.supported_parameter_type):
             parameter_tmp = self.parameter
+        elif isinstance(self.parameter, DtypeMetaData):
+            if tensor_platform == Const.MS_FRAMEWORK:
+                parameter_tmp = dtype_str_to_ms_dtype.get(self.parameter.dtype_str)
+            else:
+                parameter_tmp = dtype_str_to_torch_dtype.get(self.parameter.dtype_str)
         elif isinstance(self.parameter, MstensorMetaData):
             mstensor_meta_data = self.parameter
             ms_dtype = dtype_str_to_ms_dtype.get(mstensor_meta_data.dtype_str)
@@ -130,13 +157,13 @@ class ComputeElement:
             parameter_tmp = mindspore.Tensor(ndarray, dtype=ms_dtype)
         else:
             err_msg = "ComputeElement.get_parameter failed: self.parameter type is not in " \
-                "(int, float, str, slice, bool, torch.Tensor, mindspore.Tensor, MstensorMetaData)"
+                      "(int, float, str, slice, bool, torch.Tensor, mindspore.Tensor, MstensorMetaData)"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType))
 
         # if necessary, do transfer
         if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and tensor_platform == Const.PT_FRAMEWORK:
             parameter = self.transfer_to_torch_tensor(parameter_tmp)
-        elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and tensor_platform ==Const.MS_FRAMEWORK:
+        elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and tensor_platform == Const.MS_FRAMEWORK:
             parameter = self.transfer_to_mindspore_tensor(parameter_tmp)
         else:
             parameter = parameter_tmp
@@ -183,34 +210,38 @@ class ComputeElement:
         else:
             type_str = check_and_get_from_json_dict(compute_element_info, "type", "type field in api_info.json",
                                                     accepted_type=str, accepted_value=api_info_type_str_to_type.keys())
-
+            self.shape = tuple()
+            self.dtype_str = type_str
             if type_str == MINDSPORE_TENSOR_TYPE_STR:
                 self._init_from_mstensor_compute_element_info(compute_element_info)
-            else: # type_str in ("slice", "int", "float", "bool")
+            else:
                 value = check_and_get_from_json_dict(compute_element_info, "value", "value field in api_info.json")
-                self.shape = tuple()
-                self.dtype_str = type_str
-                self.parameter = slice(*tuple(value)) if type_str == "slice" else value
+                if type_str == MINDSPORE_DTYPE_TYPE_STR:
+                    self.parameter = DtypeMetaData(value)
+                elif type_str == SLICE_TYPE_STR:
+                    self.parameter = slice(*tuple(value))
+                else:  # type_str in ("str", "int", "float", "bool")
+                    self.parameter = value
 
     def _init_from_mstensor_compute_element_info(self, compute_element_info):
         '''
             do not load real tensor, only record meta data
         '''
         dtype_str = check_and_get_from_json_dict(compute_element_info, "dtype", "dtype field in api_info.json",
-                                            accepted_type=str, accepted_value=dtype_str_to_ms_dtype.keys())
+                                                 accepted_type=str, accepted_value=dtype_str_to_ms_dtype.keys())
         shape = check_and_get_from_json_dict(compute_element_info, "shape", "shape field in api_info.json",
-                                                accepted_type=(list,))
+                                             accepted_type=(list,))
         if global_context.get_is_constructed():
             maximum = check_and_get_from_json_dict(compute_element_info, "Max", "Max field in api_info.json",
-                                                    accepted_type=(int, float))
+                                                   accepted_type=(int, float))
             minimum = check_and_get_from_json_dict(compute_element_info, "Min", "Min field in api_info.json",
-                                                accepted_type=(int, float))
+                                                   accepted_type=(int, float))
 
             npy_path = None
         else:
             maximum, minimum = None, None
             data_name = check_and_get_from_json_dict(compute_element_info, "data_name",
-                                                "data_name field in api_info.json", accepted_type=(str,))
+                                                     "data_name field in api_info.json", accepted_type=(str,))
             npy_path = os.path.join(global_context.get_dump_data_dir(), data_name)
         mstensor_meta_data = MstensorMetaData(dtype_str, npy_path, maximum, minimum, shape)
         self.parameter = mstensor_meta_data
@@ -219,9 +250,10 @@ class ComputeElement:
 
     def _init_with_parameter(self, parameter):
         self.parameter = parameter
+        self.shape = tuple()
         if not isinstance(parameter, self.supported_parameter_type):
             err_msg = "ComputeElement._init_with_parameter failed: " \
-                "parameter type is not in (int, float, str, slice, bool, torch.Tensor, mindspore.Tensor)"
+                      "parameter type is not in (int, float, str, slice, bool, torch.Tensor, mindspore.Tensor)"
             logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType))
         if isinstance(parameter, mindspore.Tensor):
             self.shape = tuple(parameter.shape)
@@ -229,11 +261,14 @@ class ComputeElement:
         elif isinstance(parameter, torch.Tensor):
             self.shape = tuple(parameter.shape)
             self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype)
+        elif isinstance(parameter, typing.Type):
+            self.dtype_str = MINDSPORE_DTYPE_TYPE_STR
+            self.parameter = DtypeMetaData(ms_dtype_to_dtype_str.get(parameter))
+        elif isinstance(parameter, torch.dtype):
+            self.dtype_str = TORCH_DTYPE_TYPE_STR
+            self.parameter = DtypeMetaData(torch_dtype_to_dtype_str.get(parameter))
         elif isinstance(parameter, tuple):
-            self.shape = tuple()
             self.dtype_str = TUPLE_TYPE_STR
             self.parameter = tuple([ComputeElement(parameter=param) for param in parameter])
         else:
-            self.shape = tuple()
-            self.dtype_str = \
-                TUPLE_TYPE_STR if isinstance(parameter, tuple) else type_to_api_info_type_str.get(type(parameter))
+            self.dtype_str = type_to_api_info_type_str.get(type(parameter))