mindstudio-probe 1.3.0__py3-none-any.whl → 8.1.1__py3-none-any.whl

msprobe/core/compare/check.py

@@ -14,113 +14,46 @@
 # limitations under the License.
 
 from msprobe.core.common.log import logger
-from msprobe.core.compare.utils import rename_api
 from msprobe.core.common.utils import check_op_str_pattern_valid, CompareException
-from msprobe.core.common.const import CompareConst, Const
-
-dtype_mapping = {
-    "Int8": "torch.int8",
-    "UInt8": "torch.uint8",
-    "Int16": "torch.int16",
-    "UInt16": "torch.uint16",
-    "Int32": "torch.int32",
-    "UInt32": "torch.uint32",
-    "Int64": "torch.int64",
-    "UInt64": "torch.uint64",
-    "Float16": "torch.float16",
-    "Float32": "torch.float32",
-    "Float64": "torch.float64",
-    "Bool": "torch.bool",
-    "BFloat16": "torch.bfloat16",
-    "Complex64": "torch.complex64",
-    "Complex128": "torch.complex128"
+from msprobe.core.common.const import Const
+
+cross_dtype_mapping = {
+    "Int8": "int",
+    "torch.int8": "int",
+    "UInt8": "int",
+    "torch.uint8": "int",
+    "Int16": "int",
+    "torch.int16": "int",
+    "UInt16": "int",
+    "torch.uint16": "int",
+    "Int32": "int",
+    "torch.int32": "int",
+    "UInt32": "int",
+    "torch.uint32": "int",
+    "Int64": "int",
+    "torch.int64": "int",
+    "UInt64": "int",
+    "torch.uint64": "int",
+
+    "Float16": "float",
+    "torch.float16": "float",
+    "Float32": "float",
+    "torch.float32": "float",
+    "Float64": "float",
+    "torch.float64": "float",
+    "BFloat16": "float",
+    "torch.bfloat16": "float",
+
+    "Bool": "bool",
+    "torch.bool": "bool",
+
+    "Complex64": "complex",
+    "torch.complex64": "complex",
+    "Complex128": "complex",
+    "torch.complex128": "complex",
 }
 
 
-def compare_op_dict_struct(npu_dict, bench_dict):
-    return all(npu_dict.get(key) == bench_dict.get(key) for key in CompareConst.STRUCT_COMPARE_KEY)
-
-
-def check_struct_match(npu_dict, bench_dict):
-    is_match = compare_op_dict_struct(npu_dict, bench_dict)
-    if not is_match:
-        struct_match_list = []
-        try:
-            for i, key in enumerate(CompareConst.STRUCT_COMPARE_KEY):
-                # 首先额外检查input_struct是否空，input_struct不可能为空
-                if i == 0 and (not npu_dict.get(key, []) or not bench_dict.get(key, [])):
-                    return False
-                struct_match_list.append(check_type_shape_match(npu_dict.get(key, []), bench_dict.get(key, [])))
-        except CompareException as error:
-            err_msg = f'index out of bounds error occurs in npu or bench api, please check!\n' \
-                      f'npu_dict: {npu_dict}' \
-                      f'bench_dict: {bench_dict}'
-            logger.error(err_msg)
-            raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error
-        is_match = all(struct_match_list)
-    return is_match
-
-
-def check_type_shape_match(npu_struct, bench_struct):
-    """
-    further check dtypes with a dtype mapping list when dtypes are not entirely consistent.
-    """
-    if len(npu_struct) != len(bench_struct):
-        return False
-    if not npu_struct and not bench_struct:
-        return True
-
-    struct_match = False
-    for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct):
-        try:
-            npu_type = npu_type_shape[0]
-            npu_shape = npu_type_shape[1]
-            bench_type = bench_type_shape[0]
-            bench_shape = bench_type_shape[1]
-        except IndexError as error:
-            logger.error(f'length of npu_type_shape: {npu_type_shape} and bench_type_shape: {bench_type_shape} '
-                         f'should both be 2, please check!')
-            raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error
-        shape_match = npu_shape == bench_shape
-        type_match = ((npu_type == bench_type) or
-                      any(npu_type in group and bench_type in group for group in CompareConst.DTYPE_MATCH_GROUPS))
-        struct_match = shape_match and type_match
-        if not struct_match:
-            return False
-    return struct_match
-
-
-def check_graph_mode(a_op_name, b_op_name):
-    if Const.ATEN in a_op_name and Const.ATEN not in b_op_name:
-        return True
-    if Const.ATEN not in a_op_name and Const.ATEN in b_op_name:
-        return True
-    return False
-
-
-def fuzzy_check_op(npu_name_list, bench_name_list):
-    # 先检查api里的item长度是否相等，如果不是parameters_grad, 必然有input或者output，长度不可能为0
-    # 如果是parameters_grad, "parameters_grad"字段的字典不会是空字典，因此len>=1
-    if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list):
-        return False
-    is_match = True
-    for npu_name, bench_name in zip(npu_name_list, bench_name_list):
-        is_match = fuzzy_check_name(npu_name, bench_name)
-        if not is_match:
-            break
-    return is_match
-
-
-def fuzzy_check_name(npu_name, bench_name):
-    if Const.FORWARD in npu_name and Const.FORWARD in bench_name:
-        is_match = rename_api(npu_name, Const.FORWARD) == rename_api(bench_name, Const.FORWARD)
-    elif Const.BACKWARD in npu_name and Const.BACKWARD in bench_name:
-        is_match = rename_api(npu_name, Const.BACKWARD) == rename_api(bench_name, Const.BACKWARD)
-    else:
-        is_match = npu_name == bench_name
-    return is_match
-
-
 def check_dump_json_str(op_data, op_name):
     input_list = op_data.get(Const.INPUT_ARGS, None) if op_data.get(Const.INPUT_ARGS, None) else op_data.get(
         Const.INPUT, None)

msprobe/core/compare/compare_cli.py

@@ -38,6 +38,7 @@ def compare_cli(args):
     else:
         from msprobe.mindspore.compare.ms_compare import ms_compare
         from msprobe.mindspore.compare.distributed_compare import ms_compare_distributed, ms_graph_compare
+        from msprobe.mindspore.compare.common_dir_compare import common_dir_compare
 
     common_kwargs = {
         "auto_analyze": auto_analyze,
@@ -78,6 +79,9 @@ def compare_cli(args):
         if input_param.get("rank_id") is not None:
             ms_graph_compare(input_param, args.output_path)
             return
+        if input_param.get('common', False):
+            common_dir_compare(input_param, args.output_path)
+            return
         if frame_name == Const.PT_FRAMEWORK:
             compare_distributed(npu_path, bench_path, args.output_path, **kwargs)
         else:

msprobe/core/compare/config.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from msprobe.core.common.const import Const, CompareConst
+from msprobe.core.common.file_utils import load_yaml
+
+
+class ModeConfig:
+    def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.SUMMARY,
+                 compared_file_type=Const.DUMP_JSON_FILE):
+        self.stack_mode = stack_mode
+        self.auto_analyze = auto_analyze
+        self.fuzzy_match = fuzzy_match
+        self.dump_mode = dump_mode
+        self.compared_file_type = compared_file_type
+
+
+class MappingConfig:
+    def __init__(self, cell_mapping=None, api_mapping=None, data_mapping=None):
+        self.cell_mapping = cell_mapping
+        self.api_mapping = api_mapping
+        self.data_mapping = data_mapping
+
+
+class MappingDict:
+    def __init__(self, mapping_config: MappingConfig):
+        self.cell_mapping_dict = self.load_mapping_file(mapping_config.cell_mapping)
+        self.api_mapping_dict = self.load_mapping_file(mapping_config.api_mapping)
+        if mapping_config.api_mapping is not None:
+            self.ms_to_pt_mapping = self.load_internal_api()
+        self.data_mapping_dict = self.init_data_mapping(mapping_config.data_mapping)
+
+    @staticmethod
+    def load_internal_api():
+        cur_path = os.path.dirname(os.path.realpath(__file__))
+        yaml_path = os.path.abspath(os.path.join(cur_path, CompareConst.INTERNAL_API_MAPPING_FILE))
+        return load_yaml(yaml_path)
+
+    @staticmethod
+    def load_mapping_file(mapping_file):
+        if isinstance(mapping_file, str):
+            mapping_dict = load_yaml(mapping_file)
+        else:
+            mapping_dict = {}
+        return mapping_dict
+
+    def init_data_mapping(self, data_mapping):
+        """
+        初始化data_mapping_dict
+        """
+        if isinstance(data_mapping, str) or data_mapping is None:
+            data_mapping_dict = self.load_mapping_file(data_mapping)
+        elif isinstance(data_mapping, dict):
+            data_mapping_dict = data_mapping
+        else:
+            raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got "
+                            f"{type(data_mapping)}")
+        return data_mapping_dict