mindstudio-probe 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/core/compare/layer_mapping/layer_mapping.py

@@ -0,0 +1,242 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from msprobe.core.common.const import CompareConst, Const
+from msprobe.core.common.file_utils import load_json, load_yaml, save_yaml
+from msprobe.core.common.utils import (add_time_with_yaml,
+                                       detect_framework_by_dump_json,
+                                       get_stack_construct_by_dump_json_path)
+from msprobe.core.compare.layer_mapping.data_scope_parser import get_dump_data_items
+from msprobe.core.compare.utils import read_op
+
+
+class LayerTrie:
+    def __init__(self, type_name, framework=None):
+        self.type_name = type_name
+        self.data_items = []
+        self.children = {}
+        self.framework = framework
+
+    def __repr__(self):
+        return f"Layer(type_name={self.type_name}, data_number={len(self.data_items)})"
+
+    def get(self, name):
+        return self.children.get(name)
+
+    def insert(self, data_item):
+        parts = data_item.full_scope.split(Const.SEP)
+        node = self
+        scope_name_list = parts[Const.RIGHT_MOVE_INDEX:]
+
+        for name in scope_name_list:
+            if name not in node.children:
+                node.children[name] = LayerTrie(name, data_item.framework)
+            node = node.children[name]
+        node.data_items.append(data_item)
+        node.type_name = data_item.type_name
+
+    def query_data(self, scope, index, default_value=None):
+        parts = scope.split(Const.SEP)
+        node = self
+        scope_name_list = parts[1:]
+
+        for name in scope_name_list:
+            if name not in node.children:
+                return default_value
+            node = node.children[name]
+        if index >= len(node.data_items):
+            return default_value
+        return node.data_items[index]
+
+    def save_to_yaml(self, output_path):
+        result = {f"{self.type_name} @ {self}": self.convert_to_dict(self)}
+        file_name = add_time_with_yaml(f"{self.framework}_tree")
+        file_path = os.path.join(os.path.realpath(output_path), file_name)
+        save_yaml(file_path, result)
+
+    def convert_to_dict(self, node):
+        result = {}
+        result["data_item"] = [node.data_name for node in node.data_items]
+        for child_key, child_node in node.children.items():
+            key = f"{child_key} @ {child_node}"
+            result[key] = self.convert_to_dict(child_node)
+        return result
+
+
+def convert_scope(layer_trie, data_item, mapping=None):
+    if not mapping:
+        mapping = {}
+    new_scope = Const.TOP_LAYER
+    scope_list = data_item.full_scope.split(Const.SEP)
+    cur_node = layer_trie
+
+    idx = 0
+    while idx < len(scope_list) - 1:
+        child_name = scope_list[idx + 1]
+        type_name = cur_node.type_name
+        prefix_mapping = mapping.get(type_name, {})
+        mapping_list = prefix_mapping.get(child_name, [])
+        mapping_list.append((child_name, child_name, 1))
+        step = 1
+        for origin, target, level in mapping_list:
+            if Const.SEP.join(scope_list[idx + 1: idx + level + 1]) == origin:
+                new_scope = new_scope + Const.SEP + target
+                step = level
+                break
+        for _ in range(step):
+            child_node = cur_node.get(scope_list[idx + 1])
+            cur_node = child_node
+            idx += 1
+    index = -1
+    for idx, child in enumerate(cur_node.data_items):
+        if data_item.data_name == child.data_name:
+            index = idx
+    return new_scope, index
+
+
+def get_data_items_and_tree(dump_json_path, output_path):
+    framework = detect_framework_by_dump_json(dump_json_path)
+    stack, construct = get_stack_construct_by_dump_json_path(dump_json_path)
+    dump = load_json(dump_json_path)
+    dump_data_items = get_dump_data_items(dump, stack, construct, framework, output_path)
+    root = LayerTrie(Const.TOP_LAYER, framework)
+    for data_item in dump_data_items:
+        root.insert(data_item)
+    if output_path:
+        root.save_to_yaml(output_path)
+    return dump_data_items, root
+
+
+def convert_data_item(npu_tree, bench_tree, npu_data_item, mapping):
+    new_scope, index = convert_scope(npu_tree, npu_data_item, mapping)
+    bench_data_item = bench_tree.query_data(new_scope, index)
+    return bench_data_item
+
+
+def update_keys_in_place(d):
+    """
+    This function is used to compare and maintain compatibility between the old and new versions.
+    In the old version, 'Cell' was used as the top layer name, while the new version uses 'TopLayer'.
+    """
+    cell_value = d.pop(Const.CELL, None)
+
+    if cell_value is not None:
+        d[Const.TOP_LAYER] = cell_value
+
+
+def preprocess_layer_mapping(mapping):
+    """
+    before:
+        {'A': {'a.b.c': 'new_c',
+               'a.demo': 'new_demo',
+               'z': 'new_z',
+               'd.e': 'e'}}
+    after:
+        {'A': {'a': [('a.b.c', 'new_c', 3), ('a.demo', 'new_demo', 2)],
+               'z': [('z', 'new_z', 1)],
+               'd': [('d.e', 'e', 2)]}}
+    """
+    update_keys_in_place(mapping)
+    final_mapping = {}
+
+    for type_name, name_map in mapping.items():
+        final_mapping[type_name] = {}
+
+        for key, value in name_map.items():
+            key_list = key.split('.')
+            prefix = key_list[0]  # 取前缀
+            key_len = len(key_list)
+            if prefix not in final_mapping[type_name]:
+                final_mapping[type_name][prefix] = []
+            final_mapping[type_name][prefix].append((key, value, key_len))
+
+        # 前缀映射列表按规则长度排序
+        for prefix in final_mapping[type_name]:
+            final_mapping[type_name][prefix].sort(key=lambda x: -x[-1])
+
+    return final_mapping
+
+
+def convert_data_items(npu_tree, bench_tree, npu_data_items, mapping):
+    mapping = preprocess_layer_mapping(mapping)
+    api_mapping = {}
+    for npu_data_item in npu_data_items:
+        bench_data_item = convert_data_item(npu_tree, bench_tree, npu_data_item, mapping)
+        bench_name = bench_data_item.data_name if bench_data_item else CompareConst.N_A
+        npu_name = npu_data_item.data_name
+        api_mapping[npu_name] = bench_name
+    return api_mapping
+
+
+def generate_api_mapping_by_layer_mapping(npu_json_path, bench_json_path, layer_mapping_path=None, output_path=None):
+    npu_data_items, npu_root = get_data_items_and_tree(npu_json_path, output_path)
+    _, bench_root = get_data_items_and_tree(bench_json_path, output_path)
+    if isinstance(layer_mapping_path, str):
+        mapping = load_yaml(layer_mapping_path)
+    else:
+        mapping = {}
+    api_mapping = convert_data_items(npu_root, bench_root, npu_data_items, mapping)
+    if output_path:
+        file_name = add_time_with_yaml("api_mapping")
+        file_path = os.path.join(os.path.realpath(output_path), file_name)
+        save_yaml(file_path, api_mapping)
+    return api_mapping
+
+
+def generate_data_mapping(npu_json_path, bench_json_path, api_mapping, output_path=None):
+    def read_full_op_names(data, op_name):
+        op_parsed_list = read_op(data.get(op_name, {}), op_name)
+        full_op_names = [op_parsed.get('full_op_name') for op_parsed in op_parsed_list]
+        return full_op_names
+
+    def generate_op_data_mapping(npu_op_name, npu_full_op_names, bench_op_name, bench_full_op_names):
+        suffix_to_full_op_name = {}
+        op_data_mapping = {}
+        for bench_full_op_name in bench_full_op_names:
+            suffix = bench_full_op_name[len(bench_op_name):]
+            suffix_to_full_op_name[suffix] = bench_full_op_name
+
+        for npu_full_op_name in npu_full_op_names:
+            suffix = npu_full_op_name[len(npu_op_name):]
+            op_data_mapping[npu_full_op_name] = suffix_to_full_op_name.get(suffix, CompareConst.N_A)
+        return op_data_mapping
+
+    npu_data = load_json(npu_json_path).get("data", {})
+    bench_data = load_json(bench_json_path).get("data", {})
+    data_mapping = {}
+    for npu_op_name, bench_op_name in api_mapping.items():
+        if not npu_op_name:
+            continue
+        npu_full_op_names = read_full_op_names(npu_data, npu_op_name)
+        bench_full_op_names = read_full_op_names(bench_data, bench_op_name)
+        mapping = generate_op_data_mapping(npu_op_name, npu_full_op_names, bench_op_name, bench_full_op_names)
+        data_mapping.update(mapping)
+    if output_path:
+        file_name = add_time_with_yaml("data_mapping")
+        file_path = os.path.join(os.path.realpath(output_path), file_name)
+        save_yaml(file_path, data_mapping)
+    return data_mapping
+
+
+def generate_data_mapping_by_layer_mapping(input_param, layer_mapping_path=None, output_path=None):
+    npu_json_path = input_param.get("npu_json_path")
+    bench_json_path = input_param.get("bench_json_path")
+    api_mapping = generate_api_mapping_by_layer_mapping(
+        npu_json_path, bench_json_path, layer_mapping_path)
+    data_mapping = generate_data_mapping(
+        npu_json_path, bench_json_path, api_mapping, output_path)
+    return data_mapping

msprobe/core/compare/layer_mapping/postprocess_pass.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import math
+
+from msprobe.core.common.const import Const
+
+
+def postprocess_pass(data_items, name2item):
+    backward_pass(data_items, name2item)
+    renumber_index_pass(data_items, "ParallelTransformer", "layers")
+
+
+def backward_pass(data_items, name2item):
+    # 处理反向数据，反向无栈信息，沿用正向数据栈信息
+    for data_item in data_items:
+        data_name_list = data_item.data_name.split(Const.SEP)
+        if not data_name_list:
+            continue
+        if Const.BACKWARD in data_name_list[Const.SCOPE_DIRECTION_INDEX :]:
+            data_name_list[Const.SCOPE_DIRECTION_INDEX :] = [
+                s.replace(Const.BACKWARD, Const.FORWARD) for s in data_name_list[Const.SCOPE_DIRECTION_INDEX :]
+            ]
+            forward_name = Const.SEP.join(data_name_list)
+            forward_item = name2item.get(forward_name, None)
+            if not forward_item:
+                continue
+            data_item.stack_scope = forward_item.stack_scope
+            data_item.full_scope = forward_item.full_scope
+            data_item.layer_scope = forward_item.layer_scope
+
+
+def extract_next_item_last_number(data, prefix, default_result=None):
+    result = default_result
+    match = re.search(rf"^{re.escape(prefix)}\.(\S+?)(?:\.|$)", data)
+    if match:
+        next_item = match.group(1)
+        numbers = re.findall(r"\d+", next_item)
+        if numbers:
+            result = int(numbers[-1])
+    return result
+
+
+def replace_next_item_index(full_scope, prefix, index):
+    if math.isinf(index):
+        return full_scope
+    prefix_pattern = rf"^{re.escape(prefix)}\."
+    result = full_scope
+    match = re.search(rf"{prefix_pattern}(\S+?)(?:\.|$)", full_scope)
+    if match:
+        next_item = match.group(1)
+        pattern = rf"{prefix_pattern}{re.escape(next_item)}"
+        result = re.sub(pattern, f"{prefix}.{index}", full_scope, count=1)
+    return result
+
+
+def renumber_index_pass(data_items, type_name, suffix=None):
+    """
+    该函数为解决并行切分场景中编号不一致的比对问题。例如在MindSpore中ParallelTransformer层的PP切分场景，
+    MindSpore中的layers的成员编号是全局的，而在PyTorch中编号为局部的。
+    为适配此种场景，对指定层的索引进行重新编号，以确保在后续处理阶段序号对齐。
+    """
+    prefix_dict = {}  # 保存类型为type_name的前缀和最小编号的映射
+    for data_item in data_items:
+        if data_item.type_name == type_name:
+            prefix = f"{data_item.full_scope}.{suffix}" if suffix else data_item.layer_scope
+            prefix_dict[prefix] = math.inf
+
+    # 计算前缀对应的最小编号
+    for prefix in prefix_dict:
+        for data_item in data_items:
+            res = extract_next_item_last_number(data_item.full_scope, prefix, math.inf)
+            prefix_dict[prefix] = min(prefix_dict[prefix], res)
+
+    # 重新编号
+    for prefix, min_index in prefix_dict.items():
+        for data_item in data_items:
+            full_scope = data_item.full_scope
+            abs_index = extract_next_item_last_number(data_item.full_scope, prefix, math.inf)
+            rel_index = abs_index - min_index
+            full_scope = replace_next_item_index(full_scope, prefix, rel_index)
+            data_item.full_scope = full_scope

msprobe/core/compare/npy_compare.py

@@ -14,18 +14,32 @@
 # limitations under the License.
 
 import abc
+
 import numpy as np
 from msprobe.core.common.utils import format_value
 from msprobe.core.common.const import Const, CompareConst
 from msprobe.core.common.log import logger
 
+from msprobe.core.common.utils import CompareException
+
 
 def handle_inf_nan(n_value, b_value):
+    def convert_to_float(value):
+        try:
+            if isinstance(value, np.ndarray):
+                return value.astype(float)
+            else:
+                return float(value)
+        except ValueError as e:
+            logger.error('\n'.join(e.args))
+            raise CompareException(CompareException.INVALID_DATA_ERROR) from e
+
+    n_value_convert, b_value_convert = convert_to_float(n_value), convert_to_float(b_value)
     """处理inf和nan的数据"""
-    n_inf = np.isinf(n_value)
-    b_inf = np.isinf(b_value)
-    n_nan = np.isnan(n_value)
-    b_nan = np.isnan(b_value)
+    n_inf = np.isinf(n_value_convert)
+    b_inf = np.isinf(b_value_convert)
+    n_nan = np.isnan(n_value_convert)
+    b_nan = np.isnan(b_value_convert)
     n_invalid = np.any(n_inf) or np.any(n_nan)
     b_invalid = np.any(b_inf) or np.any(b_nan)
     if n_invalid or b_invalid:
@@ -50,7 +64,11 @@ def get_error_type(n_value, b_value, error_flag):
     if not n_value.shape:  # 判断数据是否为标量
         return n_value, b_value, False
 
-    n_value, b_value = handle_inf_nan(n_value, b_value)  # 判断是否有nan/inf数据
+    try:
+        n_value, b_value = handle_inf_nan(n_value, b_value)  # 判断是否有nan/inf数据
+    except CompareException:
+        logger.error('Numpy data is unreadable, please check!')
+        return CompareConst.UNREADABLE, CompareConst.UNREADABLE, True
     if n_value is CompareConst.NAN or b_value is CompareConst.NAN:
         return CompareConst.NAN, CompareConst.NAN, True
     return n_value, b_value, False
@@ -73,7 +91,9 @@ def get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=None
     """获取异常情况的错误信息"""
     if error_flag:
         if n_value == CompareConst.READ_NONE:
-            if error_file:
+            if error_file == 'no_bench_data':
+                return 'Bench does not have data file.'
+            elif error_file is not None:
                 return "Dump file: {} not found.".format(error_file)
             return CompareConst.NO_BENCH
         if n_value == CompareConst.NONE:
@@ -82,6 +102,8 @@ def get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=None
             return "Shape of NPU and bench Tensor do not match. Skipped."
         if n_value == CompareConst.NAN:
             return "The position of inf or nan in NPU and bench Tensor do not match."
+        if n_value == CompareConst.UNREADABLE:
+            return "The npy data is unable to be read or compared, please check dump data files."
     else:
         if not n_value.shape:
             return "This is type of scalar data, can not compare."
@@ -109,7 +131,11 @@ def npy_data_check(n_value, b_value):
             error_message += "Dtype of NPU and bench Tensor do not match. Skipped.\n"
 
     if not error_message:
-        n_value, b_value = handle_inf_nan(n_value, b_value)  # 判断是否有 nan/inf 数据
+        try:
+            n_value, b_value = handle_inf_nan(n_value, b_value)  # 判断是否有nan/inf数据
+        except CompareException:
+            logger.error('Numpy data is unreadable, please check!')
+            return True, 'Numpy data is unreadable, please check!'
         # handle_inf_nan 会返回'Nan'或ndarray类型，使用类型判断是否存在无法处理的nan/inf数据
         if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray):
             error_message += "The position of inf or nan in NPU and bench Tensor do not match.\n"
@@ -160,14 +186,14 @@ class GetCosineSimilarity(TensorComparisonBasic):
 
     def apply(self, n_value, b_value, error_flag, relative_err=None):
         if error_flag:
-            if n_value == CompareConst.READ_NONE:
-                return CompareConst.NONE, ''
+            if n_value == CompareConst.READ_NONE or n_value == CompareConst.UNREADABLE:
+                return CompareConst.UNSUPPORTED, ''
             if n_value == CompareConst.NONE:
                 return CompareConst.UNSUPPORTED, ''
             if n_value == CompareConst.SHAPE_UNMATCH:
                 return CompareConst.SHAPE_UNMATCH, ''
             if n_value == CompareConst.NAN:
-                return "N/A", ''
+                return CompareConst.N_A, ''
 
         if not n_value.shape:
             return CompareConst.UNSUPPORTED, ''
@@ -198,17 +224,20 @@ class GetMaxAbsErr(TensorComparisonBasic):
     """计算最大绝对误差"""
     def apply(self, n_value, b_value, error_flag, relative_err=None):
         if error_flag:
-            if n_value == CompareConst.READ_NONE:
-                return CompareConst.NONE, ""
+            if n_value == CompareConst.READ_NONE or n_value == CompareConst.UNREADABLE:
+                return CompareConst.UNSUPPORTED, ""
             if n_value == CompareConst.NONE:
                 return 0, ""
             if n_value == CompareConst.SHAPE_UNMATCH:
                 return CompareConst.SHAPE_UNMATCH, ""
             if n_value == CompareConst.NAN:
-                return "N/A", ""
+                return CompareConst.N_A, ""
 
         temp_res = n_value - b_value
         max_value = np.max(np.abs(temp_res))
+        if np.isnan(max_value):
+            message = 'Cannot compare by MaxRelativeError, the data contains nan/inf/-inf in dump data.'
+            return CompareConst.NAN, message
         return format_value(max_value), ""
 
 
@@ -228,20 +257,20 @@ class GetMaxRelativeErr(TensorComparisonBasic):
     """计算最大相对误差"""
     def apply(self, n_value, b_value, error_flag, relative_err=None):
         if error_flag:
-            if n_value == CompareConst.READ_NONE:
-                return CompareConst.NONE, ''
+            if n_value == CompareConst.READ_NONE or n_value == CompareConst.UNREADABLE:
+                return CompareConst.UNSUPPORTED, ''
             if n_value == CompareConst.NONE:
                 return 0, ''
             if n_value == CompareConst.SHAPE_UNMATCH:
                 return CompareConst.SHAPE_UNMATCH, ''
             if n_value == CompareConst.NAN:
-                return "N/A", ''
+                return CompareConst.N_A, ''
 
         if relative_err is None:
             relative_err = get_relative_err(n_value, b_value)
         max_relative_err = np.max(np.abs(relative_err))
         if np.isnan(max_relative_err):
-            message = 'Cannot compare by MaxRelativeError, the data contains nan in dump data.'
+            message = 'Cannot compare by MaxRelativeError, the data contains nan/inf/-inf in dump data.'
             return CompareConst.NAN, message
         return format_value(max_relative_err), ''
 
@@ -250,14 +279,14 @@ class GetThousandErrRatio(TensorComparisonBasic):
     """计算相对误差小于千分之一的比例"""
     def apply(self, n_value, b_value, error_flag, relative_err=None):
         if error_flag:
-            if n_value == CompareConst.READ_NONE:
-                return CompareConst.NONE, ""
+            if n_value == CompareConst.READ_NONE or n_value == CompareConst.UNREADABLE:
+                return CompareConst.UNSUPPORTED, ""
             if n_value == CompareConst.NONE:
                 return 0, ""
             if n_value == CompareConst.SHAPE_UNMATCH:
                 return CompareConst.SHAPE_UNMATCH, ""
             if n_value == CompareConst.NAN:
-                return "N/A", ""
+                return CompareConst.N_A, ""
 
         if not n_value.shape:
             return CompareConst.NAN, ""
@@ -272,14 +301,14 @@ class GetFiveThousandErrRatio(TensorComparisonBasic):
     """计算相对误差小于千分之五的比例"""
     def apply(self, n_value, b_value, error_flag, relative_err=None):
         if error_flag:
-            if n_value == CompareConst.READ_NONE:
-                return CompareConst.NONE, ""
+            if n_value == CompareConst.READ_NONE or n_value == CompareConst.UNREADABLE:
+                return CompareConst.UNSUPPORTED, ""
             if n_value == CompareConst.NONE:
                 return 0, ""
             if n_value == CompareConst.SHAPE_UNMATCH:
                 return CompareConst.SHAPE_UNMATCH, ""
             if n_value == CompareConst.NAN:
-                return "N/A", ""
+                return CompareConst.N_A, ""
 
         if not n_value.shape:
             return CompareConst.NAN, ""