mindstudio-probe 1.1.1__py3-none-any.whl → 1.2.2__py3-none-any.whl

msprobe/visualization/builder/graph_builder.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -14,16 +14,24 @@
 # limitations under the License.
 
 import re
+
+from msprobe.core.common.const import Const
+from msprobe.core.common.file_utils import load_json
+from msprobe.visualization.builder.msprobe_adapter import get_input_output
+from msprobe.visualization.builder.msprobe_adapter import op_patterns
 from msprobe.visualization.graph.graph import Graph
 from msprobe.visualization.graph.node_op import NodeOp
 from msprobe.visualization.utils import save_json_file, GraphConst
-from msprobe.visualization.builder.msprobe_adapter import get_input_output
-from msprobe.core.common.file_utils import load_json
 
 
 class GraphBuilder:
+    backward_pattern = re.compile(r"(\.backward\.)(\d+)$")
+    forward_pattern = re.compile(r"(\.forward\.)(\d+)$")
+    # 匹配以大写字母开头，后接任意字母，并以Template(结尾
+    template_pattern = re.compile(r'\b[A-Z][a-zA-Z]*Template\(')
+
     @staticmethod
-    def build(construct_path, data_path, stack_path, model_name='DefaultModel'):
+    def build(construct_path, data_path, stack_path, model_name='DefaultModel', complete_stack=False):
         """
         GraphBuilder的对外提供的构图方法
         Args:
@@ -31,11 +39,14 @@ class GraphBuilder:
             data_path: dump.json路径
             stack_path: stack.json路径
             model_name: 模型名字，依赖外部输入
+            complete_stack: 完整的堆栈信息
         Returns: Graph，代表图的数据结构
         """
         construct_dict = load_json(construct_path)
         dump_dict = load_json(data_path)
         stack_dict = load_json(stack_path)
+        if not complete_stack:
+            GraphBuilder._simplify_stack(stack_dict)
         data_dict = dump_dict.get(GraphConst.DATA_KEY, {})
         graph = Graph(model_name, data_path=dump_dict.get('dump_data_dir', ''), dump_data=data_dict)
         GraphBuilder._init_nodes(graph, construct_dict, data_dict, stack_dict)
@@ -61,20 +72,59 @@ class GraphBuilder:
             result[GraphConst.MICRO_STEPS] = config.micro_steps
         if config.task:
             result[GraphConst.JSON_TASK_KEY] = config.task
+        result[GraphConst.OVERFLOW_CHECK] = config.overflow_check
         save_json_file(filename, result)
 
+    @staticmethod
+    def _simplify_stack(stack_dict):
+        """
+        精简堆栈内容，模块级保留包含"模块名("的堆栈，api级保留"xxxTemplate("的下一行堆栈
+
+        例如模块 Module.layer3.0.bn2.BatchNorm2d.forward.0，模块名为bn2，匹配"bn2("，
+        保留堆栈"File /home/models/resnet.py, line 97, in forward, \n out = self.bn2(out)"
+
+        例如Api Tensor.__iadd__.4.forward，堆栈为：
+        "File /home/wrap_tensor.py, line 61,  return TensorOPTemplate(op_name, hook)(*args, **kwargs)",
+        "File /home/torchvision/models/resnet.py, line 102, in forward, \n out += identity",
+        匹配到第一行的"TensorOPTemplate("，保留下一行堆栈
+        """
+        module_pattern = re.compile(op_patterns[0])
+        for dump_name, stack_list in stack_dict.items():
+            if not isinstance(stack_list, list):
+                continue
+            if module_pattern.match(dump_name):
+                parts = dump_name.split(Const.SEP)
+                if len(parts) < abs(Const.LAYER_NAME_INDEX):
+                    continue
+                module_name = parts[Const.LAYER_NAME_INDEX]
+                for stack in stack_list:
+                    if re.search(module_name + r'\(', stack):
+                        stack_list = [stack]
+                        break
+            else:
+                for index, stack in enumerate(stack_list):
+                    if GraphBuilder.template_pattern.search(stack) and index < len(stack_list) - 1:
+                        stack_list = [stack_list[index + 1]]
+                        break
+            stack_dict[dump_name] = stack_list
+
     @staticmethod
     def _handle_backward_upnode_missing(construct_dict, subnode_id, upnode_id):
         """
         如果backward节点的父级节点是null，则尝试从同名的forward节点寻找父级节点
         """
         # 匹配以.backward.后跟一个或多个数字结尾的模式
-        backward_pattern = r"(\.backward\.)(\d+)$"
-        forward_pattern = r"(\.forward\.)(\d+)$"
-        if re.search(backward_pattern, subnode_id) and not upnode_id:
-            forward_upnode_id = construct_dict.get(re.sub(backward_pattern, r".forward.\2", subnode_id))
+        if GraphBuilder.backward_pattern.search(subnode_id) and not upnode_id:
+            forward_upnode_id = construct_dict.get(GraphBuilder.backward_pattern.sub(r".forward.\2", subnode_id))
+            if forward_upnode_id:
+                new_upnode_id = GraphBuilder.forward_pattern.sub(r".backward.\2", forward_upnode_id)
+                if new_upnode_id in construct_dict:
+                    return new_upnode_id
+        # 匹配以.backward结尾的节点
+        if subnode_id.endswith(Const.SEP + Const.BACKWARD) and not upnode_id:
+            forward_upnode_id = construct_dict.get(subnode_id.replace(Const.BACKWARD, Const.FORWARD))
             if forward_upnode_id:
-                new_upnode_id = re.sub(forward_pattern, r".backward.\2", forward_upnode_id)
+                new_upnode_id = forward_upnode_id.replace(Const.FORWARD, Const.BACKWARD)
                 if new_upnode_id in construct_dict:
                     return new_upnode_id
         return upnode_id
@@ -104,11 +154,42 @@ class GraphBuilder:
             input_data, output_data = get_input_output(node_data, node.id)
             # 更新数据
             node.set_input_output(input_data, output_data)
+            if GraphConst.BATCH_P2P in name:
+                GraphBuilder._extract_batch_p2p_info(node, node_data)
+            # 反向节点使用对应前向节点的堆栈信息
+            # 模块命名举例：Module.module.module.GPTModel.backward.0; API命名举例：Tensor.permute.1.backward
+            if (not node_stack_info and
+                    (GraphBuilder.backward_pattern.search(name) or name.endswith(f'{Const.SEP}{Const.BACKWARD}'))):
+                forward_node = graph.get_node(
+                    # 同名模块全局唯一，无论调用几次堆栈信息都一致，直接使用编号0的同名模块堆栈信息，避免遗漏
+                    GraphBuilder.backward_pattern.sub(f'{Const.SEP}{Const.FORWARD}{Const.SEP}0', name)) \
+                    if GraphBuilder.backward_pattern.search(name) \
+                    else graph.get_node(name.replace(Const.BACKWARD, Const.FORWARD))
+                node_stack_info = forward_node.stack_info if forward_node \
+                    else ['This backward node cannot find the forward node and cannot retrieve stack information.']
             node.stack_info = node_stack_info
         # 添加节点
         node.add_upnode(upnode)
         return node
 
+    @staticmethod
+    def _is_valid_batch_p2p_output(param_list):
+        if not isinstance(param_list, list) or not param_list:
+            return False
+        if not isinstance(param_list[0], list) or not param_list[0]:
+            return False
+        return True
+
+    @staticmethod
+    def _extract_batch_p2p_info(node, node_data):
+        param_list = node_data.get(Const.OUTPUT, [])
+        # 数据格式："output": [[{param1}, {param2}, ...]]
+        if GraphBuilder._is_valid_batch_p2p_output(param_list):
+            for param in param_list[0]:
+                info = {GraphConst.OP: param.get(GraphConst.OP), GraphConst.PEER: param.get(GraphConst.PEER),
+                        GraphConst.GROUP_ID: param.get(GraphConst.GROUP_ID)}
+                node.batch_p2p_info.append(info)
+
     @staticmethod
     def _collect_apis_between_modules(graph):
         """
@@ -156,10 +237,12 @@ class GraphBuilder:
 
 
 class GraphExportConfig:
-    def __init__(self, graph_n, graph_b=None, tool_tip=None, node_colors=None, micro_steps=None, task=''):
+    def __init__(self, graph_n, graph_b=None, tool_tip=None, node_colors=None, micro_steps=None, task='',
+                 overflow_check=False):
         self.graph_n = graph_n
         self.graph_b = graph_b
         self.tool_tip = tool_tip
         self.node_colors = node_colors
         self.micro_steps = micro_steps
         self.task = task
+        self.overflow_check = overflow_check

msprobe/visualization/builder/msprobe_adapter.py

@@ -18,11 +18,12 @@ from msprobe.core.compare.acc_compare import read_op, merge_tensor, get_accuracy
 from msprobe.core.common.utils import set_dump_path, get_dump_mode
 from msprobe.visualization.utils import GraphConst
 from msprobe.core.common.const import Const
+from msprobe.core.compare.acc_compare import ModeConfig
 
 # 用于将节点名字解析成对应的NodeOp的规则
 op_patterns = [
     # NodeOp.module
-    r'^(Module.|Cell.)',
+    r'^(Module.|Cell.|optimizer|clip_grad)',
     # NodeOp.function_api
     r'^(Tensor.|Torch.|Functional.|NPU.|VF.|Distributed.|Aten.|Mint.|Primitive.|Jit.|MintFunctional.)'
 ]
@@ -50,12 +51,14 @@ def run_real_data(dump_path_param, csv_path, framework, is_cross_frame=False):
         framework: 框架类型, pytorch或mindspore
         is_cross_frame: 是否进行跨框架比对，仅支持mindspore比pytorch, 其中pytorch为标杆
     """
+    mode_config = ModeConfig(stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.ALL)
+
     if framework == Const.PT_FRAMEWORK:
         from msprobe.pytorch.compare.pt_compare import PTComparator
-        return PTComparator().do_multi_process(dump_path_param, csv_path)
+        return PTComparator(mode_config).do_multi_process(dump_path_param, csv_path)
     else:
-        from msprobe.mindspore.compare.ms_compare import MSComparator
-        ms_comparator = MSComparator()
+        from msprobe.mindspore.compare.ms_compare import MSComparator, MappingConfig
+        ms_comparator = MSComparator(mode_config, MappingConfig())
         ms_comparator.cross_frame = is_cross_frame
         return ms_comparator.do_multi_process(dump_path_param, csv_path)
 
@@ -105,11 +108,25 @@ def compare_data(data_dict_list1, data_dict_list2):
     return True
 
 
-def format_node_data(data_dict):
+def compare_data_fuzzy(data_dict_list1, data_dict_list2):
+    """
+    模糊匹配，仅校验参数shape是否一致
+    """
+    for x, y in zip(data_dict_list1.values(), data_dict_list2.values()):
+        x_shape = x.get(Const.SHAPE)
+        y_shape = y.get(Const.SHAPE)
+        if x_shape != y_shape:
+            return False
+    return True
+
+
+def format_node_data(data_dict, node_id=None):
     """
-    批量进行节点数据的输出
+    删除节点数据中不需要展示的字段
     """
     del_list = ['requires_grad', 'full_op_name']
+    if node_id and GraphConst.BATCH_P2P in node_id:
+        del_list.extend(['op', 'peer', 'tag', 'group_id'])
     for _, value in data_dict.items():
         if not isinstance(value, dict):
             continue
@@ -179,6 +196,13 @@ def _format_data(data_dict):
     """
     pattern = r'^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)$'
     all_null = False
+
+    keys_to_keep = ['type', 'group_ranks', 'group_id', 'data_name']
+    if data_dict.get('type') == 'torch.ProcessGroup':
+        keys_to_remove = [key for key in data_dict if key not in keys_to_keep]
+        for key in keys_to_remove:
+            del data_dict[key]
+
     for key, value in data_dict.items():
         if isinstance(value, str):
             # 将单引号删掉，None换成null避免前端解析错误

msprobe/visualization/compare/graph_comparator.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 from msprobe.visualization.builder.msprobe_adapter import compare_node, get_compare_mode, run_real_data
 from msprobe.visualization.utils import GraphConst, load_json_file, load_data_json_file, get_csv_df
 from msprobe.visualization.graph.graph import Graph, NodeOp
@@ -22,18 +23,23 @@ from msprobe.core.common.const import Const
 
 
 class GraphComparator:
-    def __init__(self, graphs, dump_path_param, output_path, framework=Const.PT_FRAMEWORK, mapping_dict=None):
+    def __init__(self, graphs, dump_path_param, args, mapping_dict=None):
         self.graph_n = graphs[0]
         self.graph_b = graphs[1]
-        self._parse_param(dump_path_param, output_path)
-        self.framework = framework
+        self._parse_param(dump_path_param, args.output_path)
+        self.framework = args.framework
         self.mapping_dict = mapping_dict
+        self.fuzzy_match = args.fuzzy_match
+        self.pattern = re.compile(r'\.\d+\.')
 
     def compare(self):
         """
         比较函数，初始化结束后单独调用。比较结果写入graph_n
         """
-        self._compare_nodes(self.graph_n.root)
+        if self.fuzzy_match:
+            self._compare_nodes_fuzzy(self.graph_n.root)
+        else:
+            self._compare_nodes(self.graph_n.root)
         self._postcompare()
     
     def add_compare_result_to_node(self, node, compare_result_list):
@@ -60,8 +66,6 @@ class GraphComparator:
             self.ma.parse_result(node, [compare_in_dict, compare_out_dict]))
         node.data[GraphConst.JSON_INDEX_KEY] = precision_index
         node.data.update(other_dict)
-        if NodeColors.get_node_error_status(self.ma.compare_mode, precision_index):
-            node.get_suggestions()
     
     def _parse_param(self, dump_path_param, output_path):
         self.dump_path_param = dump_path_param
@@ -82,8 +86,6 @@ class GraphComparator:
         for node in self.ma.compare_nodes:
             precision_index, _ = self.ma.parse_result(node, [compare_data_dict])
             node.data[GraphConst.JSON_INDEX_KEY] = precision_index
-            if NodeColors.get_node_error_status(self.ma.compare_mode, precision_index):
-                node.get_suggestions()
 
     def _handle_api_collection_index(self):
         """
@@ -120,11 +122,59 @@ class GraphComparator:
                 node_n.add_link(node_b, ancestors)
         if node_b:
             # 真实数据比对只会得到基本信息，并没有精度指标，需要调用多进程对比接口
-            compare_result_list = compare_node([node_n.id, node_b.id],
-                                               [self.data_n_dict, self.data_b_dict],
-                                               self.stack_json_data, self.ma.compare_mode)
-            if compare_result_list:
-                self.ma.add_csv_data(compare_result_list)
-                self.add_compare_result_to_node(node_n, compare_result_list)
+            self._get_and_add_result(node_n, node_b)
         for subnode in node_n.subnodes:
             self._compare_nodes(subnode)
+
+    def _compare_nodes_fuzzy(self, node_n):
+        if node_n.op != NodeOp.function_api:
+            # 模块经过模糊匹配
+            node_b, ancestors_n, ancestors_b = Graph.fuzzy_match(node_n, self.graph_b.node_map.get(node_n.id))
+            if node_b:
+                self._process_matched_nodes(node_n, node_b, ancestors_n, ancestors_b)
+                # 匹配上的两个模块中的所有api, 忽略dump调用次数，按照名称一致+模块中的调用顺序进行匹配
+                recount_result_n = self._recount_api_node(node_n)
+                recount_result_b = self._recount_api_node(node_b)
+                for recount_node_id, node_id_n in recount_result_n.items():
+                    api_node_n = self.graph_n.node_map.get(node_id_n)
+                    if not api_node_n:
+                        continue
+                    api_node_b, ancestors_n, ancestors_b = Graph.fuzzy_match(
+                        api_node_n, self.graph_b.node_map.get(recount_result_b.get(recount_node_id)))
+                    if api_node_b:
+                        self._process_matched_nodes(api_node_n, api_node_b, ancestors_n, ancestors_b)
+        for sub_node in node_n.subnodes:
+            self._compare_nodes_fuzzy(sub_node)
+
+    def _get_and_add_result(self, node_n, node_b):
+        compare_result_list = compare_node([node_n.id, node_b.id],
+                                           [self.data_n_dict, self.data_b_dict],
+                                           self.stack_json_data, self.ma.compare_mode)
+        if compare_result_list:
+            self.ma.add_csv_data(compare_result_list)
+            self.add_compare_result_to_node(node_n, compare_result_list)
+
+    def _recount_api_node(self, node):
+        """
+        两个匹配上的模块, 忽略各自模块下所有api的dump调用次数, 并赋予模块中的调用顺序
+        Return:
+            {赋予模块中的调用顺序的node_id: 原始node_id}
+        """
+        recount_result = {}
+        node_count = {}
+        for sub_node in node.subnodes:
+            if sub_node.op == NodeOp.function_api:
+                # 忽略dump调用次数
+                count_removed_id = self.pattern.sub(Const.SEP, sub_node.id)
+                node_count[count_removed_id] = node_count.get(count_removed_id, 0) + 1
+                # 赋予模块中的调用顺序
+                recount_node_id = count_removed_id + str(node_count.get(count_removed_id))
+                recount_result[recount_node_id] = sub_node.id
+        return recount_result
+
+    def _process_matched_nodes(self, node_n, node_b, ancestors_n, ancestors_b):
+        ancestors_n.append(node_n.id)
+        ancestors_b.append(node_b.id)
+        node_n.matched_node_link = ancestors_b
+        node_b.matched_node_link = ancestors_n
+        self._get_and_add_result(node_n, node_b)

msprobe/visualization/compare/mode_adapter.py

@@ -83,27 +83,13 @@ class ModeAdapter:
                 continue
             compare_data = compare_data_dict.get(key)
             if compare_data:
-                dtype = data_info.get(Const.DTYPE)
                 # 对应比对结果csv的列
                 key_list = GraphConst.SUMMARY_INDEX_LIST
                 headers = CompareConst.SUMMARY_COMPARE_RESULT_HEADER
                 id_list = [headers.index(x) for x in key_list]
                 ModeAdapter._match_data(data_info, compare_data, key_list, id_list)
-                for index, item in enumerate(key_list[4:]):
-                    value = data_info.get(GraphConst.VALUE_INDEX_LIST[index])
-                    value_diff = data_info.get(key_list[index])
+                for item in key_list[4:]:
                     relative_err = str2float(data_info.get(item))
-                    if isinstance(value, float) and isinstance(value_diff, float) \
-                            and dtype in GraphConst.SMALL_VALUES.keys():
-                        small_value = GraphConst.SMALL_VALUES.get(dtype)
-                        # 小值域
-                        if abs(value) <= small_value:
-                            data_info[item] = ToolTip.SMALL_VALUE_TIP.format(data_info.get(item),
-                                                                             GraphConst.VALUE_INDEX_LIST[index],
-                                                                             small_value)
-                            relative_err = GraphConst.MIN_INDEX_KEY \
-                                if abs(value_diff) <= GraphConst.SMALL_VALUES_ABS_ERROR.get(dtype) \
-                                else GraphConst.MAX_INDEX_KEY
                     max_relative_err = max(max_relative_err, relative_err)
                 node_data[key] = data_info
         max_relative_err = 1 if max_relative_err > 1 else max_relative_err

msprobe/visualization/graph/base_node.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 from msprobe.core.overflow_check.level import OverflowLevel
 from msprobe.visualization.graph.node_op import NodeOp
-from msprobe.visualization.utils import Suggestions, GraphConst
-from msprobe.visualization.builder.msprobe_adapter import format_node_data, compare_data
+from msprobe.visualization.utils import GraphConst
+from msprobe.visualization.builder.msprobe_adapter import format_node_data, compare_data, compare_data_fuzzy
 
 
 class BaseNode:
@@ -33,6 +33,8 @@ class BaseNode:
         self.stack_info = []
         self.micro_step_id = None
         self.overflow_level = None
+        self.matched_distributed = {}
+        self.batch_p2p_info = []
 
     def __str__(self):
         info = f'id:\t{self.id}'
@@ -48,16 +50,12 @@ class BaseNode:
             return False
         return True
 
-    def get_suggestions(self):
-        """
-        精度疑似有问题时，提供一些建议
-        """
-        if self.op == NodeOp.module:
-            self.suggestions[GraphConst.SUGGEST_KEY] = Suggestions.Module
-            self.suggestions[Suggestions.DUMP] = Suggestions.DUMP_URL
-        elif self.op == NodeOp.function_api:
-            self.suggestions[GraphConst.SUGGEST_KEY] = Suggestions.API
-            self.suggestions[Suggestions.API_ACCURACY_CHECKER] = Suggestions.API_ACCURACY_CHECKER_URL
+    def fuzzy_eq(self, other):
+        if not compare_data_fuzzy(self.input_data, other.input_data):
+            return False
+        if not compare_data_fuzzy(self.output_data, other.output_data):
+            return False
+        return True
 
     def set_input_output(self, input_data, output_data):
         self.input_data = input_data
@@ -67,6 +65,7 @@ class BaseNode:
         if not level or not isinstance(level, OverflowLevel):
             return
         self.overflow_level = level
+        self.data[GraphConst.OVERFLOW_LEVEL] = self.overflow_level.value
 
     def add_upnode(self, node):
         """
@@ -94,8 +93,8 @@ class BaseNode:
         result = {
             'id': self.id,
             'node_type': self.op.value,
-            'output_data': format_node_data(self.output_data),
-            'input_data': format_node_data(self.input_data),
+            'output_data': format_node_data(self.output_data, self.id),
+            'input_data': format_node_data(self.input_data, self.id),
             'upnode': self.upnode.id if self.upnode else 'None',
             'subnodes': [node.id for node in self.subnodes],
             'matched_node_link': self.matched_node_link,
@@ -104,12 +103,9 @@ class BaseNode:
         }
         if self.micro_step_id is not None:
             result['micro_step_id'] = self.micro_step_id
-        # 是否存在overflow，并保存结果
-        if self.overflow_level and isinstance(self.overflow_level, OverflowLevel):
-            if self.data is None:
-                self.data = dict()
-            self.data['overflow_level'] = self.overflow_level.value
         result['data'] = self.data
+        if self.matched_distributed:
+            result[GraphConst.MATCHED_DISTRIBUTED] = self.matched_distributed
         return result
 
     def get_ancestors(self):