mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/visualization/builder/graph_builder.py

@@ -0,0 +1,222 @@
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from msprobe.core.common.const import Const
+from msprobe.core.common.file_utils import load_json
+from msprobe.visualization.builder.msprobe_adapter import get_input_output
+from msprobe.visualization.builder.msprobe_adapter import op_patterns
+from msprobe.visualization.graph.graph import Graph
+from msprobe.visualization.graph.node_op import NodeOp
+from msprobe.visualization.utils import save_json_file, GraphConst
+
+
+class GraphBuilder:
+    backward_pattern = re.compile(r"(\.backward\.)(\d+)$")
+    # 匹配以大写字母开头，后接任意字母，并以Template(结尾
+    template_pattern = re.compile(r'\b[A-Z][a-zA-Z]*Template\(')
+
+    @staticmethod
+    def build(construct_path, data_path, stack_path, model_name='DefaultModel', complete_stack=False):
+        """
+        GraphBuilder的对外提供的构图方法
+        Args:
+            construct_path: construct.json路径
+            data_path: dump.json路径
+            stack_path: stack.json路径
+            model_name: 模型名字，依赖外部输入
+            complete_stack: 完整的堆栈信息
+        Returns: Graph，代表图的数据结构
+        """
+        construct_dict = load_json(construct_path)
+        dump_dict = load_json(data_path)
+        stack_dict = load_json(stack_path)
+        if not complete_stack:
+            GraphBuilder._simplify_stack(stack_dict)
+        data_dict = dump_dict.get(GraphConst.DATA_KEY, {})
+        graph = Graph(model_name, data_path=dump_dict.get('dump_data_dir', ''), dump_data=data_dict)
+        GraphBuilder._init_nodes(graph, construct_dict, data_dict, stack_dict)
+        GraphBuilder._collect_apis_between_modules(graph)
+        return graph
+
+    @staticmethod
+    def to_json(filename, config):
+        """
+        将graph导出成.vis文件的接口
+        """
+        result = {}
+        if config.graph_b:
+            result[GraphConst.JSON_NPU_KEY] = config.graph_n.to_dict()
+            result[GraphConst.JSON_BENCH_KEY] = config.graph_b.to_dict()
+        else:
+            result = config.graph_n.to_dict()
+        if config.tool_tip:
+            result[GraphConst.JSON_TIP_KEY] = config.tool_tip
+        if config.node_colors:
+            result[GraphConst.COLORS] = config.node_colors
+        if config.micro_steps:
+            result[GraphConst.MICRO_STEPS] = config.micro_steps
+        if config.task:
+            result[GraphConst.JSON_TASK_KEY] = config.task
+        result[GraphConst.OVERFLOW_CHECK] = config.overflow_check
+        save_json_file(filename, result)
+
+    @staticmethod
+    def _simplify_stack(stack_dict):
+        """
+        精简堆栈内容，模块级保留包含"模块名("的堆栈，api级保留"xxxTemplate("的下一行堆栈
+
+        例如模块 Module.layer3.0.bn2.BatchNorm2d.forward.0，模块名为bn2，匹配"bn2("，
+        保留堆栈"File /home/models/resnet.py, line 97, in forward, \n out = self.bn2(out)"
+
+        例如Api Tensor.__iadd__.4.forward，堆栈为：
+        "File /home/wrap_tensor.py, line 61,  return TensorOPTemplate(op_name, hook)(*args, **kwargs)",
+        "File /home/torchvision/models/resnet.py, line 102, in forward, \n out += identity",
+        匹配到第一行的"TensorOPTemplate("，保留下一行堆栈
+        """
+        module_pattern = re.compile(op_patterns[0])
+        for dump_name, stack_list in stack_dict.items():
+            if not isinstance(stack_list, list):
+                continue
+            if module_pattern.match(dump_name):
+                parts = dump_name.split(Const.SEP)
+                if len(parts) < abs(Const.LAYER_NAME_INDEX):
+                    continue
+                module_name = parts[Const.LAYER_NAME_INDEX]
+                for stack in stack_list:
+                    if re.search(module_name + r'\(', stack):
+                        stack_list = [stack]
+                        break
+            else:
+                for index, stack in enumerate(stack_list):
+                    if GraphBuilder.template_pattern.search(stack) and index < len(stack_list) - 1:
+                        stack_list = [stack_list[index + 1]]
+                        break
+            stack_dict[dump_name] = stack_list
+
+    @staticmethod
+    def _handle_backward_upnode_missing(construct_dict, subnode_id, upnode_id):
+        """
+        如果backward节点的父级节点是null，则尝试从同名的forward节点寻找父级节点
+        """
+        # 匹配以.backward.后跟一个或多个数字结尾的模式
+        backward_pattern = r"(\.backward\.)(\d+)$"
+        forward_pattern = r"(\.forward\.)(\d+)$"
+        if re.search(backward_pattern, subnode_id) and not upnode_id:
+            forward_upnode_id = construct_dict.get(re.sub(backward_pattern, r".forward.\2", subnode_id))
+            if forward_upnode_id:
+                new_upnode_id = re.sub(forward_pattern, r".backward.\2", forward_upnode_id)
+                if new_upnode_id in construct_dict:
+                    return new_upnode_id
+        return upnode_id
+
+    @staticmethod
+    def _init_nodes(graph, construct_dict, data_dict, stack_dict):
+        for subnode_id, upnode_id in construct_dict.items():
+            upnode_id = GraphBuilder._handle_backward_upnode_missing(construct_dict, subnode_id, upnode_id)
+            if upnode_id:
+                upnode_op = NodeOp.get_node_op(upnode_id)
+                upnode = GraphBuilder._create_or_get_node(graph, [data_dict, stack_dict], upnode_op, upnode_id)
+            else:
+                upnode = graph.root
+            node_op = NodeOp.get_node_op(subnode_id)
+            GraphBuilder._create_or_get_node(graph, [data_dict, stack_dict], node_op, subnode_id, upnode)
+
+    @staticmethod
+    def _create_or_get_node(graph, data_stack_list, op, name, upnode=None):
+        if name in graph.node_map:
+            node = graph.get_node(name)
+        else:
+            graph.add_node(op, name, upnode)
+            node = graph.get_node(name)
+            node_data = data_stack_list[0].get(name, {})
+            node_stack_info = data_stack_list[1].get(name, [])
+            # 添加输入输出数据
+            input_data, output_data = get_input_output(node_data, node.id)
+            # 更新数据
+            node.set_input_output(input_data, output_data)
+            # 反向节点使用对应前向节点的堆栈信息
+            # 模块命名举例：Module.module.module.GPTModel.backward.0; API命名举例：Tensor.permute.1.backward
+            if (not node_stack_info and
+                    (GraphBuilder.backward_pattern.search(name) or name.endswith(f'{Const.SEP}{Const.BACKWARD}'))):
+                forward_node = graph.get_node(
+                    # 同名模块全局唯一，无论调用几次堆栈信息都一致，直接使用编号0的同名模块堆栈信息，避免遗漏
+                    GraphBuilder.backward_pattern.sub(f'{Const.SEP}{Const.FORWARD}{Const.SEP}0', name)) \
+                    if GraphBuilder.backward_pattern.search(name) \
+                    else graph.get_node(name.replace(Const.BACKWARD, Const.FORWARD))
+                node_stack_info = forward_node.stack_info if forward_node \
+                    else ['This backward node cannot find the forward node and cannot retrieve stack information.']
+            node.stack_info = node_stack_info
+        # 添加节点
+        node.add_upnode(upnode)
+        return node
+
+    @staticmethod
+    def _collect_apis_between_modules(graph):
+        """
+        图首次展开，这些首层节点包含许多module和api，api数量很多导致图被拉得很长严重影响查阅，因此将module之间的apis收集起来成为节点
+        Args:
+            graph: 模型结构
+
+        Returns: None
+        """
+        i = 0
+        output = []
+        node_list = graph.root.subnodes
+        while i < len(node_list):
+            current_node = node_list[i]
+
+            # 当前节点为api，检查后续是否还有api
+            if current_node.op == NodeOp.function_api:
+                temp_nodes = [current_node]
+                i += 1
+                while i < len(node_list) and node_list[i].op == NodeOp.function_api:
+                    temp_nodes.append(node_list[i])
+                    i += 1
+
+                # 检查api节点是否大于等于2个
+                if len(temp_nodes) >= 2:
+                    # 创建新节点，将这些api节点放入新节点的subnodes属性
+                    node_id = graph.add_node(NodeOp.api_collection, GraphConst.APIS_BETWEEN_MODULES,
+                                             id_accumulation=True)
+                    api_collection_node = graph.get_node(node_id)
+                    api_collection_node.subnodes = temp_nodes
+                    # 重新确立父子关系
+                    for node in temp_nodes:
+                        node.upnode = api_collection_node
+                    api_collection_node.upnode = graph.root
+                    output.append(api_collection_node)
+                else:
+                    # 如果连续的api节点不足2个，将它们原样添加到输出列表
+                    output.extend(temp_nodes)
+            else:
+                # 如果当前节点为module，直接添加到输出列表
+                output.append(current_node)
+                i += 1
+
+        graph.root.subnodes = output
+
+
+class GraphExportConfig:
+    def __init__(self, graph_n, graph_b=None, tool_tip=None, node_colors=None, micro_steps=None, task='',
+                 overflow_check=False):
+        self.graph_n = graph_n
+        self.graph_b = graph_b
+        self.tool_tip = tool_tip
+        self.node_colors = node_colors
+        self.micro_steps = micro_steps
+        self.task = task
+        self.overflow_check = overflow_check

msprobe/visualization/builder/msprobe_adapter.py

@@ -0,0 +1,227 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import math
+from msprobe.core.compare.acc_compare import read_op, merge_tensor, get_accuracy
+from msprobe.core.common.utils import set_dump_path, get_dump_mode
+from msprobe.visualization.utils import GraphConst
+from msprobe.core.common.const import Const
+from msprobe.core.compare.acc_compare import ModeConfig
+
+# 用于将节点名字解析成对应的NodeOp的规则
+op_patterns = [
+    # NodeOp.module
+    r'^(Module.|Cell.)',
+    # NodeOp.function_api
+    r'^(Tensor.|Torch.|Functional.|NPU.|VF.|Distributed.|Aten.|Mint.|Primitive.|Jit.|MintFunctional.)'
+]
+
+
+def get_compare_mode(dump_path_param):
+    """
+    获得比较模式，包括summary、MD5和真实数据三种模式
+    Args:
+        dump_path_param: 调用acc_compare接口所依赖的参数
+    Returns: 0 summary mode, 1 md5 mode, 2 true data mode
+    """
+    set_dump_path(dump_path_param)
+    dump_mode = get_dump_mode(dump_path_param)
+    compare_mode = GraphConst.DUMP_MODE_TO_GRAPHCOMPARE_MODE_MAPPING.get(dump_mode)
+    return compare_mode
+
+
+def run_real_data(dump_path_param, csv_path, framework, is_cross_frame=False):
+    """
+    多进程运行生成真实数据
+    Args:
+        dump_path_param: 调用acc_compare接口所依赖的参数
+        csv_path: 生成文件路径
+        framework: 框架类型, pytorch或mindspore
+        is_cross_frame: 是否进行跨框架比对，仅支持mindspore比pytorch, 其中pytorch为标杆
+    """
+    mode_config = ModeConfig(stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.ALL)
+
+    if framework == Const.PT_FRAMEWORK:
+        from msprobe.pytorch.compare.pt_compare import PTComparator
+        return PTComparator(mode_config).do_multi_process(dump_path_param, csv_path)
+    else:
+        from msprobe.mindspore.compare.ms_compare import MSComparator
+        ms_comparator = MSComparator(mode_config)
+        ms_comparator.cross_frame = is_cross_frame
+        return ms_comparator.do_multi_process(dump_path_param, csv_path)
+
+
+def get_input_output(node_data, node_id):
+    """
+    将dump的原始数据进行拆解，分解为output和input两个数据
+    Args:
+        node_data: 属于单个节点的dump数据
+        node_id: 节点名字
+    """
+    input_data = {}
+    output_data = {}
+    op_parsed_list = read_op(node_data, node_id)
+    for item in op_parsed_list:
+        full_op_name = item.get('full_op_name', '')
+        if not full_op_name:
+            continue
+        if GraphConst.OUTPUT in full_op_name and GraphConst.INPUT not in full_op_name:
+            output_data[full_op_name] = item
+        else:
+            name = item.get('data_name')
+            # 节点参数名称尽量使用落盘数据的名称
+            if isinstance(name, str) and name != '-1':
+                input_data[name.rsplit(Const.SEP, 1)[0]] = item
+            else:
+                input_data[full_op_name] = item
+    return input_data, output_data
+
+
+def compare_data(data_dict_list1, data_dict_list2):
+    """
+    比较get_input_output中输出的结果是否结构一致，比较一致返回True
+    """
+    if len(data_dict_list1) != len(data_dict_list2):
+        return False
+    # 用于比较两个节点是否相等的关键字段
+    tag_keys = ['type', 'shape']
+    for key1, key2 in zip(data_dict_list1, data_dict_list2):
+        dict1 = data_dict_list1[key1]
+        dict2 = data_dict_list2[key2]
+        for tag_key in tag_keys:
+            tag_value1 = dict1.get(tag_key, None)
+            tag_value2 = dict2.get(tag_key, None)
+            if tag_value1 != tag_value2:
+                return False
+    return True
+
+
+def compare_data_fuzzy(data_dict_list1, data_dict_list2):
+    """
+    模糊匹配，仅校验参数shape是否一致
+    """
+    for x, y in zip(data_dict_list1.values(), data_dict_list2.values()):
+        x_shape = x.get(Const.SHAPE)
+        y_shape = y.get(Const.SHAPE)
+        if x_shape != y_shape:
+            return False
+    return True
+
+
+def format_node_data(data_dict):
+    """
+    批量进行节点数据的输出
+    """
+    del_list = ['requires_grad', 'full_op_name']
+    for _, value in data_dict.items():
+        if not isinstance(value, dict):
+            continue
+        for item in del_list:
+            if item in value:
+                del value[item]
+        _format_data(value)
+    return data_dict
+
+
+def compare_node(node_ids, data_dicts, stack_json_data, compare_mode):
+    """
+    调用acc_compare.py中的get_accuracy获得精度对比指标
+    真实数据对比模式无法获得精度对比指标，需要调用多进程比对接口
+    Returns: 包含参数信息和对比指标（真实数据对比模式除外）的list
+    """
+    merge_n = _parse_node(node_ids[0], data_dicts[0], stack_json_data, compare_mode)
+    merge_b = _parse_node(node_ids[1], data_dicts[1], stack_json_data, compare_mode)
+    result = []
+    dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode)
+    get_accuracy(result, merge_n, merge_b, dump_mode)
+    return result
+
+
+def _parse_node(node_id, data_dict, stack_json_data, compare_mode):
+    """
+    转换节点，使其能够作为acc_compare.py中的get_accuracy的入参
+    """
+    dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode)
+    op_parsed_list = read_op(data_dict.get(node_id, {}), node_id)
+    if node_id in stack_json_data:
+        op_parsed_list.append(
+            {'full_op_name': node_id, 'full_info': stack_json_data[node_id]})
+    else:
+        op_parsed_list.append({'full_op_name': node_id, 'full_info': None})
+    result = merge_tensor(op_parsed_list, dump_mode)
+    if not result:
+        result['op_name'] = []
+    return result
+
+
+def _format_decimal_string(s):
+    """
+    使用正则表达式匹配包含数字、小数点和可选的百分号的字符串
+    """
+    pattern = re.compile(r'\d{1,20}\.\d{1,20}%?')
+    matches = pattern.findall(s)
+    for match in matches:
+        is_percent = match.endswith('%')
+        number_str = match.rstrip('%')
+        decimal_part = number_str.split('.')[1]
+        # 如果小数位数大于6，进行处理
+        if len(decimal_part) > GraphConst.ROUND_TH:
+            number_float = float(number_str)
+            formatted_number = f"{number_float:.{GraphConst.ROUND_TH}f}"
+            # 如果原来是百分数，加回百分号
+            if is_percent:
+                formatted_number += '%'
+            # 替换原字符串中的数值部分
+            s = s.replace(match, formatted_number)
+    return s
+
+
+def _format_data(data_dict):
+    """
+    格式化数据，小数保留6位，处理一些异常值
+    """
+    pattern = r'^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)$'
+    all_null = False
+
+    keys_to_keep = ['type', 'group_ranks', 'group_id', 'data_name']
+    if data_dict.get('type') == 'torch.ProcessGroup':
+        keys_to_remove = [key for key in data_dict if key not in keys_to_keep]
+        for key in keys_to_remove:
+            del data_dict[key]
+
+    for key, value in data_dict.items():
+        if isinstance(value, str):
+            # 将单引号删掉，None换成null避免前端解析错误
+            value = value.replace("'", "").replace(GraphConst.NONE, GraphConst.NULL)
+            value = _format_decimal_string(value)
+        elif value is None or value == ' ':
+            value = GraphConst.NULL
+        # 科学计数法1.123123123123e-11，格式化为1.123123e-11
+        elif isinstance(value, float) and len(str(value)) < GraphConst.STR_MAX_LEN and re.match(pattern, str(value)):
+            value = "{:.6e}".format(value)
+        elif isinstance(value, float):
+            value = round(value, GraphConst.ROUND_TH)
+        # Inf会走入这里，确保转成Inf。另外给其他不符合预期的类型做兜底方案
+        if key != GraphConst.ERROR_KEY:
+            # 除了error_key不转str，其他都转str, 避免前端解析错误
+            value = str(value)
+        # max为null, 意味着这个参数值为null
+        if key == Const.MAX and value == GraphConst.NULL:
+            all_null = True
+        data_dict[key] = value
+    # 字典里的value全null，只保留一个null
+    if all_null:
+        data_dict.clear()
+        data_dict[GraphConst.VALUE] = GraphConst.NULL

msprobe/visualization/compare/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

msprobe/visualization/compare/graph_comparator.py

@@ -0,0 +1,180 @@
+# Copyright (c) 2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from msprobe.visualization.builder.msprobe_adapter import compare_node, get_compare_mode, run_real_data
+from msprobe.visualization.utils import GraphConst, load_json_file, load_data_json_file, get_csv_df
+from msprobe.visualization.graph.graph import Graph, NodeOp
+from msprobe.visualization.graph.node_colors import NodeColors
+from msprobe.visualization.compare.mode_adapter import ModeAdapter
+from msprobe.core.common.const import Const
+
+
+class GraphComparator:
+    def __init__(self, graphs, dump_path_param, args, mapping_dict=None):
+        self.graph_n = graphs[0]
+        self.graph_b = graphs[1]
+        self._parse_param(dump_path_param, args.output_path)
+        self.framework = args.framework
+        self.mapping_dict = mapping_dict
+        self.fuzzy_match = args.fuzzy_match
+        self.pattern = re.compile(r'\.\d+\.')
+
+    def compare(self):
+        """
+        比较函数，初始化结束后单独调用。比较结果写入graph_n
+        """
+        if self.fuzzy_match:
+            self._compare_nodes_fuzzy(self.graph_n.root)
+        else:
+            self._compare_nodes(self.graph_n.root)
+        self._postcompare()
+    
+    def add_compare_result_to_node(self, node, compare_result_list):
+        """
+        将比对结果添加到节点的输入输出数据中
+        Args:
+            node: 节点
+            compare_result_list: 包含参数信息和对比指标（真实数据对比模式除外）的list
+        """
+        # 真实数据比对，先暂存节点，在多进程对比得到精度指标后，再将指标添加到节点中
+        if self.ma.prepare_real_data(node):
+            return
+        compare_in_dict = {}
+        compare_out_dict = {}
+        # input和output对比数据分开
+        for item in compare_result_list:
+            if not isinstance(item, (list, tuple)) or not item:
+                continue
+            if '.output.' in item[0]:
+                compare_out_dict[item[0]] = item
+            else:
+                compare_in_dict[item[0]] = item
+        precision_index, other_dict = (
+            self.ma.parse_result(node, [compare_in_dict, compare_out_dict]))
+        node.data[GraphConst.JSON_INDEX_KEY] = precision_index
+        node.data.update(other_dict)
+    
+    def _parse_param(self, dump_path_param, output_path):
+        self.dump_path_param = dump_path_param
+        self.output_path = output_path
+        compare_mode = get_compare_mode(self.dump_path_param)
+        self.ma = ModeAdapter(compare_mode)
+        self.data_n_dict = load_data_json_file(dump_path_param.get('npu_json_path'))
+        self.data_b_dict = load_data_json_file(dump_path_param.get('bench_json_path'))
+        self.stack_json_data = load_json_file(dump_path_param.get('stack_json_path'))
+
+    def _postcompare(self):
+        self._handle_api_collection_index()
+        if not self.ma.compare_mode == GraphConst.REAL_DATA_COMPARE:
+            return
+        df = get_csv_df(True, self.ma.csv_data, self.ma.compare_mode)
+        df = run_real_data(self.dump_path_param, df, self.framework, True if self.mapping_dict else False)
+        compare_data_dict = {row[0]: row.tolist() for _, row in df.iterrows()}
+        for node in self.ma.compare_nodes:
+            precision_index, _ = self.ma.parse_result(node, [compare_data_dict])
+            node.data[GraphConst.JSON_INDEX_KEY] = precision_index
+
+    def _handle_api_collection_index(self):
+        """
+        api集合的指标, md5模式使用集合中所有api最小的指标，statistics和tensor模式使用集合中所有api最大的指标
+        md5模式下指标为0代表最差，statistics和tensor模式下指标为1代表最差
+        """
+        for node in self.graph_n.root.subnodes:
+            if node.op == NodeOp.api_collection:
+                precision_index = GraphConst.MAX_INDEX_KEY if self.ma.compare_mode == GraphConst.MD5_COMPARE \
+                    else GraphConst.MIN_INDEX_KEY
+                for api in node.subnodes:
+                    precision_index = min(precision_index,
+                                          api.data.get(GraphConst.JSON_INDEX_KEY, GraphConst.MAX_INDEX_KEY)) \
+                        if self.ma.compare_mode == GraphConst.MD5_COMPARE \
+                        else max(precision_index, api.data.get(GraphConst.JSON_INDEX_KEY, GraphConst.MIN_INDEX_KEY))
+                node.data[GraphConst.JSON_INDEX_KEY] = precision_index
+
+    def _compare_nodes(self, node_n):
+        """
+        递归遍历NPU树中的节点，如果在Bench中找到具有相同名称的节点，检查他们的祖先和参数信息，检查一致则及逆行精度数据对比
+        这里采用先序遍历，好处在于当这个节点被比较时，他的先序已经被匹配，这可以为后续的模糊匹配提供重要信息
+        """
+        if self.mapping_dict:
+            node_b, ancestors_n, ancestors_b = Graph.mapping_match(node_n, self.graph_b, self.mapping_dict)
+            if node_b:
+                ancestors_n.append(node_n.id)
+                ancestors_b.append(node_b.id)
+                node_n.matched_node_link = ancestors_b
+                node_b.matched_node_link = ancestors_n
+        else:
+            node_b, ancestors = Graph.match(self.graph_n, node_n, self.graph_b)
+            if node_b:
+                ancestors.append(node_b.id)
+                node_n.add_link(node_b, ancestors)
+        if node_b:
+            # 真实数据比对只会得到基本信息，并没有精度指标，需要调用多进程对比接口
+            self._get_and_add_result(node_n, node_b)
+        for subnode in node_n.subnodes:
+            self._compare_nodes(subnode)
+
+    def _compare_nodes_fuzzy(self, node_n):
+        if node_n.op != NodeOp.function_api:
+            # 模块经过模糊匹配
+            node_b, ancestors_n, ancestors_b = Graph.fuzzy_match(node_n, self.graph_b.node_map.get(node_n.id))
+            if node_b:
+                self._process_matched_nodes(node_n, node_b, ancestors_n, ancestors_b)
+                # 匹配上的两个模块中的所有api, 忽略dump调用次数，按照名称一致+模块中的调用顺序进行匹配
+                recount_result_n = self._recount_api_node(node_n)
+                recount_result_b = self._recount_api_node(node_b)
+                for recount_node_id, node_id_n in recount_result_n.items():
+                    api_node_n = self.graph_n.node_map.get(node_id_n)
+                    if not api_node_n:
+                        continue
+                    api_node_b, ancestors_n, ancestors_b = Graph.fuzzy_match(
+                        api_node_n, self.graph_b.node_map.get(recount_result_b.get(recount_node_id)))
+                    if api_node_b:
+                        self._process_matched_nodes(api_node_n, api_node_b, ancestors_n, ancestors_b)
+        for sub_node in node_n.subnodes:
+            self._compare_nodes_fuzzy(sub_node)
+
+    def _get_and_add_result(self, node_n, node_b):
+        compare_result_list = compare_node([node_n.id, node_b.id],
+                                           [self.data_n_dict, self.data_b_dict],
+                                           self.stack_json_data, self.ma.compare_mode)
+        if compare_result_list:
+            self.ma.add_csv_data(compare_result_list)
+            self.add_compare_result_to_node(node_n, compare_result_list)
+
+    def _recount_api_node(self, node):
+        """
+        两个匹配上的模块, 忽略各自模块下所有api的dump调用次数, 并赋予模块中的调用顺序
+        Return:
+            {赋予模块中的调用顺序的node_id: 原始node_id}
+        """
+        recount_result = {}
+        node_count = {}
+        for sub_node in node.subnodes:
+            if sub_node.op == NodeOp.function_api:
+                # 忽略dump调用次数
+                count_removed_id = self.pattern.sub(Const.SEP, sub_node.id)
+                node_count[count_removed_id] = node_count.get(count_removed_id, 0) + 1
+                # 赋予模块中的调用顺序
+                recount_node_id = count_removed_id + str(node_count.get(count_removed_id))
+                recount_result[recount_node_id] = sub_node.id
+        return recount_result
+
+    def _process_matched_nodes(self, node_n, node_b, ancestors_n, ancestors_b):
+        ancestors_n.append(node_n.id)
+        ancestors_b.append(node_b.id)
+        node_n.matched_node_link = ancestors_b
+        node_b.matched_node_link = ancestors_n
+        self._get_and_add_result(node_n, node_b)