mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/visualization/compare/mode_adapter.py

@@ -0,0 +1,197 @@
+# Copyright (c) 2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import math
+from msprobe.core.common.const import CompareConst, Const
+from msprobe.visualization.utils import ToolTip, GraphConst, str2float
+
+
+class ModeAdapter:
+    def __init__(self, compare_mode):
+        self.compare_mode = compare_mode
+        self.csv_data = []
+        self.compare_nodes = []
+
+    @staticmethod
+    def _add_md5_compare_data(node_data, compare_data_dict):
+        precision_index = GraphConst.MAX_INDEX_KEY
+        for key, value in node_data.items():
+            if not isinstance(value, dict):
+                continue
+            compare_data = compare_data_dict.get(key)
+            if compare_data:
+                headers = CompareConst.MD5_COMPARE_RESULT_HEADER
+                id_list = [headers.index(x) for x in GraphConst.MD5_INDEX_LIST]
+                ModeAdapter._match_data(value, compare_data, GraphConst.MD5_INDEX_LIST, id_list)
+                # md5比对是否通过
+                if value.get(CompareConst.RESULT) != CompareConst.PASS:
+                    precision_index = GraphConst.MIN_INDEX_KEY
+                node_data[key] = value
+        return precision_index
+
+    @staticmethod
+    def _add_real_compare_data(node_data, compare_data_dict):
+        min_thousandth = float(1)
+        numbers = []
+        for key, value in node_data.items():
+            if not isinstance(value, dict):
+                continue
+            compare_data = compare_data_dict.get(key)
+            if compare_data:
+                headers = CompareConst.COMPARE_RESULT_HEADER
+                id_list = [headers.index(x) for x in GraphConst.REAL_DATA_INDEX_LIST]
+                ModeAdapter._match_data(value, compare_data, GraphConst.REAL_DATA_INDEX_LIST, id_list)
+                # 跳过scalar data，因为无法计算双千指标，会得到Nan
+                if not value.get(Const.SHAPE):
+                    continue
+                # 获取一个节点所有的输入或输出最小的双千指标
+                thousandth = value.get(CompareConst.ONE_THOUSANDTH_ERR_RATIO)
+                # 可能是None，可能是非数字内容str
+                try:
+                    thousandth = float(thousandth)
+                except (ValueError, TypeError):
+                    thousandth = None
+                if thousandth is not None:
+                    numbers.append(thousandth)
+                node_data[key] = value
+        # 双千指标都是None的异常情况
+        if not numbers:
+            min_thousandth = None
+        else:
+            min_thousandth = min(numbers + [min_thousandth])
+        return min_thousandth
+
+    @staticmethod
+    def _add_summary_compare_data(node_data, compare_data_dict):
+        max_relative_err = GraphConst.MIN_INDEX_KEY
+        # data_info: {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [2, 536320], 'Max': 9.66036224, ...}
+        for key, data_info in node_data.items():
+            if not isinstance(data_info, dict):
+                continue
+            compare_data = compare_data_dict.get(key)
+            if compare_data:
+                # 对应比对结果csv的列
+                key_list = GraphConst.SUMMARY_INDEX_LIST
+                headers = CompareConst.SUMMARY_COMPARE_RESULT_HEADER
+                id_list = [headers.index(x) for x in key_list]
+                ModeAdapter._match_data(data_info, compare_data, key_list, id_list)
+                for item in key_list[4:]:
+                    relative_err = str2float(data_info.get(item))
+                    max_relative_err = max(max_relative_err, relative_err)
+                node_data[key] = data_info
+        max_relative_err = 1 if max_relative_err > 1 else max_relative_err
+        return max_relative_err
+
+    @staticmethod
+    def _match_data(data_dict, compare_data, key_list, id_list):
+        """
+        绑定精度指标到node的input_data和output_data
+        """
+        if len(key_list) != len(id_list):
+            return
+        for id_val, key in zip(id_list, key_list):
+            data_dict[key] = compare_data[id_val]
+
+    @staticmethod
+    def _check_list_len(data_list, len_num):
+        if len(data_list) < len_num:
+            raise ValueError(f"compare_data_dict_list must contain at least {len_num} items.")
+
+    def parse_result(self, node, compare_data_dict_list):
+        """
+        根据结果返回数据，分别是precision_index，和附加数据
+        """
+
+        other_dict = {}
+        if self.compare_mode == GraphConst.MD5_COMPARE:
+            ModeAdapter._check_list_len(compare_data_dict_list, 2)
+            precision_index_in = ModeAdapter._add_md5_compare_data(node.input_data, compare_data_dict_list[0])
+            precision_index_out = ModeAdapter._add_md5_compare_data(node.output_data, compare_data_dict_list[1])
+            # 所有输入输出md5对比通过，这个节点才算通过
+            precision_index = min(precision_index_in, precision_index_out)
+            other_result = CompareConst.PASS if precision_index == GraphConst.MAX_INDEX_KEY else CompareConst.DIFF
+            other_dict[CompareConst.RESULT] = other_result
+        elif self.compare_mode == GraphConst.SUMMARY_COMPARE:
+            ModeAdapter._check_list_len(compare_data_dict_list, 2)
+            ModeAdapter._add_summary_compare_data(node.input_data, compare_data_dict_list[0])
+            precision_index_out = ModeAdapter._add_summary_compare_data(node.output_data, compare_data_dict_list[1])
+            precision_index = precision_index_out
+        else:
+            ModeAdapter._check_list_len(compare_data_dict_list, 1)
+            min_thousandth_in = ModeAdapter._add_real_compare_data(node.input_data, compare_data_dict_list[0])
+            min_thousandth_out = ModeAdapter._add_real_compare_data(node.output_data, compare_data_dict_list[0])
+            if min_thousandth_in is not None and min_thousandth_out is not None:
+                change_percentage = min_thousandth_in - min_thousandth_out
+            else:
+                change_percentage = GraphConst.MIN_INDEX_KEY
+            change_percentage = GraphConst.MIN_INDEX_KEY if change_percentage < GraphConst.MIN_INDEX_KEY \
+                else change_percentage
+            precision_index = GraphConst.MAX_INDEX_KEY \
+                if change_percentage > GraphConst.MAX_INDEX_KEY else change_percentage
+        return precision_index, other_dict
+
+    def prepare_real_data(self, node):
+        """
+        为真实数据比较模式准备节点信息
+        """
+        if self.compare_mode == GraphConst.REAL_DATA_COMPARE:
+            self.compare_nodes.append(node)
+            return True
+        return False
+
+    def add_csv_data(self, compare_result_list):
+        if self.compare_mode != GraphConst.REAL_DATA_COMPARE:
+            return
+        self.csv_data.extend(compare_result_list)
+
+    def add_error_key(self, node_data):
+        """
+        根据不同的模式进行提供不同错误信息
+        """
+        for key, value in node_data.items():
+            if not isinstance(value, dict):
+                continue
+            if self.compare_mode == GraphConst.SUMMARY_COMPARE:
+                message = [CompareConst.MAX_RELATIVE_ERR, CompareConst.MIN_RELATIVE_ERR,
+                           CompareConst.MEAN_RELATIVE_ERR, CompareConst.NORM_RELATIVE_ERR]
+            elif self.compare_mode == GraphConst.REAL_DATA_COMPARE:
+                message = [CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO]
+            else:
+                # 输出件优化
+                message = []
+            value[GraphConst.ERROR_KEY] = message
+            node_data[key] = value
+
+    def get_tool_tip(self):
+        """
+        用于前端展示字段的具体含义
+        """
+        if self.compare_mode == GraphConst.SUMMARY_COMPARE:
+            tips = {
+                CompareConst.MAX_DIFF: ToolTip.MAX_DIFF,
+                CompareConst.MIN_DIFF: ToolTip.MIN_DIFF,
+                CompareConst.MEAN_DIFF: ToolTip.MEAN_DIFF,
+                CompareConst.NORM_DIFF: ToolTip.NORM_DIFF}
+        elif self.compare_mode == GraphConst.MD5_COMPARE:
+            tips = {Const.MD5: ToolTip.MD5}
+        else:
+            tips = {
+                CompareConst.ONE_THOUSANDTH_ERR_RATIO: ToolTip.ONE_THOUSANDTH_ERR_RATIO,
+                CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: ToolTip.FIVE_THOUSANDTHS_ERR_RATIO,
+                CompareConst.COSINE: ToolTip.COSINE,
+                CompareConst.MAX_ABS_ERR: ToolTip.MAX_ABS_ERR,
+                CompareConst.MAX_RELATIVE_ERR: ToolTip.MAX_RELATIVE_ERR}
+        return json.dumps(tips)

msprobe/visualization/graph/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

msprobe/visualization/graph/base_node.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from msprobe.core.overflow_check.level import OverflowLevel
+from msprobe.visualization.graph.node_op import NodeOp
+from msprobe.visualization.utils import GraphConst
+from msprobe.visualization.builder.msprobe_adapter import format_node_data, compare_data, compare_data_fuzzy
+
+
+class BaseNode:
+    def __init__(self, node_op, node_id, up_node=None):
+        self.op = node_op
+        self.id = node_id
+        self.data = {}
+        self.output_data = {}
+        self.input_data = {}
+        self.upnode = None
+        self.add_upnode(up_node)
+        self.subnodes = []
+        self.matched_node_link = []
+        self.suggestions = {}
+        self.stack_info = []
+        self.micro_step_id = None
+        self.overflow_level = None
+        self.matched_distributed = {}
+
+    def __str__(self):
+        info = f'id:\t{self.id}'
+        return info
+
+    def __eq__(self, other):
+        """
+        用来判断两个节点是否可以被匹配上，认为结构上是否一致
+        """
+        if not compare_data(self.input_data, other.input_data):
+            return False
+        if not compare_data(self.output_data, other.output_data):
+            return False
+        return True
+
+    def fuzzy_eq(self, other):
+        if not compare_data_fuzzy(self.input_data, other.input_data):
+            return False
+        if not compare_data_fuzzy(self.output_data, other.output_data):
+            return False
+        return True
+
+    def set_input_output(self, input_data, output_data):
+        self.input_data = input_data
+        self.output_data = output_data
+
+    def set_overflow_level(self, level):
+        if not level or not isinstance(level, OverflowLevel):
+            return
+        self.overflow_level = level
+        self.data[GraphConst.OVERFLOW_LEVEL] = self.overflow_level.value
+
+    def add_upnode(self, node):
+        """
+        绑定upnode，用于对两个节点进行上下级关联
+        """
+        if not node or node.id == self.id or self.upnode:
+            return
+        self.upnode = node
+        node.subnodes.append(self)
+
+    def add_link(self, node, ancestors):
+        """
+        在节点匹配成功后进行匹配数据的录入
+        Args:
+            node: 和self相互匹配的节点
+            ancestors: 对面节点的祖先信息
+        """
+        self.matched_node_link = ancestors
+        node.matched_node_link = ancestors
+
+    def to_dict(self):
+        """
+        输出数据
+        """
+        result = {
+            'id': self.id,
+            'node_type': self.op.value,
+            'output_data': format_node_data(self.output_data),
+            'input_data': format_node_data(self.input_data),
+            'upnode': self.upnode.id if self.upnode else 'None',
+            'subnodes': [node.id for node in self.subnodes],
+            'matched_node_link': self.matched_node_link,
+            'suggestions': self.suggestions,
+            'stack_info': self.stack_info
+        }
+        if self.micro_step_id is not None:
+            result['micro_step_id'] = self.micro_step_id
+        result['data'] = self.data
+        if self.matched_distributed:
+            result[GraphConst.MATCHED_DISTRIBUTED] = self.matched_distributed
+        return result
+
+    def get_ancestors(self):
+        """
+        获取节点所有祖先的列表
+        """
+        ancestors = []
+        current_node = self.upnode
+        while current_node:
+            ancestors.append(current_node.id)
+            current_node = current_node.upnode
+        return list(reversed(ancestors))

msprobe/visualization/graph/distributed_analyzer.py

@@ -0,0 +1,318 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from enum import Enum
+from msprobe.visualization.utils import GraphConst
+from msprobe.core.common.const import Const, CompareConst
+from msprobe.core.common.log import logger
+
+
+class CommunicationType(Enum):
+    """
+    通信类型：发送、接收、发送接收
+    """
+    SEND = 'send'
+    RECEIVE = 'receive'
+    SEND_RECEIVE = 'send_receive'
+
+
+class DistributedType(Enum):
+    """
+    分布式类型：点对点通信、集体通信
+    """
+    P2P = 'p2p'
+    COLLECTIVE = 'collective'
+
+
+CANNOT_MATCH = 'cannot match distributed node in rank'
+
+
+class DistributedAnalyzer:
+
+    def __init__(self, graphs: dict, overflow_check: bool):
+        self.graphs = graphs
+        self.overflow_check = overflow_check
+        self.config = {
+            # 当前通信api名称: 匹配目标通信api名称, 获取rank信息的位置参数或关键字参数, 通信类型, 分布式类型
+            'send': ['recv', GraphConst.DST, CommunicationType.SEND.value, DistributedType.P2P],
+            'isend': ['irecv', GraphConst.DST, CommunicationType.SEND.value, DistributedType.P2P],
+            'recv': ['send', GraphConst.SRC, CommunicationType.RECEIVE.value, DistributedType.P2P],
+            'irecv': ['isend', GraphConst.SRC, CommunicationType.RECEIVE.value, DistributedType.P2P],
+            'broadcast': ['broadcast', '1', CommunicationType.SEND.value, DistributedType.COLLECTIVE],
+            'scatter': ['scatter', GraphConst.SRC, CommunicationType.SEND.value, DistributedType.COLLECTIVE],
+            'gather': ['gather', GraphConst.DST, CommunicationType.RECEIVE.value, DistributedType.COLLECTIVE],
+            'reduce': ['reduce', '1', CommunicationType.RECEIVE.value, DistributedType.COLLECTIVE]
+        }
+        self.group_node_mapping = {}
+        self._make_group_node_mapping()
+
+    @staticmethod
+    def _get_opposite_communication_type(action):
+        if action == CommunicationType.SEND.value:
+            return CommunicationType.RECEIVE.value
+        elif action == CommunicationType.RECEIVE.value:
+            return CommunicationType.SEND.value
+        return action
+
+    @staticmethod
+    def _node_output_all_equal(data: dict, target_data: dict):
+        keys_to_compare = [Const.DTYPE, Const.SHAPE, Const.MAX, Const.MIN, Const.MEAN, Const.NORM]
+        return all(data.get(key) == target_data.get(key) for key in keys_to_compare)
+
+    @staticmethod
+    def _get_target_rank(node, rank, parameter):
+        """
+        点对点通信, 从输出数据参数src或dst, 获取通信目标rank
+        一对多通信和多对一通信, 从输出数据参数src或dst或位置参数, 获取发送或接收的rank源头
+        :param node: 当前节点
+        :param rank: 当前rank
+        :param parameter: 输出数据参数
+        :return: 目标rank
+        """
+        target_rank = node.input_data.get(f'{node.id}{GraphConst.INPUT}{parameter}', {}).get('value')
+        if target_rank is None:
+            logger.warning(f'The parameter {parameter} of node {node.id} does not exist, {CANNOT_MATCH}{rank}')
+        return target_rank
+
+    @staticmethod
+    def _get_group_info(node, rank):
+        """
+        获取当前通信节点的group参数中的group_ranks和group_id
+        :param node: 当前通信节点
+        :param rank: 当前rank
+        :return: group_ranks和group_id
+        """
+        group = node.input_data.get(f'{node.id}{GraphConst.INPUT}group', {})
+        if not group:
+            logger.warning(f'The kwarg group of node {node.id} does not exist, {CANNOT_MATCH}{rank}')
+            return None, None
+        group_ranks = group.get('group_ranks')
+        if not group_ranks:
+            logger.warning(f'The group_ranks of node {node.id} does not exist, {CANNOT_MATCH}{rank}')
+            return None, None
+        group_id = group.get('group_id')
+        if not group_id:
+            logger.warning(f'The group_id of node {node.id} does not exist, {CANNOT_MATCH}{rank}')
+            return None, None
+        return group_ranks, group_id
+
+    def distributed_match(self):
+        for rank, graph in self.graphs.items():
+            nodes = graph.node_map
+            for node_id, node in nodes.items():
+                # 不是通信节点或者已经匹配过了
+                if not node_id.startswith(Const.DISTRIBUTED) or node.matched_distributed:
+                    continue
+                api_name, distributed_type = self._get_distributed_name_and_type(node_id)
+                if distributed_type == DistributedType.P2P:
+                    self._p2p_match(node, rank, api_name)
+                else:
+                    self._collective_match(node, rank, api_name)
+
+    def _make_group_node_mapping(self):
+        """
+        建立通信节点的全局唯一标识映射
+        key: rank号, value: unique_group_id与node_id之间的映射
+        {
+            "0": {
+                "unique_group_id1": "node_id1",
+                "unique_group_id2": "node_id2",
+                "node_id1": "unique_group_id1",
+                "node_id2": "unique_group_id2"
+            },
+            "1": {},
+            "2": {}
+        }
+        """
+        for rank, graph in self.graphs.items():
+            group_count = {}
+            group_info = {}
+            nodes = graph.node_map
+            for node_id, node in nodes.items():
+                if not node_id.startswith(Const.DISTRIBUTED):
+                    continue
+                api_name, distributed_type = self._get_distributed_name_and_type(node_id)
+                if distributed_type == DistributedType.P2P:
+                    config_info = self.config.get(api_name)
+                    target_rank = self._get_target_rank(node, rank, config_info[1])
+                    if target_rank is None:
+                        continue
+                    # p2p通信节点，api名称+传输目标rank作为group_id
+                    group_id = api_name + Const.RANK + str(target_rank)
+                else:
+                    # 其他通信节点直接获取group_id, 并拼接api名称
+                    _, group_id = self._get_group_info(node, rank)
+                    if not group_id:
+                        continue
+                    group_id += api_name
+                # 同group_id的调用次数累计
+                group_count[group_id] = group_count.get(group_id, 0) + 1
+                # group_id+同group_id的调用次数作为唯一的unique_group_id
+                unique_group_id = group_id + Const.REPLACEMENT_CHARACTER + str(group_count.get(group_id))
+                group_info[unique_group_id] = node_id
+                group_info[node_id] = unique_group_id
+            self.group_node_mapping[rank] = group_info
+
+    def _get_distributed_name_and_type(self, node_id):
+        if Const.SEP not in node_id:
+            raise ValueError(f'Invalid node id {node_id}.')
+        api_name = node_id.split(Const.SEP)[1]
+        if api_name in self.config:
+            return api_name, self.config.get(api_name)[3]
+        return api_name, DistributedType.COLLECTIVE
+
+    def _get_target_node(self, rank, unique_group_id, api_name, target_rank, target_api_name=None):
+        """
+        获取名称匹配上的目标节点
+        :param rank: 当前rank
+        :param unique_group_id: 当前节点唯一group id
+        :param api_name: 当前节点的api名称, 例如Distributed.isend.0.forward, api名称为isend
+        :param target_rank: 与当前节点产生通信的rank
+        :param target_api_name: 与当前节点产生通信的节点api名称, 仅p2p通信需要配置
+        :return: 目标节点
+        """
+        target_graph = self.graphs.get(target_rank)
+        if not target_graph:
+            logger.warning(f'Graph data does not exist, {CANNOT_MATCH}{target_rank}')
+            return None
+        target_group_mapping = self.group_node_mapping.get(target_rank)
+        # p2p通信，想要获取目标节点，需要替换unique_group_id中的rank和api name,
+        # 例如isend发送到rank1，对应的irecv接收自rank0, isend_rank1与irecv_rank0对应
+        target_unique_group_id = (unique_group_id
+                                  .replace(Const.RANK + str(target_rank), Const.RANK + str(rank))
+                                  .replace(api_name, target_api_name)) if target_api_name else unique_group_id
+        target_node_id = target_group_mapping.get(target_unique_group_id, '')
+        target_node = target_graph.node_map.get(target_node_id)
+        if not target_node:
+            logger.warning(f'Node {target_node_id} does not exist, {CANNOT_MATCH}{target_rank}')
+            return None
+        return target_node
+
+    def _add_node_matched_distributed(self, node, target_node, api_name, target_rank, reversal_type=False):
+        """
+        给当前节点添加matched_distributed字段信息
+        :param node: 当前节点
+        :param target_node: 匹配上的目标节点
+        :param api_name: 当前节点的api名称
+        :param target_rank: 匹配上的目标rank
+        :param reversal_type: 是否需要反转通信类型，例如broadcast在rank0通信类型是发送，但在其他rank通信类型是接收
+        """
+        communications_type = self.config.get(api_name)[2]
+        communications_type = self._get_opposite_communication_type(communications_type) if reversal_type \
+            else communications_type
+        index = target_node.data.get(GraphConst.OVERFLOW_LEVEL, CompareConst.NAN) if self.overflow_check \
+            else target_node.data.get(GraphConst.JSON_INDEX_KEY, CompareConst.NAN)
+        matched_distributed = {
+            'communications_type': communications_type,
+            'nodes_info': {target_rank: [str(index), target_node.id]}
+        }
+        node.matched_distributed = matched_distributed
+
+    def _p2p_match(self, node, rank, api_name):
+        """
+        点对点通信匹配
+
+        根据当前点对点通信节点的输出数据中的src或dst参数, 确定目标rank, 并从目标rank中找到对应的点对点通信节点, 校验输出数据是否一致，
+        校验通过则在两个匹配节点增加匹配信息
+        Args:
+            node: 当前点对点通信节点
+            rank: 当前节点所属rank
+            api_name: 当前节点的api名称
+        Returns:
+        """
+        config_info = self.config.get(api_name)
+        target_api_name = config_info[0]
+        #
+        target_rank = self._get_target_rank(node, rank, config_info[1])
+        if target_rank is None:
+            return
+        unique_group_id = self.group_node_mapping.get(rank, {}).get(node.id, '')
+        target_node = self._get_target_node(rank, unique_group_id, api_name, target_rank, target_api_name)
+        if not target_node:
+            return
+        target_config_info = self.config.get(target_api_name)
+        source_rank = (target_node.input_data.get(f'{target_node.id}{GraphConst.INPUT}{target_config_info[1]}', {})
+                       .get('value'))
+        if source_rank is None:
+            logger.warning(
+                f'The kwarg {target_config_info[1]} of node {target_node.id} does not exist, '
+                f'{CANNOT_MATCH}{source_rank}')
+            return
+        if source_rank != rank:
+            # 点对点通信，待匹配目标节点包含的rank信息要与当前rank一致
+            logger.warning(
+                f'{node.id} of rank{rank} is expected to communicate with {target_node.id} of rank{target_rank}, '
+                f'but the data shows that {target_node.id} communicates with rank{source_rank}.'
+                f'The rank is inconsistent, cannot match distributed node')
+            return
+
+        # 点对点通信，两个匹配节点的输出数据要一致
+        if not DistributedAnalyzer._node_output_all_equal(node.output_data.get(node.id + '.output.0'),
+                                                          target_node.output_data.get(target_node.id + '.output.0')):
+            logger.warning(f'{node.id} output of rank{rank} is different from the {target_node.id} '
+                           f'output of rank{target_rank}, cannot match distributed node')
+            return
+
+        self._add_node_matched_distributed(node, target_node, api_name, target_rank)
+        self._add_node_matched_distributed(target_node, node, target_api_name, rank)
+
+    def _collective_match(self, node, rank, api_name):
+        """
+        集体通信匹配
+
+        一对多通信和多对一通信, 需要先获取节点输出数据中的src或dst或位置参数, 确定发送源或接收源, 多对多通信不需要
+        :param node: 当前集体通信节点
+        :param rank: 当前节点所属rank
+        :param api_name: 当前节点的api名称
+        :return:
+        """
+        communications_type = CommunicationType.SEND_RECEIVE.value
+        config_info = self.config.get(api_name)
+        if config_info:
+            # 此时为一对多通信或多对一通信
+            source_rank = self._get_target_rank(node, rank, config_info[1])
+            if source_rank is None or str(source_rank) != str(rank):
+                return
+            communications_type = config_info[2]
+        group_ranks, group_id = self._get_group_info(node, rank)
+        if not group_ranks or not group_id:
+            return
+        unique_group_id = self.group_node_mapping.get(rank, {}).get(node.id, '')
+        matched_distributed = {'communications_type': communications_type}
+        nodes_info = {}
+        for target_rank in group_ranks:
+            if str(target_rank) == str(rank):
+                continue
+            target_node = self._get_target_node(rank, unique_group_id, api_name, target_rank)
+            if not target_node:
+                continue
+            _, target_group_id = self._get_group_info(target_node, target_rank)
+            if not target_group_id:
+                continue
+            if group_id != target_group_id:
+                logger.warning(
+                    f'{node.id} of rank{rank} is expected to communicate with {target_node.id} of rank{target_rank}'
+                    f', but the data shows that the group id of the two nodes are different, '
+                    f'cannot match distributed node')
+                continue
+            # 给当前通信节点添加matched_distributed字段信息
+            index = target_node.data.get(GraphConst.OVERFLOW_LEVEL, CompareConst.NAN) if self.overflow_check \
+                else target_node.data.get(GraphConst.JSON_INDEX_KEY, CompareConst.NAN)
+            nodes_info[target_rank] = [str(index), target_node.id]
+            if config_info:
+                # 给匹配上的目标节点也添加matched_distributed字段信息
+                self._add_node_matched_distributed(target_node, node, api_name, rank, True)
+        if nodes_info:
+            matched_distributed['nodes_info'] = nodes_info
+            node.matched_distributed = matched_distributed