mindstudio-probe 1.3.0__py3-none-any.whl → 8.1.1__py3-none-any.whl

msprobe/nan_analyze/analyzer.py

@@ -0,0 +1,255 @@
+# Copyright (c) 2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from collections import defaultdict
+import os
+from itertools import dropwhile, chain
+
+from msprobe.core.common.file_utils import check_file_or_directory_path, save_json, make_dir
+from msprobe.core.common.log import logger
+from msprobe.core.common.const import Const
+from msprobe.nan_analyze.utils import (RankPath, FileCache, is_communication_op, is_ignore_op, NanAnalyseConst,
+                                       analyze_anomaly_in_group)
+from msprobe.nan_analyze.graph import DataNode, CommunicationNode
+
+
+class NanAnalyzer:
+    def __init__(self, input_path, output_path):
+        self._input_path = input_path
+        self._output_path = output_path
+        self._paths = {}
+        self._resolve_input_path()
+        self._anomaly_nodes = []  # 记录所有异常节点
+        self._cache = FileCache()
+        self._first_comm_nodes = {}  # 记录各rank下首个通信节点的node_id
+        self._after_comm_anomalies = {}  # 记录各rank下发生在通信节点之后的异常计算节点
+        self._rank_comm_nodes_dict = {}  # 记录各rank的通信节点
+
+    def analyze(self):
+        for analyze_func in [self._pre_analyze, self._analyze, self._post_analyze]:
+            analyze_func()
+            if self._anomaly_nodes:
+                self._gen_analyze_info()
+                return
+        logger.info('Cannot find any anomaly node, no need to generate analyze file.')
+
+    def _resolve_input_path(self):
+        contents = os.listdir(self._input_path)
+        for path in contents:
+            if not path.startswith('rank'):
+                continue
+            rank_str = path.strip('rank')
+            if not rank_str:
+                rank = 0
+            elif not rank_str.isdigit():
+                continue
+            else:
+                rank = int(rank_str)
+            dump_path = os.path.join(self._input_path, path, NanAnalyseConst.DUMP_FILE)
+            construct_path = os.path.join(self._input_path, path, NanAnalyseConst.CONSTRUCT_FILE)
+            stack_path = os.path.join(self._input_path, path, NanAnalyseConst.STACK_FILE)
+            self._paths[rank] = RankPath(rank, dump_path, construct_path, stack_path)
+
+    def _pre_analyze(self):
+        logger.info('Start searching anomaly node before communication.')
+        for path in self._paths.values():
+            dump_data = self._cache.load_json(path.dump_path).get('data')
+            if not dump_data:
+                logger.warning(f'Rank {path.rank} has no dump data!')
+                continue
+            for op_name, op_data in dump_data.items():
+                if is_communication_op(op_name):
+                    self._first_comm_nodes[path.rank] = op_name
+                    break
+                data_node = DataNode(op_name, path.rank, op_data)
+                if data_node.is_anomaly():
+                    self._anomaly_nodes.append(data_node)
+                    break
+
+    def _analyze(self):
+        logger.info('Start searching anomaly node during communication.')
+        self._rank_comm_nodes_dict = {rank: self._analyze_comm_nodes(rank) for rank in self._paths}
+        self._connect_comm_nodes()
+        self._pruning()
+        self._search_first_anomaly()
+
+    def _post_analyze(self):
+        logger.info('Start searching anomaly node after communication.')
+        for nodes in self._after_comm_anomalies.values():
+            if nodes:
+                self._anomaly_nodes.append(nodes[0])
+
+    def _gen_analyze_info(self):
+        if not os.path.exists(self._output_path):
+            make_dir(self._output_path)
+        file_name = f'anomaly_analyze_{time.time_ns()}.json'
+        result_file = os.path.join(self._output_path, file_name)
+        result_content = defaultdict(list)
+        for node in self._anomaly_nodes:
+            result_content[f'rank_{node.rank}'].append(node.gen_node_info(self._paths[node.rank]))
+        save_json(result_file, result_content, 2)
+        logger.info(f"The analyze result is saved in: {result_file}")
+
+    def _analyze_comm_nodes(self, rank):
+        path = self._paths[rank]
+        data = self._cache.load_json(path.dump_path).get('data')
+        communication_nodes = {}
+        if rank not in self._first_comm_nodes:  # 此rank没有通信节点
+            return communication_nodes
+        last_node_id = None  # 记录上一个通信节点的node_id
+        compute_ops = []  # 记录两个通信节点之间的计算节点
+        sub_layer = 0  # 记录两个通信算子之间异常计算节点的调用序数
+        for op_name in dropwhile(lambda k: k != self._first_comm_nodes[rank], data):
+            node_id = f'{rank}.{op_name}'
+            op_data = data[op_name]
+            if is_communication_op(op_name):
+                comm_node = CommunicationNode(node_id, rank, DataNode(op_name, rank, op_data, sub_layer=sub_layer),
+                                              compute_ops=compute_ops)
+                if last_node_id:
+                    communication_nodes.get(last_node_id).add_next(comm_node)
+                communication_nodes[node_id] = comm_node
+                last_node_id = node_id
+                compute_ops = []
+                sub_layer = 0
+            elif not is_ignore_op(op_name):
+                data_node = DataNode(op_name, rank, op_data, sub_layer=sub_layer)
+                if data_node.is_anomaly():
+                    compute_ops.append(data_node)
+                sub_layer += 1
+        if compute_ops:
+            self._after_comm_anomalies[rank] = compute_ops
+        return communication_nodes
+
+    def _connect_comm_nodes(self):
+        searched_ranks = set()
+        for rank, nodes in list(self._rank_comm_nodes_dict.items())[:-1]:
+            searched_ranks.add(rank)
+            seen_nodes = set()
+            for cur_node in nodes.values():
+                conn_info = cur_node.find_connected_nodes()
+                if not conn_info.get('ranks'):
+                    conn_info['ranks'] = self._rank_comm_nodes_dict.keys()
+                if not self._find_connection(conn_info, cur_node, searched_ranks, seen_nodes):
+                    logger.info(f'Cannot find connected communication node for "{cur_node.node_id}".')
+
+    def _find_connection(self, conn_info, cur_node, searched_ranks, seen_nodes):
+        def connect():
+            seen_nodes.add(search_node.node_id)
+            if search_node.type == NanAnalyseConst.DST:
+                cur_node.add_dst(search_node)
+            elif search_node.type == NanAnalyseConst.SRC:
+                search_node.layer = cur_node.layer
+                search_node.add_dst(cur_node)
+            else:
+                cur_node.add_link(search_node)
+
+        found = cur_node.connected
+        for connected_rank in conn_info['ranks']:
+            if connected_rank in searched_ranks:
+                continue
+            tar_id_prefix = f'{connected_rank}.{conn_info["api"]}'
+            for search_id, search_node in self._rank_comm_nodes_dict[connected_rank].items():
+                if search_id in seen_nodes:
+                    continue
+                if not (search_id.startswith(tar_id_prefix) and search_node.type == conn_info.get('type')):
+                    continue
+                search_conn_ranks = search_node.find_connected_nodes().get('ranks')
+                if ((not search_conn_ranks and search_node.api not in NanAnalyseConst.DIRECTED_API) or
+                    cur_node.rank in search_conn_ranks):  # 有些无向通信算子没有填ProcessGroup，默认连接所有rank
+                    connect()
+                    found = True
+                    break
+        return found
+
+    def _pruning(self):
+        deleted_node_id = []
+        for nodes in self._rank_comm_nodes_dict.values():
+            for node_id in list(nodes.keys()):
+                node = nodes[node_id]
+                if node.has_nan_inf() or node.compute_ops:
+                    continue
+                deleted_node_id.append(node_id)
+                node.delete()
+                del nodes[node_id]
+        logger.debug(f'After pruning, following nodes are removed: [{", ".join(deleted_node_id)}]')
+
+    def _search_first_anomaly(self):
+        nodes_queues = []
+        for comm_nodes in self._rank_comm_nodes_dict.values():
+            nodes_queues.append(sorted(list(comm_nodes.values()), key=lambda x: x.layer))
+        seen_nodes = set()
+
+        def get_next_node(node_list):
+            while node_list:
+                next_node = node_list.pop(0)
+                if next_node.node_id not in seen_nodes:
+                    return next_node
+            return None
+
+        def find_all_members(ori_node):
+            ids = get_relative_ids(ori_node)
+            id_queue = list(chain(*[get_relative_ids(self._get_node_by_id(n_id)).difference(ids) for n_id in ids]))
+            while id_queue:
+                new_id = id_queue.pop(0)
+                ids.add(new_id)
+                id_queue.extend(get_relative_ids(self._get_node_by_id(new_id)).difference(ids))
+            return ids
+
+        def get_relative_ids(ori_node):
+            if not ori_node:
+                return set()
+            return ({ori_node.node_id} | ori_node.link_nodes.keys() | ori_node.src_nodes.keys() |
+                    ori_node.dst_nodes.keys())
+
+        while any(nodes_queues):
+            groups = []
+            all_ids_in_groups = set()
+            for nodes in nodes_queues:
+                node = get_next_node(nodes)
+                if not node:
+                    continue
+                if not groups or node.node_id in all_ids_in_groups:
+                    new_group = find_all_members(node)
+                    groups.append(new_group)
+                    all_ids_in_groups.update(new_group)
+            for group in groups:
+                seen_nodes.update(group)
+                self._anomaly_nodes.extend(analyze_anomaly_in_group([self._get_node_by_id(n_id) for n_id in group]))
+            if self._anomaly_nodes:
+                self._anomaly_nodes = [min(self._anomaly_nodes, key=lambda x: (x.layer, x.sub_layer))]
+                return
+
+    def _get_node_by_id(self, node_id):
+        splits = node_id.split(Const.SEP, 1)
+        if len(splits) < 2 or not splits[0].isdigit():
+            logger.error(f'invalid node_id {node_id}')
+            raise RuntimeError(f'invalid node_id {node_id}')
+        rank = int(splits[0])
+        return self._rank_comm_nodes_dict.get(rank, {}).get(node_id)
+
+
+def _nan_analyze_parser(parser):
+    parser.add_argument("-i", "--input_path", dest="input_path", default="", type=str,
+                        help="<Required> The dump file path, over step level. eg: \"xxx/step_0/\".",
+                        required=True)
+    parser.add_argument("-o", "--output_path", dest="output_path", default="./output", type=str,
+                        help="<optional> The nan inf analyze result output file path.",
+                        required=False)
+
+
+def _run_nan_analyze(args):
+    check_file_or_directory_path(args.input_path, True)
+    NanAnalyzer(args.input_path, args.output_path).analyze()

msprobe/nan_analyze/graph.py

@@ -0,0 +1,189 @@
+# Copyright (c) 2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from msprobe.core.common.const import Const
+from msprobe.core.common.log import logger
+from msprobe.nan_analyze.utils import FileCache, RankPath, is_ignore_op, check_item_anomaly, NanAnalyseConst
+
+
+@dataclass
+class DataNode:
+    op_name: str
+    rank: int
+    inputs: list
+    input_args: list
+    input_kwargs: dict
+    outputs: dict
+    layer: int = 0  # 和communication_node的layer保持一致
+    sub_layer: int = 0  # 调用顺序，越小表示越先调用
+
+    def __init__(self, op_name, rank, op_data, **kwargs):
+        self.op_name = op_name
+        self.rank = rank
+        self.inputs = op_data.get(Const.INPUT, [])
+        self.input_args = op_data.get(Const.INPUT_ARGS, [])
+        self.input_kwargs = op_data.get(Const.INPUT_KWARGS, {})
+        self.outputs = op_data.get(Const.OUTPUT, {})
+        self.sub_layer = kwargs.get('sub_layer', 0)
+
+    @staticmethod
+    def find_complete_construct(construct_info, op_name):
+        construct = [op_name]
+        seen = set(op_name)
+        while True:
+            op_name = construct_info.get(op_name)
+            if not op_name or op_name in seen:
+                return construct
+            construct.insert(0, op_name)
+            seen.add(op_name)
+
+    def find_stack(self, stack_info):
+        for item in stack_info.values():
+            if len(item) >= 2 and self.op_name in item[0]:
+                return item[1]
+        return {}
+
+    def is_anomaly(self) -> bool:
+        if is_ignore_op(self.op_name):
+            return False
+        is_input_anomaly = (check_item_anomaly(self.inputs) or check_item_anomaly(self.input_args) or
+                            check_item_anomaly(self.input_kwargs))
+        is_output_anomaly = check_item_anomaly(self.outputs)
+        return (not is_input_anomaly) and is_output_anomaly
+
+    def gen_node_info(self, path: RankPath):
+        cache = FileCache()
+        construct = cache.load_json(path.construct_path)
+        stack = cache.load_json(path.stack_path)
+        if Const.FORWARD in self.op_name:
+            data_info_list = {Const.INPUT_ARGS: self.input_args, Const.INPUT_KWARGS: self.input_kwargs,
+                              Const.OUTPUT: self.outputs}
+        else:
+            data_info_list = {Const.INPUT: self.inputs, Const.OUTPUT: self.outputs}
+        return {'op_name': self.op_name,
+                'data_info': data_info_list,
+                'construct_info': self.find_complete_construct(construct, self.op_name),
+                'stack_info': self.find_stack(stack)}
+
+
+class CommunicationNode:
+    def __init__(self, node_id, rank, data: DataNode, layer=0, **kwargs):
+        self.node_id = node_id
+        self.rank = rank
+        self.data = data
+        self.layer = layer
+        op_name_split = self.data.op_name.split(Const.SEP)
+        if len(op_name_split) < 4:
+            logger.error(f'invalid op_name: {self.data.op_name}')
+            raise RuntimeError(f'invalid op_name: {self.data.op_name}')
+        self.api = op_name_split[1]
+        self.call_cnt = op_name_split[2]
+        self.pre_node = kwargs.get('pre_node')
+        self.link_nodes = kwargs.get('link_nodes', {})
+        self.dst_nodes = kwargs.get('dst_nodes', {})
+        self.src_nodes = kwargs.get('src_nodes', {})
+        self.next_nodes = kwargs.get('next_nodes', {})
+        self.compute_ops = kwargs.get('compute_ops', [])
+        self.type = self._resolve_type()
+        self.connected = False
+
+    def add_next(self, node):
+        self.next_nodes[node.node_id] = node
+        node.pre_node = self
+        node.layer = self.layer + 1
+        node.data.layer = node.layer
+
+    def add_link(self, node):
+        self.link_nodes[node.node_id] = node
+        node.link_nodes[self.node_id] = self
+        node.layer = self.layer
+        node.data.layer = node.layer
+        self.connected = True
+        node.connected = True
+
+    def add_dst(self, node):
+        self.dst_nodes[node.node_id] = node
+        node.src_nodes[self.node_id] = self
+        node.layer = self.layer
+        node.data.layer = node.layer
+        self.connected = True
+        node.connected = True
+
+    def delete(self):
+        for node in self.next_nodes.values():
+            node.pre_node = None
+        for node in self.dst_nodes.values():
+            node.src_nodes.pop(self.node_id)
+        for node in self.src_nodes.values():
+            node.dst_nodes.pop(self.node_id)
+        for node in self.link_nodes.values():
+            node.link_nodes.pop(self.node_id)
+        if self.pre_node:
+            self.pre_node.next_nodes.pop(self.node_id)
+
+    def has_nan_inf(self):
+        return self.input_has_nan_inf() or check_item_anomaly(self.data.outputs)
+    
+    def input_has_nan_inf(self):
+        return check_item_anomaly(self.data.input_args) or check_item_anomaly(self.data.input_kwargs)
+
+    def find_connected_nodes(self):
+        """
+        根据 api/类型/入参/调用次数 确定相连接的node的op_name
+        """
+        tar_api = NanAnalyseConst.P2P_API_MAPPING.get(self.api, self.api)
+        ranks = set()
+        for dst in [NanAnalyseConst.DST, NanAnalyseConst.DST_GROUP]:
+            if dst in self.data.input_kwargs:
+                dst_value = self.data.input_kwargs.get(dst)
+                if dst_value:
+                    ranks.add(dst_value.get('value'))
+                break
+        for src in [NanAnalyseConst.SRC, NanAnalyseConst.SRC_GROUP]:
+            if src in self.data.input_kwargs:
+                src_value = self.data.input_kwargs.get(src)
+                if src_value:
+                    ranks.add(src_value.get('value'))
+                break
+        if not ranks:
+            for item in self.data.input_args:
+                if isinstance(item, dict) and item.get(Const.TYPE) == 'int':
+                    ranks.add(item.get('value'))
+        group = self.data.input_kwargs.get('group')
+        if group:
+            ranks.update(group.get('group_ranks'))
+        return {'ranks': ranks, 'api': f'Distributed.{tar_api}',
+                'type': NanAnalyseConst.OPPOSITE_DIR.get(self.type, NanAnalyseConst.LINK)}
+
+    def _resolve_type(self):
+        for src in [NanAnalyseConst.SRC, NanAnalyseConst.SRC_GROUP]:
+            if src in self.data.input_kwargs and self.data.input_kwargs[src]:
+                if self.data.input_kwargs[src].get('value') == self.rank:
+                    return NanAnalyseConst.SRC
+                else:
+                    return NanAnalyseConst.DST
+        for dst in [NanAnalyseConst.DST, NanAnalyseConst.DST_GROUP]:
+            if dst in self.data.input_kwargs and self.data.input_kwargs[dst]:
+                if self.data.input_kwargs[dst].get('value') == self.rank:
+                    return NanAnalyseConst.DST
+                else:
+                    return NanAnalyseConst.SRC
+        if self.api in NanAnalyseConst.DIRECTED_API:
+            for item in self.data.input_args:
+                if item.get(Const.TYPE) == 'int':
+                    node_type = NanAnalyseConst.DIRECTED_API[self.api]
+                    return node_type if item.get('value') == self.rank else NanAnalyseConst.OPPOSITE_DIR[node_type]
+        return NanAnalyseConst.LINK

msprobe/nan_analyze/utils.py

@@ -0,0 +1,211 @@
+# Copyright (c) 2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+from dataclasses import dataclass
+import sys
+import time
+import psutil
+
+from msprobe.core.common.const import CompareConst
+from msprobe.core.common.file_utils import check_file_or_directory_path, load_json
+
+
+@dataclass
+class RankPath:
+    rank: int
+    dump_path: str
+    construct_path: str
+    stack_path: str
+
+    def __init__(self, rank, dump_path, construct_path, stack_path):
+        self.rank = rank
+        check_file_or_directory_path(dump_path)
+        self.dump_path = dump_path
+        check_file_or_directory_path(construct_path)
+        self.construct_path = construct_path
+        check_file_or_directory_path(stack_path)
+        self.stack_path = stack_path
+
+
+class FileCache:
+    """
+    lazy load file
+    """
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if not cls._instance:
+            cls._instance = super().__new__(cls, *args, **kwargs)
+        return cls._instance
+
+    def __init__(self):
+        self._max_memory_usage = psutil.virtual_memory().available / 4  # 最大占用当前可用内存空间的1/4
+        self._cache = OrderedDict()
+        self._access_cnt = {}
+        self._access_time = {}
+        self._size = {}
+
+    @staticmethod
+    def _sizeof(obj):
+        seen = set()
+        objs = [obj]
+        size = 0
+        while objs:
+            obj = objs.pop()
+            obj_id = id(obj)
+            if obj_id in seen:
+                continue
+            seen.add(obj_id)
+            size += sys.getsizeof(obj)
+            if isinstance(obj, dict):
+                objs.extend(obj.keys())
+                objs.extend(obj.values())
+            elif isinstance(obj, (list, tuple, set, frozenset)):
+                objs.extend(obj)
+        return size
+
+    def load_json(self, json_path):
+        if json_path in self._cache:
+            self._access_cnt[json_path] += 1
+            self._access_time[json_path] = time.monotonic()
+            self._cache.move_to_end(json_path)
+            return self._cache[json_path]
+        self._cleanup()
+        return self._load(json_path)
+
+    def _load(self, json_path):
+        data = load_json(json_path)
+        self._add_to_cache(json_path, data)
+        return data
+
+    def _add_to_cache(self, key, data):
+        if key in self._cache:
+            self._cache.move_to_end(key)
+        else:
+            self._cache[key] = data
+            self._access_cnt[key] = 0
+            self._access_time[key] = time.monotonic()
+            self._size[key] = self._sizeof(data)
+
+    def _calc_cache_size(self):
+        return sys.getsizeof(self._cache) + sum(self._size.values())
+
+    def _cleanup(self):
+        while self._calc_cache_size() > self._max_memory_usage and self._cache:
+            least_frequent_key = min(self._access_cnt.keys(), key=lambda k: self._access_cnt[k])
+            least_recent_key = min(self._access_time.keys(), key=lambda k: self._access_time[k])
+            largest_key = max(self._cache.keys(), key=lambda k: self._size[k])
+            key_to_rm = min([least_frequent_key, least_recent_key, largest_key],
+                            key=lambda k: (self._access_cnt[k], self._access_time[k], -self._size[k]))
+            del self._cache[key_to_rm]
+            del self._access_cnt[key_to_rm]
+            del self._access_time[key_to_rm]
+            del self._size[key_to_rm]
+
+
+def is_communication_op(op_name):
+    # 定义通信算子的关键字，覆盖各种通信操作，如all_reduce, send, broadcast等
+    # 从wrap文件中读取，先硬编码在文件中
+    return (op_name.startswith('Distributed.') and
+            any(keyword in op_name for keyword in NanAnalyseConst.COMMUNICATION_KEYWORDS))
+
+
+def is_ignore_op(op_name):
+    ignore_keywords = [
+        'Torch.empty',
+        'Torch.fill'
+    ]
+    return any(keyword in op_name for keyword in ignore_keywords)
+
+
+def check_item_anomaly(param):
+    def has_nan_inf(dict_obj, key):
+        return str(dict_obj.get(key)).lower() in CompareConst.OVERFLOW_LIST
+
+    items = []
+    if isinstance(param, list):
+        items = param
+    elif isinstance(param, dict):
+        items = param.values()
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        if has_nan_inf(item, 'Max') or has_nan_inf(item, 'Min'):
+            return True
+    return False
+
+
+def analyze_anomaly_in_group(nodes_group):
+    anomaly_nodes = []
+
+    def get_compute_ops_from_comm_nodes(comm_nodes):
+        for comm_node in comm_nodes:
+            for op_node in comm_node.compute_ops:
+                op_node.layer = comm_node.layer
+                anomaly_nodes.append(op_node)
+
+    def get_comm_ops(comm_nodes):
+        for node in comm_nodes:
+            node.data.layer = node.layer
+            anomaly_nodes.append(node.data)
+
+    # 先看src或link中input是否有异常
+    src_list = list(filter(lambda node: node.type in [NanAnalyseConst.SRC, NanAnalyseConst.LINK], nodes_group))
+    input_anomaly_nodes = list(filter(lambda node: node.input_has_nan_inf(), src_list))
+    # 如果有异常回溯计算节点找到异常来源
+    # 使用cpu模拟节点进行计算，查看结果是否有问题。需要对所有计算节点录入/映射，暂不实现。
+    get_compute_ops_from_comm_nodes(input_anomaly_nodes)
+    # 筛选入参没问题但出参有问题的通信节点
+    output_anomaly_nodes = list(filter(lambda node: node.data.is_anomaly(), nodes_group))
+    get_comm_ops(output_anomaly_nodes)
+    return anomaly_nodes
+
+
+class NanAnalyseConst:
+    COMMUNICATION_KEYWORDS = {
+        'send',  # send 算子
+        'recv',  # recv 算子
+        'broadcast',  # broadcast 算子
+        'all_reduce',  # all_reduce 算子
+        'reduce',  # reduce 算子
+        'all_gather',  # all_gather 算子
+        'gather',  # gather 算子
+        'isend',  # isend 算子
+        'irecv',  # irecv 算子
+        'scatter',  # scatter 算子
+        'reduce_scatter',  # reduce_scatter 算子
+        '_reduce_scatter_base',  # _reduce_scatter_base 算子
+        '_all_gather_base',  # _all_gather_base 算子
+        'all_to_all_single',  # all_to_all_single 算子
+        'all_to_all',  # all_to_all 算子
+        'all_gather_into_tensor',  # all_gather_into_tensor 算子
+        'reduce_scatter_tensor',  # reduce_scatter_tensor 算子
+        'send_object_list',  # send_object_list 算子
+        'recv_object_list'  # recv_object_list 算子
+    }
+    P2P_API_MAPPING = {'send': 'recv', 'recv': 'send', 'isend': 'irecv', 'irecv': 'isend',
+                       'send_object_list': 'recv_object_list', 'recv_object_list': 'send_object_list'}
+    SRC = 'src'
+    DST = 'dst'
+    SRC_GROUP = 'src_group'
+    DST_GROUP = 'dst_group'
+    LINK = 'link'
+    DIRECTED_API = {'send': DST, 'recv': SRC, 'isend': DST, 'irecv': SRC, 'broadcast': SRC, 'scatter': SRC,
+                    'gather': DST, 'send_object_list': DST, 'recv_object_list': SRC}
+    OPPOSITE_DIR = {SRC: DST, DST: SRC}
+    DUMP_FILE = "dump.json"
+    CONSTRUCT_FILE = "construct.json"
+    STACK_FILE = "stack.json"

msprobe/pytorch/api_accuracy_checker/common/config.py

@@ -125,8 +125,8 @@ class CheckerConfig:
             save_error_data=config_params.get('save_error_data'),
             is_continue_run_ut=config_params.get('is_continue_run_ut'),
             real_data_path=config_params.get('real_data_path'),
-            white_list=self.white_list,
-            black_list=self.black_list,
+            white_list=self.white_list.copy() if self.white_list else [],
+            black_list=self.black_list.copy() if self.black_list else [],
             error_data_path=config_params.get('error_data_path'),
             online_config=self.get_online_config()
         )