mindstudio-probe 1.3.0__py3-none-any.whl → 8.1.1__py3-none-any.whl

msprobe/visualization/graph/graph.py

@@ -146,7 +146,7 @@ class Graph:
         """
         return self.node_map.get(node_id, None)
 
-    def to_dict(self):
+    def to_dict(self, compare_mode=None):
         """
         用于数据输出
         """
@@ -155,7 +155,7 @@ class Graph:
         result[GraphConst.JSON_DATA_KEY] = self.data_path
         result[GraphConst.JSON_NODE_KEY] = {}
         for node_id in self.node_map:
-            info = self.node_map.get(node_id).to_dict()
+            info = self.node_map.get(node_id).to_dict(compare_mode)
             result[GraphConst.JSON_NODE_KEY][node_id] = info
         return result
 

msprobe/visualization/graph_service.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -15,83 +15,93 @@
 
 import os
 import time
-import json
+from copy import deepcopy
+from multiprocessing import cpu_count, Pool
 from msprobe.core.common.file_utils import (check_file_type, create_directory, FileChecker,
                                             check_file_or_directory_path, load_json)
 from msprobe.core.common.const import FileCheckConst, Const
-from msprobe.core.common.utils import CompareException
-from msprobe.core.overflow_check.checker import AnomalyDetector
+from msprobe.core.common.utils import CompareException, get_dump_mode
 from msprobe.visualization.compare.graph_comparator import GraphComparator
-from msprobe.visualization.utils import GraphConst, check_directory_content
-from msprobe.visualization.builder.graph_builder import GraphBuilder, GraphExportConfig
+from msprobe.visualization.utils import GraphConst, check_directory_content, SerializableArgs
+from msprobe.visualization.builder.graph_builder import GraphBuilder, GraphExportConfig, GraphInfo, BuildGraphTaskInfo
 from msprobe.core.common.log import logger
 from msprobe.visualization.graph.node_colors import NodeColors
 from msprobe.core.compare.layer_mapping import generate_api_mapping_by_layer_mapping
 from msprobe.core.compare.utils import check_and_return_dir_contents
+from msprobe.core.common.utils import detect_framework_by_dump_json
 from msprobe.visualization.graph.distributed_analyzer import DistributedAnalyzer
 
 current_time = time.strftime("%Y%m%d%H%M%S")
 
 
-def _compare_graph(input_param, args):
-    logger.info('Start building model graphs...')
-    # 对两个数据进行构图
-    dump_path_n = input_param.get('npu_path')
-    dump_path_b = input_param.get('bench_path')
-    construct_path_n = FileChecker(os.path.join(dump_path_n, GraphConst.CONSTRUCT_FILE),
-                                   FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check()
-    construct_path_b = FileChecker(os.path.join(dump_path_b, GraphConst.CONSTRUCT_FILE),
-                                   FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check()
-    data_path_n = FileChecker(os.path.join(dump_path_n, GraphConst.DUMP_FILE), FileCheckConst.FILE,
-                              FileCheckConst.READ_ABLE).common_check()
-    data_path_b = FileChecker(os.path.join(dump_path_b, GraphConst.DUMP_FILE), FileCheckConst.FILE,
-                              FileCheckConst.READ_ABLE).common_check()
-    stack_path_n = FileChecker(os.path.join(dump_path_n, GraphConst.STACK_FILE), FileCheckConst.FILE,
-                               FileCheckConst.READ_ABLE).common_check()
-    stack_path_b = FileChecker(os.path.join(dump_path_b, GraphConst.STACK_FILE), FileCheckConst.FILE,
-                               FileCheckConst.READ_ABLE).common_check()
-    graph_n = GraphBuilder.build(construct_path_n, data_path_n, stack_path_n, complete_stack=args.complete_stack)
-    graph_b = GraphBuilder.build(construct_path_b, data_path_b, stack_path_b, complete_stack=args.complete_stack)
-    logger.info('Model graphs built successfully, start Comparing graphs...')
-    # 基于graph、stack和data进行比较
+def _compare_graph(graph_n: GraphInfo, graph_b: GraphInfo, input_param, args):
     dump_path_param = {
-        'npu_json_path': data_path_n,
-        'bench_json_path': data_path_b,
-        'stack_json_path': stack_path_n,
+        'npu_json_path': graph_n.data_path,
+        'bench_json_path': graph_b.data_path,
+        'stack_json_path': graph_n.stack_path,
         'is_print_compare_log': input_param.get("is_print_compare_log", True)
     }
-    mapping_dict = None
+    mapping_dict = {}
     if args.layer_mapping:
-        yaml_path = FileChecker(args.layer_mapping, FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check()
         try:
-            mapping_dict = generate_api_mapping_by_layer_mapping(data_path_n, data_path_b, yaml_path)
+            mapping_dict = generate_api_mapping_by_layer_mapping(graph_n.data_path, graph_b.data_path,
+                                                                 args.layer_mapping)
         except Exception:
             logger.warning('The layer mapping file parsing failed, please check file format, mapping is not effective.')
-    graph_comparator = GraphComparator([graph_n, graph_b], dump_path_param, args, mapping_dict=mapping_dict)
+    is_cross_framework = detect_framework_by_dump_json(graph_n.data_path) != \
+                         detect_framework_by_dump_json(graph_b.data_path)
+    if is_cross_framework and not args.layer_mapping:
+        logger.error('The cross_frame graph comparison failed. '
+                     'Please specify -lm or --layer_mapping when performing cross_frame graph comparison.')
+        raise CompareException(CompareException.CROSS_FRAME_ERROR)
+
+    graph_comparator = GraphComparator([graph_n.graph, graph_b.graph], dump_path_param, args, is_cross_framework,
+                                       mapping_dict=mapping_dict)
     graph_comparator.compare()
-    micro_steps = graph_n.paging_by_micro_step(graph_b)
+    return graph_comparator
+
+
+def _compare_graph_result(input_param, args):
+    logger.info('Start building model graphs...')
+    # 对两个数据进行构图
+    graph_n = _build_graph_info(input_param.get('npu_path'), args)
+    graph_b = _build_graph_info(input_param.get('bench_path'), args)
+    logger.info('Model graphs built successfully, start Comparing graphs...')
+    # 基于graph、stack和data进行比较
+    graph_comparator = _compare_graph(graph_n, graph_b, input_param, args)
+    # 增加micro step标记
+    micro_steps = graph_n.graph.paging_by_micro_step(graph_b.graph)
     # 开启溢出检测
     if args.overflow_check:
-        graph_n.overflow_check()
-        graph_b.overflow_check()
+        graph_n.graph.overflow_check()
+        graph_b.graph.overflow_check()
 
-    return CompareGraphResult(graph_n, graph_b, graph_comparator, micro_steps)
+    return CompareGraphResult(graph_n.graph, graph_b.graph, graph_comparator, micro_steps)
 
 
-def _export_compare_graph_result(args, graphs, graph_comparator, micro_steps,
-                                 output_file_name=f'compare_{current_time}.vis'):
-    create_directory(args.output_path)
+def _export_compare_graph_result(args, result):
+    graphs = [result.graph_n, result.graph_b]
+    graph_comparator = result.graph_comparator
+    micro_steps = result.micro_steps
+    output_file_name = result.output_file_name
+    if not output_file_name:
+        output_file_name = f'compare_{current_time}.vis'
+    logger.info(f'Start exporting compare graph result, file name: {output_file_name}...')
     output_path = os.path.join(args.output_path, output_file_name)
     task = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(graph_comparator.ma.compare_mode)
     export_config = GraphExportConfig(graphs[0], graphs[1], graph_comparator.ma.get_tool_tip(),
                                       NodeColors.get_node_colors(graph_comparator.ma.compare_mode), micro_steps, task,
-                                      args.overflow_check)
-    GraphBuilder.to_json(output_path, export_config)
-    logger.info(f'Model graphs compared successfully, the result file is saved in {output_path}')
+                                      args.overflow_check, graph_comparator.ma.compare_mode)
+    try:
+        GraphBuilder.to_json(output_path, export_config)
+        logger.info(f'Exporting compare graph result successfully, the result file is saved in {output_path}')
+        return ''
+    except RuntimeError as e:
+        logger.error(f'Failed to export compare graph result, file: {output_file_name}, error: {e}')
+        return output_file_name
 
 
-def _build_graph(dump_path, args):
-    logger.info('Start building model graph...')
+def _build_graph_info(dump_path, args):
     construct_path = FileChecker(os.path.join(dump_path, GraphConst.CONSTRUCT_FILE), FileCheckConst.FILE,
                                  FileCheckConst.READ_ABLE).common_check()
     data_path = FileChecker(os.path.join(dump_path, GraphConst.DUMP_FILE), FileCheckConst.FILE,
@@ -99,6 +109,13 @@ def _build_graph(dump_path, args):
     stack_path = FileChecker(os.path.join(dump_path, GraphConst.STACK_FILE), FileCheckConst.FILE,
                              FileCheckConst.READ_ABLE).common_check()
     graph = GraphBuilder.build(construct_path, data_path, stack_path, complete_stack=args.complete_stack)
+    return GraphInfo(graph, construct_path, data_path, stack_path)
+
+
+def _build_graph_result(dump_path, args):
+    logger.info('Start building model graphs...')
+    graph = _build_graph_info(dump_path, args).graph
+    # 增加micro step标记
     micro_steps = graph.paging_by_micro_step()
     # 开启溢出检测
     if args.overflow_check:
@@ -106,15 +123,128 @@ def _build_graph(dump_path, args):
     return BuildGraphResult(graph, micro_steps)
 
 
-def _export_build_graph_result(out_path, graph, micro_steps, overflow_check,
-                               output_file_name=f'build_{current_time}.vis'):
-    create_directory(out_path)
+def _run_build_graph_compare(input_param, args, nr, br):
+    logger.info(f'Start building graph for {nr}...')
+    graph_n = _build_graph_info(input_param.get('npu_path'), args)
+    graph_b = _build_graph_info(input_param.get('bench_path'), args)
+    logger.info(f'Building graph for {nr} finished.')
+    return BuildGraphTaskInfo(graph_n, graph_b, nr, br, current_time)
+
+
+def _run_build_graph_single(dump_ranks_path, rank, step, args):
+    logger.info(f'Start building graph for {rank}...')
+    dump_path = os.path.join(dump_ranks_path, rank)
+    output_file_name = f'build_{step}_{rank}_{current_time}.vis' if step else f'build_{rank}_{current_time}.vis'
+    result = _build_graph_result(dump_path, args)
+    result.output_file_name = output_file_name
+    if rank != Const.RANK:
+        try:
+            result.rank = int(rank.replace(Const.RANK, ""))
+        except Exception as e:
+            logger.error('The folder name format is incorrect, expected rank+number.')
+            raise CompareException(CompareException.INVALID_PATH_ERROR) from e
+    logger.info(f'Building graph for step: {step}, rank: {rank} finished.')
+    return result
+
+
+def _run_graph_compare(graph_task_info, input_param, args, output_file_name):
+    logger.info(f'Start comparing data for {graph_task_info.npu_rank}...')
+    graph_n = graph_task_info.graph_info_n
+    graph_b = graph_task_info.graph_info_b
+    nr = graph_task_info.npu_rank
+    graph_comparator = _compare_graph(graph_n, graph_b, input_param, args)
+    micro_steps = graph_n.graph.paging_by_micro_step(graph_b.graph)
+    # 开启溢出检测
+    if args.overflow_check:
+        graph_n.graph.overflow_check()
+        graph_b.graph.overflow_check()
+    graph_result = CompareGraphResult(graph_n.graph, graph_b.graph, graph_comparator, micro_steps)
+    graph_result.output_file_name = output_file_name
+    if nr != Const.RANK:
+        try:
+            graph_result.rank = int(nr.replace(Const.RANK, ""))
+        except Exception as e:
+            logger.error('The folder name format is incorrect, expected rank+number.')
+            raise CompareException(CompareException.INVALID_PATH_ERROR) from e
+    logger.info(f'Comparing data for {graph_task_info.npu_rank} finished.')
+    return graph_result
+
+
+def _export_build_graph_result(args, result):
+    out_path = args.output_path
+    graph = result.graph
+    micro_steps = result.micro_steps
+    overflow_check = args.overflow_check
+    output_file_name = result.output_file_name
+    if not output_file_name:
+        output_file_name = f'build_{current_time}.vis'
+    logger.info(f'Start exporting graph for {output_file_name}...')
     output_path = os.path.join(out_path, output_file_name)
-    GraphBuilder.to_json(output_path, GraphExportConfig(graph, micro_steps=micro_steps, overflow_check=overflow_check))
-    logger.info(f'Model graph built successfully, the result file is saved in {output_path}')
+    try:
+        GraphBuilder.to_json(output_path, GraphExportConfig(graph, micro_steps=micro_steps,
+                                                            overflow_check=overflow_check))
+        logger.info(f'Model graph exported successfully, the result file is saved in {output_path}')
+        return None
+    except RuntimeError as e:
+        logger.error(f'Failed to export model graph, file: {output_file_name}, error: {e}')
+        return output_file_name
+
+
+def is_real_data_compare(input_param, npu_ranks, bench_ranks):
+    dump_rank_n = input_param.get('npu_path')
+    dump_rank_b = input_param.get('bench_path')
+    has_real_data = False
+    for nr, br in zip(npu_ranks, bench_ranks):
+        dump_path_param = {
+            'npu_json_path': FileChecker(os.path.join(dump_rank_n, nr, GraphConst.DUMP_FILE), FileCheckConst.FILE,
+                                         FileCheckConst.READ_ABLE).common_check(),
+            'bench_json_path': FileChecker(os.path.join(dump_rank_b, br, GraphConst.DUMP_FILE), FileCheckConst.FILE,
+                                           FileCheckConst.READ_ABLE).common_check()
+        }
+        has_real_data |= get_dump_mode(dump_path_param) == Const.ALL
+    return has_real_data
+
+
+def _mp_compare(input_param, serializable_args, output_file_name, nr, br):
+    graph_task_info = _run_build_graph_compare(input_param, serializable_args, nr, br)
+    return _run_graph_compare(graph_task_info, input_param, serializable_args, output_file_name)
 
 
 def _compare_graph_ranks(input_param, args, step=None):
+    with Pool(processes=max(int((cpu_count() + 1) // 4), 1)) as pool:
+        def err_call(err):
+            logger.error(f'Error occurred while comparing graph ranks: {err}')
+            try:
+                pool.close()
+            except OSError as e:
+                logger.error(f'Error occurred while terminating the pool: {e}')
+
+        serializable_args = SerializableArgs(args)
+        # 暂存所有rank的graph，用于匹配rank间的分布式节点
+        compare_graph_results = _get_compare_graph_results(input_param, serializable_args, step, pool, err_call)
+
+        # 匹配rank间的分布式节点
+        if len(compare_graph_results) > 1:
+            DistributedAnalyzer({obj.rank: obj.graph_n for obj in compare_graph_results},
+                                args.overflow_check).distributed_match()
+            DistributedAnalyzer({obj.rank: obj.graph_b for obj in compare_graph_results},
+                                args.overflow_check).distributed_match()
+
+        export_res_task_list = []
+        create_directory(args.output_path)
+        for result in compare_graph_results:
+            export_res_task_list.append(pool.apply_async(_export_compare_graph_result,
+                                                         args=(serializable_args, result),
+                                                         error_callback=err_call))
+        export_res_list = [res.get() for res in export_res_task_list]
+        if any(export_res_list):
+            failed_names = list(filter(lambda x: x, export_res_list))
+            logger.error(f'Unable to export compare graph results: {", ".join(failed_names)}.')
+        else:
+            logger.info('Successfully exported compare graph results.')
+
+
+def _get_compare_graph_results(input_param, serializable_args, step, pool, err_call):
     dump_rank_n = input_param.get('npu_path')
     dump_rank_b = input_param.get('bench_path')
     npu_ranks = sorted(check_and_return_dir_contents(dump_rank_n, Const.RANK))
@@ -123,32 +253,33 @@ def _compare_graph_ranks(input_param, args, step=None):
         logger.error('The number of ranks in the two runs are different. Unable to match the ranks.')
         raise CompareException(CompareException.INVALID_PATH_ERROR)
     compare_graph_results = []
-    for nr, br in zip(npu_ranks, bench_ranks):
-        logger.info(f'Start processing data for {nr}...')
-        input_param['npu_path'] = os.path.join(dump_rank_n, nr)
-        input_param['bench_path'] = os.path.join(dump_rank_b, br)
-        output_file_name = f'compare_{step}_{nr}_{current_time}.vis' if step else f'compare_{nr}_{current_time}.vis'
-        result = _compare_graph(input_param, args)
-        result.output_file_name = output_file_name
-        if nr != Const.RANK:
-            try:
-                result.rank = int(nr.replace(Const.RANK, ""))
-            except Exception as e:
-                logger.error('The folder name format is incorrect, expected rank+number.')
-                raise CompareException(CompareException.INVALID_PATH_ERROR) from e
-        # 暂存所有rank的graph，用于匹配rank间的分布式节点
-        compare_graph_results.append(result)
-
-    # 匹配rank间的分布式节点
-    if len(compare_graph_results) > 1:
-        DistributedAnalyzer({obj.rank: obj.graph_n for obj in compare_graph_results},
-                            args.overflow_check).distributed_match()
-        DistributedAnalyzer({obj.rank: obj.graph_b for obj in compare_graph_results},
-                            args.overflow_check).distributed_match()
-
-    for result in compare_graph_results:
-        _export_compare_graph_result(args, [result.graph_n, result.graph_b], result.graph_comparator,
-                                     result.micro_steps, output_file_name=result.output_file_name)
+    if is_real_data_compare(input_param, npu_ranks, bench_ranks):
+        mp_task_dict = {}
+        for nr, br in zip(npu_ranks, bench_ranks):
+            input_param['npu_path'] = os.path.join(dump_rank_n, nr)
+            input_param['bench_path'] = os.path.join(dump_rank_b, br)
+            output_file_name = f'compare_{step}_{nr}_{current_time}.vis' if step else f'compare_{nr}_{current_time}.vis'
+            input_param_copy = deepcopy(input_param)
+            mp_task_dict[output_file_name] = pool.apply_async(_run_build_graph_compare,
+                                                              args=(input_param_copy, serializable_args, nr, br),
+                                                              error_callback=err_call)
+
+        mp_res_dict = {k: v.get() for k, v in mp_task_dict.items()}
+        for output_file_name, mp_res in mp_res_dict.items():
+            compare_graph_results.append(_run_graph_compare(mp_res, input_param, serializable_args, output_file_name))
+    else:
+        compare_graph_tasks = []
+        for nr, br in zip(npu_ranks, bench_ranks):
+            input_param['npu_path'] = os.path.join(dump_rank_n, nr)
+            input_param['bench_path'] = os.path.join(dump_rank_b, br)
+            output_file_name = f'compare_{step}_{nr}_{current_time}.vis' if step else f'compare_{nr}_{current_time}.vis'
+            input_param_copy = deepcopy(input_param)
+            compare_graph_tasks.append(pool.apply_async(_mp_compare,
+                                                        args=(input_param_copy, serializable_args, output_file_name, nr,
+                                                              br),
+                                                        error_callback=err_call))
+        compare_graph_results = [task.get() for task in compare_graph_tasks]
+    return compare_graph_results
 
 
 def _compare_graph_steps(input_param, args):
@@ -172,28 +303,39 @@ def _compare_graph_steps(input_param, args):
 
 def _build_graph_ranks(dump_ranks_path, args, step=None):
     ranks = sorted(check_and_return_dir_contents(dump_ranks_path, Const.RANK))
-    build_graph_results = []
-    for rank in ranks:
-        logger.info(f'Start processing data for {rank}...')
-        dump_path = os.path.join(dump_ranks_path, rank)
-        output_file_name = f'build_{step}_{rank}_{current_time}.vis' if step else f'build_{rank}_{current_time}.vis'
-        result = _build_graph(dump_path, args)
-        result.output_file_name = output_file_name
-        if rank != Const.RANK:
+    serializable_args = SerializableArgs(args)
+    with Pool(processes=max(int((cpu_count() + 1) // 4), 1)) as pool:
+        def err_call(err):
+            logger.error(f'Error occurred while comparing graph ranks: {err}')
             try:
-                result.rank = int(rank.replace(Const.RANK, ""))
-            except Exception as e:
-                logger.error('The folder name format is incorrect, expected rank+number.')
-                raise CompareException(CompareException.INVALID_PATH_ERROR) from e
-        build_graph_results.append(result)
-
-    if len(build_graph_results) > 1:
-        DistributedAnalyzer({obj.rank: obj.graph for obj in build_graph_results},
-                            args.overflow_check).distributed_match()
+                pool.close()
+            except OSError as e:
+                logger.error(f'Error occurred while terminating the pool: {e}')
+
+        build_graph_tasks = []
+        for rank in ranks:
+            build_graph_tasks.append(pool.apply_async(_run_build_graph_single,
+                                                      args=(dump_ranks_path, rank, step, serializable_args),
+                                                      error_callback=err_call))
+        build_graph_results = [task.get() for task in build_graph_tasks]
+
+        if len(build_graph_results) > 1:
+            DistributedAnalyzer({obj.rank: obj.graph for obj in build_graph_results},
+                                args.overflow_check).distributed_match()
+
+        create_directory(args.output_path)
+        export_build_graph_tasks = []
+        for result in build_graph_results:
+            export_build_graph_tasks.append(pool.apply_async(_export_build_graph_result,
+                                                             args=(serializable_args, result),
+                                                             error_callback=err_call))
+        export_build_graph_result = [task.get() for task in export_build_graph_tasks]
+        if any(export_build_graph_result):
+            failed_names = list(filter(lambda x: x, export_build_graph_result))
+            logger.error(f'Unable to export build graph results: {failed_names}.')
+        else:
+            logger.info(f'Successfully exported build graph results.')
 
-    for result in build_graph_results:
-        _export_build_graph_result(args.output_path, result.graph, result.micro_steps, args.overflow_check,
-                                   result.output_file_name)
 
 
 def _build_graph_steps(dump_steps_path, args):
@@ -209,7 +351,7 @@ def _graph_service_parser(parser):
                         help="<Required> The compare input path, a dict json.", required=True)
     parser.add_argument("-o", "--output_path", dest="output_path", type=str,
                         help="<Required> The compare task result out path.", required=True)
-    parser.add_argument("-lm", "--layer_mapping", dest="layer_mapping", type=str,
+    parser.add_argument("-lm", "--layer_mapping", dest="layer_mapping", type=str, nargs='?', const=True,
                         help="<Optional> The layer mapping file path.", required=False)
     parser.add_argument("-oc", "--overflow_check", dest="overflow_check", action="store_true",
                         help="<Optional> whether open overflow_check for graph.", required=False)
@@ -233,8 +375,11 @@ def _graph_service_command(args):
         elif content == GraphConst.STEPS:
             _build_graph_steps(npu_path, args)
         else:
-            result = _build_graph(npu_path, args)
-            _export_build_graph_result(args.output_path, result.graph, result.micro_steps, args.overflow_check)
+            result = _build_graph_result(npu_path, args)
+            create_directory(args.output_path)
+            file_name = _export_build_graph_result(args, result)
+            if file_name:
+                logger.error('Failed to export model build graph.')
     elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR:
         content_n = check_directory_content(npu_path)
         content_b = check_directory_content(bench_path)
@@ -245,9 +390,11 @@ def _graph_service_command(args):
         elif content_n == GraphConst.STEPS:
             _compare_graph_steps(input_param, args)
         else:
-            result = _compare_graph(input_param, args)
-            _export_compare_graph_result(args, [result.graph_n, result.graph_b],
-                                         result.graph_comparator, result.micro_steps)
+            result = _compare_graph_result(input_param, args)
+            create_directory(args.output_path)
+            file_name = _export_compare_graph_result(args, result)
+            if file_name:
+                logger.error('Failed to export model compare graph.')
     else:
         logger.error("The npu_path or bench_path should be a folder.")
         raise CompareException(CompareException.INVALID_COMPARE_MODE)

msprobe/visualization/utils.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -16,9 +16,10 @@
 import os
 import re
 import json
+import pickle
 from msprobe.core.common.file_utils import FileOpen
 from msprobe.core.common.const import CompareConst, Const
-from msprobe.core.compare.acc_compare import Comparator, ModeConfig
+from msprobe.core.common.log import logger
 
 
 def load_json_file(file_path):
@@ -42,15 +43,6 @@ def load_data_json_file(file_path):
     return load_json_file(file_path).get(GraphConst.DATA_KEY, {})
 
 
-def get_csv_df(stack_mode, csv_data, compare_mode):
-    """
-    调用acc接口写入csv
-    """
-    dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode)
-    mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=dump_mode)
-    return Comparator(mode_config).make_result_table(csv_data)
-
-
 def str2float(percentage_str):
     """
     百分比字符串转换转换为浮点型
@@ -192,3 +184,27 @@ class GraphConst:
     OP = 'op'
     PEER = 'peer'
     GROUP_ID = 'group_id'
+    
+    IGNORE_PRECISION_INDEX = {'empty', 'empty_like', 'empty_with_format', 'new_empty_strided', 'new_empty',
+                              'empty_strided'}
+
+
+def is_serializable(obj):
+    """
+    Check if an object is serializable
+    """
+    try:
+        pickle.dumps(obj)
+        return True
+    except (pickle.PicklingError, AttributeError, TypeError):
+        return False
+    except Exception as e:
+        logger.error('Unexpected error occurred while pickling obj.')
+        raise RuntimeError('Unexpected error occurred while pickling obj.') from e
+
+
+class SerializableArgs:
+    def __init__(self, args):
+        for k, v in vars(args).items():
+            if is_serializable(v):
+                setattr(self, k, v)

msprobe/mindspore/dym_loader/hook_dynamic_loader.cc

@@ -1,106 +0,0 @@
-/**
- * Copyright 2024 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "hook_dynamic_loader.h"
-#include <sys/stat.h>
-#include <cstdlib>
-#include <cstring>
-#include <pybind11/embed.h>
-#include "utils/log_adapter.h"
-
-namespace py = pybind11;
-
-HookDynamicLoader &HookDynamicLoader::GetInstance() {
-  static HookDynamicLoader instance;
-  return instance;
-}
-
-bool HookDynamicLoader::loadFunction(void *handle, const std::string &functionName) {
-  void *func = dlsym(handle, functionName.c_str());
-  if (!func) {
-    MS_LOG(WARNING) << "Could not load function: " << functionName << ", error: " << dlerror();
-    return false;
-  }
-  funcMap_[functionName] = func;
-  return true;
-}
-
-bool HookDynamicLoader::LoadLibrary() {
-  std::string msprobePath = "";
-  // 获取gil锁
-  py::gil_scoped_acquire acquire;
-  try {
-    py::module msprobeMod = py::module::import("msprobe.lib._msprobe_c");
-		if (!py::hasattr(msprobeMod, "__file__")) {
-			MS_LOG(WARNING) << "Adump mod not found";
-			return false;
-		}
-		msprobePath = msprobeMod.attr("__file__").cast<std::string>();
-  } catch (const std::exception& e) {
-		MS_LOG(WARNING) << "Adump mod path unable to get: " << e.what();
-		return false;
-	}
-  std::lock_guard<std::mutex> lock(mutex_);
-  if (handle_) {
-    MS_LOG(WARNING) << "Hook library already loaded!";
-    return false;
-  }
-	if (msprobePath == "") {
-		MS_LOG(WARNING) << "Adump path not loaded";
-		return false;
-	}
-  handle_ = dlopen(msprobePath.c_str(), RTLD_LAZY | RTLD_LOCAL);
-  if (!handle_) {
-    MS_LOG(WARNING) << "Failed to load Hook library: " << dlerror();
-    return false;
-  }
-
-  for (const auto &functionName : functionList_) {
-    if (!loadFunction(handle_, functionName)) {
-      MS_LOG(WARNING) << "Failed to load adump function";
-      dlclose(handle_);
-      handle_ = nullptr;
-      return false;
-    }
-  }
-
-  MS_LOG(INFO) << "Hook library loaded successfully.";
-  return true;
-}
-
-bool HookDynamicLoader::UnloadLibrary() {
-  std::lock_guard<std::mutex> lock(mutex_);
-  if (!handle_) {
-    MS_LOG(WARNING) << "Hook library hasn't been loaded.";
-    return false;
-  }
-
-  dlclose(handle_);
-  handle_ = nullptr;
-  funcMap_.clear();
-  MS_LOG(INFO) << "Library unloaded successfully.";
-  return true;
-}
-
-void *HookDynamicLoader::GetHooker(const std::string &funcName) {
-  std::lock_guard<std::mutex> lock(mutex_);
-  auto iter = funcMap_.find(funcName);
-  if (iter == funcMap_.end()) {
-    MS_LOG(WARNING) << "Function not found: " << funcName;
-    return nullptr;
-  }
-  return iter->second;
-}