mindstudio-probe 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/parse_tool/lib/utils.py

@@ -28,7 +28,7 @@ from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException
 from msprobe.core.common.file_utils import change_mode, check_other_user_writable,\
     check_path_executable, check_path_owner_consistent
 from msprobe.core.common.const import FileCheckConst
-from msprobe.core.common.file_utils import check_file_or_directory_path, remove_path, check_file_type
+from msprobe.core.common.file_utils import check_file_or_directory_path, remove_path, check_file_type, os_walk_for_files
 from msprobe.pytorch.common.log import logger
 
 
@@ -81,16 +81,8 @@ class Util:
 
     @staticmethod
     def get_subfiles_count(directory):
-        file_count = 0
-        for root, _, files in os.walk(directory, topdown=True):
-            check_file_or_directory_path(root, isdir=True)
-            file_count += len(files)
-            path_depth = root.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield root, _, files
-            else:
-                _[:] = []
-        return file_count
+        files = os_walk_for_files(directory, Const.MAX_TRAVERSAL_DEPTH)
+        return len(files)
 
     @staticmethod
     def get_sorted_subdirectories_names(directory):
@@ -146,16 +138,10 @@ class Util:
     
     @staticmethod
     def dir_contains_only(path, endfix):
-        for root, _, files in os.walk(path, topdown=True):
-            check_file_or_directory_path(root, isdir=True)
-            for file in files:
-                if not file.endswith(endfix):
-                    return False
-            path_depth = root.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield root, _, files
-            else:
-                _[:] = []
+        files = os_walk_for_files(path, Const.MAX_TRAVERSAL_DEPTH)
+        for file in files:
+            if not file['file'].endswith(endfix):
+                return False
         return True
     
     @staticmethod
@@ -273,20 +259,15 @@ class Util:
         self.check_path_valid(path)
         file_list = {}
         re_pattern = re.compile(pattern)
-        for dir_path, _, file_names in os.walk(path, topdown=True):
-            check_file_or_directory_path(dir_path, isdir=True)
-            for name in file_names:
-                match = re_pattern.match(name)
-                if not match:
-                    continue
-                if extern_pattern != '' and re_pattern.match(extern_pattern) and not re.match(extern_pattern, name):
-                    continue
-                file_list[name] = gen_info_func(name, match, dir_path)
-            path_depth = dir_path.count(os.sep)
-            if path_depth <= Const.MAX_TRAVERSAL_DEPTH:
-                yield dir_path, _, file_names
-            else:
-                _[:] = []
+        files = os_walk_for_files(path, Const.MAX_TRAVERSAL_DEPTH)
+        for file in files:
+            name = file["file"]
+            match = re_pattern.match(name)
+            if not match:
+                continue
+            if extern_pattern != '' and re_pattern.match(extern_pattern) and not re.match(extern_pattern, name):
+                continue
+            file_list[name] = gen_info_func(name, match, file["root"])
         return file_list
 
     def check_file_path_format(self, path, suffix):

msprobe/pytorch/parse_tool/lib/visualization.py

@@ -65,6 +65,8 @@ class Visualization:
                     self.util.log.error("%s %s in line %s" % ("JSONDecodeError", str(e), pkl_line))
                     self.util.log.warning("Please check the pkl file")
                     raise ParseException(ParseException.PARSE_JSONDECODE_ERROR) from e
+                if not isinstance(msg, list) or len(msg) == 0:
+                    break
                 info_prefix = msg[0]
                 if not info_prefix.startswith(api_name):
                     continue

msprobe/pytorch/pt_config.py

@@ -14,11 +14,13 @@
 # limitations under the License.
 
 import os
+import re
 
 from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import MsprobeException
-from msprobe.core.common.file_utils import FileOpen, load_json
+from msprobe.core.common.file_utils import FileOpen, load_json, check_file_or_directory_path, check_crt_valid
 from msprobe.core.common.log import logger
+from msprobe.core.common.utils import is_int
 from msprobe.core.common_config import BaseConfig, CommonConfig
 from msprobe.core.grad_probe.constant import level_adp
 from msprobe.core.grad_probe.utils import check_bounds
@@ -38,17 +40,38 @@ class TensorConfig(BaseConfig):
         self.host = json_config.get("host", "")
         self.port = json_config.get("port", -1)
         self.tls_path = json_config.get("tls_path", "./")
+        self.online_run_ut_recompute = json_config.get("online_run_ut_recompute", False)
         self.check_config()
         self._check_file_format()
-        self._check_tls_path_config()
+        if self.online_run_ut:
+            self._check_online_run_ut()
 
     def _check_file_format(self):
         if self.file_format is not None and self.file_format not in ["npy", "bin"]:
             raise Exception("file_format is invalid")
 
-    def _check_tls_path_config(self):
-        if self.tls_path and not os.path.exists(self.tls_path):
-            raise Exception("tls_path: %s does not exist" % self.tls_path)
+    def _check_online_run_ut(self):
+        if not isinstance(self.online_run_ut, bool):
+            raise Exception(f"online_run_ut: {self.online_run_ut} is invalid.")
+
+        if not isinstance(self.online_run_ut_recompute, bool):
+            raise Exception(f"online_run_ut_recompute: {self.online_run_ut_recompute} is invalid.")
+
+        if self.nfs_path:
+            check_file_or_directory_path(self.nfs_path, isdir=True)
+            return
+
+        if self.tls_path:
+            check_file_or_directory_path(self.tls_path, isdir=True)
+            check_file_or_directory_path(os.path.join(self.tls_path, "client.key"))
+            check_file_or_directory_path(os.path.join(self.tls_path, "client.crt"))
+            check_crt_valid(os.path.join(self.tls_path, "client.crt"))
+
+        if not isinstance(self.host, str) or not re.match(Const.ipv4_pattern, self.host):
+            raise Exception(f"host: {self.host} is invalid.")
+
+        if not isinstance(self.port, int) or not (0 < self.port <= 65535):
+            raise Exception(f"port: {self.port} is invalid, port range 0-65535.")
 
 
 class StatisticsConfig(BaseConfig):
@@ -70,7 +93,7 @@ class OverflowCheckConfig(BaseConfig):
         self.check_overflow_config()
 
     def check_overflow_config(self):
-        if self.overflow_nums is not None and not isinstance(self.overflow_nums, int):
+        if self.overflow_nums is not None and not is_int(self.overflow_nums):
             raise Exception("overflow_num is invalid")
         if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]:
             raise Exception("check_mode is invalid")
@@ -170,7 +193,7 @@ class FreeBenchmarkCheckConfig(BaseConfig):
             )
 
     def _check_preheat_config(self):
-        if not isinstance(self.preheat_step, int):
+        if not is_int(self.preheat_step):
             msg = "preheat_step is invalid, it should be an integer"
             logger.error_log_with_exp(
                 msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg)
@@ -180,7 +203,7 @@ class FreeBenchmarkCheckConfig(BaseConfig):
             logger.error_log_with_exp(
                 msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg)
             )
-        if not isinstance(self.max_sample, int):
+        if not is_int(self.max_sample):
             msg = "max_sample is invalid, it should be an integer"
             logger.error_log_with_exp(
                 msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg)

msprobe/pytorch/service.py

@@ -15,8 +15,8 @@
 
 import functools
 import os
-
 from collections import namedtuple
+
 import torch
 from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException
@@ -25,13 +25,14 @@ from msprobe.core.common.utils import print_tools_ends_info
 from msprobe.core.data_dump.data_collector import build_data_collector
 from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs
 from msprobe.core.data_dump.scope import BaseScope
+from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData
 from msprobe.pytorch.common.log import logger
 from msprobe.pytorch.common.utils import get_rank_if_initialized
+from msprobe.pytorch.dump.kernel_dump.kernel_config import create_kernel_config_json
 from msprobe.pytorch.hook_module import remove_dropout
 from msprobe.pytorch.hook_module.api_registry import api_register
 from msprobe.pytorch.hook_module.hook_module import HOOKModule
 from msprobe.pytorch.module_processer import ModuleProcesser
-from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData
 
 torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0'
 if torch_version_above_or_equal_2:
@@ -159,10 +160,10 @@ class Service:
         if api_origin:
             api_register.api_modularity()
         if self.config.online_run_ut and torch_version_above_or_equal_2:
-            run_ut_dispatch(self.attl, True)
+            run_ut_dispatch(self.attl, True, self.config.online_run_ut_recompute)
         self.switch = True
         logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ")
-        if self.config.level != "L2" and not self.config.online_run_ut:
+        if not self.config.online_run_ut:
             self.create_dirs()
             logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.")
 
@@ -177,7 +178,7 @@ class Service:
             return
         self.switch = False
         if self.config.online_run_ut and torch_version_above_or_equal_2:
-            run_ut_dispatch(self.attl, False)
+            run_ut_dispatch(self.attl, False, self.config.online_run_ut_recompute)
             return
         self.data_collector.write_json()
 
@@ -191,6 +192,9 @@ class Service:
         HOOKModule.reset_module_stats()
         self.data_collector.data_writer.reset_cache()
 
+        if self.config.level == Const.LEVEL_L2:
+            self.data_collector.data_processor.reset_status()
+
     def need_stop_service(self):
         if self.should_stop_service:
             return True
@@ -221,6 +225,12 @@ class Service:
         create_directory(self.config.dump_path)
         self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}")
         cur_rank = self.current_rank if self.current_rank is not None else ''
+        if self.config.level == Const.LEVEL_L2:
+            create_directory(self.dump_iter_dir)
+            kernel_config_path = create_kernel_config_json(self.dump_iter_dir, cur_rank)
+            self.config.kernel_config_path = kernel_config_path
+            return
+
         dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}")
         create_directory(dump_dir)
         if self.config.task in self.data_collector.tasks_need_tensor_data:

msprobe/visualization/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

msprobe/visualization/builder/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

msprobe/visualization/builder/graph_builder.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from msprobe.visualization.graph.graph import Graph
+from msprobe.visualization.graph.node_op import NodeOp
+from msprobe.visualization.utils import save_json_file, GraphConst
+from msprobe.visualization.builder.msprobe_adapter import get_input_output
+from msprobe.core.common.file_utils import load_json
+
+
+class GraphBuilder:
+    @staticmethod
+    def build(construct_path, data_path, stack_path, model_name='DefaultModel'):
+        """
+        GraphBuilder的对外提供的构图方法
+        Args:
+            construct_path: construct.json路径
+            data_path: dump.json路径
+            stack_path: stack.json路径
+            model_name: 模型名字，依赖外部输入
+        Returns: Graph，代表图的数据结构
+        """
+        construct_dict = load_json(construct_path)
+        dump_dict = load_json(data_path)
+        stack_dict = load_json(stack_path)
+        data_dict = dump_dict.get(GraphConst.DATA_KEY, {})
+        graph = Graph(model_name, data_path=dump_dict.get('dump_data_dir', ''), dump_data=data_dict)
+        GraphBuilder._init_nodes(graph, construct_dict, data_dict, stack_dict)
+        GraphBuilder._collect_apis_between_modules(graph)
+        return graph
+
+    @staticmethod
+    def to_json(filename, config):
+        """
+        将graph导出成.vis文件的接口
+        """
+        result = {}
+        if config.graph_b:
+            result[GraphConst.JSON_NPU_KEY] = config.graph_n.to_dict()
+            result[GraphConst.JSON_BENCH_KEY] = config.graph_b.to_dict()
+        else:
+            result = config.graph_n.to_dict()
+        if config.tool_tip:
+            result[GraphConst.JSON_TIP_KEY] = config.tool_tip
+        if config.node_colors:
+            result[GraphConst.COLORS] = config.node_colors
+        if config.micro_steps:
+            result[GraphConst.MICRO_STEPS] = config.micro_steps
+        if config.task:
+            result[GraphConst.JSON_TASK_KEY] = config.task
+        save_json_file(filename, result)
+
+    @staticmethod
+    def _handle_backward_upnode_missing(construct_dict, subnode_id, upnode_id):
+        """
+        如果backward节点的父级节点是null，则尝试从同名的forward节点寻找父级节点
+        """
+        # 匹配以.backward.后跟一个或多个数字结尾的模式
+        backward_pattern = r"(\.backward\.)(\d+)$"
+        forward_pattern = r"(\.forward\.)(\d+)$"
+        if re.search(backward_pattern, subnode_id) and not upnode_id:
+            forward_upnode_id = construct_dict.get(re.sub(backward_pattern, r".forward.\2", subnode_id))
+            if forward_upnode_id:
+                new_upnode_id = re.sub(forward_pattern, r".backward.\2", forward_upnode_id)
+                if new_upnode_id in construct_dict:
+                    return new_upnode_id
+        return upnode_id
+
+    @staticmethod
+    def _init_nodes(graph, construct_dict, data_dict, stack_dict):
+        for subnode_id, upnode_id in construct_dict.items():
+            upnode_id = GraphBuilder._handle_backward_upnode_missing(construct_dict, subnode_id, upnode_id)
+            if upnode_id:
+                upnode_op = NodeOp.get_node_op(upnode_id)
+                upnode = GraphBuilder._create_or_get_node(graph, [data_dict, stack_dict], upnode_op, upnode_id)
+            else:
+                upnode = graph.root
+            node_op = NodeOp.get_node_op(subnode_id)
+            GraphBuilder._create_or_get_node(graph, [data_dict, stack_dict], node_op, subnode_id, upnode)
+
+    @staticmethod
+    def _create_or_get_node(graph, data_stack_list, op, name, upnode=None):
+        if name in graph.node_map:
+            node = graph.get_node(name)
+        else:
+            graph.add_node(op, name, upnode)
+            node = graph.get_node(name)
+            node_data = data_stack_list[0].get(name, {})
+            node_stack_info = data_stack_list[1].get(name, [])
+            # 添加输入输出数据
+            input_data, output_data = get_input_output(node_data, node.id)
+            # 更新数据
+            node.set_input_output(input_data, output_data)
+            node.stack_info = node_stack_info
+        # 添加节点
+        node.add_upnode(upnode)
+        return node
+
+    @staticmethod
+    def _collect_apis_between_modules(graph):
+        """
+        图首次展开，这些首层节点包含许多module和api，api数量很多导致图被拉得很长严重影响查阅，因此将module之间的apis收集起来成为节点
+        Args:
+            graph: 模型结构
+
+        Returns: None
+        """
+        i = 0
+        output = []
+        node_list = graph.root.subnodes
+        while i < len(node_list):
+            current_node = node_list[i]
+
+            # 当前节点为api，检查后续是否还有api
+            if current_node.op == NodeOp.function_api:
+                temp_nodes = [current_node]
+                i += 1
+                while i < len(node_list) and node_list[i].op == NodeOp.function_api:
+                    temp_nodes.append(node_list[i])
+                    i += 1
+
+                # 检查api节点是否大于等于2个
+                if len(temp_nodes) >= 2:
+                    # 创建新节点，将这些api节点放入新节点的subnodes属性
+                    node_id = graph.add_node(NodeOp.api_collection, GraphConst.APIS_BETWEEN_MODULES,
+                                             id_accumulation=True)
+                    api_collection_node = graph.get_node(node_id)
+                    api_collection_node.subnodes = temp_nodes
+                    # 重新确立父子关系
+                    for node in temp_nodes:
+                        node.upnode = api_collection_node
+                    api_collection_node.upnode = graph.root
+                    output.append(api_collection_node)
+                else:
+                    # 如果连续的api节点不足2个，将它们原样添加到输出列表
+                    output.extend(temp_nodes)
+            else:
+                # 如果当前节点为module，直接添加到输出列表
+                output.append(current_node)
+                i += 1
+
+        graph.root.subnodes = output
+
+
+class GraphExportConfig:
+    def __init__(self, graph_n, graph_b=None, tool_tip=None, node_colors=None, micro_steps=None, task=''):
+        self.graph_n = graph_n
+        self.graph_b = graph_b
+        self.tool_tip = tool_tip
+        self.node_colors = node_colors
+        self.micro_steps = micro_steps
+        self.task = task

msprobe/visualization/builder/msprobe_adapter.py

@@ -0,0 +1,205 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import math
+from msprobe.core.compare.acc_compare import read_op, merge_tensor, get_accuracy
+from msprobe.core.common.utils import set_dump_path, get_dump_mode
+from msprobe.visualization.utils import GraphConst
+from msprobe.core.common.const import Const
+
+# 用于将节点名字解析成对应的NodeOp的规则
+op_patterns = [
+    # NodeOp.module
+    r'^(Module.|Cell.)',
+    # NodeOp.function_api
+    r'^(Tensor.|Torch.|Functional.|NPU.|VF.|Distributed.|Aten.|Mint.|Primitive.|Jit.|MintFunctional.)'
+]
+
+
+def get_compare_mode(dump_path_param):
+    """
+    获得比较模式，包括summary、MD5和真实数据三种模式
+    Args:
+        dump_path_param: 调用acc_compare接口所依赖的参数
+    Returns: 0 summary mode, 1 md5 mode, 2 true data mode
+    """
+    set_dump_path(dump_path_param)
+    dump_mode = get_dump_mode(dump_path_param)
+    compare_mode = GraphConst.DUMP_MODE_TO_GRAPHCOMPARE_MODE_MAPPING.get(dump_mode)
+    return compare_mode
+
+
+def run_real_data(dump_path_param, csv_path, framework, is_cross_frame=False):
+    """
+    多进程运行生成真实数据
+    Args:
+        dump_path_param: 调用acc_compare接口所依赖的参数
+        csv_path: 生成文件路径
+        framework: 框架类型, pytorch或mindspore
+        is_cross_frame: 是否进行跨框架比对，仅支持mindspore比pytorch, 其中pytorch为标杆
+    """
+    if framework == Const.PT_FRAMEWORK:
+        from msprobe.pytorch.compare.pt_compare import PTComparator
+        return PTComparator().do_multi_process(dump_path_param, csv_path)
+    else:
+        from msprobe.mindspore.compare.ms_compare import MSComparator
+        ms_comparator = MSComparator()
+        ms_comparator.cross_frame = is_cross_frame
+        return ms_comparator.do_multi_process(dump_path_param, csv_path)
+
+
+def get_input_output(node_data, node_id):
+    """
+    将dump的原始数据进行拆解，分解为output和input两个数据
+    Args:
+        node_data: 属于单个节点的dump数据
+        node_id: 节点名字
+    """
+    input_data = {}
+    output_data = {}
+    op_parsed_list = read_op(node_data, node_id)
+    for item in op_parsed_list:
+        full_op_name = item.get('full_op_name', '')
+        if not full_op_name:
+            continue
+        if GraphConst.OUTPUT in full_op_name and GraphConst.INPUT not in full_op_name:
+            output_data[full_op_name] = item
+        else:
+            name = item.get('data_name')
+            # 节点参数名称尽量使用落盘数据的名称
+            if isinstance(name, str) and name != '-1':
+                input_data[name.rsplit(Const.SEP, 1)[0]] = item
+            else:
+                input_data[full_op_name] = item
+    return input_data, output_data
+
+
+def compare_data(data_dict_list1, data_dict_list2):
+    """
+    比较get_input_output中输出的结果是否结构一致，比较一致返回True
+    """
+    if len(data_dict_list1) != len(data_dict_list2):
+        return False
+    # 用于比较两个节点是否相等的关键字段
+    tag_keys = ['type', 'shape']
+    for key1, key2 in zip(data_dict_list1, data_dict_list2):
+        dict1 = data_dict_list1[key1]
+        dict2 = data_dict_list2[key2]
+        for tag_key in tag_keys:
+            tag_value1 = dict1.get(tag_key, None)
+            tag_value2 = dict2.get(tag_key, None)
+            if tag_value1 != tag_value2:
+                return False
+    return True
+
+
+def format_node_data(data_dict):
+    """
+    批量进行节点数据的输出
+    """
+    del_list = ['requires_grad', 'full_op_name']
+    for _, value in data_dict.items():
+        if not isinstance(value, dict):
+            continue
+        for item in del_list:
+            if item in value:
+                del value[item]
+        _format_data(value)
+    return data_dict
+
+
+def compare_node(node_ids, data_dicts, stack_json_data, compare_mode):
+    """
+    调用acc_compare.py中的get_accuracy获得精度对比指标
+    真实数据对比模式无法获得精度对比指标，需要调用多进程比对接口
+    Returns: 包含参数信息和对比指标（真实数据对比模式除外）的list
+    """
+    merge_n = _parse_node(node_ids[0], data_dicts[0], stack_json_data, compare_mode)
+    merge_b = _parse_node(node_ids[1], data_dicts[1], stack_json_data, compare_mode)
+    result = []
+    dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode)
+    get_accuracy(result, merge_n, merge_b, dump_mode)
+    return result
+
+
+def _parse_node(node_id, data_dict, stack_json_data, compare_mode):
+    """
+    转换节点，使其能够作为acc_compare.py中的get_accuracy的入参
+    """
+    dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode)
+    op_parsed_list = read_op(data_dict.get(node_id, {}), node_id)
+    if node_id in stack_json_data:
+        op_parsed_list.append(
+            {'full_op_name': node_id, 'full_info': stack_json_data[node_id]})
+    else:
+        op_parsed_list.append({'full_op_name': node_id, 'full_info': None})
+    result = merge_tensor(op_parsed_list, dump_mode)
+    if not result:
+        result['op_name'] = []
+    return result
+
+
+def _format_decimal_string(s):
+    """
+    使用正则表达式匹配包含数字、小数点和可选的百分号的字符串
+    """
+    pattern = re.compile(r'\d{1,20}\.\d{1,20}%?')
+    matches = pattern.findall(s)
+    for match in matches:
+        is_percent = match.endswith('%')
+        number_str = match.rstrip('%')
+        decimal_part = number_str.split('.')[1]
+        # 如果小数位数大于6，进行处理
+        if len(decimal_part) > GraphConst.ROUND_TH:
+            number_float = float(number_str)
+            formatted_number = f"{number_float:.{GraphConst.ROUND_TH}f}"
+            # 如果原来是百分数，加回百分号
+            if is_percent:
+                formatted_number += '%'
+            # 替换原字符串中的数值部分
+            s = s.replace(match, formatted_number)
+    return s
+
+
+def _format_data(data_dict):
+    """
+    格式化数据，小数保留6位，处理一些异常值
+    """
+    pattern = r'^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)$'
+    all_null = False
+    for key, value in data_dict.items():
+        if isinstance(value, str):
+            # 将单引号删掉，None换成null避免前端解析错误
+            value = value.replace("'", "").replace(GraphConst.NONE, GraphConst.NULL)
+            value = _format_decimal_string(value)
+        elif value is None or value == ' ':
+            value = GraphConst.NULL
+        # 科学计数法1.123123123123e-11，格式化为1.123123e-11
+        elif isinstance(value, float) and len(str(value)) < GraphConst.STR_MAX_LEN and re.match(pattern, str(value)):
+            value = "{:.6e}".format(value)
+        elif isinstance(value, float):
+            value = round(value, GraphConst.ROUND_TH)
+        # Inf会走入这里，确保转成Inf。另外给其他不符合预期的类型做兜底方案
+        if key != GraphConst.ERROR_KEY:
+            # 除了error_key不转str，其他都转str, 避免前端解析错误
+            value = str(value)
+        # max为null, 意味着这个参数值为null
+        if key == Const.MAX and value == GraphConst.NULL:
+            all_null = True
+        data_dict[key] = value
+    # 字典里的value全null，只保留一个null
+    if all_null:
+        data_dict.clear()
+        data_dict[GraphConst.VALUE] = GraphConst.NULL

msprobe/visualization/compare/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.