mindstudio-probe 1.2.2__py3-none-any.whl → 8.1.0__py3-none-any.whl

msprobe/mindspore/common/utils.py

@@ -13,19 +13,63 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import inspect
 import os
 import random
+import types
 
 import mindspore as ms
-
 from mindspore import ops
+from mindspore.common.jit_config import JitConfig
 from mindspore.mint import nn
 
+from msprobe.core.common.const import Const
+from msprobe.core.common.decorator import recursion_depth_decorator
 from msprobe.core.common.exceptions import DistributedNotInitializedError
 from msprobe.core.common.file_utils import path_len_exceeds_limit, check_path_exists, save_npy
 from msprobe.core.common.log import logger
-from msprobe.core.common.const import Const
-from msprobe.core.common.utils import CompareException, check_seed_all
+from msprobe.core.common.utils import CompareException, check_seed_all, is_save_variable_valid
+from msprobe.mindspore.common.const import Const as MsConst
+
+try:
+    from mindspore._c_expression import _set_init_iter
+except ImportError:
+    enable_dynamic_kbyk_dump = False
+else:
+    enable_dynamic_kbyk_dump = True
+
+mindtorch_check_result = None
+register_backward_hook_functions = {}
+kwargs_exist_in_forward_hook = None
+
+
+class MsprobeStep(ms.train.Callback):
+    def __init__(self, debugger):
+        super(MsprobeStep, self).__init__()
+        self.debugger = debugger
+
+    def on_train_begin(self, run_context):
+        self.debugger.start()
+        if enable_dynamic_kbyk_dump:
+            _set_init_iter(0)
+
+    def on_train_step_begin(self, run_context):
+        self.debugger.start()
+
+    def on_train_step_end(self, run_context):
+        self.debugger.stop()
+        self.debugger.step()
+
+
+class MsprobeInitStep(ms.train.Callback):
+    def on_train_begin(self, run_context):
+        try:
+            from ms._c_expression import _set_init_iter
+        except ImportError:
+            logger.warning('MsprobeInitStep does not work on this version of MindSpore.')
+            return
+        cb_params = run_context.original_args()
+        _set_init_iter(cb_params.cur_step_num)
 
 
 def get_rank_if_initialized():
@@ -58,8 +102,8 @@ def convert_to_int(value):
 
 
 def clean_input_kwargs(cell):
-    if hasattr(cell, 'input_kwargs'):
-        del cell.input_kwargs
+    if hasattr(cell, 'msprobe_input_kwargs'):
+        del cell.msprobe_input_kwargs
 
 
 def list_lowest_level_directories(root_dir):
@@ -93,20 +137,6 @@ def seed_all(seed=1234, mode=False, rm_dropout=True):
         remove_dropout()
 
 
-class MsprobeStep(ms.train.Callback):
-
-    def __init__(self, debugger):
-        super(MsprobeStep, self).__init__()
-        self.debugger = debugger
-
-    def on_train_step_begin(self, run_context):
-        self.debugger.start()
-
-    def on_train_step_end(self, run_context):
-        self.debugger.stop()
-        self.debugger.step()
-
-
 class Dropout(ops.Dropout):
     def __init__(self, keep_prob=0.5, seed0=0, seed1=1):
         super().__init__(1., seed0, seed1)
@@ -142,9 +172,6 @@ def remove_dropout():
     nn.functional.dropout = dropout_ext
 
 
-mindtorch_check_result = None
-
-
 def is_mindtorch():
     global mindtorch_check_result
     if mindtorch_check_result is None:
@@ -159,17 +186,17 @@ def is_mindtorch():
     return mindtorch_check_result
 
 
-register_backward_hook_functions = {}
-
-
 def set_register_backward_hook_functions():
     global register_backward_hook_functions
+    if register_backward_hook_functions:
+        return
+
     if is_mindtorch():
         import torch
         from msprobe.mindspore.mindtorch import (_call_impl,
                                                  register_full_backward_pre_hook,
                                                  register_full_backward_hook)
-        if not hasattr(torch, "register_full_backward_hook"):
+        if not hasattr(torch.nn.Module, "register_full_backward_hook"):
             setattr(torch.nn.Module, "_call_impl", _call_impl)
             setattr(torch.nn.Module, "register_full_backward_pre_hook", register_full_backward_pre_hook)
             setattr(torch.nn.Module, "register_full_backward_hook", register_full_backward_hook)
@@ -182,9 +209,11 @@ def set_register_backward_hook_functions():
 
 def check_save_param(variable, name, save_backward):
     # try catch this api to skip invalid call
-    if not isinstance(variable, (list, dict, ms.Tensor, int, float, str)):
+    valid_data_types = (ms.Tensor, int, float, str)
+    if not is_save_variable_valid(variable, valid_data_types):
+        valid_data_types_with_nested_types = valid_data_types + (dict, tuple, list)
         logger.warning("PrecisionDebugger.save variable type not valid, "
-                       "should be one of list, dict, ms.Tensor, int, float or string. "
+                       f"should be one of {valid_data_types_with_nested_types}"
                        "Skip current save process.")
         raise ValueError
     if not isinstance(name, str):
@@ -196,4 +225,103 @@ def check_save_param(variable, name, save_backward):
         logger.warning("PrecisionDebugger.save_backward name not valid, "
                        "should be bool. "
                        "Skip current save process.")
-        raise ValueError
+        raise ValueError
+
+
+def is_graph_mode_cell_dump_allowed(config):
+    if config.task not in [Const.TENSOR, Const.STATISTICS] or is_mindtorch() or not hasattr(ops, 'DumpGradient'):
+        return False
+    valid_mix_level = [MsConst.CELL_AND_API, Const.LEVEL_MIX]
+    if config.level in valid_mix_level and config.execution_mode == MsConst.PYNATIVE_MODE:
+        return True
+    return config.level == MsConst.CELL or config.level == Const.LEVEL_L0
+
+
+@recursion_depth_decorator('msprobe.mindspore.common.utils.is_decorated_by_jit')
+def is_decorated_by_jit(func):
+    closure = getattr(func, '__closure__', [])
+    if closure:
+        for obj in closure:
+            if isinstance(obj.cell_contents, JitConfig):
+                return True
+            elif isinstance(obj.cell_contents, types.FunctionType) and hasattr(obj.cell_contents, '__closure__'):
+                if is_decorated_by_jit(obj.cell_contents):
+                    return True
+    return False
+
+
+@recursion_depth_decorator('msprobe.mindspore.common.utils.get_cells_and_names')
+def get_cells_and_names(model, cells_set=None, name_prefix=''):
+    cells_set = cells_set if cells_set else set()
+    if model in cells_set:
+        return
+
+    cells_set.add(model)
+    jit_decorated = is_decorated_by_jit(model.construct)
+    yield name_prefix, model, jit_decorated
+    if jit_decorated:
+        return
+
+    children_cells = getattr(model, '_cells')
+    for name, cell in children_cells.items():
+        if cell:
+            cells_name_prefix = f'{name_prefix}{Const.SEP}{name}' if name_prefix else name
+            jit_decorated = is_decorated_by_jit(model.construct)
+            if jit_decorated:
+                yield cells_name_prefix, cell, jit_decorated
+            else:
+                for ele in get_cells_and_names(cell, cells_set, cells_name_prefix):
+                    yield ele
+
+
+def get_cells_and_names_with_index(models):
+    cells_with_index_in_pynative_mode = {}
+    cells_with_index_in_graph_mode = {}
+
+    def distinguish_cells(cells):
+        cells_in_pynative_mode = []
+        cells_in_graph_mode = []
+        for name, cell, jit_decorated in cells:
+            if jit_decorated:
+                cells_in_graph_mode.append((name, cell))
+            else:
+                cells_in_pynative_mode.append((name, cell))
+        return cells_in_pynative_mode, cells_in_graph_mode
+
+    if is_mindtorch():
+        if isinstance(models, (list, tuple)):
+            for index, model in enumerate(models):
+                cells_with_index_in_pynative_mode[str(index)] = model.named_modules()
+        else:
+            cells_with_index_in_pynative_mode["-1"] = models.named_modules()
+    else:
+        if isinstance(models, (list, tuple)):
+            for index, model in enumerate(models):
+                cells = get_cells_and_names(model)
+                cells_in_pynative_mode, cells_in_graph_mode = distinguish_cells(cells)
+                cells_with_index_in_pynative_mode[str(index)] = cells_in_pynative_mode
+                cells_with_index_in_graph_mode[str(index)] = cells_in_graph_mode
+        else:
+            cells = get_cells_and_names(models)
+            cells_in_pynative_mode, cells_in_graph_mode = distinguish_cells(cells)
+            cells_with_index_in_pynative_mode["-1"] = cells_in_pynative_mode
+            cells_with_index_in_graph_mode["-1"] = cells_in_graph_mode
+
+    return cells_with_index_in_pynative_mode, cells_with_index_in_graph_mode
+
+
+def has_kwargs_in_forward_hook():
+    global kwargs_exist_in_forward_hook
+
+    if kwargs_exist_in_forward_hook is None:
+        if is_mindtorch():
+            kwargs_exist_in_forward_hook = True
+            return kwargs_exist_in_forward_hook
+
+        try:
+            func_params = inspect.signature(nn.Cell.register_forward_hook).parameters
+            kwargs_exist_in_forward_hook = 'with_kwargs' in func_params
+        except Exception:
+            kwargs_exist_in_forward_hook = False
+
+    return kwargs_exist_in_forward_hook

msprobe/mindspore/compare/common_dir_compare.py

@@ -0,0 +1,382 @@
+# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.import functools
+
+import os
+import multiprocessing
+from dataclasses import dataclass
+from typing import Dict, List, Tuple, Optional, Any
+from concurrent.futures import ProcessPoolExecutor
+from functools import partial
+from pathlib import Path
+
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+
+from msprobe.core.common.log import logger
+from msprobe.core.common.utils import CompareException
+from msprobe.core.common.exceptions import FileCheckException
+from msprobe.core.common.file_utils import check_file_or_directory_path, write_df_to_csv, create_directory, \
+                                           check_path_before_create, load_npy
+from msprobe.core.common.const import CompareConst, FileCheckConst
+from msprobe.core.compare.npy_compare import compare_ops_apply
+from msprobe.core.compare.multiprocessing_compute import check_accuracy
+
+
+def common_dir_compare(input_params: Dict, output_dir: str) -> Optional[pd.DataFrame]:
+    """
+    高级目录比对函数，完全镜像输入目录结构
+    
+    Args:
+        input_params: 包含npu_path和bench_path的字典
+        output_dir: 输出根目录
+        
+    Returns:
+        当输入目录是平铺npy文件时返回DataFrame，否则返回None
+    """
+    npu_root = Path(input_params.get('npu_path'))
+    bench_root = Path(input_params.get('bench_path'))
+    name_map_dict = input_params.get('map_dict', {})
+    file_tree = build_mirror_file_tree(npu_root, bench_root)
+    
+    # 处理文件比对
+    with ProcessPoolExecutor() as executor:
+        results = list(tqdm(
+            executor.map(
+                partial(process_directory_pair, name_map_dict=name_map_dict, output_dir=output_dir),
+                file_tree.items()
+            ),
+            total=len(file_tree),
+            desc="Processing directories"
+        ))
+    return
+
+
+def process_directory_pair(item: Tuple[Path, Tuple[Path, Path]], name_map_dict: Dict, output_dir: str):
+    """
+    处理一个目录对
+    
+    Args:
+        item: (相对路径, (npu目录, bench目录))元组
+        output_dir: 输出根目录
+        
+    Returns:
+        比对结果的DataFrame（仅平铺结构时返回）
+    """
+    rel_path, (npu_dir, bench_dir) = item
+    
+    # 创建镜像输出目录
+    output_path = Path(output_dir) / rel_path
+    create_directory(output_path)
+    
+    # 生成文件映射
+    npu_files = find_npy_files(npu_dir)
+    bench_files = find_npy_files(bench_dir)
+    map_dict = generate_map_dict(npu_files, bench_files, name_map_dict)
+    
+    if not map_dict:
+        logger.warning(f"No file pairs found in {rel_path}")
+        return None
+    
+    # 执行比对
+    result_df = do_multi_process(process_chunk, map_dict)
+    check_path_before_create(output_path)
+    # 保存结果
+    result_path = os.path.join(output_path, 'result.csv')
+    write_df_to_csv(result_df, result_path)
+    logger.info(f"Results saved to {result_path}")
+    return None
+
+
+def build_mirror_file_tree(npu_root: Path, bench_root: Path) -> Dict[Path, Tuple[Path, Path]]:
+    """
+    构建镜像文件树，键为相对路径，值为(npu_path, bench_path)元组
+    
+    Args:
+        npu_root: NPU数据根目录
+        bench_root: 基准数据根目录
+        
+    Returns:
+        文件树字典
+    """
+    file_tree = {}
+    
+    # 遍历NPU目录构建树结构
+    for npu_path in npu_root.rglob('*.npy'):
+        dir_path = npu_path.relative_to(npu_root).parent
+        npu_dir_pair = os.path.join(npu_root, dir_path)
+        bench_dir_pair = os.path.join(bench_root, dir_path)
+        try:
+            check_file_or_directory_path(bench_dir_pair, isdir=True)
+        except FileCheckException:
+            continue
+        # 添加到文件树
+        if dir_path not in file_tree:
+            file_tree[dir_path] = (npu_dir_pair, bench_dir_pair)
+    
+    return file_tree
+
+
+def find_npy_files(directory):
+    npy_files_dict = {}
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".npy"):
+                # 分割文件名并去掉最后两个元素
+                file_name = file.split('_')
+                if len(file_name) < 2:
+                    continue
+                key = '_'.join(file_name[:-2])
+                # 文件的完整路径
+                value = os.path.join(root, file)
+                # 添加到字典中
+                if not npy_files_dict.get(key):
+                    npy_files_dict[key] = []
+                npy_files_dict[key].append(value)
+    return npy_files_dict
+
+
+def generate_map_dict(npu_file_dict, bench_file_dict, name_map_dict=None):
+    for k, npu_file_list in npu_file_dict.items():
+        bench_file_list = bench_file_dict.get(k)
+        if not bench_file_list and k in name_map_dict:
+            bench_file_list = bench_file_dict.get(name_map_dict.get(k))
+        bench_length = len(bench_file_list)
+        if not (bench_file_list and bench_length):
+            continue
+        result_dict = {}
+        for i, npu_file in enumerate(npu_file_list):
+            if i >= bench_length:
+                break
+            bench_file = bench_file_list[i]
+            result_dict[f"{k}_{i}"] = (npu_file, bench_file)
+    return result_dict
+
+
+def do_multi_process(func, map_dict):
+    lock = multiprocessing.Manager().RLock()
+    result_len = len(map_dict)
+    process_num = max(int((multiprocessing.cpu_count() + 1) // 4), 1)
+    # every block size
+    df_chunk_size = result_len // process_num
+
+    # generate the same len of map_dict df
+    result_df = initialize_result_df(result_len)
+    if df_chunk_size > 0:
+        df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)]
+    else:
+        df_chunks = [result_df]
+        process_num = 1
+    logger.info(f"Using {process_num} processes with chunk size {df_chunk_size}")
+    
+    # 分割字典
+    map_chunks = split_dict(map_dict, df_chunk_size)
+    
+    # 创建结果列表和进程池
+    results = []
+    pool = multiprocessing.Pool(process_num)
+    
+    progress_bar = tqdm(total=len(result_df), desc="API/Module Item Compare Process", unit="row", ncols=100)
+
+    def update_progress(size, progress_lock, extra_param=None):
+        with progress_lock:
+            progress_bar.update(size)
+
+    def err_call(args):
+        logger.error('multiprocess compare failed! Reason: {}'.format(args))
+        try:
+            pool.close()
+        except OSError as e:
+            logger.error(f'pool terminate failed: {str(e)}')
+    results = []
+    try:
+        # 提交任务到进程池
+        for process_idx, (df_chunk, map_chunk) in enumerate(zip(df_chunks, map_chunks)):
+            start_idx = df_chunk_size * process_idx
+            result = pool.apply_async(
+                func,
+                args=(df_chunk, start_idx, map_chunk, lock),
+                error_callback=err_call,
+                callback=partial(update_progress, len(map_chunk), lock)
+            )
+            results.append(result)
+
+        final_results = [r.get() for r in results]
+        # 等待所有任务完成
+        pool.close()
+        pool.join()
+        return pd.concat(final_results, ignore_index=True)
+    except Exception as e:
+        logger.error(f"\nMain process error: {str(e)}")
+        pool.terminate()
+        return pd.DataFrame({})
+    finally:
+        pool.close()
+
+
+def initialize_result_df(total_size):
+    """预分配结果DataFrame"""
+    columns = [
+        CompareConst.NAME,
+        CompareConst.NPU_DTYPE,
+        CompareConst.BENCH_DTYPE,
+        CompareConst.NPU_SHAPE,
+        CompareConst.BENCH_SHAPE,
+        CompareConst.COSINE,
+        CompareConst.EUC_DIST,
+        CompareConst.MAX_ABS_ERR,
+        CompareConst.MAX_RELATIVE_ERR,
+        CompareConst.ONE_THOUSANDTH_ERR_RATIO,
+        CompareConst.FIVE_THOUSANDTHS_ERR_RATIO,
+        CompareConst.NPU_MAX,
+        CompareConst.NPU_MIN,
+        CompareConst.NPU_MEAN,
+        CompareConst.NPU_NORM,
+        CompareConst.BENCH_MAX,
+        CompareConst.BENCH_MIN,
+        CompareConst.BENCH_MEAN,
+        CompareConst.BENCH_NORM,
+        CompareConst.ACCURACY,
+        CompareConst.ERROR_MESSAGE,
+        CompareConst.DATA_NAME
+    ]
+    return pd.DataFrame(index=range(total_size), columns=columns)
+
+
+def split_dict(input_dict, chunk_size):
+    """将字典按指定chunk_size分割"""
+    items = list(input_dict.items())
+    if chunk_size > 0:
+        return [dict(items[i:i + chunk_size]) for i in range(0, len(items), chunk_size)]
+    return [input_dict]
+
+
+def get_tensor_stats(tensor: np.ndarray) -> Tuple[float, float, float, float]:
+    """获取张量的统计信息"""
+    t_max = np.max(tensor)
+    t_min = np.min(tensor)
+    t_mean = np.mean(tensor)
+    t_l2norm = np.linalg.norm(tensor)
+    return t_max, t_min, t_mean, t_l2norm
+
+
+def process_chunk(df, start_idx, map_chunk, lock):
+    """处理一个数据块"""
+    err_mess = []
+    results = []
+    for name, file_pair in map_chunk.items():
+        err_msg = ""
+        npu_file, bench_file = file_pair
+        n_value = load_npy(npu_file)
+        # if need to support cross frame b_value need to add load_pt
+        b_value = load_npy(bench_file)
+        error_flag = False
+
+        err_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg)
+        cos_sim, euc_dist, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio = err_list
+        a_max, a_min, a_mean, a_l2norm = get_tensor_stats(n_value)
+        b_max, b_min, b_mean, b_l2norm = get_tensor_stats(b_value)
+        err_mess.append(err_msg)
+        # 使用示例
+        result = ComparisonResult(
+            name=name,  # CompareConst.NAME
+            npu_dtype=n_value.dtype,  # CompareConst.NPU_DTYPE
+            bench_dtype=b_value.dtype,  # CompareConst.BENCH_DTYPE
+            npu_shape=n_value.shape,  # CompareConst.NPU_SHAPE
+            bench_shape=b_value.shape,  # CompareConst.BENCH_SHAPE
+            cosine=cos_sim,  # CompareConst.COSINE
+            euc_dist=euc_dist,  # CompareConst.EUC_DIST
+            max_abs_err=max_abs_err,  # CompareConst.MAX_ABS_ERR
+            max_relative_err=max_relative_err,  # CompareConst.MAX_RELATIVE_ERR
+            one_thousandth_err_ratio=one_thousand_err_ratio,  # CompareConst.ONE_THOUSANDTH_ERR_RATIO
+            five_thousandth_err_ratio=five_thousand_err_ratio,  # CompareConst.FIVE_THOUSANDTHS_ERR_RATIO
+            npu_max=a_max,  # CompareConst.NPU_MAX
+            npu_min=a_min,  # CompareConst.NPU_MIN
+            npu_mean=a_mean,  # CompareConst.NPU_MEAN
+            npu_norm=a_l2norm,  # CompareConst.NPU_NORM
+            bench_max=b_max,  # CompareConst.BENCH_MAX
+            bench_min=b_min,  # CompareConst.BENCH_MIN
+            bench_mean=b_mean,  # CompareConst.BENCH_MEAN
+            bench_norm=b_l2norm,  # CompareConst.BENCH_NORM
+            accuracy=check_accuracy(cos_sim, max_abs_err),  # CompareConst.ACCURACY
+            error_message=err_msg,  # CompareConst.ERROR_MESSAGE
+            data_name=[npu_file, bench_file]  # CompareConst.DATA_NAME
+        )
+        results.append(result)
+    return _save_part_df(df, start_idx, results, lock)
+
+
+@dataclass
+class ComparisonResult:
+    name: str  # CompareConst.NAME
+    npu_dtype: Any  # CompareConst.NPU_DTYPE
+    bench_dtype: Any  # CompareConst.BENCH_DTYPE
+    npu_shape: Tuple[int, ...]  # CompareConst.NPU_SHAPE
+    bench_shape: Tuple[int, ...]  # CompareConst.BENCH_SHAPE
+    cosine: float  # Cons   t.COSINE
+    euc_dist: float  # CompareConst.EUC_DIST
+    max_abs_err: float  # CompareConst.MAX_ABS_ERR
+    max_relative_err: float  # CompareConst.MAX_RELATIVE_ERR
+    one_thousandth_err_ratio: float  # CompareConst.ONE_THOUSANDTH_ERR_RATIO
+    five_thousandth_err_ratio: float  # CompareConst.FIVE_THOUSANDTHS_ERR_RATIO
+    npu_max: float  # CompareConst.NPU_MAX
+    npu_min: float  # CompareConst.NPU_MIN
+    npu_mean: float  # CompareConst.NPU_MEAN
+    npu_norm: float  # CompareConst.NPU_NORM
+    bench_max: float  # CompareConst.BENCH_MAX
+    bench_min: float  # CompareConst.BENCH_MIN
+    bench_mean: float  # CompareConst.BENCH_MEAN
+    bench_norm: float  # CompareConst.BENCH_NORM
+    accuracy: bool  # CompareConst.ACCURACY
+    error_message: str  # CompareConst.ERROR_MESSAGE
+    data_name: List[str]  # CompareConst.DATA_NAME
+
+
+def _save_part_df(df, start_idx, results, lock):
+    lock.acquire()
+    try:
+        for i, result in enumerate(results):
+            process_index = i + start_idx
+            df.loc[process_index, CompareConst.NAME] = result.name
+            df.loc[process_index, CompareConst.NPU_DTYPE] = result.npu_dtype
+            df.loc[process_index, CompareConst.BENCH_DTYPE] = result.bench_dtype
+            df.loc[process_index, CompareConst.NPU_SHAPE] = str(result.npu_shape)  # 通常将tuple转为字符串存储
+            df.loc[process_index, CompareConst.BENCH_SHAPE] = str(result.bench_shape)
+            df.loc[process_index, CompareConst.COSINE] = result.cosine
+            df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist
+            df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_abs_err
+            df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err
+            df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousandth_err_ratio
+            df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousandth_err_ratio
+            df.loc[process_index, CompareConst.NPU_MAX] = result.npu_max
+            df.loc[process_index, CompareConst.NPU_MIN] = result.npu_min
+            df.loc[process_index, CompareConst.NPU_MEAN] = result.npu_mean
+            df.loc[process_index, CompareConst.NPU_NORM] = result.npu_norm
+            df.loc[process_index, CompareConst.BENCH_MAX] = result.bench_max
+            df.loc[process_index, CompareConst.BENCH_MIN] = result.bench_min
+            df.loc[process_index, CompareConst.BENCH_MEAN] = result.bench_mean
+            df.loc[process_index, CompareConst.BENCH_NORM] = result.bench_norm
+            df.loc[process_index, CompareConst.ACCURACY] = result.accuracy
+            df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.error_message
+            df.loc[process_index, CompareConst.DATA_NAME] = str(result.data_name)  # 列表转为字符串存储
+        return df
+    except ValueError as e:
+        logger.error('result dataframe is not found.')
+        raise CompareException(CompareException.INVALID_DATA_ERROR) from e
+    except IndexError as e:
+        logger.error('result dataframe elements can not be access.')
+        raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e
+    finally:
+        lock.release()

msprobe/mindspore/compare/distributed_compare.py

@@ -13,41 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 from msprobe.core.common.utils import CompareException
 from msprobe.core.common.file_utils import create_directory
 from msprobe.core.common.exceptions import FileCheckException
 from msprobe.mindspore.common.log import logger
 from msprobe.mindspore.compare.ms_compare import ms_compare
-from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json
+from msprobe.core.compare.utils import compare_distributed_inner
 from msprobe.mindspore.compare.ms_graph_compare import GraphMSComparator
 
 
 def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs):
-    if kwargs.get('suffix'):
-        logger.error("Argument 'suffix' is not supported for compare_distributed.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    is_print_compare_log = kwargs.get('is_print_compare_log', True)
-    # get the ranks and match by order
-    npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank'))
-    bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank'))
-    if len(npu_ranks) != len(bench_ranks):
-        logger.error('The number of ranks in the two runs are different. '
-                     'Unable to match the ranks. Please use another folder to compare '
-                     'or use compare() api and manually match the ranks.')
-        raise CompareException(CompareException.INVALID_PATH_ERROR)
-    for nr, br in zip(npu_ranks, bench_ranks):
-        npu_data_dir = os.path.join(npu_dump_dir, nr)
-        bench_data_dir = os.path.join(bench_dump_dir, br)
-        npu_path = extract_json(npu_data_dir, stack_json=False)
-        bench_path = extract_json(bench_data_dir, stack_json=False)
-
-        dump_result_param = {
-            'npu_json_path': npu_path,
-            'bench_json_path': bench_path,
-            'is_print_compare_log': is_print_compare_log
-        }
-        ms_compare(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}-{br}', **kwargs)
+    compare_distributed_inner(npu_dump_dir, bench_dump_dir, output_path, ms_compare, **kwargs)
 
 
 def ms_graph_compare(inputs, outputs):