mindstudio-probe 8.1.2__py3-none-any.whl → 8.2.1__py3-none-any.whl

msprobe/core/compare/acc_compare.py

@@ -25,16 +25,18 @@ from tqdm import tqdm
 from msprobe.core.advisor.advisor import Advisor
 from msprobe.core.common.const import CompareConst, Const
 from msprobe.core.common.exceptions import FileCheckException
-from msprobe.core.common.file_utils import load_json, remove_path, create_directory
+from msprobe.core.common.file_utils import load_json, remove_path, create_directory, save_excel, save_json
 from msprobe.core.common.log import logger
 from msprobe.core.common.utils import CompareException, add_time_with_xlsx, check_op_str_pattern_valid, \
-    set_dump_path, get_dump_mode, check_compare_param, check_configuration_param, load_stack_json, get_file_type
-from msprobe.core.compare.check import check_dump_json_str, check_stack_json_str, cross_dtype_mapping
-from msprobe.core.compare.utils import merge_tensor, print_compare_ends_info, read_op, \
-    reorder_op_x_list, set_stack_json_path, check_api_info_len
+    set_dump_path, get_dump_mode, check_compare_param, load_stack_json, get_file_type, add_time_with_json
+from msprobe.core.compare.check import check_dump_json_str, check_stack_json_str, cross_dtype_mapping, \
+    check_configuration_param
+from msprobe.core.compare.utils import merge_tensor, print_compare_ends_info, read_op, set_stack_json_path, \
+    reorder_index
 from msprobe.core.compare.config import ModeConfig, MappingConfig, MappingDict
 from msprobe.core.compare.multiprocessing_compute import CompareRealData
 from msprobe.core.compare.highlight import HighLight
+from msprobe.core.compare.diff_analyze.first_diff_analyze import FirstDiffAnalyze
 
 
 @dataclass
@@ -43,12 +45,15 @@ class ComparisonConfig:
     stack_mode: bool
     auto_analyze: bool
     fuzzy_match: bool
+    highlight: bool
     data_mapping: dict
     suffix: str
     cell_mapping: dict
     api_mapping: dict
     layer_mapping: dict
     compared_file_type: str
+    first_diff_analyze: bool
+    is_print_compare_log: bool
 
 
 class Comparator:
@@ -57,17 +62,18 @@ class Comparator:
         self.mode_config = mode_config
         self.mapping_config = mapping_config
         self.cross_frame = is_cross_framework
-
         self.mapping_dict = MappingDict(mapping_config)
 
-    @staticmethod
-    def process_output_file(output_path, suffix, compared_file_type):
+    def process_output_file(self, output_path, suffix, compared_file_type):
         file_name_prefix_mapping = {
             Const.DUMP_JSON_FILE: "compare_result",
             Const.DEBUG_JSON_FILE: "debug_compare_result"
         }
         file_name_prefix = file_name_prefix_mapping.get(compared_file_type, "compare_result")
-        file_name = add_time_with_xlsx(file_name_prefix + suffix)
+        if self.mode_config.first_diff_analyze:
+            file_name = add_time_with_json("compare_result" + suffix)
+        else:
+            file_name = add_time_with_xlsx(file_name_prefix + suffix)
         file_path = os.path.join(os.path.realpath(output_path), file_name)
         if os.path.exists(file_path):
             logger.warning(f"{file_path} will be deleted.")
@@ -95,6 +101,7 @@ class Comparator:
 
         # get kwargs or set default value
         suffix = kwargs.get('suffix', '')
+        rank = suffix[1:]
 
         # process output file
         file_path = self.process_output_file(output_path, suffix, self.mode_config.compared_file_type)
@@ -103,22 +110,45 @@ class Comparator:
         npu_json = input_param.get("npu_json_path")
         bench_json = input_param.get("bench_json_path")
         stack_json = input_param.get("stack_json_path")
-        result_df = self.compare_statistics([npu_json, bench_json, stack_json])
+        parse_data = ParseData(self.mode_config, rank)  # load and parse json data
+        npu_df, bench_df = parse_data.parse([npu_json, bench_json, stack_json])
+        result_df = self.compare_statistics(npu_df, bench_df)
         if not result_df.values.tolist():
             logger.warning("Can`t match any op. No compare result file generated.")
             return
 
+        if self.mode_config.first_diff_analyze:
+            # add P2POp additional info from npu_df and bench_df to result_df
+            result_df['NPU P2POp op'] = npu_df['op']
+            result_df['Bench P2POp op'] = bench_df['op']
+            result_df['NPU P2POp peer'] = npu_df['peer']
+            result_df['Bench P2POp peer'] = bench_df['peer']
+
+            first_diff_analyze = FirstDiffAnalyze(self.mode_config, rank)
+            check_result = first_diff_analyze.check(result_df)
+            save_json(file_path, check_result, indent=4)
+            logger.info(f"Saving json file to disk: {file_path}")
+            return
+
         # compare real data
         if self.mode_config.dump_mode == Const.ALL:
             compare_real_data = CompareRealData(self.file_reader, self.mode_config, self.cross_frame)
             result_df = compare_real_data.do_multi_process(input_param, result_df)
 
-        # highlight suspicious API
-        highlight_dict = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []}
-        highlight = HighLight(self.mode_config)
-        if self.mode_config.compared_file_type == Const.DUMP_JSON_FILE:
-            highlight.find_compare_result_error_rows(result_df, highlight_dict)
-        highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path)
+        # save result excel file
+        logger.info(f'Saving result excel file in progress. The file path is: {file_path}.')
+        if self.mode_config.highlight and len(result_df) <= CompareConst.MAX_EXCEL_LENGTH:
+            # highlight if not too long
+            highlight_dict = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []}
+            highlight = HighLight(self.mode_config, rank)
+            if self.mode_config.compared_file_type == Const.DUMP_JSON_FILE:
+                highlight.find_compare_result_error_rows(result_df, highlight_dict)
+            result_df.drop(columns=['state', 'api_origin_name'], inplace=True)  # 删除中间数据，两列不落盘
+            highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path)
+        else:
+            # fallback to simple save without highlight
+            result_df.drop(columns=['state', 'api_origin_name'], inplace=True)  # 删除中间数据，两列不落盘
+            save_excel(file_path, result_df)
 
         # output compare analysis suggestions
         if self.mode_config.auto_analyze:
@@ -127,11 +157,7 @@ class Comparator:
 
         print_compare_ends_info()
 
-    def compare_statistics(self, file_list):
-        # load and parse json data
-        parse_data = ParseData(self.mode_config)
-        npu_df, bench_df = parse_data.parse(file_list)
-
+    def compare_statistics(self, npu_df, bench_df):
         npu_df[[Const.DTYPE, Const.SHAPE]] = npu_df[[Const.DTYPE, Const.SHAPE]].astype(str)
         bench_df[[Const.DTYPE, Const.SHAPE]] = bench_df[[Const.DTYPE, Const.SHAPE]].astype(str)
 
@@ -149,6 +175,8 @@ class Comparator:
         match_result.loc[~match.gen_dtype_condition(match_result), bench_columns] = CompareConst.N_A
 
         # organize compare result table by renaming columns
+        if self.mode_config.dump_mode == Const.ALL and self.mode_config.first_diff_analyze:
+            self.mode_config.dump_mode = Const.SUMMARY
         create_table = CreateTable(self.mode_config)
         result_df, header = create_table.make_result_df(match_result)
 
@@ -158,8 +186,9 @@ class Comparator:
 
 
 class ParseData:
-    def __init__(self, mode_config: ModeConfig):
+    def __init__(self, mode_config: ModeConfig, rank):
         self.mode_config = mode_config
+        self.rank = rank
 
     def parse(self, file_list):
         npu_json_path, bench_json_path, stack_json_path = file_list
@@ -168,21 +197,24 @@ class ParseData:
         stack_json_data = load_stack_json(stack_json_path) if self.mode_config.stack_mode else None
 
         # parse json data and generate df
-        npu_df = self.gen_data_df(npu_json_data, stack_json_data)
-        bench_df = self.gen_data_df(bench_json_data, stack_json_data)
+        npu_df = self.gen_data_df(npu_json_data, stack_json_data, 'NPU')
+        bench_df = self.gen_data_df(bench_json_data, stack_json_data, 'Bench')
 
         return npu_df, bench_df
 
-    def gen_data_df(self, data_json, stack_json_data):
+    def gen_data_df(self, data_json, stack_json_data, device: str):
         result = {
             CompareConst.OP_NAME: [],
             Const.DTYPE: [],
             Const.SHAPE: [],
             Const.SUMMARY: [],
-            Const.STACK_INFO: []
+            Const.STACK_INFO: [],
+            Const.STATE: [],
+            Const.API_ORIGIN_NAME: [],
+            Const.REQ_GRAD: []
         }
         if self.mode_config.dump_mode == Const.ALL:
-            result['data_name'] = []
+            result[Const.DATA_NAME] = []
         elif self.mode_config.dump_mode == Const.MD5:
             result[Const.MD5] = []
 
@@ -192,56 +224,50 @@ class ParseData:
             return pd.DataFrame(result)
 
         api_nums = len(apis_data)
-        progress_bar = tqdm(total=api_nums, desc="API/Module Read Progress", unit="api/module", ncols=100)
+        default_bar_desc = f'{device} API/Module Read Progress'
+        bar_desc_add_rank = f'[{self.rank}]' + default_bar_desc if self.rank else default_bar_desc
+        progress_bar = tqdm(total=api_nums, desc=bar_desc_add_rank, unit="api/module", ncols=100)
 
         # 从json中循环解析API数据，遍历所有API
         for data_name in apis_data:
             check_op_str_pattern_valid(data_name)
-            merge_list = self.gen_merge_list(data_json, data_name, stack_json_data)
-            if not merge_list:
+            op_parsed_list = self.gen_merge_list(data_json, data_name, stack_json_data)
+            if not op_parsed_list:
                 continue
-
-            op_name_list = merge_list.get(CompareConst.OP_NAME)
-            summary_list = merge_list.get(Const.SUMMARY)
-            data_name_list = merge_list.get('data_name')
-            op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list,
-                                                                                    summary_list,
-                                                                                    data_name_list)
-            # 遍历单个API的所有item
-            for index, op_name in enumerate(op_name_reorder):
-                result[CompareConst.OP_NAME].append(op_name)
-                if (CompareConst.INPUT_PATTERN in op_name) or (CompareConst.KWARGS_PATTERN in op_name):
-                    info_list = merge_list[CompareConst.INPUT_STRUCT]
-                elif CompareConst.OUTPUT_PATTERN in op_name:
-                    info_list = merge_list[CompareConst.OUTPUT_STRUCT]
-                elif CompareConst.PARAMS_PATTERN in op_name:
-                    info_list = merge_list[CompareConst.PARAMS_STRUCT]
-                elif CompareConst.PARAMS_GRAD_PATTERN in op_name:
-                    info_list = merge_list[CompareConst.PARAMS_GRAD_STRUCT]
-                else:
-                    info_list = merge_list[CompareConst.DEBUG_STRUCT]
-                check_api_info_len(op_name, info_list, 1)
-                struct = info_list.pop(0)
-
-                check_api_info_len(op_name, struct, 2)
-                result[Const.DTYPE].append(struct[0])
-                result[Const.SHAPE].append(struct[1])
+            reordered_index_list = reorder_index(op_parsed_list)
+            for i, index in enumerate(reordered_index_list):
+                op_item = op_parsed_list[index]
+
+                # common key
+                result[CompareConst.OP_NAME].append(op_item.get('full_op_name'))
+                result[Const.DTYPE].append(op_item.get(Const.DTYPE))
+                result[Const.SHAPE].append(op_item.get(Const.SHAPE))
+                result[Const.STATE].append(op_item.get(Const.STATE))
+                result[Const.REQ_GRAD].append(op_item.get(Const.REQ_GRAD))
+                result[Const.API_ORIGIN_NAME].append(data_name)
+                summary_data = [
+                    str(op_item.get(key)) if op_item.get(key) is None else op_item.get(key)
+                    for key in Const.SUMMARY_METRICS_LIST
+                ]
+                result[Const.SUMMARY].append(summary_data)
+
+                # dump_mode differ key
                 if self.mode_config.dump_mode == Const.MD5:
-                    check_api_info_len(op_name, struct, 3)
-                    result[Const.MD5].append(struct[2])
-
-                check_api_info_len(op_name, summary_reorder, 1)
-                result[Const.SUMMARY].append(summary_reorder.pop(0))
+                    result[Const.MD5].append(op_parsed_list[index].get(Const.MD5))
+                if self.mode_config.dump_mode == Const.ALL:
+                    result[Const.DATA_NAME].append(op_item.get(Const.DATA_NAME))
 
-                if index == 0 and self.mode_config.stack_mode:
-                    check_api_info_len(op_name, merge_list[Const.STACK_INFO], 1)
-                    result[Const.STACK_INFO].append(merge_list[Const.STACK_INFO][0])
+                # mode_config stack_mode addition key
+                if i == 0 and self.mode_config.stack_mode:
+                    result[Const.STACK_INFO].append(op_parsed_list[-1].get('full_info'))
                 else:
                     result[Const.STACK_INFO].append(None)
 
-                if self.mode_config.dump_mode == Const.ALL:
-                    check_api_info_len(op_name, data_name_reorder, 1)
-                    result['data_name'].append(data_name_reorder.pop(0))
+                # mode_config first_diff_analyze addition key
+                if self.mode_config.first_diff_analyze:
+                    result.setdefault('op', []).append(op_item.get('op', str(None)))
+                    result.setdefault('peer', []).append(op_item.get('peer', str(None)))
+
             progress_bar.update(1)
         progress_bar.close()
         return pd.DataFrame(result)
@@ -256,14 +282,14 @@ class ParseData:
             stack_info = stack_json_data.get(op_name)
             if stack_info is not None:
                 check_stack_json_str(stack_info, op_name)
-            # append only when stack_mode is True,
-            op_parsed_list.append({
-                'full_op_name': op_name,
-                'full_info': stack_info
-            })
-
-        merge_list = merge_tensor(op_parsed_list, self.mode_config.dump_mode)
-        return merge_list
+        else:
+            stack_info = None
+        # always add stack_info whether stack_mode is True
+        op_parsed_list.append({
+            'full_op_name': op_name,
+            'full_info': stack_info
+        })
+        return op_parsed_list
 
 
 class ProcessDf:
@@ -327,13 +353,17 @@ class ProcessDf:
             return npu_op_name
 
     def modify_compare_data_with_user_mapping(self, npu_df, bench_df):
+        def remove_prefix(string, prefix):
+            if string.startswith(prefix):
+                return string[len(prefix):]
+            return string
+
         def gen_input_compare_key(pattern, term):
             is_unmatched = True
             for i, prefix in enumerate(mapping_dict.get(f'ms_{term}')):
-                if op_name.split(pattern)[1].startswith(str(prefix)):
+                if remove_prefix(op_name, api_origin_name + pattern) == str(prefix):
                     npu_df.loc[index, CompareConst.CMP_KEY] = (
-                        op_name.replace(pattern + str(prefix),
-                                        pattern + str(mapping_dict.get(f'pt_{term}')[i])))
+                        op_name.replace(pattern + str(prefix), pattern + str(mapping_dict.get(f'pt_{term}')[i])))
                     is_unmatched = False
             return is_unmatched
 
@@ -355,15 +385,17 @@ class ProcessDf:
                 continue
             for index in ms_api_indices_dict.get(ms_api):
                 op_name = npu_df.loc[index, CompareConst.OP_NAME].replace(ms_api, pt_api, 1)
-                if CompareConst.INPUT_PATTERN in op_name:
+                state = npu_df.loc[index, Const.STATE]
+                api_origin_name = npu_df.loc[index, Const.API_ORIGIN_NAME].replace(ms_api, pt_api, 1)
+                if state == Const.INPUT:
                     is_abandoned = gen_input_compare_key(CompareConst.INPUT_PATTERN, 'args')
-                elif CompareConst.KWARGS_PATTERN in op_name:
+                elif state == Const.KWARGS:
                     is_abandoned = gen_input_compare_key(CompareConst.KWARGS_PATTERN, 'args')
-                elif CompareConst.OUTPUT_PATTERN in op_name:
+                elif state == Const.OUTPUT:
                     is_abandoned = gen_input_compare_key(CompareConst.OUTPUT_PATTERN, 'output')
-                elif CompareConst.PARAMS_PATTERN in op_name:
+                elif state == Const.PARAMS:
                     is_abandoned = gen_input_compare_key(CompareConst.PARAMS_PATTERN, 'parameters')
-                elif CompareConst.PARAMS_GRAD_PATTERN in op_name:
+                elif state == Const.PARAMS_GRAD:
                     is_abandoned = gen_input_compare_key(CompareConst.PARAMS_GRAD_PATTERN, 'parameters_grad')
                 else:
                     logger.error(f'Excepted op_name: {op_name}')
@@ -413,8 +445,8 @@ class Match:
     @staticmethod
     def put_unmatched_in_table(match_result, npu_op_item):
         npu_columns = npu_op_item.index.tolist()[:-2]
-        new_columns = [name[:-1] + 'y' for name in npu_columns]
-        na_series = pd.Series([CompareConst.N_A] * len(new_columns), index=new_columns)
+        bench_columns = [name + '_y' for name in npu_columns]
+        na_series = pd.Series([CompareConst.N_A] * len(bench_columns), index=bench_columns)
         new_result_item = pd.concat([npu_op_item, na_series]).to_frame().T
         new_result_item.columns = CompareConst.MATCH_RESULT_COLUMNS
         match_result = pd.concat([match_result, new_result_item])
@@ -610,12 +642,21 @@ class CreateTable:
                                'md5_x': CompareConst.NPU_MD5,
                                'md5_y': CompareConst.BENCH_MD5,
                                'data_name_x': CompareConst.DATA_NAME,
-                               'stack_info_x': CompareConst.STACK}, inplace=True)
+                               'stack_info_x': CompareConst.STACK,
+                               'state_x': Const.STATE,
+                               'api_origin_name_x': Const.API_ORIGIN_NAME,
+                               'requires_grad_x': CompareConst.NPU_REQ_GRAD,
+                               'requires_grad_y': CompareConst.BENCH_REQ_GRAD
+                               },
+                      inplace=True)
 
         # process summary data
         npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM]
         bench_summary = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN,
                          CompareConst.BENCH_NORM]
+        # process requires_grad
+        result[CompareConst.REQ_GRAD_CONSIST] = result[CompareConst.NPU_REQ_GRAD] == result[CompareConst.BENCH_REQ_GRAD]
+
         if result.empty:
             result[npu_summary] = pd.DataFrame(columns=npu_summary)
             result[bench_summary] = pd.DataFrame(columns=bench_summary)
@@ -623,6 +664,7 @@ class CreateTable:
             result[npu_summary] = result['summary_x'].apply(self.set_summary).tolist()
             result[bench_summary] = result['summary_y'].apply(self.set_summary).tolist()
 
+        header.extend([Const.STATE, Const.API_ORIGIN_NAME])
         result_df = pd.DataFrame(columns=header)
         for h in header:
             if h in result.columns:
@@ -667,13 +709,13 @@ class CalcStatsDiff:
         result_df.loc[cond_nan_diff, [diff_name, rel_err_name]] = CompareConst.NAN
 
         cond_not_nan_diff = cond_valid_stat & ~cond_diff_nan
-        condition_pt_zero = bench_val == 0
+        condition_pt_zero = self.get_number(bench_val) == 0
         result_df.loc[cond_not_nan_diff & condition_pt_zero, rel_err_name] = CompareConst.N_A
 
         # 相对误差转成百分比字符串
         cond_ref_err = cond_not_nan_diff & ~condition_pt_zero
         result_df.loc[cond_ref_err, rel_err_name] = (
-                result_df.loc[cond_ref_err, diff_name] / bench_val[cond_ref_err] * 100)
+                result_df.loc[cond_ref_err, diff_name] / bench_val[cond_ref_err].astype(float) * 100)
         result_df.loc[cond_ref_err, rel_err_name] = (result_df.loc[cond_ref_err, rel_err_name].abs().astype(str) + '%')
 
         magnitude = self.get_number(result_df[diff_name]).abs() / (pd.Series(
@@ -685,12 +727,13 @@ class CalcStatsDiff:
         condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A
         result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A)
         result_df.loc[condition_no_bench, CompareConst.ERROR_MESSAGE] = CompareConst.NO_BENCH
+        condition_req_grad_consist = result_df[CompareConst.NPU_REQ_GRAD] == result_df[CompareConst.BENCH_REQ_GRAD]
 
         if self.mode_config.dump_mode == Const.MD5:
             condition_md5_equal = result_df[CompareConst.NPU_MD5] == result_df[CompareConst.BENCH_MD5]
             result_df.loc[condition_md5_equal, CompareConst.RESULT] = CompareConst.PASS
             result_df.loc[~condition_md5_equal & ~condition_no_bench, CompareConst.RESULT] = CompareConst.DIFF
-        elif self.mode_config.dump_mode == Const.SUMMARY:
+        elif self.mode_config.first_diff_analyze or self.mode_config.dump_mode == Const.SUMMARY:
             warning_list = [
                 self.calc_summary_diff(result_df, condition_no_bench, stats_index)
                 for stats_index in ['max', 'min', 'mean', 'l2norm']
@@ -698,14 +741,16 @@ class CalcStatsDiff:
             warning_flag = pd.DataFrame(warning_list).any()
             result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = ''
             result_df.loc[warning_flag, CompareConst.RESULT] = CompareConst.WARNING
-            result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy.'
+            result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy. '
+            result_df.loc[~condition_req_grad_consist, CompareConst.ERROR_MESSAGE] += 'Requires_grad inconsistent. '
         else:
             fill_cols = [CompareConst.COSINE, CompareConst.EUC_DIST,
                          CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR,
                          CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO,
                          CompareConst.ERROR_MESSAGE]
-            result_df.loc[~condition_no_bench, fill_cols] = ''
+            result_df.loc[~condition_no_bench, fill_cols] = ''  # 默认填充'', df默认省缺值为nan，不便后续处理，容易出现意外情况
             result_df.loc[~condition_no_bench, CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES
+            result_df.loc[~condition_req_grad_consist, CompareConst.ERROR_MESSAGE] = 'Requires_grad inconsistent. '
 
         return result_df[header]
 
@@ -718,12 +763,15 @@ def setup_comparison(input_param, output_path, **kwargs) -> ComparisonConfig:
             stack_mode=False,
             auto_analyze=kwargs.get('auto_analyze', True),
             fuzzy_match=kwargs.get('fuzzy_match', False),
+            highlight=kwargs.get('highlight', False),
             data_mapping=kwargs.get('data_mapping', {}),
             suffix=kwargs.get('suffix', ''),
             cell_mapping=kwargs.get('cell_mapping', {}),
             api_mapping=kwargs.get('api_mapping', {}),
             layer_mapping=kwargs.get('layer_mapping', {}),
+            first_diff_analyze=kwargs.get('first_diff_analyze', False),
             compared_file_type='',
+            is_print_compare_log=input_param.get('is_print_compare_log', True)
         )
 
         set_dump_path(input_param)
@@ -736,8 +784,7 @@ def setup_comparison(input_param, output_path, **kwargs) -> ComparisonConfig:
         else:
             config.stack_mode = set_stack_json_path(input_param)
 
-        check_configuration_param(config.stack_mode, config.auto_analyze, config.fuzzy_match,
-                                  input_param.get('is_print_compare_log', True))
+        check_configuration_param(config)
         create_directory(output_path)
         check_compare_param(input_param, output_path, config.dump_mode, config.stack_mode)
 

msprobe/core/compare/check.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 from msprobe.core.common.log import logger
 from msprobe.core.common.utils import check_op_str_pattern_valid, CompareException
 from msprobe.core.common.const import Const
@@ -106,3 +108,14 @@ def check_stack_json_str(stack_info, op_name):
     else:
         logger.error(f"Expected stack_info to be a list, but got {type(stack_info).__name__} for '{op_name}'")
         raise CompareException(CompareException.INVALID_OBJECT_TYPE_ERROR)
+
+
+def check_configuration_param(config):
+    arg_list = [config.stack_mode, config.auto_analyze, config.fuzzy_match,
+                config.highlight, config.first_diff_analyze, config.is_print_compare_log]
+    arg_names = ['stack_mode', 'auto_analyze', 'fuzzy_match',
+                 'highlight', 'first_diff_analyze', 'is_print_compare_log']
+    for arg, name in zip(arg_list, arg_names):
+        if not isinstance(arg, bool):
+            logger.error(f"Invalid input parameter, {name} which should be only bool type.")
+            raise CompareException(CompareException.INVALID_PARAM_ERROR)

msprobe/core/compare/compare_cli.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -13,28 +13,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
+import os
+
 from msprobe.core.common.file_utils import check_file_type, load_json, check_file_or_directory_path
 from msprobe.core.common.const import FileCheckConst, Const
 from msprobe.core.common.utils import CompareException
 from msprobe.core.common.log import logger
+from msprobe.core.compare.utils import get_paired_dirs
+
 
+def compare_cli(args, depth=1):
+    if depth > 2:
+        logger.error("Recursive compare error, depth exceeds 2.")
+        raise CompareException(CompareException.RECURSION_LIMIT_ERROR)
 
-def compare_cli(args):
-    input_param = load_json(args.input_path)
+    if isinstance(args.input_path, dict):  # special for dyn-graph mix compare
+        input_param = args.input_path
+    else:
+        input_param = load_json(args.input_path)
     if not isinstance(input_param, dict):
         logger.error("input_param should be dict, please check!")
         raise CompareException(CompareException.INVALID_OBJECT_TYPE_ERROR)
+
     npu_path = input_param.get("npu_path", None)
     bench_path = input_param.get("bench_path", None)
     if not npu_path:
-        logger.error(f"Missing npu_path in configuration file {args.input_path}, please check!")
+        logger.error(f"Missing npu_path in input configuration file, please check!")
         raise CompareException(CompareException.INVALID_PATH_ERROR)
     if not bench_path:
-        logger.error(f"Missing bench_path in configuration file {args.input_path}, please check!")
+        logger.error(f"Missing bench_path in input configuration file, please check!")
         raise CompareException(CompareException.INVALID_PATH_ERROR)
+
     frame_name = args.framework
     auto_analyze = not args.compare_only
+
     if frame_name == Const.PT_FRAMEWORK:
         from msprobe.pytorch.compare.pt_compare import compare
         from msprobe.pytorch.compare.distributed_compare import compare_distributed
@@ -46,7 +58,9 @@ def compare_cli(args):
     common_kwargs = {
         "auto_analyze": auto_analyze,
         "fuzzy_match": args.fuzzy_match,
+        "highlight": args.highlight,
         "data_mapping": args.data_mapping,
+        "diff_analyze": args.diff_analyze
     }
 
     if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE:
@@ -75,6 +89,12 @@ def compare_cli(args):
     elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR:
         check_file_or_directory_path(npu_path, isdir=True)
         check_file_or_directory_path(bench_path, isdir=True)
+
+        if depth == 1:
+            mix_compare_success = mix_compare(args, input_param, depth)
+            if mix_compare_success:
+                return
+
         kwargs = {
             **common_kwargs,
             "stack_mode": args.stack_mode,
@@ -90,6 +110,13 @@ def compare_cli(args):
         if isinstance(common, bool) and common:
             common_dir_compare(input_param, args.output_path)
             return
+
+        if common_kwargs.get('diff_analyze', False):
+            logger.info("Start finding first diff node......")
+            from msprobe.core.compare.find_first.analyzer import DiffAnalyzer
+            DiffAnalyzer(npu_path, bench_path, args.output_path, frame_name).analyze()
+            return
+
         if frame_name == Const.PT_FRAMEWORK:
             compare_distributed(npu_path, bench_path, args.output_path, **kwargs)
         else:
@@ -97,3 +124,34 @@ def compare_cli(args):
     else:
         logger.error("The npu_path and bench_path need to be of the same type.")
         raise CompareException(CompareException.INVALID_COMPARE_MODE)
+
+
+def mix_compare(args, input_param, depth):
+    npu_path = input_param.get("npu_path", None)
+    bench_path = input_param.get("bench_path", None)
+
+    npu_bench_same_dirs_set = set(get_paired_dirs(npu_path, bench_path))
+    compare_cross_set = npu_bench_same_dirs_set & Const.MIX_DUMP_NAMES
+
+    if compare_cross_set:
+        logger.info("Start mix compare.")
+        origin_output = args.output_path
+
+        for folder_name in list(compare_cross_set):
+            new_npu_path = os.path.join(npu_path, folder_name)
+            new_bench_path = os.path.join(bench_path, folder_name)
+            paired_steps = get_paired_dirs(new_npu_path, new_bench_path)
+
+            for step_name in paired_steps:
+                logger.info(f"[mix compare] Start comparing {folder_name}/{step_name}")
+                npu_dir = os.path.join(new_npu_path, step_name)
+                bench_dir = os.path.join(new_bench_path, step_name)
+                args.input_path = {
+                    "npu_path": npu_dir,
+                    "bench_path": bench_dir,
+                    "is_print_compare_log": input_param.get("is_print_compare_log", True)
+                }
+                args.output_path = os.path.join(origin_output, folder_name, step_name)
+                compare_cli(args, depth + 1)
+        return True
+    return False

msprobe/core/compare/config.py

@@ -20,13 +20,15 @@ from msprobe.core.common.file_utils import load_yaml
 
 
 class ModeConfig:
-    def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.SUMMARY,
-                 compared_file_type=Const.DUMP_JSON_FILE):
-        self.stack_mode = stack_mode
-        self.auto_analyze = auto_analyze
-        self.fuzzy_match = fuzzy_match
-        self.dump_mode = dump_mode
-        self.compared_file_type = compared_file_type
+    def __init__(self, **kwargs):
+        self.stack_mode = kwargs.get('stack_mode', False)
+        self.auto_analyze = kwargs.get('auto_analyze', True)
+        self.fuzzy_match = kwargs.get('fuzzy_match', False)
+        self.highlight = kwargs.get('highlight', False)
+        self.dump_mode = kwargs.get('dump_mode', Const.SUMMARY)
+        self.first_diff_analyze = kwargs.get('first_diff_analyze', False)
+        self.diff_analyze = kwargs.get('diff_analyze', False)
+        self.compared_file_type = kwargs.get('compared_file_type', Const.DUMP_JSON_FILE)
 
 
 class MappingConfig:
@@ -69,4 +71,4 @@ class MappingDict:
         else:
             raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got "
                             f"{type(data_mapping)}")
-        return data_mapping_dict
+        return data_mapping_dict

msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml

@@ -0,0 +1,14 @@
+compare_metrics:
+  - MaxRelativeErr
+  - MinRelativeErr
+  - MeanRelativeErr
+  - NormRelativeErr
+
+MaxRelativeErr:
+  - 0.5
+MinRelativeErr:
+  - 0.5
+MeanRelativeErr:
+  - 0.5
+NormRelativeErr:
+  - 0.5