mindstudio-probe 1.1.1__py3-none-any.whl → 1.2.2__py3-none-any.whl

msprobe/core/common/const.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -29,6 +29,7 @@ class Const:
     SEP = "."
     REGEX_PREFIX_MAX_LENGTH = 20
     REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$"
+    REGEX_FORWARD_BACKWARD = r'\.(forward|backward)\.'
     FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
     STRING_BLACKLIST = r"^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]"
     COMMA = ","
@@ -65,6 +66,7 @@ class Const:
     ONLINE_DUMP_MODE = [ALL, LIST, AUTO, OFF]
     SUMMARY = "summary"
     MD5 = "md5"
+    VALUE = "value"
     SUMMARY_MODE = [ALL, SUMMARY, MD5]
 
     WRITE_FLAGS = os.O_WRONLY | os.O_CREAT
@@ -73,6 +75,7 @@ class Const:
 
     PKL_SUFFIX = ".pkl"
     NUMPY_SUFFIX = ".npy"
+    NUMPY_PATTERN = "*.npy"
     PT_SUFFIX = ".pt"
     ONE_GB = 1073741824  # 1 * 1024 * 1024 * 1024
     TEN_GB = 10737418240  # 10 * 1024 * 1024 * 1024
@@ -87,6 +90,8 @@ class Const:
     INPUT_KWARGS = 'input_kwargs'
     GRAD_INPUT = 'grad_input'
     GRAD_OUTPUT = 'grad_output'
+    PARAMS = 'parameters'
+    PARAMS_GRAD = 'parameters_grad'
     START = "start"
     STOP = "stop"
     ENV_ENABLE = "1"
@@ -98,20 +103,23 @@ class Const:
     FREE_BENCHMARK = "free_benchmark"
     RUN_UT = "run_ut"
     GRAD_PROBE = "grad_probe"
-    TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE]
-    DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR]
+    STRUCTURE = "structure"
+    TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE, STRUCTURE]
+    DUMP_DATA_COLLECTION_LIST = [STATISTICS, TENSOR, STRUCTURE]
     DUMP_DATA_MODE_LIST = [ALL, INPUT, OUTPUT, FORWARD, BACKWARD]
     LEVEL_L0 = "L0"
     LEVEL_L1 = "L1"
     LEVEL_L2 = "L2"
     LEVEL_MIX = "mix"
-    LEVEL_LIST = [LEVEL_L0, LEVEL_L1, LEVEL_L2, LEVEL_MIX]
+    LEVEL_DEBUG = "debug"
+    LEVEL_LIST = [LEVEL_L0, LEVEL_L1, LEVEL_L2, LEVEL_MIX, LEVEL_DEBUG]
     ATTR_NAME_PREFIX = "wrap_"
     ATTR_NAME_PREFIX_LEN = len(ATTR_NAME_PREFIX)
     KERNEL_DUMP = "kernel_dump"
     DATA = "data"
     PT_FRAMEWORK = "pytorch"
     MS_FRAMEWORK = "mindspore"
+    MT_FRAMEWORK = "mindtorch"
     UNKNOWN_FRAMEWORK = "unknown"
     DIRECTORY_LENGTH = 4096
     FILE_NAME_LENGTH = 255
@@ -122,9 +130,12 @@ class Const:
     NPU_LOWERCASE = 'npu'
     CPU_LOWERCASE = 'cpu'
     CUDA_LOWERCASE = 'cuda'
+    DEVICE = 'device'
     DISTRIBUTED = 'Distributed'
-    DUMP_PREFIX = ["Distributed", "Functional", "Torch", "Tensor", "Mint", "MintFunctional", "Primitive", 
+    DUMP_PREFIX = ["Distributed", "Functional", "Torch", "Tensor", "Mint", "MintFunctional", "Primitive",
                    "Aten", "VF", "NPU", "Jit"]
+    MODULE_PREFIX = ["Module", "Cell"]
+    FORWARD_NAME_SUFFIX = ".forward"
 
     # struct json param
     ORIGIN_DATA = "origin_data"
@@ -145,10 +156,13 @@ class Const:
     SCOPE_ID_INDEX = -1
     SCOPE_DIRECTION_INDEX = -2
     TYPE_NAME_INDEX = -3
+    PARAMS_GRAD_TYPE_NAME_INDEX = -2
     LAYER_NAME_INDEX = -4
+    PARAMS_GRAD_NAME_INDEX = -3
     API_TYPE_INDEX = 0
     LEFT_MOVE_INDEX = -1
     RIGHT_MOVE_INDEX = 1
+    LAST_INDEX = -1
 
     TOP_LAYER = "TopLayer"
     CELL = "Cell"
@@ -162,12 +176,16 @@ class Const:
 
     CONVERT = {
         "int32_to_int64": ["torch.int32", "torch.int64"],
+        "int64_to_fp32": ["torch.int64", "torch.float32"]
     }
 
     CONVERT_API = {
-        "int32_to_int64": ["cross_entropy"]
+        "int32_to_int64": ["cross_entropy"],
+        "int64_to_fp32": ["histc"]
     }
 
+    FA_SPECIAL_SPARSE_MODE = [2, 3, 4]
+
     FILL_CHAR_NUMS = 50
     TOOL_ENDS_SUCCESSFULLY = f"{TOOL_NAME} ends successfully."
     WITHOUT_CALL_STACK = "The call stack retrieval failed."
@@ -179,6 +197,8 @@ class Const:
     STEP_RANK_MAXIMUM_VALUE = int(1e6)
 
     # data type const
+    TORCH_INT_DTYPE = ["torch.int8", "torch.int32", "torch.int64"]
+    TORCH_FLOAT_DTYPE = ["torch.bfloat16", "torch.float16", "torch.float32", "torch.float64"]
     FLOAT16 = "Float16"
     FLOAT32 = "Float32"
     BFLOAT16 = "BFloat16"
@@ -193,6 +213,23 @@ class Const:
     MEAN = 'Mean'
     NORM = 'Norm'
 
+    CODE_STACK = 'Code Stack'
+    OP_NAME = 'Op Name'
+    SCOPE_NAME = 'Scope Name'
+    CODE_STACKS = 'Code Stacks'
+    FILE_PATH = 'File Path'
+    NEW_LINE = '\n'
+    CSV_NEWLINE_SEPARATOR = ',\n'
+    # 分隔符常量
+    SCOPE_SEPARATOR = "/"
+    REPLACEMENT_CHARACTER = "_"
+
+    OPTIMIZER = "optimizer"
+    CLIP_GRAD = "clip_grad"
+    END_PREFIX = "end_"
+
+    TENSOR_STAT_LEN = 2
+
 
 class CompareConst:
     """
@@ -239,13 +276,58 @@ class CompareConst:
     INPUT_STRUCT = "input_struct"
     KWARGS_STRUCT = "kwargs_struct"
     OUTPUT_STRUCT = "output_struct"
+    PARAMS_STRUCT = "params_struct"
+    PARAMS_GRAD_STRUCT = "params_grad_struct"
     SUMMARY = "summary"
+    COMPARE_RESULT = "compare_result"
+    COMPARE_MESSAGE = "compare_message"
     MAX_EXCEL_LENGTH = 1048576
     YES = "Yes"
     NO = "No"
     STATISTICS_INDICATOR_NUM = 4
     EPSILON = 1e-10
     COMPARE_ENDS_SUCCESSFULLY = "msprobe compare ends successfully."
+    DEFAULT_RATIO_VALUE = 10000
+    THOUSANDTH_PASS_VALUE = 0.999
+    ZERO_SHAPE = '(0,)'
+
+    BENCHMARK_COMPARE_ALGORITHM_NAME = "标杆比对法"
+    ULP_COMPARE_ALGORITHM_NAME = "ULP误差比对法"
+    BINARY_CONSISTENCY_ALGORITHM_NAME = "二进制一致法"
+    ABSOLUTE_THRESHOLD_ALGORITHM_NAME = "绝对阈值法"
+    THOUSANDTH_STANDARD_ALGORITHM_NAME = "双千指标法"
+    ACCUMULATIVE_ERROR_COMPARE_ALGORITHM_NAME = "累积误差比对法"
+
+    ABSOLUTE_THRESHOLD = 'absolute_threshold'
+    BINARY_CONSISTENCY = 'binary_consistency'
+    ULP_COMPARE = 'ulp_compare'
+    THOUSANDTH_STANDARD = 'thousandth_threshold'
+    BENCHMARK = 'benchmark'
+    ACCUMULATIVE_ERROR_COMPARE = 'accumulative_error_compare'
+
+    SMALL_VALUE_ERR_RATIO = "small_value_err_ratio"
+    RMSE_RATIO = "rmse_ratio"
+    MAX_REL_ERR_RATIO = "max_rel_err_ratio"
+    MEAN_REL_ERR_RATIO = "mean_rel_err_ratio"
+    EB_RATIO = "eb_ratio"
+
+    SMALL_VALUE = "small_value"
+    RMSE = "rmse"
+    MAX_REL_ERR = "max_rel_err"
+    MEAN_REL_ERR = "mean_rel_err"
+    EB = "eb"
+
+    SMALL_VALUE_ERR_STATUS = "small_value_err_status"
+    RMSE_STATUS = "rmse_status"
+    MAX_REL_ERR_STATUS = "max_rel_err_status"
+    MEAN_REL_ERR_STATUS = "mean_rel_err_status"
+    EB_STATUS = "eb_status"
+
+    MEAN_ULP_ERR = "mean_ulp_err"
+    ULP_ERR_PROPORTION = "ulp_err_proportion"
+    ULP_ERR_PROPORTION_RATIO = "ulp_err_proportion_ratio"
+
+    ULP_ERR_STATUS = "ulp_err_status"
 
     COMPARE_RESULT_HEADER = [
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR,
@@ -263,12 +345,57 @@ class CompareConst:
         NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_MD5, BENCH_MD5, RESULT
     ]
 
+    COMPARE_RESULT_HEADER_STACK = COMPARE_RESULT_HEADER + [STACK]
+
+    SUMMARY_COMPARE_RESULT_HEADER_STACK = SUMMARY_COMPARE_RESULT_HEADER + [STACK]
+
+    MD5_COMPARE_RESULT_HEADER_STACK = MD5_COMPARE_RESULT_HEADER + [STACK]
+
     HEAD_OF_COMPARE_MODE = {
         Const.ALL: COMPARE_RESULT_HEADER,
         Const.SUMMARY: SUMMARY_COMPARE_RESULT_HEADER,
         Const.MD5: MD5_COMPARE_RESULT_HEADER
     }
 
+    ALL_COMPARE_INDEX = [COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR, ONE_THOUSANDTH_ERR_RATIO, FIVE_THOUSANDTHS_ERR_RATIO]
+    SUMMARY_COMPARE_INDEX = [MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF,
+                             MAX_RELATIVE_ERR, MIN_RELATIVE_ERR, MEAN_RELATIVE_ERR, NORM_RELATIVE_ERR]
+
+    # dtype match
+    MS_TYPE = [
+        [Const.FLOAT16, Const.FLOAT32], [Const.FLOAT32, Const.FLOAT16],
+        [Const.FLOAT16, Const.BFLOAT16], [Const.BFLOAT16, Const.FLOAT16]
+    ]
+    TORCH_TYPE = [
+        [Const.TORCH_FLOAT16, Const.TORCH_FLOAT32], [Const.TORCH_FLOAT32, Const.TORCH_FLOAT16],
+        [Const.TORCH_FLOAT16, Const.TORCH_BFLOAT16], [Const.TORCH_BFLOAT16, Const.TORCH_FLOAT16]
+    ]
+
+    # read_op
+    IO_NAME_MAPPING = {
+        Const.INPUT_ARGS: '.input',
+        Const.INPUT_KWARGS: '.input',
+        Const.INPUT: '.input',
+        Const.OUTPUT: '.output',
+        Const.PARAMS: '.parameters'
+    }
+
+    # state to struct mapping
+    STATE_TO_STRUCT_MAPPING = {
+        Const.INPUT: INPUT_STRUCT,
+        Const.KWARGS: INPUT_STRUCT,
+        Const.OUTPUT: OUTPUT_STRUCT,
+        Const.PARAMS: PARAMS_STRUCT,
+        Const.PARAMS_GRAD: PARAMS_GRAD_STRUCT
+    }
+
+    STRUCT_COMPARE_KEY = [
+        INPUT_STRUCT,
+        OUTPUT_STRUCT,
+        PARAMS_STRUCT,
+        PARAMS_GRAD_STRUCT
+    ]
+
     # compare standard
     HUNDRED_RATIO_THRESHOLD = 0.01
     THOUSAND_RATIO_THRESHOLD = 0.001
@@ -350,6 +477,8 @@ class CompareConst:
     INPUT_PATTERN = Const.SEP + Const.INPUT + Const.SEP
     KWARGS_PATTERN = Const.SEP + Const.KWARGS + Const.SEP
     OUTPUT_PATTERN = Const.SEP + Const.OUTPUT + Const.SEP
+    PARAMS_PATTERN = Const.SEP + Const.PARAMS + Const.SEP
+    PARAMS_GRAD_PATTERN = Const.SEP + Const.PARAMS_GRAD + Const.SEP
     COMPARE_KEY = 'compare_key'
     COMPARE_SHAPE = 'compare_shape'
     INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml'
@@ -372,13 +501,17 @@ class FileCheckConst:
     JSON_SUFFIX = ".json"
     PT_SUFFIX = ".pt"
     CSV_SUFFIX = ".csv"
+    XLSX_SUFFIX = ".xlsx"
     YAML_SUFFIX = ".yaml"
+    IR_SUFFIX = ".ir"
     MAX_PKL_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_NUMPY_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_PT_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
     MAX_CSV_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_XLSX_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     MAX_YAML_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_IR_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
     COMMOM_FILE_SIZE = 1048576  # 1 * 1024 * 1024
     DIR = "dir"
     FILE = "file"
@@ -390,7 +523,9 @@ class FileCheckConst:
         JSON_SUFFIX: MAX_JSON_SIZE,
         PT_SUFFIX: MAX_PT_SIZE,
         CSV_SUFFIX: MAX_CSV_SIZE,
-        YAML_SUFFIX: MAX_YAML_SIZE
+        XLSX_SUFFIX: MAX_XLSX_SIZE,
+        YAML_SUFFIX: MAX_YAML_SIZE,
+        IR_SUFFIX: MAX_IR_SIZE
     }
     CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]'
 
@@ -410,9 +545,24 @@ class MsCompareConst:
     TENSOR_API = "Tensor"
 
     API_NAME_STR_LENGTH = 4
+    MAX_RECURSION_DEPTH = 20
+
+    # Mindtorch api_info field
+    MINDTORCH_TENSOR = "Tensor"
+    MINDTORCH = "Torch"
+    MINDTORCH_FUNC = "Functional"
+    MINDTORCH_NPU = "NPU"
+    MINDTORCH_DIST = "Distributed"
+
+
+
+    MT_VALID_API_TYPES = [
+        MINDTORCH, MINDTORCH_FUNC, MINDTORCH_TENSOR
+    ]
 
     TASK_FIELD = "task"
     STATISTICS_TASK = "statistics"
+    FRAMEWORK = "framework"
     TENSOR_TASK = "tensor"
     DUMP_DATA_DIR_FIELD = "dump_data_dir"
     DATA_FIELD = "data"
@@ -437,6 +587,11 @@ class MsCompareConst:
 
     EPSILON = 1e-8
 
+    class ProcessStatus:
+        SUCCESS = "success"
+        API_NOT_FOUND = "api_not_found"
+        EXCEPTION_SKIP = "exception_skip"
+
 
 class MsgConst:
     """
@@ -474,29 +629,48 @@ class MonitorConst:
     """
     Class for monitor const
     """
-    OP_LIST = ["min", "max", "norm", "zeros", "nans", "id", "mean"]
+    OP_LIST = ["norm", "min", "max", "zeros", "nans", "id", "mean"]
     MONITOR_OUTPUT_DIR = "MONITOR_OUTPUT_DIR"
     DEFAULT_MONITOR_OUTPUT_DIR = "./monitor_output"
     DATABASE = "database"
     EMAIL = "email"
     OPT_TY = ['Megatron_DistributedOptimizer', 'Megatron_Float16OptimizerWithFloat16Params']
-    DEEPSPEED_OPT_TY = ("DeepSpeedZeroOptimizer_Stage0", "DeepSpeedZeroOptimizer_Stage1_or_2", "DeepSpeedZeroOptimizer_Stage3")
+    DEEPSPEED_OPT_TY = (
+        "DeepSpeedZeroOptimizer_Stage0",
+        "DeepSpeedZeroOptimizer_Stage1_or_2",
+        "DeepSpeedZeroOptimizer_Stage3"
+    )
+    DEEPSPEED_ZERO_OPT_FILTER = "DeepSpeedZeroOptimizer"
     RULE_NAME = ['AnomalyTurbulence']
 
+    SLICE_SIZE = 20480
+    # used for name
     DOT = "."
-    VPP_SEP = ":"
+    NAME_SEP = ":"
+    INPUT_GRAD = "input_grad"
+    OUTPUT_GRAD = "output_grad"
     ACTV_IN = "input"
     ACTV_OUT = "output"
     ACTVGRAD_IN = "input_grad"
     ACTVGRAD_OUT = "output_grad"
+    # used for tasks
+    ACTV = "actv"
+    ACTVGRAD = "actv_grad"
     POST_GRAD = "post_grad"
     PRE_GRAD = "pre_grad"
+    ACC_GRAD = "acc_grad"
     PREFIX_POST = "post"
     PREFIX_PRE = "pre"
+    EXP_AVG = "exp_avg"
+    EXP_AVG_SQ = "exp_avg_sq"
+    PARAM = "param"
 
-
+    CSV_HEADER = ["vpp_stage", "name", "step"]
+    CSV_HEADER_XY = ["vpp_stage", "name", "step", "micro_step"]
+    OUTPUT_DIR_PATTERN = r"([\w-]{0,20})-rank(\d{1,5})-"
     ANOMALY_JSON = "anomaly.json"
     ANALYSE_JSON = "anomaly_analyse.json"
     TENSORBOARD = "tensorboard"
     CSV = "csv"
     API = "api"
+    HEADER_NAME = 'name'

msprobe/core/common/exceptions.py

@@ -27,11 +27,13 @@ class MsprobeException(CodedException):
     INVALID_PARAM_ERROR = 0
     OVERFLOW_NUMS_ERROR = 1
     RECURSION_LIMIT_ERROR = 2
+    INTERFACE_USAGE_ERROR = 3
 
     err_strs = {
         INVALID_PARAM_ERROR: "[msprobe] 无效参数：",
         OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：",
-        RECURSION_LIMIT_ERROR: "[msprobe] 递归调用超过限制："
+        RECURSION_LIMIT_ERROR: "[msprobe] 递归调用超过限制：",
+        INTERFACE_USAGE_ERROR: "[msprobe] Invalid interface usage: "
     }
 
 

msprobe/core/common/file_utils.py

@@ -22,7 +22,6 @@ import re
 import shutil
 from datetime import datetime, timezone
 from dateutil import parser
-import OpenSSL
 import yaml
 import numpy as np
 import pandas as pd
@@ -419,20 +418,36 @@ def save_yaml(yaml_path, data):
 
 
 def save_excel(path, data):
+    def validate_data(data):
+        """Validate that the data is a DataFrame or a list of (DataFrame, sheet_name) pairs."""
+        if isinstance(data, pd.DataFrame):
+            return "single"
+        elif isinstance(data, list):
+            if all(isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], pd.DataFrame) for item in data):
+                return "list"
+        raise ValueError("Data must be a DataFrame or a list of (DataFrame, sheet_name) pairs.")
+
     check_path_before_create(path)
     path = os.path.realpath(path)
+
+    # 验证数据类型
+    data_type = validate_data(data)
+
     try:
-        if isinstance(data, pd.DataFrame):
+        if data_type == "single":
             data.to_excel(path, index=False)
-        else:
-            logger.error(f'unsupported data type.')
-            return
+        elif data_type == "list":
+            with pd.ExcelWriter(path) as writer:
+                for data_df, sheet_name in data:
+                    data_df.to_excel(writer, sheet_name=sheet_name, index=False)
     except Exception as e:
         logger.error(f'Save excel file "{os.path.basename(path)}" failed.')
         raise RuntimeError(f"Save excel file {path} failed.") from e
     change_mode(path, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
+
+
 def move_file(src_path, dst_path):
     check_file_or_directory_path(src_path)
     check_path_before_create(dst_path)
@@ -522,11 +537,11 @@ def write_csv(data, filepath, mode="a+", malicious_check=False):
     change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
 
 
-def read_csv(filepath, as_pd=True):
+def read_csv(filepath, as_pd=True, header='infer'):
     check_file_or_directory_path(filepath)
     try:
         if as_pd:
-            csv_data = pd.read_csv(filepath)
+            csv_data = pd.read_csv(filepath, header=header)
         else:
             with FileOpen(filepath, 'r', encoding='utf-8-sig') as f:
                 csv_reader = csv.reader(f, delimiter=',')
@@ -630,6 +645,7 @@ def check_crt_valid(pem_path):
     Raises:
     RuntimeError: If the SSL certificate is invalid or expired.
     """
+    import OpenSSL
     try:
         with FileOpen(pem_path, "r") as f:
             pem_data = f.read()
@@ -645,3 +661,13 @@ def check_crt_valid(pem_path):
     now_utc = datetime.now(tz=timezone.utc)
     if cert.has_expired() or not (pem_start <= now_utc <= pem_end):
         raise RuntimeError(f"The SSL certificate has expired and needs to be replaced, {pem_path}")
+
+
+def read_xlsx(file_path):
+    check_file_or_directory_path(file_path)
+    try:
+        result_df = pd.read_excel(file_path, keep_default_na=False)
+    except Exception as e:
+        logger.error(f"The xlsx file failed to load. Please check the path: {file_path}.")
+        raise RuntimeError(f"Read xlsx file {file_path} failed.") from e
+    return result_df

msprobe/core/common/inplace_ops.yaml

@@ -157,6 +157,9 @@ inplace_tensor_op:
   - trunc_
   - unsqueeze_
   - xlogy_
+  - bitwise_left_shift_
+  - bitwise_right_shift_
+  - arctan2_
 
 inplace_torch_op:
   - _add_relu_
@@ -247,5 +250,6 @@ inplace_distributed_op:
   - all_to_all
   - all_gather_into_tensor
   - reduce_scatter_tensor
+  - batch_isend_irecv
 
 

msprobe/core/common/utils.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -29,6 +29,7 @@ from msprobe.core.common.const import Const, CompareConst
 from msprobe.core.common.log import logger
 from msprobe.core.common.exceptions import MsprobeException
 
+
 device = collections.namedtuple('device', ['type', 'index'])
 prefixes = ['api_stack', 'list', 'range', 'acl']
 
@@ -71,6 +72,9 @@ class MsprobeBaseException(Exception):
     BACKWARD_DATA_COLLECTION_ERROR = 30
     INVALID_KEY_ERROR = 31
     MISSING_HEADER_ERROR = 32
+    MERGE_COMPARE_RESULT_ERROR = 33
+    NAMES_STRUCTS_MATCH_ERROR = 34
+    INVALID_STATE_ERROR = 35
 
     def __init__(self, code, error_info: str = ""):
         super(MsprobeBaseException, self).__init__()
@@ -109,7 +113,7 @@ def is_json_file(file_path):
         return False
 
 
-def check_compare_param(input_param, output_path, dump_mode):
+def check_compare_param(input_param, output_path, dump_mode, stack_mode):
     if not isinstance(input_param, dict):
         logger.error(f"Invalid input parameter 'input_param', the expected type dict but got {type(input_param)}.")
         raise CompareException(CompareException.INVALID_PARAM_ERROR)
@@ -127,7 +131,8 @@ def check_compare_param(input_param, output_path, dump_mode):
 
     check_json_path("npu_json_path")
     check_json_path("bench_json_path")
-    check_json_path("stack_json_path")
+    if stack_mode:
+        check_json_path("stack_json_path")
 
     if dump_mode == Const.ALL:
         check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True)
@@ -135,9 +140,12 @@ def check_compare_param(input_param, output_path, dump_mode):
     check_file_or_directory_path(output_path, True)
 
     with FileOpen(input_param.get("npu_json_path"), "r") as npu_json, \
-            FileOpen(input_param.get("bench_json_path"), "r") as bench_json, \
-            FileOpen(input_param.get("stack_json_path"), "r") as stack_json:
-        check_json_file(input_param, npu_json, bench_json, stack_json)
+            FileOpen(input_param.get("bench_json_path"), "r") as bench_json:
+        _check_json(npu_json, input_param.get("npu_json_path"))
+        _check_json(bench_json, input_param.get("bench_json_path"))
+    if stack_mode:
+        with FileOpen(input_param.get("stack_json_path"), "r") as stack_json:
+            _check_json(stack_json, input_param.get("stack_json_path"))
 
 
 def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False, is_print_compare_log=True):
@@ -231,6 +239,8 @@ def md5_find(data):
                 for data_detail in data[key_op][api_info]:
                     if data_detail and 'md5' in data_detail:
                         return True
+            if isinstance(data[key_op][api_info], bool):
+                continue
             elif data[key_op][api_info] and 'md5' in data[key_op][api_info]:
                 return True
     return False
@@ -295,6 +305,9 @@ def get_dump_mode(input_param):
     if npu_task == Const.TENSOR:
         return Const.ALL
 
+    if npu_task == Const.STRUCTURE:
+        return Const.STRUCTURE
+
     if npu_task == Const.STATISTICS:
         npu_md5_compare = md5_find(npu_json_data['data'])
         bench_md5_compare = md5_find(bench_json_data['data'])
@@ -395,20 +408,23 @@ def get_real_step_or_rank(step_or_rank_input, obj):
         if not is_int(element) and not isinstance(element, str):
             raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
                                    f"{obj} element {element} must be an integer or string.")
-        if isinstance(element, int) and element < 0:
-            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
-                                   f"Each element of {obj} must be non-negative, currently it is {element}.")
-        if isinstance(element, int) and Const.STEP_RANK_MINIMUM_VALUE <= element <= Const.STEP_RANK_MAXIMUM_VALUE:
+        if is_int(element):
+            if not Const.STEP_RANK_MINIMUM_VALUE <= element <= Const.STEP_RANK_MAXIMUM_VALUE:
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR,
+                    f"Each element of {obj} must be between {Const.STEP_RANK_MINIMUM_VALUE} and "
+                    f"{Const.STEP_RANK_MAXIMUM_VALUE}, currently it is {element}."
+                )
             real_step_or_rank.append(element)
-        elif isinstance(element, str) and Const.HYPHEN in element:
-            continual_step_or_rank = get_step_or_rank_from_string(element, obj)
-            real_step_or_rank.extend(continual_step_or_rank)
+            continue
+        continual_step_or_rank = get_step_or_rank_from_string(element, obj)
+        real_step_or_rank.extend(continual_step_or_rank)
     real_step_or_rank = list(set(real_step_or_rank))
     real_step_or_rank.sort()
     return real_step_or_rank
 
 
-def check_seed_all(seed, mode):
+def check_seed_all(seed, mode, rm_dropout):
     if is_int(seed):
         if seed < 0 or seed > Const.MAX_SEED_VALUE:
             logger.error(f"Seed must be between 0 and {Const.MAX_SEED_VALUE}.")
@@ -419,6 +435,9 @@ def check_seed_all(seed, mode):
     if not isinstance(mode, bool):
         logger.error("seed_all mode must be bool.")
         raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
+    if not isinstance(rm_dropout, bool):
+        logger.error("The rm_dropout parameter must be bool.")
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
 
 
 def safe_get_value(container, index, container_name, key=None):
@@ -482,3 +501,12 @@ def check_str_param(param):
     if not re.match(Const.REGEX_PREFIX_PATTERN, param):
         logger.error('The parameter {} contains special characters.'.format(param))
         raise MsprobeBaseException(MsprobeBaseException.INVALID_CHAR_ERROR)
+
+
+class DumpPathAggregation:
+    dump_file_path = None
+    stack_file_path = None
+    construct_file_path = None
+    dump_tensor_data_dir = None
+    free_benchmark_file_path = None
+    debug_file_path = None

msprobe/core/common_config.py

@@ -27,6 +27,7 @@ class CommonConfig:
         self.step = get_real_step_or_rank(json_config.get('step'), Const.STEP)
         self.level = json_config.get('level')
         self.enable_dataloader = json_config.get('enable_dataloader', False)
+        self.async_dump = json_config.get("async_dump", False)
         self._check_config()
 
     def _check_config(self):
@@ -42,6 +43,11 @@ class CommonConfig:
         if not isinstance(self.enable_dataloader, bool):
             logger.error_log_with_exp("enable_dataloader is invalid, it should be a boolean",
                                       MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
+        if not isinstance(self.async_dump, bool):
+            logger.error_log_with_exp("async_dump is invalid, it should be a boolean",
+                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
+        elif self.async_dump:
+            logger.warning("async_dump is True, it may cause OOM when dumping large tensor.")
 
 
 class BaseConfig: