mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/core/common/utils.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-# Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,28 +12,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
+
 import collections
 import os
 import re
 import subprocess
 import time
-import json
+from collections import defaultdict
 from datetime import datetime, timezone
+from functools import wraps
 
-from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path)
+import numpy as np
+
+from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path, load_json)
 from msprobe.core.common.const import Const, CompareConst
 from msprobe.core.common.log import logger
-
+from msprobe.core.common.exceptions import MsprobeException
 
 device = collections.namedtuple('device', ['type', 'index'])
 prefixes = ['api_stack', 'list', 'range', 'acl']
 
 
-class CompareException(Exception):
+class MsprobeBaseException(Exception):
     """
-    Class for Accuracy Compare Exception
+    Base class for all custom exceptions.
     """
+    # 所有的错误代码
     NONE_ERROR = 0
     INVALID_PATH_ERROR = 1
     OPEN_FILE_ERROR = 2
@@ -57,10 +60,20 @@ class CompareException(Exception):
     INVALID_SUMMARY_MODE = 19
     INVALID_TASK_ERROR = 20
     DETACH_ERROR = 21
-
+    INVALID_OBJECT_TYPE_ERROR = 22
+    INVALID_CHAR_ERROR = 23
+    RECURSION_LIMIT_ERROR = 24
+    INVALID_ATTRIBUTE_ERROR = 25
+    OUTPUT_HOOK_ERROR = 26
+    INPUT_HOOK_ERROR = 27
+    FUNCTION_CALL_ERROR = 28
+    FORWARD_DATA_COLLECTION_ERROR = 29
+    BACKWARD_DATA_COLLECTION_ERROR = 30
+    INVALID_KEY_ERROR = 31
+    MISSING_HEADER_ERROR = 32
 
     def __init__(self, code, error_info: str = ""):
-        super(CompareException, self).__init__()
+        super(MsprobeBaseException, self).__init__()
         self.code = code
         self.error_info = error_info
 
@@ -68,80 +81,55 @@ class CompareException(Exception):
         return self.error_info
 
 
-class DumpException(CompareException):
-    pass
-
-
-def check_mode_valid(mode, scope=None, api_list=None):
-    if scope is None:
-        scope = []
-    if api_list is None:
-        api_list = []
-    if not isinstance(scope, list):
-        raise ValueError("scope param set invalid, it's must be a list.")
-    if not isinstance(api_list, list):
-        raise ValueError("api_list param set invalid, it's must be a list.")
-    mode_check = {
-        Const.ALL: lambda: None,
-        Const.RANGE: lambda:  ValueError("set_dump_switch, scope param set invalid, it's must be [start, end].") if len(scope) != 2 else None,
-        Const.LIST: lambda:  ValueError("set_dump_switch, scope param set invalid, it's should not be an empty list.") if len(scope) == 0 else None,
-        Const.STACK: lambda:  ValueError("set_dump_switch, scope param set invalid, it's must be [start, end] or [].") if len(scope) > 2 else None,
-        Const.ACL: lambda:  ValueError("set_dump_switch, scope param set invalid, only one api name is supported in acl mode.") if len(scope) != 1 else None,
-        Const.API_LIST: lambda:  ValueError("Current dump mode is 'api_list', but the content of api_list parameter is empty or valid.") if len(api_list) < 1 else None,
-        Const.API_STACK: lambda: None,
-    }
-    if mode not in Const.DUMP_MODE:
-        msg = "Current mode '%s' is not supported. Please use the field in %s" % \
-              (mode, Const.DUMP_MODE)
-        raise CompareException(CompareException.INVALID_DUMP_MODE, msg)
-
-    if mode_check.get(mode)() is not None:
-        raise mode_check.get(mode)()
-
-
-def check_switch_valid(switch):
-    if switch not in ["ON", "OFF"]:
-        logger.error("Please set switch with 'ON' or 'OFF'.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
+class CompareException(MsprobeBaseException):
+    """
+    Class for Accuracy Compare Exception
+    """
 
+    def __init__(self, code, error_info: str = ""):
+        super(CompareException, self).__init__(code, error_info)
 
-def check_dump_mode_valid(dump_mode):
-    if not isinstance(dump_mode, list):
-        logger.warning("Please set dump_mode as a list.")
-        dump_mode = [dump_mode]
-    if not all(mode in ["all", "forward", "backward", "input", "output"] for mode in dump_mode):
-        raise ValueError("Please set dump_mode as a list containing one or more of the following: 'all', 'forward', 'backward', 'input', 'output'.")
-    if 'input' not in dump_mode and 'output' not in dump_mode:
-        dump_mode.extend(['input', 'output'])
-    if 'forward' not in dump_mode and 'backward' not in dump_mode:
-        dump_mode.extend(['forward', 'backward'])
-    if 'all' in dump_mode or set(["forward", "backward", "input", "output"]).issubset(set(dump_mode)):
-        return ["forward", "backward", "input", "output"]
-    return dump_mode
 
+class DumpException(MsprobeBaseException):
+    """
+    Class for Dump Exception
+    """
 
-def check_summary_mode_valid(summary_mode):
-    if summary_mode not in Const.SUMMARY_MODE:
-        msg = "The summary_mode is not valid"
-        raise CompareException(CompareException.INVALID_SUMMARY_MODE, msg)
+    def __init__(self, code, error_info: str = ""):
+        super(DumpException, self).__init__(code, error_info)
 
+    def __str__(self):
+        return f"Dump Error Code {self.code}: {self.error_info}"
 
-def check_summary_only_valid(summary_only):
-    if not isinstance(summary_only, bool):
-        logger.error("Params summary_only only support True or False.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    return summary_only
+
+def is_json_file(file_path):
+    if isinstance(file_path, str) and file_path.lower().endswith('.json'):
+        return True
+    else:
+        return False
 
 
-def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False):
-    if not (isinstance(input_param, dict) and isinstance(output_path, str)):
-        logger.error("Invalid input parameters")
+def check_compare_param(input_param, output_path, dump_mode):
+    if not isinstance(input_param, dict):
+        logger.error(f"Invalid input parameter 'input_param', the expected type dict but got {type(input_param)}.")
         raise CompareException(CompareException.INVALID_PARAM_ERROR)
+    if not isinstance(output_path, str):
+        logger.error(f"Invalid input parameter 'output_path', the expected type str but got {type(output_path)}.")
+        raise CompareException(CompareException.INVALID_PARAM_ERROR)
+
+    def check_json_path(json_path_str):
+        json_path = input_param.get(json_path_str)
+        check_file_or_directory_path(json_path, False)
+        json_type_check = is_json_file(json_path)
+        if not json_type_check:
+            logger.error(f"Invalid {json_path_str}: {json_path}, please check!")
+            raise CompareException(CompareException.INVALID_PATH_ERROR)
+
+    check_json_path("npu_json_path")
+    check_json_path("bench_json_path")
+    check_json_path("stack_json_path")
 
-    check_file_or_directory_path(input_param.get("npu_json_path"), False)
-    check_file_or_directory_path(input_param.get("bench_json_path"), False)
-    check_file_or_directory_path(input_param.get("stack_json_path"), False)
-    if not summary_compare and not md5_compare:
+    if dump_mode == Const.ALL:
         check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True)
         check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True)
     check_file_or_directory_path(output_path, True)
@@ -152,15 +140,12 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com
         check_json_file(input_param, npu_json, bench_json, stack_json)
 
 
-
-def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False):
-    if not (isinstance(stack_mode, bool) and isinstance(auto_analyze, bool) and isinstance(fuzzy_match, bool)):
-        logger.error("Invalid input parameters which should be only bool type.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-
-
-def is_starts_with(string, prefix_list):
-    return any(string.startswith(prefix) for prefix in prefix_list)
+def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False, is_print_compare_log=True):
+    arg_list = [stack_mode, auto_analyze, fuzzy_match, is_print_compare_log]
+    for arg in arg_list:
+        if not isinstance(arg, bool):
+            logger.error(f"Invalid input parameter, {arg} which should be only bool type.")
+            raise CompareException(CompareException.INVALID_PARAM_ERROR)
 
 
 def _check_json(json_file_handle, file_name):
@@ -198,28 +183,6 @@ def check_regex_prefix_format_valid(prefix):
         raise ValueError(f"prefix contains invalid characters, prefix pattern {Const.REGEX_PREFIX_PATTERN}")
 
 
-def get_dump_data_path(dump_dir):
-    """
-    Function Description:
-        traverse directories and obtain the absolute path of dump data
-    Parameter:
-        dump_dir: dump data directory
-    Return Value:
-        dump data path,file is exist or file is not exist
-    """
-    dump_data_path = None
-    file_is_exist = False
-
-    check_file_or_directory_path(dump_dir, True)
-    for dir_path, _, files in os.walk(dump_dir):
-        if len(files) != 0:
-            dump_data_path = dir_path
-            file_is_exist = True
-            break
-        dump_data_path = dir_path
-    return dump_data_path, file_is_exist
-
-
 def execute_command(cmd):
     """
     Function Description:
@@ -235,28 +198,12 @@ def execute_command(cmd):
         line = process.stdout.readline()
         line = line.strip()
         if line:
-            print(line)
+            logger.info(line)
     if process.returncode != 0:
         logger.error('Failed to execute command:%s' % " ".join(cmd))
         raise CompareException(CompareException.INVALID_DATA_ERROR)
 
 
-def parse_value_by_comma(value):
-    """
-    parse value by comma, like '1,2,4,8'
-    """
-    value_list = []
-    value_str_list = value.split(Const.COMMA)
-    for value_str in value_str_list:
-        value_str = value_str.strip()
-        if value_str.isdigit() or value_str == '-1':
-            value_list.append(int(value_str))
-        else:
-            logger.error("please check your input shape.")
-            raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    return value_list
-
-
 def add_time_as_suffix(name):
     return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
 
@@ -265,6 +212,10 @@ def add_time_with_xlsx(name):
     return '{}_{}.xlsx'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
 
 
+def add_time_with_yaml(name):
+    return '{}_{}.yaml'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
+
+
 def get_time():
     return datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S")
 
@@ -273,61 +224,6 @@ def format_value(value):
     return float('{:.12f}'.format(value))
 
 
-def check_seed_all(seed, mode):
-    if isinstance(seed, int):
-        if seed < 0 or seed > Const.MAX_SEED_VALUE:
-            logger.error(f"Seed must be between 0 and {Const.MAX_SEED_VALUE}.")
-            raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    else:
-        logger.error(f"Seed must be integer.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    if not isinstance(mode, bool):
-        logger.error(f"seed_all mode must be bool.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-
-
-def get_process_rank(model):
-    logger.info("Rank id is not provided. Trying to get the rank id of the model.")
-    try:
-        local_device = next(model.parameters()).device
-    except StopIteration:
-        logger.warning('There is no parameter in the model. Fail to get rank id.')
-        return 0, False
-    if local_device.type == 'cpu':
-        logger.warning("Warning: the debugger is unable to get the rank id. "
-            "This may cause the dumpped data to be corrupted in the "
-            "case of distributed training. (You may ignore this if you are using only one card.) "
-            "Transfer the model to npu or gpu before register_hook() to avoid this warning.")
-        return 0, False
-    else:
-        return local_device.index, True
-
-
-def generate_compare_script(dump_path, pkl_file_path, dump_switch_mode):
-    template_path = os.path.join(os.path.dirname(__file__), "compare_script.template")
-    pkl_dir = os.path.dirname(pkl_file_path)
-    compare_script_path = os.path.join(pkl_dir, "compare_data.py")
-    is_api_stack = "True" if dump_switch_mode == Const.API_STACK else "False"
-
-    try:
-        with FileOpen(template_path, 'r') as ftemp, \
-           os.fdopen(os.open(compare_script_path, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as fout:
-            code_temp = ftemp.read()
-            fout.write(code_temp % (pkl_file_path, dump_path, is_api_stack))
-    except OSError:
-        logger.error(f"Failed to open file. Please check file {template_path} or path {pkl_dir}.")
-
-    logger.info(f"Generate compare script successfully which is {compare_script_path}.")
-
-
-def check_inplace_op(prefix):
-    if len(prefix) > Const.DISTRIBUTED_PREFIX_LENGTH:
-        return False
-    match_op = re.findall(r"Distributed\.(.+?)\.\d", prefix)
-    op_name = match_op[0] if match_op else None
-    return op_name in Const.INPLACE_LIST
-
-
 def md5_find(data):
     for key_op in data:
         for api_info in data[key_op]:
@@ -335,46 +231,89 @@ def md5_find(data):
                 for data_detail in data[key_op][api_info]:
                     if data_detail and 'md5' in data_detail:
                         return True
-            elif 'md5' in data[key_op][api_info]:
+            elif data[key_op][api_info] and 'md5' in data[key_op][api_info]:
                 return True
     return False
 
 
-def task_dumppath_get(input_param):
+def detect_framework_by_dump_json(file_path):
+    pattern_ms = r'"type":\s*"mindspore'
+    pattern_pt = r'"type":\s*"torch'
+    with FileOpen(file_path, 'r') as file:
+        for line in file:
+            if re.search(pattern_ms, line):
+                return Const.MS_FRAMEWORK
+            if re.search(pattern_pt, line):
+                return Const.PT_FRAMEWORK
+    logger.error(f"{file_path} must be based on the MindSpore or PyTorch framework.")
+    raise CompareException(CompareException.INVALID_PARAM_ERROR)
+
+
+def get_stack_construct_by_dump_json_path(dump_json_path):
+    if not dump_json_path:
+        logger.error("The path is empty. Please enter a valid path.")
+        raise CompareException(CompareException.INVALID_PATH_ERROR)
+    directory = os.path.dirname(dump_json_path)
+    check_file_or_directory_path(directory, True)
+    stack_json = os.path.join(directory, "stack.json")
+    construct_json = os.path.join(directory, "construct.json")
+
+    stack = load_json(stack_json)
+    construct = load_json(construct_json)
+    return stack, construct
+
+
+def set_dump_path(input_param):
     npu_path = input_param.get("npu_json_path", None)
     bench_path = input_param.get("bench_json_path", None)
-    if not npu_path or not bench_path:
-        logger.error(f"Please check the json path is valid.")
+    npu_path_valid = npu_path is not None and npu_path.endswith("dump.json")
+    bench_path_valid = bench_path is not None and bench_path.endswith("dump.json")
+    if not npu_path_valid or not bench_path_valid:
+        logger.error(f"Please check the json path is valid. npu_path: {npu_path}, bench_path: {bench_path}")
         raise CompareException(CompareException.INVALID_PATH_ERROR)
-    with FileOpen(npu_path, 'r') as npu_f:
-        npu_json_data = json.load(npu_f)
-    with FileOpen(bench_path, 'r') as bench_f:
-        bench_json_data = json.load(bench_f)
-    if npu_json_data['task'] != bench_json_data['task']:
+    input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA)
+    input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA)
+
+
+def get_dump_mode(input_param):
+    npu_path = input_param.get("npu_json_path", None)
+    bench_path = input_param.get("bench_json_path", None)
+    npu_json_data = load_json(npu_path)
+    bench_json_data = load_json(bench_path)
+
+    npu_task = npu_json_data.get('task', None)
+    bench_task = bench_json_data.get('task', None)
+
+    if not npu_task or not bench_task:
+        logger.error(f"Please check the dump task is correct, npu's task is {npu_task}, bench's task is {bench_task}.")
+        raise CompareException(CompareException.INVALID_TASK_ERROR)
+
+    if npu_task != bench_task:
         logger.error(f"Please check the dump task is consistent.")
         raise CompareException(CompareException.INVALID_TASK_ERROR)
-    if npu_json_data['task'] == Const.TENSOR:
-        summary_compare = False
-        md5_compare = False
-    elif npu_json_data['task'] == Const.STATISTICS:
-        md5_compare = md5_find(npu_json_data['data'])
-        if md5_compare:
-            summary_compare = False
+
+    if npu_task == Const.TENSOR:
+        return Const.ALL
+
+    if npu_task == Const.STATISTICS:
+        npu_md5_compare = md5_find(npu_json_data['data'])
+        bench_md5_compare = md5_find(bench_json_data['data'])
+        if npu_md5_compare == bench_md5_compare:
+            return Const.MD5 if npu_md5_compare else Const.SUMMARY
         else:
-            summary_compare = True
-    else:
-        logger.error(f"Compare is not required for overflow_check or free_benchmark.")
-        raise CompareException(CompareException.INVALID_TASK_ERROR)
-    input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA)
-    input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA)
-    return summary_compare, md5_compare
+            logger.error(f"Please check the dump task is consistent, "
+                         f"dump mode of npu and bench should both be statistics or md5.")
+            raise CompareException(CompareException.INVALID_TASK_ERROR)
 
+    logger.error(f"Compare applies only to task is tensor or statistics")
+    raise CompareException(CompareException.INVALID_TASK_ERROR)
 
-def get_header_index(header_name, summary_compare=False):
-    if summary_compare:
-        header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:]
-    else:
-        header = CompareConst.COMPARE_RESULT_HEADER[:]
+
+def get_header_index(header_name, dump_mode):
+    header = CompareConst.HEAD_OF_COMPARE_MODE.get(dump_mode)
+    if not header:
+        logger.error(f"{dump_mode} not in {CompareConst.HEAD_OF_COMPARE_MODE}")
+        raise CompareException(CompareException.INVALID_PARAM_ERROR)
     if header_name not in header:
         logger.error(f"{header_name} not in data name")
         raise CompareException(CompareException.INVALID_PARAM_ERROR)
@@ -382,4 +321,164 @@ def get_header_index(header_name, summary_compare=False):
 
 
 def convert_tuple(data):
-    return data if isinstance(data, tuple) else (data, )
+    return data if isinstance(data, tuple) else (data,)
+
+
+def check_op_str_pattern_valid(string, op_name=None, stack=False):
+    if isinstance(string, str) and is_invalid_pattern(string):
+        if stack:
+            message = f"stack info of {op_name} contains special characters, please check!"
+        elif not op_name:
+            message = f"api name contains special characters, please check!"
+        else:
+            message = f"data info of {op_name} contains special characters, please check!"
+        logger.error(message)
+        raise CompareException(CompareException.INVALID_CHAR_ERROR)
+
+
+def is_invalid_pattern(string):
+    pattern = Const.STRING_BLACKLIST
+    return re.search(pattern, string)
+
+
+def is_int(x):
+    return isinstance(x, int) and not isinstance(x, bool)
+
+
+def print_tools_ends_info():
+    total_len = len(Const.TOOL_ENDS_SUCCESSFULLY) + Const.FILL_CHAR_NUMS
+    logger.info('*' * total_len)
+    logger.info(f"*{Const.TOOL_ENDS_SUCCESSFULLY.center(total_len - 2)}*")
+    logger.info('*' * total_len)
+
+
+def get_step_or_rank_from_string(step_or_rank, obj):
+    splited = step_or_rank.split(Const.HYPHEN)
+    if len(splited) == 2:
+        try:
+            borderlines = int(splited[0]), int(splited[1])
+        except (ValueError, IndexError) as e:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   "The hyphen(-) must start and end with decimal numbers.") from e
+    else:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                               f'The string parameter for {obj} only supports formats like "3-5". '
+                               f'Now string parameter for {obj} is "{step_or_rank}".')
+    if all(Const.STEP_RANK_MINIMUM_VALUE <= b <= Const.STEP_RANK_MAXIMUM_VALUE for b in borderlines):
+        if borderlines[0] <= borderlines[1]:
+            continual_step_or_rank = list(range(borderlines[0], borderlines[1] + 1))
+        else:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f'For the hyphen(-) in {obj}, the left boundary ({borderlines[0]}) cannot be '
+                                   f'greater than the right boundary ({borderlines[1]}).')
+    else:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                               f"The boundaries must fall within the range of "
+                               f"[{Const.STEP_RANK_MINIMUM_VALUE}, {Const.STEP_RANK_MAXIMUM_VALUE}].")
+    return continual_step_or_rank
+
+
+def get_real_step_or_rank(step_or_rank_input, obj):
+    if obj not in [Const.STEP, Const.RANK]:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                               f"Only support parsing {[Const.STEP, Const.RANK]}, the current parsing object is {obj}.")
+    if step_or_rank_input is None:
+        return []
+    if not isinstance(step_or_rank_input, list):
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"{obj} is invalid, it should be a list")
+    if len(step_or_rank_input) > Const.STEP_RANK_MAXIMUM_VALUE:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                               f"{obj} is invalid, its length cannot exceed {Const.STEP_RANK_MAXIMUM_VALUE}")
+
+    real_step_or_rank = []
+    for element in step_or_rank_input:
+        if not is_int(element) and not isinstance(element, str):
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"{obj} element {element} must be an integer or string.")
+        if isinstance(element, int) and element < 0:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   f"Each element of {obj} must be non-negative, currently it is {element}.")
+        if isinstance(element, int) and Const.STEP_RANK_MINIMUM_VALUE <= element <= Const.STEP_RANK_MAXIMUM_VALUE:
+            real_step_or_rank.append(element)
+        elif isinstance(element, str) and Const.HYPHEN in element:
+            continual_step_or_rank = get_step_or_rank_from_string(element, obj)
+            real_step_or_rank.extend(continual_step_or_rank)
+    real_step_or_rank = list(set(real_step_or_rank))
+    real_step_or_rank.sort()
+    return real_step_or_rank
+
+
+def check_seed_all(seed, mode):
+    if is_int(seed):
+        if seed < 0 or seed > Const.MAX_SEED_VALUE:
+            logger.error(f"Seed must be between 0 and {Const.MAX_SEED_VALUE}.")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
+    else:
+        logger.error("Seed must be integer.")
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
+    if not isinstance(mode, bool):
+        logger.error("seed_all mode must be bool.")
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
+
+
+def safe_get_value(container, index, container_name, key=None):
+    try:
+        # 处理字典情况
+        if isinstance(container, dict):
+            return container.get(key)[index]
+        # 处理列表、元组、numpy情况
+        elif isinstance(container, (list, tuple, np.ndarray)):
+            return container[index]
+        else:
+            err_msg = f"Unsupported container type for '{container_name}': {type(container)}"
+            logger.error(err_msg)
+            raise MsprobeBaseException(MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR)
+    except IndexError as e:
+        err_msg = "index out of bounds error occurs, please check!\n" \
+                  f"{container_name} is {container}\n" \
+                  f"index is {index}"
+        logger.error(err_msg)
+        raise MsprobeBaseException(MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) from e
+    except TypeError as e:
+        err_msg = "wrong type, please check!\n" \
+                  f"{container_name} is {container}\n" \
+                  f"index is {index}\n" \
+                  f"key is {key}"
+        logger.error(err_msg)
+        raise MsprobeBaseException(MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) from e
+
+
+# 记录工具函数递归的深度
+recursion_depth = defaultdict(int)
+
+
+# 装饰一个函数，当函数递归调用超过限制时，抛出异常并打印函数信息。
+def recursion_depth_decorator(func_info):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            func_id = id(func)
+            recursion_depth[func_id] += 1
+            if recursion_depth[func_id] > Const.MAX_DEPTH:
+                msg = f"call {func_info} exceeds the recursion limit."
+                logger.error_log_with_exp(
+                    msg,
+                    MsprobeException(
+                        MsprobeException.RECURSION_LIMIT_ERROR, msg
+                    ),
+                )
+            try:
+                result = func(*args, **kwargs)
+            finally:
+                recursion_depth[func_id] -= 1
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+def check_str_param(param):
+    if not re.match(Const.REGEX_PREFIX_PATTERN, param):
+        logger.error('The parameter {} contains special characters.'.format(param))
+        raise MsprobeBaseException(MsprobeBaseException.INVALID_CHAR_ERROR)