mindstudio-probe 1.0.4__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/core/common/utils.py

@@ -22,19 +22,21 @@ import time
 import json
 from datetime import datetime, timezone
 
-from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path)
+from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path, load_json)
 from msprobe.core.common.const import Const, CompareConst
 from msprobe.core.common.log import logger
+from msprobe.core.common.exceptions import MsprobeException
 
 
 device = collections.namedtuple('device', ['type', 'index'])
 prefixes = ['api_stack', 'list', 'range', 'acl']
 
 
-class CompareException(Exception):
+class MsprobeBaseException(Exception):
     """
-    Class for Accuracy Compare Exception
+    Base class for all custom exceptions.
     """
+    # 所有的错误代码
     NONE_ERROR = 0
     INVALID_PATH_ERROR = 1
     OPEN_FILE_ERROR = 2
@@ -57,10 +59,18 @@ class CompareException(Exception):
     INVALID_SUMMARY_MODE = 19
     INVALID_TASK_ERROR = 20
     DETACH_ERROR = 21
-
+    INVALID_OBJECT_TYPE_ERROR = 22
+    INVALID_CHAR_ERROR = 23
+    RECURSION_LIMIT_ERROR = 24
+    INVALID_ATTRIBUTE_ERROR = 25
+    OUTPUT_HOOK_ERROR = 26
+    INPUT_HOOK_ERROR = 27
+    FUNCTION_CALL_ERROR = 28
+    FORWARD_DATA_COLLECTION_ERROR = 29
+    BACKWARD_DATA_COLLECTION_ERROR = 30
 
     def __init__(self, code, error_info: str = ""):
-        super(CompareException, self).__init__()
+        super(MsprobeBaseException, self).__init__()
         self.code = code
         self.error_info = error_info
 
@@ -68,74 +78,33 @@ class CompareException(Exception):
         return self.error_info
 
 
-class DumpException(CompareException):
-    pass
-
-
-def check_mode_valid(mode, scope=None, api_list=None):
-    if scope is None:
-        scope = []
-    if api_list is None:
-        api_list = []
-    if not isinstance(scope, list):
-        raise ValueError("scope param set invalid, it's must be a list.")
-    if not isinstance(api_list, list):
-        raise ValueError("api_list param set invalid, it's must be a list.")
-    mode_check = {
-        Const.ALL: lambda: None,
-        Const.RANGE: lambda:  ValueError("set_dump_switch, scope param set invalid, it's must be [start, end].") if len(scope) != 2 else None,
-        Const.LIST: lambda:  ValueError("set_dump_switch, scope param set invalid, it's should not be an empty list.") if len(scope) == 0 else None,
-        Const.STACK: lambda:  ValueError("set_dump_switch, scope param set invalid, it's must be [start, end] or [].") if len(scope) > 2 else None,
-        Const.ACL: lambda:  ValueError("set_dump_switch, scope param set invalid, only one api name is supported in acl mode.") if len(scope) != 1 else None,
-        Const.API_LIST: lambda:  ValueError("Current dump mode is 'api_list', but the content of api_list parameter is empty or valid.") if len(api_list) < 1 else None,
-        Const.API_STACK: lambda: None,
-    }
-    if mode not in Const.DUMP_MODE:
-        msg = "Current mode '%s' is not supported. Please use the field in %s" % \
-              (mode, Const.DUMP_MODE)
-        raise CompareException(CompareException.INVALID_DUMP_MODE, msg)
-
-    if mode_check.get(mode)() is not None:
-        raise mode_check.get(mode)()
-
-
-def check_switch_valid(switch):
-    if switch not in ["ON", "OFF"]:
-        logger.error("Please set switch with 'ON' or 'OFF'.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-
+class CompareException(MsprobeBaseException):
+    """
+    Class for Accuracy Compare Exception
+    """
 
-def check_dump_mode_valid(dump_mode):
-    if not isinstance(dump_mode, list):
-        logger.warning("Please set dump_mode as a list.")
-        dump_mode = [dump_mode]
-    if not all(mode in ["all", "forward", "backward", "input", "output"] for mode in dump_mode):
-        raise ValueError("Please set dump_mode as a list containing one or more of the following: 'all', 'forward', 'backward', 'input', 'output'.")
-    if 'input' not in dump_mode and 'output' not in dump_mode:
-        dump_mode.extend(['input', 'output'])
-    if 'forward' not in dump_mode and 'backward' not in dump_mode:
-        dump_mode.extend(['forward', 'backward'])
-    if 'all' in dump_mode or set(["forward", "backward", "input", "output"]).issubset(set(dump_mode)):
-        return ["forward", "backward", "input", "output"]
-    return dump_mode
+    def __init__(self, code, error_info: str = ""):
+        super(CompareException, self).__init__(code, error_info)
 
 
-def check_summary_mode_valid(summary_mode):
-    if summary_mode not in Const.SUMMARY_MODE:
-        msg = "The summary_mode is not valid"
-        raise CompareException(CompareException.INVALID_SUMMARY_MODE, msg)
+class DumpException(MsprobeBaseException):
+    """
+    Class for Dump Exception
+    """
 
+    def __init__(self, code, error_info: str = ""):
+        super(DumpException, self).__init__(code, error_info)
 
-def check_summary_only_valid(summary_only):
-    if not isinstance(summary_only, bool):
-        logger.error("Params summary_only only support True or False.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    return summary_only
+    def __str__(self):
+        return f"Dump Error Code {self.code}: {self.error_info}"
 
 
 def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False):
-    if not (isinstance(input_param, dict) and isinstance(output_path, str)):
-        logger.error("Invalid input parameters")
+    if not isinstance(input_param, dict):
+        logger.error(f"Invalid input parameter 'input_param', the expected type dict but got {type(input_param)}.")
+        raise CompareException(CompareException.INVALID_PARAM_ERROR)
+    if not isinstance(output_path, str):
+        logger.error(f"Invalid input parameter 'output_path', the expected type str but got {type(output_path)}.")
         raise CompareException(CompareException.INVALID_PARAM_ERROR)
 
     check_file_or_directory_path(input_param.get("npu_json_path"), False)
@@ -152,15 +121,12 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com
         check_json_file(input_param, npu_json, bench_json, stack_json)
 
 
-
-def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False):
-    if not (isinstance(stack_mode, bool) and isinstance(auto_analyze, bool) and isinstance(fuzzy_match, bool)):
-        logger.error("Invalid input parameters which should be only bool type.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-
-
-def is_starts_with(string, prefix_list):
-    return any(string.startswith(prefix) for prefix in prefix_list)
+def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False, is_print_compare_log=True):
+    arg_list = [stack_mode, auto_analyze, fuzzy_match, is_print_compare_log]
+    for arg in arg_list:
+        if not isinstance(arg, bool):
+            logger.error(f"Invalid input parameter, {arg} which should be only bool type.")
+            raise CompareException(CompareException.INVALID_PARAM_ERROR)
 
 
 def _check_json(json_file_handle, file_name):
@@ -198,28 +164,6 @@ def check_regex_prefix_format_valid(prefix):
         raise ValueError(f"prefix contains invalid characters, prefix pattern {Const.REGEX_PREFIX_PATTERN}")
 
 
-def get_dump_data_path(dump_dir):
-    """
-    Function Description:
-        traverse directories and obtain the absolute path of dump data
-    Parameter:
-        dump_dir: dump data directory
-    Return Value:
-        dump data path,file is exist or file is not exist
-    """
-    dump_data_path = None
-    file_is_exist = False
-
-    check_file_or_directory_path(dump_dir, True)
-    for dir_path, _, files in os.walk(dump_dir):
-        if len(files) != 0:
-            dump_data_path = dir_path
-            file_is_exist = True
-            break
-        dump_data_path = dir_path
-    return dump_data_path, file_is_exist
-
-
 def execute_command(cmd):
     """
     Function Description:
@@ -241,22 +185,6 @@ def execute_command(cmd):
         raise CompareException(CompareException.INVALID_DATA_ERROR)
 
 
-def parse_value_by_comma(value):
-    """
-    parse value by comma, like '1,2,4,8'
-    """
-    value_list = []
-    value_str_list = value.split(Const.COMMA)
-    for value_str in value_str_list:
-        value_str = value_str.strip()
-        if value_str.isdigit() or value_str == '-1':
-            value_list.append(int(value_str))
-        else:
-            logger.error("please check your input shape.")
-            raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    return value_list
-
-
 def add_time_as_suffix(name):
     return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
 
@@ -265,6 +193,10 @@ def add_time_with_xlsx(name):
     return '{}_{}.xlsx'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
 
 
+def add_time_with_yaml(name):
+    return '{}_{}.yaml'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())))
+
+
 def get_time():
     return datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S")
 
@@ -273,61 +205,6 @@ def format_value(value):
     return float('{:.12f}'.format(value))
 
 
-def check_seed_all(seed, mode):
-    if isinstance(seed, int):
-        if seed < 0 or seed > Const.MAX_SEED_VALUE:
-            logger.error(f"Seed must be between 0 and {Const.MAX_SEED_VALUE}.")
-            raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    else:
-        logger.error(f"Seed must be integer.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-    if not isinstance(mode, bool):
-        logger.error(f"seed_all mode must be bool.")
-        raise CompareException(CompareException.INVALID_PARAM_ERROR)
-
-
-def get_process_rank(model):
-    logger.info("Rank id is not provided. Trying to get the rank id of the model.")
-    try:
-        local_device = next(model.parameters()).device
-    except StopIteration:
-        logger.warning('There is no parameter in the model. Fail to get rank id.')
-        return 0, False
-    if local_device.type == 'cpu':
-        logger.warning("Warning: the debugger is unable to get the rank id. "
-            "This may cause the dumpped data to be corrupted in the "
-            "case of distributed training. (You may ignore this if you are using only one card.) "
-            "Transfer the model to npu or gpu before register_hook() to avoid this warning.")
-        return 0, False
-    else:
-        return local_device.index, True
-
-
-def generate_compare_script(dump_path, pkl_file_path, dump_switch_mode):
-    template_path = os.path.join(os.path.dirname(__file__), "compare_script.template")
-    pkl_dir = os.path.dirname(pkl_file_path)
-    compare_script_path = os.path.join(pkl_dir, "compare_data.py")
-    is_api_stack = "True" if dump_switch_mode == Const.API_STACK else "False"
-
-    try:
-        with FileOpen(template_path, 'r') as ftemp, \
-           os.fdopen(os.open(compare_script_path, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as fout:
-            code_temp = ftemp.read()
-            fout.write(code_temp % (pkl_file_path, dump_path, is_api_stack))
-    except OSError:
-        logger.error(f"Failed to open file. Please check file {template_path} or path {pkl_dir}.")
-
-    logger.info(f"Generate compare script successfully which is {compare_script_path}.")
-
-
-def check_inplace_op(prefix):
-    if len(prefix) > Const.DISTRIBUTED_PREFIX_LENGTH:
-        return False
-    match_op = re.findall(r"Distributed\.(.+?)\.\d", prefix)
-    op_name = match_op[0] if match_op else None
-    return op_name in Const.INPLACE_LIST
-
-
 def md5_find(data):
     for key_op in data:
         for api_info in data[key_op]:
@@ -340,6 +217,29 @@ def md5_find(data):
     return False
 
 
+def struct_json_get(input_param, framework):
+    if framework == Const.PT_FRAMEWORK:
+        prefix = "bench"
+    elif framework == Const.MS_FRAMEWORK:
+        prefix = "npu"
+    else:
+        logger.error("Error framework found.")
+        raise CompareException(CompareException.INVALID_PARAM_ERROR)
+
+    frame_json_path = input_param.get(f"{prefix}_json_path", None)
+    if not frame_json_path:
+        logger.error(f"Please check the json path is valid.")
+        raise CompareException(CompareException.INVALID_PATH_ERROR)
+    directory = os.path.dirname(frame_json_path)
+    check_file_or_directory_path(directory, True)
+    stack_json = os.path.join(directory, "stack.json")
+    construct_json = os.path.join(directory, "construct.json")
+
+    stack = load_json(stack_json)
+    construct = load_json(construct_json)
+    return stack, construct
+
+
 def task_dumppath_get(input_param):
     npu_path = input_param.get("npu_json_path", None)
     bench_path = input_param.get("bench_json_path", None)
@@ -383,3 +283,89 @@ def get_header_index(header_name, summary_compare=False):
 
 def convert_tuple(data):
     return data if isinstance(data, tuple) else (data, )
+
+
+def check_op_str_pattern_valid(string, op_name=None, stack=False):
+    if isinstance(string, str) and is_invalid_pattern(string):
+        if stack:
+            message = f"stack info of {op_name} contains special characters, please check!"
+        elif not op_name:
+            message = f"api name contains special characters, please check!"
+        else:
+            message = f"data info of {op_name} contains special characters, please check!"
+        logger.error(message)
+        raise CompareException(CompareException.INVALID_CHAR_ERROR)
+
+
+def is_invalid_pattern(string):
+    pattern = Const.STRING_BLACKLIST
+    return re.search(pattern, string)
+
+
+def print_tools_ends_info():
+    total_len = len(Const.TOOL_ENDS_SUCCESSFULLY) + Const.FILL_CHAR_NUMS
+    logger.info('*' * total_len)
+    logger.info(f"*{Const.TOOL_ENDS_SUCCESSFULLY.center(total_len - 2)}*")
+    logger.info('*' * total_len)
+
+
+def get_step_or_rank_from_string(step_or_rank, obj):
+    splited = step_or_rank.split(Const.HYPHEN)
+    if len(splited) == 2:
+        try:
+            borderlines = int(splited[0]), int(splited[1])
+        except (ValueError, IndexError) as e:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                                   "The hyphen(-) must start and end with decimal numbers.") from e
+    else:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                               f'The string parameter for {obj} only supports formats like "3-5". Now string parameter for {obj} is "{step_or_rank}".')
+    if all(Const.STEP_RANK_MAXIMUM_RANGE[0] <= b <= Const.STEP_RANK_MAXIMUM_RANGE[1] for b in borderlines):
+        if borderlines[0] <= borderlines[1]:
+            continual_step_or_rank = list(range(borderlines[0], borderlines[1] + 1))
+        else:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                               f'For the hyphen(-) in {obj}, the left boundary ({borderlines[0]}) cannot be greater than the right boundary ({borderlines[1]}).')
+    else:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                               f"The boundaries must fall within the range of [{Const.STEP_RANK_MAXIMUM_RANGE[0]}, {Const.STEP_RANK_MAXIMUM_RANGE[1]}].")
+    return continual_step_or_rank
+
+
+def get_real_step_or_rank(step_or_rank_input, obj):
+    if obj not in [Const.STEP, Const.RANK]:
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                               f"Only support parsing {[Const.STEP, Const.RANK]}, the current parsing object is {obj}.")
+    if step_or_rank_input is None:
+        return []
+    if not isinstance(step_or_rank_input, list):
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"{obj} is invalid, it should be a list")
+    real_step_or_rank = []
+    for element in step_or_rank_input:
+        if not isinstance(element, (int, str)):
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                                   f"{obj} element {element} must be an integer or string.")
+        if isinstance(element, int) and element < 0:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, 
+                                   f"Each element of {obj} must be non-negative, currently it is {element}.")
+        if isinstance(element, int) and Const.STEP_RANK_MAXIMUM_RANGE[0] <= element <= Const.STEP_RANK_MAXIMUM_RANGE[1]:
+            real_step_or_rank.append(element)
+        elif isinstance(element, str) and Const.HYPHEN in element:
+            continual_step_or_rank = get_step_or_rank_from_string(element, obj)
+            real_step_or_rank.extend(continual_step_or_rank)
+    real_step_or_rank = list(set(real_step_or_rank))
+    real_step_or_rank.sort()
+    return real_step_or_rank
+
+
+def check_seed_all(seed, mode):
+    if isinstance(seed, int):
+        if seed < 0 or seed > Const.MAX_SEED_VALUE:
+            logger.error(f"Seed must be between 0 and {Const.MAX_SEED_VALUE}.")
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
+    else:
+        logger.error("Seed must be integer.")
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)
+    if not isinstance(mode, bool):
+        logger.error("seed_all mode must be bool.")
+        raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR)

msprobe/core/common_config.py

@@ -2,18 +2,17 @@ from msprobe.core.common.const import Const, FileCheckConst
 from msprobe.core.common.log import logger
 from msprobe.core.common.exceptions import MsprobeException
 from msprobe.core.common.file_utils import FileChecker
+from msprobe.core.common.utils import get_real_step_or_rank
 
 
 class CommonConfig:
     def __init__(self, json_config):
         self.task = json_config.get('task')
         self.dump_path = json_config.get('dump_path')
-        self.rank = json_config.get('rank')
-        self.step = json_config.get('step')
+        self.rank = get_real_step_or_rank(json_config.get('rank'), Const.RANK)
+        self.step = get_real_step_or_rank(json_config.get('step'), Const.STEP)
         self.level = json_config.get('level')
-        self.seed = json_config.get('seed')
         self.acl_config = json_config.get('acl_config')
-        self.is_deterministic = json_config.get('is_deterministic', False)
         self.enable_dataloader = json_config.get('enable_dataloader', False)
         self._check_config()
 
@@ -24,21 +23,9 @@ class CommonConfig:
         if self.dump_path is not None and not isinstance(self.dump_path, str):
             logger.error_log_with_exp("dump_path is invalid, it should be a string",
                                       MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
-        if self.rank is not None and not isinstance(self.rank, list):
-            logger.error_log_with_exp("rank is invalid, it should be a list",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
-        if self.step is not None and not isinstance(self.step, list):
-            logger.error_log_with_exp("step is invalid, it should be a list",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
         if self.level and self.level not in Const.LEVEL_LIST:
             logger.error_log_with_exp("level is invalid, it should be one of {}".format(Const.LEVEL_LIST),
                                       MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
-        if self.seed is not None and not isinstance(self.seed, int):
-            logger.error_log_with_exp("seed is invalid, it should be an integer",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
-        if not isinstance(self.is_deterministic, bool):
-            logger.error_log_with_exp("is_deterministic is invalid, it should be a boolean",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
         if not isinstance(self.enable_dataloader, bool):
             logger.error_log_with_exp("enable_dataloader is invalid, it should be a boolean",
                                       MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
@@ -73,13 +60,19 @@ class BaseConfig:
         self.preheat_step = json_config.get("preheat_step")
         self.max_sample = json_config.get("max_sample")
 
+    @staticmethod
+    def _check_str_list_config(config_item, config_name):
+        if config_item is not None:
+            if not isinstance(config_item, list):
+                logger.error_log_with_exp(f"{config_name} is invalid, it should be a list[str]",
+                                          MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
+            for name in config_item:
+                if not isinstance(name, str):
+                    logger.error_log_with_exp(f"{config_name} is invalid, it should be a list[str]",
+                                              MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
+
     def check_config(self):
-        if self.scope is not None and not isinstance(self.scope, list):
-            logger.error_log_with_exp("scope is invalid, it should be a list",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
-        if self.list is not None and not isinstance(self.list, list):
-            logger.error_log_with_exp("list is invalid, it should be a list",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
-        if self.data_mode is not None and not isinstance(self.data_mode, list):
-            logger.error_log_with_exp("data_mode is invalid, it should be a list",
-                                      MsprobeException(MsprobeException.INVALID_PARAM_ERROR))
+        self._check_str_list_config(self.scope, "scope")
+        self._check_str_list_config(self.list, "list")
+        self._check_str_list_config(self.data_mode, "data_mode")
+        self._check_str_list_config(self.backward_input, "backward_input")