mindstudio-probe 1.0.1__py3-none-any.whl

msprobe/core/common/const.py

@@ -0,0 +1,241 @@
+import os
+import stat
+import numpy as np
+
+class Const:
+    """
+    Class for const
+    """
+    SEP = "."
+    REGEX_PREFIX_MAX_LENGTH = 20
+    REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$"
+    FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    COMMA = ","
+    FLOAT_EPSILON = np.finfo(float).eps
+    OFF = 'OFF'
+    BACKWARD = 'backward'
+    FORWARD = 'forward'
+
+    # dump mode
+    ALL = "all"
+    LIST = "list"
+    RANGE = "range"
+    STACK = "stack"
+    ACL = "acl"
+    API_LIST = "api_list"
+    API_STACK = "api_stack"
+    DUMP_MODE = [ALL, LIST, RANGE, STACK, ACL, API_LIST, API_STACK]
+    SUMMARY = "summary"
+    MD5 = "md5"
+    SUMMARY_MODE = [ALL, SUMMARY, MD5]
+
+    WRITE_FLAGS = os.O_WRONLY | os.O_CREAT
+    WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR
+    OVERWRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+
+    PKL_SUFFIX = ".pkl"
+    NUMPY_SUFFIX = ".npy"
+    ONE_GB = 1073741824  # 1 * 1024 * 1024 * 1024
+    TEN_GB = 10737418240  # 10 * 1024 * 1024 * 1024
+    FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    DISTRIBUTED_PREFIX_LENGTH = 60
+    # env dump path
+    KWARGS = 'kwargs'
+    INPUT = 'input'
+    OUTPUT = 'output'
+    INPUT_ARGS = 'input_args'
+    INPUT_KWARGS = 'input_kwargs'
+    GRAD_INPUT = 'grad_input'
+    GRAD_OUTPUT = 'grad_output'
+    START = "start"
+    STOP = "stop"
+    ENV_ENABLE = "1"
+    ENV_DISABLE = "0"
+    MAX_SEED_VALUE = 4294967295  # 2**32 - 1
+    TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark"]
+    LEVEL_LIST = ["L0", "L1", "L2", "mix"]
+    STATISTICS = "statistics"
+    TENSOR = "tensor"
+    OVERFLOW_CHECK = "overflow_check"
+    FREE_BENCHMARK = "free_benchmark"
+    ATTR_NAME_PREFIX = "wrap_"
+    KERNEL_DUMP = "kernel_dump"
+    DATA = "data"
+    PT_FRAMEWORK = "pytorch"
+    MS_FRAMEWORK = "mindspore"
+    DIRECTORY_LENGTH = 4096
+    FILE_NAME_LENGTH = 255
+    FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16]
+    BOOL_TYPE = [bool, np.uint8]
+    INT_TYPE = [np.int32, np.int64]
+    NPU = 'NPU'
+    DISTRIBUTED = 'Distributed'
+    
+    INPLACE_LIST = [
+        "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter",
+        "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single"
+    ]
+    
+    CONVERT = {
+        "int32_to_int64": ["torch.int32", "torch.int64"],
+    }
+
+    CONVERT_API = {
+        "int32_to_int64": ["cross_entropy"]
+    }
+
+class CompareConst:
+    """
+    Class for compare module const
+    """
+    SPACE = " "
+    # compare result column name
+    NPU_NAME = "NPU Name"
+    BENCH_NAME = "Bench Name"
+    NPU_DTYPE = "NPU Dtype"
+    BENCH_DTYPE = "Bench Dtype"
+    NPU_SHAPE = "NPU Tensor Shape"
+    BENCH_SHAPE = "Bench Tensor Shape"
+    NPU_MAX = "NPU max"
+    NPU_MIN = "NPU min"
+    NPU_MEAN = "NPU mean"
+    NPU_NORM = "NPU l2norm"
+    BENCH_MAX = "Bench max"
+    BENCH_MIN = "Bench min"
+    BENCH_MEAN = "Bench mean"
+    BENCH_NORM = "Bench l2norm"
+    MAX_DIFF = "Max diff"
+    MIN_DIFF = "Min diff"
+    MEAN_DIFF = "Mean diff"
+    NORM_DIFF = "L2norm diff"
+    COSINE = "Cosine"
+    MAX_ABS_ERR = "MaxAbsErr"
+    MAX_RELATIVE_ERR = "MaxRelativeErr"
+    MIN_RELATIVE_ERR = "MinRelativeErr"
+    MEAN_RELATIVE_ERR = "MeanRelativeErr"
+    NORM_RELATIVE_ERR = "NormRelativeErr"
+    ACCURACY = "Accuracy Reached or Not"
+    STACK = "NPU_Stack_Info"
+    DATA_NAME = "Data_name"
+    ERROR_MESSAGE = "Err_message"
+    ONE_THOUSANDTH_ERR_RATIO = "One Thousandth Err Ratio"
+    FIVE_THOUSANDTHS_ERR_RATIO = "Five Thousandths Err Ratio"
+    NPU_MD5 = "NPU MD5"
+    BENCH_MD5 = "BENCH MD5"
+    RESULT = "Result"
+
+    COMPARE_RESULT_HEADER = [
+        NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR,
+        ONE_THOUSANDTH_ERR_RATIO, FIVE_THOUSANDTHS_ERR_RATIO,
+        NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, ACCURACY, ERROR_MESSAGE
+    ]
+
+    SUMMARY_COMPARE_RESULT_HEADER = [
+        NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF,
+        MAX_RELATIVE_ERR, MIN_RELATIVE_ERR, MEAN_RELATIVE_ERR, NORM_RELATIVE_ERR,
+        NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, RESULT, ERROR_MESSAGE
+    ]
+
+    MD5_COMPARE_RESULT_HEADER = [
+        NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, NPU_MD5, BENCH_MD5, RESULT
+    ]
+
+    # compare standard
+    HUNDRED_RATIO_THRESHOLD = 0.01
+    THOUSAND_RATIO_THRESHOLD = 0.001
+    FIVE_THOUSAND_RATIO_THRESHOLD = 0.005
+    TEN_THOUSAND_RATIO_THRESHOLD = 0.0001
+    COSINE_THRESHOLD = 0.9999
+    ULP_FLOAT32_THRESHOLD = 32
+    ULP_FLOAT16_THRESHOLD = 1
+
+    # compare result data
+    READ_NONE = 'No data'
+    NONE = 'None'
+    SHAPE_UNMATCH = 'shape unmatched'
+    DIFF = 'Different'
+    UNSUPPORTED = 'unsupported'
+    NAN = 'Nan'
+    PASS = 'pass'
+    WARNING = 'Warning'
+    ERROR = 'error'
+    SKIP = 'SKIP'
+    BFLOAT16_MIN = -3.3895313892515355e+38
+    BFLOAT16_MAX = 3.3895313892515355e+38
+    BFLOAT16_EPS = 3.90625e-3  # 2 ** -8
+
+    # accuracy standards
+    COS_THRESHOLD = 0.99
+    MAX_ABS_ERR_THRESHOLD = 0.001
+    COS_MAX_THRESHOLD = 0.9
+    MAX_ABS_ERR_MAX_THRESHOLD = 1
+    ACCURACY_CHECK_YES = "Yes"
+    ACCURACY_CHECK_NO = "No"
+    ACCURACY_CHECK_UNMATCH = "Unmatched"
+
+    # error message
+    NO_BENCH = "No bench data matched."
+
+    # compare const
+    FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble]
+
+    # highlight xlsx color const
+    RED = "FFFF0000"
+    YELLOW = "FFFF00"
+    BLUE = "0000FF"
+
+    # highlight rules const
+    OVERFLOW_LIST = ['nan\t', 'inf\t', '-inf\t', 'nan', 'inf', '-inf']
+    MAX_DIFF_RED = 1e+10
+    ORDER_MAGNITUDE_DIFF_YELLOW = 1
+    ONE_THOUSAND_ERROR_IN_RED = 0.9
+    ONE_THOUSAND_ERROR_OUT_RED = 0.6
+    ONE_THOUSAND_ERROR_DIFF_YELLOW = 0.1
+    COSINE_DIFF_YELLOW = 0.1
+    MAX_RELATIVE_OUT_RED = 0.5
+    MAX_RELATIVE_OUT_YELLOW = 0.1
+    MAX_RELATIVE_IN_YELLOW = 0.01
+
+class FileCheckConst:
+    """
+    Class for file check const
+    """
+    READ_ABLE = "read"
+    WRITE_ABLE = "write"
+    READ_WRITE_ABLE = "read and write"
+    DIRECTORY_LENGTH = 4096
+    FILE_NAME_LENGTH = 255
+    FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$"
+    FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$'
+    PKL_SUFFIX = ".pkl"
+    NUMPY_SUFFIX = ".npy"
+    JSON_SUFFIX = ".json"
+    PT_SUFFIX = ".pt"
+    CSV_SUFFIX = ".csv"
+    YAML_SUFFIX = ".yaml"
+    MAX_PKL_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_NUMPY_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
+    MAX_JSON_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_PT_SIZE = 10737418240  # 10 * 1024 * 1024 * 1024
+    MAX_CSV_SIZE = 1073741824  # 1 * 1024 * 1024 * 1024
+    MAX_YAML_SIZE = 1048576  # 10 * 1024 * 1024
+    DIR = "dir"
+    FILE = "file"
+    DATA_DIR_AUTHORITY = 0o750
+    DATA_FILE_AUTHORITY = 0o640
+    FILE_SIZE_DICT = {
+        PKL_SUFFIX: MAX_PKL_SIZE,
+        NUMPY_SUFFIX: MAX_NUMPY_SIZE,
+        JSON_SUFFIX: MAX_JSON_SIZE,
+        PT_SUFFIX: MAX_PT_SIZE,
+        CSV_SUFFIX: MAX_CSV_SIZE,
+        YAML_SUFFIX: MAX_YAML_SIZE
+    }
+
+class OverflowConst:
+    """
+    Class for Overflow
+    """
+    OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE"
+    OVERFLOW_ORIGINAL_MODE = 0
+    OVERFLOW_DEBUG_MODE = 1

msprobe/core/common/exceptions.py

@@ -0,0 +1,88 @@
+class CodedException(Exception):
+    def __init__(self, code, error_info=''):
+        super().__init__()
+        self.code = code
+        self.error_info = self.err_strs.get(code) + error_info
+
+    def __str__(self):
+        return self.error_info
+
+
+class MsaccException(CodedException):
+    INVALID_PARAM_ERROR = 0
+    OVERFLOW_NUMS_ERROR = 1
+
+    err_strs = {
+        INVALID_PARAM_ERROR: "[msacc] 无效参数： ",
+        OVERFLOW_NUMS_ERROR: "[msacc] 超过预设溢出次数 当前溢出次数:"
+    }
+
+
+class FileCheckException(CodedException):
+    INVALID_FILE_ERROR = 0
+    FILE_PERMISSION_ERROR = 1
+    SOFT_LINK_ERROR = 2
+    ILLEGAL_PATH_ERROR = 3
+    ILLEGAL_PARAM_ERROR = 4
+    FILE_TOO_LARGE_ERROR = 5
+
+    err_strs = {
+        SOFT_LINK_ERROR: "[msacc] 检测到软链接： ",
+        FILE_PERMISSION_ERROR: "[msacc] 文件权限错误： ",
+        INVALID_FILE_ERROR: "[msacc] 无效文件： ",
+        ILLEGAL_PATH_ERROR: "[msacc] 非法文件路径： ",
+        ILLEGAL_PARAM_ERROR: "[msacc] 非法打开方式： ",
+        FILE_TOO_LARGE_ERROR: "[msacc] 文件过大： "
+    }
+
+
+class ParseJsonException(CodedException):
+    UnexpectedNameStruct = 0
+    InvalidDumpJson = 1
+    err_strs = {
+        UnexpectedNameStruct: "[msacc] Unexpected name in json: ",
+        InvalidDumpJson: "[msacc] json格式不正确: ",
+    }
+
+
+class ScopeException(CodedException):
+    InvalidApiStr = 0
+    InvalidScope = 1
+    ArgConflict = 2
+    err_strs = {
+        InvalidApiStr: "[msacc] Invalid api_list: ",
+        InvalidScope: "[msacc] Invalid scope: ",
+        ArgConflict: "[msacc] Scope and api_list conflict: ",
+    }
+
+
+class RepairException(CodedException):
+    InvalidRepairType = 0
+    err_strs = {
+        InvalidRepairType: "[msacc] Invalid repair_type: "
+    }
+
+
+class StepException(CodedException):
+    InvalidPostProcess = 0
+    err_strs = {
+        InvalidPostProcess: "[msacc] 错误的step后处理配置: ",
+    }
+
+
+class FreeBenchmarkException(CodedException):
+    UnsupportedType = 0
+    InvalidGrad = 1
+    err_strs = {
+        UnsupportedType: "[msacc] Free benchmark get unsupported type: ",
+        InvalidGrad: "[msacc] Free benchmark gradient invalid: ",
+    }
+
+
+class DistributedNotInitializedError(Exception):
+    def __init__(self, msg):
+        super().__init__()
+        self.msg = msg
+
+    def __str__(self):
+        return self.msg

msprobe/core/common/file_check.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+import os
+import re
+
+from msprobe.core.common.log import logger
+from msprobe.core.common.exceptions import FileCheckException
+from msprobe.core.common.const import FileCheckConst
+
+
+class FileChecker:
+    """
+    The class for check file.
+
+    Attributes:
+        file_path: The file or dictionary path to be verified.
+        path_type: file or dictionary
+        ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability
+        file_type(str): The correct file type for file
+    """
+    def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True):
+        self.file_path = file_path
+        self.path_type = self._check_path_type(path_type)
+        self.ability = ability
+        self.file_type = file_type
+        self.is_script = is_script
+
+    @staticmethod
+    def _check_path_type(path_type):
+        if path_type not in [FileCheckConst.DIR, FileCheckConst.FILE]:
+            logger.error(f'The path_type must be {FileCheckConst.DIR} or {FileCheckConst.FILE}.')
+            raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
+        return path_type
+
+    def common_check(self):
+        """
+        功能：用户校验基本文件权限：软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符
+        注意：文件后缀的合法性，非通用操作，可使用其他独立接口实现
+        """
+        check_path_exists(self.file_path)
+        check_link(self.file_path)
+        self.file_path = os.path.realpath(self.file_path)
+        check_path_length(self.file_path)
+        check_path_type(self.file_path, self.path_type)
+        self.check_path_ability()
+        if self.is_script:
+            check_path_owner_consistent(self.file_path)
+        check_path_pattern_vaild(self.file_path)
+        check_common_file_size(self.file_path)
+        check_file_suffix(self.file_path, self.file_type)
+        return self.file_path
+
+    def check_path_ability(self):
+        if self.ability == FileCheckConst.WRITE_ABLE:
+            check_path_writability(self.file_path)
+        if self.ability == FileCheckConst.READ_ABLE:
+            check_path_readability(self.file_path)
+        if self.ability == FileCheckConst.READ_WRITE_ABLE:
+            check_path_readability(self.file_path)
+            check_path_writability(self.file_path)
+
+
+class FileOpen:
+    """
+    The class for open file by a safe way.
+
+    Attributes:
+        file_path: The file or dictionary path to be opened.
+        mode(str): The file open mode
+    """
+    SUPPORT_READ_MODE = ["r", "rb"]
+    SUPPORT_WRITE_MODE = ["w", "wb", "a", "ab"]
+    SUPPORT_READ_WRITE_MODE = ["r+", "rb+", "w+", "wb+", "a+", "ab+"]
+
+    def __init__(self, file_path, mode, encoding='utf-8'):
+        self.file_path = file_path
+        self.mode = mode
+        self.encoding = encoding
+        self._handle = None
+
+    def __enter__(self):
+        self.check_file_path()
+        binary_mode = "b"
+        if binary_mode not in self.mode:
+            self._handle = open(self.file_path, self.mode, encoding=self.encoding)
+        else:
+            self._handle = open(self.file_path, self.mode)
+        return self._handle
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._handle:
+            self._handle.close()
+
+    def check_file_path(self):
+        support_mode = self.SUPPORT_READ_MODE + self.SUPPORT_WRITE_MODE + self.SUPPORT_READ_WRITE_MODE
+        if self.mode not in support_mode:
+            logger.error("File open not support %s mode" % self.mode)
+            raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
+        check_link(self.file_path)
+        self.file_path = os.path.realpath(self.file_path)
+        check_path_length(self.file_path)
+        self.check_ability_and_owner()
+        check_path_pattern_vaild(self.file_path)
+        if os.path.exists(self.file_path):
+            check_common_file_size(self.file_path)
+
+    def check_ability_and_owner(self):
+        if self.mode in self.SUPPORT_READ_MODE:
+            check_path_exists(self.file_path)
+            check_path_readability(self.file_path)
+            check_path_owner_consistent(self.file_path)
+        if self.mode in self.SUPPORT_WRITE_MODE and os.path.exists(self.file_path):
+            check_path_writability(self.file_path)
+            check_path_owner_consistent(self.file_path)
+        if self.mode in self.SUPPORT_READ_WRITE_MODE and os.path.exists(self.file_path):
+            check_path_readability(self.file_path)
+            check_path_writability(self.file_path)
+            check_path_owner_consistent(self.file_path)
+
+
+def check_link(path):
+    abs_path = os.path.abspath(path)
+    if os.path.islink(abs_path):
+        logger.error('The file path {} is a soft link.'.format(path))
+        raise FileCheckException(FileCheckException.SOFT_LINK_ERROR)
+
+
+def check_path_length(path, name_length=None):
+    file_max_name_length = name_length if name_length else FileCheckConst.FILE_NAME_LENGTH
+    if len(path) > FileCheckConst.DIRECTORY_LENGTH or \
+            len(os.path.basename(path)) > file_max_name_length:
+        logger.error('The file path length exceeds limit.')
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR)
+
+
+def check_path_exists(path):
+    if not os.path.exists(path):
+        logger.error('The file path %s does not exist.' % path)
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR)
+
+
+def check_path_readability(path):
+    if not os.access(path, os.R_OK):
+        logger.error('The file path %s is not readable.' % path)
+        raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
+
+
+def check_path_writability(path):
+    if not os.access(path, os.W_OK):
+        logger.error('The file path %s is not writable.' % path)
+        raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
+
+
+def check_path_executable(path):
+    if not os.access(path, os.X_OK):
+        logger.error('The file path %s is not executable.' % path)
+        raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
+
+
+def check_other_user_writable(path):
+    st = os.stat(path)
+    if st.st_mode & 0o002:
+        logger.error('The file path %s may be insecure because other users have write permissions. ' % path)
+        raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
+
+
+def check_path_owner_consistent(path):
+    file_owner = os.stat(path).st_uid
+    if file_owner != os.getuid():
+        logger.error('The file path %s may be insecure because is does not belong to you.' % path)
+        raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR)
+
+
+def check_path_pattern_vaild(path):
+    if not re.match(FileCheckConst.FILE_VALID_PATTERN, path):
+        logger.error('The file path %s contains special characters.' %(path))
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR)
+
+
+def check_file_size(file_path, max_size):
+    file_size = os.path.getsize(file_path)
+    if file_size >= max_size:
+        logger.error(f'The size of file path {file_path} exceeds {max_size} bytes.')
+        raise FileCheckException(FileCheckException.FILE_TOO_LARGE_ERROR)
+
+
+def check_common_file_size(file_path):
+    if os.path.isfile(file_path):
+        for suffix, max_size in FileCheckConst.FILE_SIZE_DICT.items():
+            if file_path.endswith(suffix):
+                check_file_size(file_path, max_size)
+                break
+
+
+def check_file_suffix(file_path, file_suffix):
+    if file_suffix:
+        if not file_path.endswith(file_suffix):
+            logger.error(f"The {file_path} should be a {file_suffix} file!")
+            raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
+
+
+def check_path_type(file_path, file_type):
+    if file_type == FileCheckConst.FILE:
+        if not os.path.isfile(file_path):
+            logger.error(f"The {file_path} should be a file!")
+            raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
+    if file_type == FileCheckConst.DIR:
+        if not os.path.isdir(file_path):
+            logger.error(f"The {file_path} should be a dictionary!")
+            raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
+
+
+def create_directory(dir_path):
+    """
+    Function Description:
+        creating a directory with specified permissions
+    Parameter:
+        dir_path: directory path
+    Exception Description:
+        when invalid data throw exception
+    """
+    dir_path = os.path.realpath(dir_path)
+    try:
+        os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True)
+    except OSError as ex:
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR,
+            'Failed to create {}. Please check the path permission or disk space .{}'.format(dir_path, str(ex))) from ex
+
+
+def check_path_before_create(path):
+    if path_len_exceeds_limit(path):
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, 'The file path length exceeds limit.')
+
+    if not re.match(FileCheckConst.FILE_PATTERN, os.path.realpath(path)):
+        raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR,
+                                 'The file path {} contains special characters.'.format(path))
+
+
+def change_mode(path, mode):
+    if not os.path.exists(path) or os.path.islink(path):
+        return
+    try:
+        os.chmod(path, mode)
+    except PermissionError as ex:
+        raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR,
+                                 'Failed to change {} authority. {}'.format(path, str(ex))) from ex
+
+
+def path_len_exceeds_limit(file_path):
+    return len(os.path.realpath(file_path)) > FileCheckConst.DIRECTORY_LENGTH or \
+        len(os.path.basename(file_path)) > FileCheckConst.FILE_NAME_LENGTH

msprobe/core/common/log.py

@@ -0,0 +1,55 @@
+import os
+import time
+import sys
+
+class BaseLogger:
+    def __init__(self):
+        self.warning_level = "WARNING"
+        self.error_level = "ERROR"
+        self.info_level = "INFO"
+        self.rank = None
+
+    @staticmethod
+    def _print_log(level, msg, end='\n'):
+        current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        pid = os.getpid()
+        full_msg = f"{current_time} ({pid}) [{level}] {msg}"
+        print(full_msg, end=end)
+        sys.stdout.flush()
+
+    def get_rank(self):
+        return self.rank
+
+    def info(self, msg):
+        self._print_log(self.info_level, msg)
+
+    def error(self, msg):
+        self._print_log(self.error_level, msg)
+
+    def warning(self, msg):
+        self._print_log(self.warning_level, msg)
+
+    def on_rank_0(self, func):
+        def func_rank_0(*args, **kwargs):
+            current_rank = self.get_rank()
+            if current_rank is None or current_rank == 0:
+                return func(*args, **kwargs)
+            else:
+                return None
+        return func_rank_0
+
+    def info_on_rank_0(self, msg):
+        return self.on_rank_0(self.info)(msg)
+
+    def error_on_rank_0(self, msg):
+        return self.on_rank_0(self.error)(msg)
+
+    def warning_on_rank_0(self, msg):
+        return self.on_rank_0(self.warning)(msg)
+    
+    def error_log_with_exp(self, msg, exception):
+        self.error(msg)
+        raise exception
+
+
+logger = BaseLogger()