mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

msprobe/msaccucmp/cmp_utils/log.py

@@ -0,0 +1,257 @@
+# coding=utf-8
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+"""
+Function:
+This file mainly involves the print function.
+"""
+
+import os
+import logging
+from functools import wraps
+from msprobe.msaccucmp.cmp_utils.constant.const_manager import ConstManager
+
+
+def filter_special_chars(func):
+    @wraps(func)
+    def func_level(msg, **kwargs):
+        if isinstance(msg, str):
+            for char in ConstManager.SPECIAL_CHAR:
+                msg = msg.replace(char, '_')
+        return func(msg, **kwargs)
+
+    return func_level
+
+
+def _setting_config(message: str) -> str:
+    pid = os.getpid()
+    cur_format = '%(asctime)s (' + str(pid) + ') - [%(levelname)s] %(message)s'
+    logging.basicConfig(format=cur_format, datefmt="%Y-%m-%d %H:%M:%S",
+                        level=logging.INFO)
+    return message
+
+
+@filter_special_chars
+def print_error_log(error_msg: str) -> None:
+    """
+    print error log
+    :param error_msg: the error message
+    """
+    message = _setting_config(error_msg)
+    logging.error(message)
+
+
+@filter_special_chars
+def print_warn_log(warn_msg: str) -> None:
+    """
+    print warn log
+    :param warn_msg: the warn message
+    """
+    message = _setting_config(warn_msg)
+    logging.warning(message)
+
+
+@filter_special_chars
+def print_info_log(info_msg: str) -> None:
+    """
+    print info log
+    :param info_msg: the info message
+    """
+    message = _setting_config(info_msg)
+    logging.info(message)
+
+
+def print_no_left_dump_file_error(op_name: str, op_type: str, is_error: bool = False) -> str:
+    """
+    Print warn or error log for no my output dump file error
+    :param op_name: the op name
+    :param op_type: the op type
+    :param is_error: the log lever is error
+    :return: message
+    """
+    message = '[%s] There is no dump file for my output operator "%s". The type is %s.' \
+          % (op_name, op_name, op_type)
+    if is_error:
+        print_error_log(message)
+    else:
+        print_warn_log(message)
+    return message
+
+
+def print_no_right_dump_file_error(op_name: str, tensor_id: str, is_error: bool = False) -> str:
+    """
+    Print warn or error log for no right dump file error
+    :param op_name: the op name
+    :param tensor_id: the tensor id
+    :param is_error: the log lever is error
+    :return message
+    """
+    message = '[%s] There is no the ground truth dump file for %s.' % (op_name, tensor_id)
+    if is_error:
+        print_error_log(message)
+    else:
+        print_warn_log(message)
+    return message
+
+
+def print_invalid_nz_dump_data(message: str, op_name: str = None, is_error: bool = False) -> str:
+    if not is_error and op_name:
+        message = "[%s] %s" % (op_name, message)
+    if is_error:
+        print_error_log(message)
+    else:
+        print_warn_log(message)
+    return message
+
+
+def print_start_to_compare_op(op_name: str) -> None:
+    """
+    Print info log for start to compare op
+    :param op_name: the op name
+    """
+    message = '[%s] Start to compare op "%s".' % (op_name, op_name)
+    print_info_log(message)
+
+
+def print_open_file_error(path: str, io_error: any) -> None:
+    """
+    Print error log for open file error
+    :param path: the path
+    :param io_error: error info
+    """
+    message = 'Failed to open "%r". %s' % (path, str(io_error))
+    print_error_log(message)
+
+
+def print_write_result_info(prefix: str, path: str) -> None:
+    """
+    Print info log for write result to file
+    :param prefix: the info
+    :param path: the path
+    """
+    message = 'The %s have been written to "%r".' % (prefix, path)
+    print_info_log(message)
+
+
+def print_only_support_error(prefix: str, value: any, support_info: list) -> None:
+    """
+    Print error log for only supports error
+    :param prefix: the info
+    :param value: the value no support
+    :param support_info: the support info
+    """
+    message = "The %s '%s' is invalid. It only supports '%s'." % (prefix, str(value), str(support_info))
+    print_error_log(message)
+
+
+def print_not_match_error(op_name: str, prefix: str, left_value: str, right_value: str, tensor_id: str = None) -> str:
+    """
+    Print not match error
+    :param op_name: the op name
+    :param prefix: the info
+    :param left_value: the left value
+    :param right_value: the right value
+    :param tensor_id: the tensor id
+    :return message
+    """
+    line = '[%s] The %s does not match (%s vs %s)' % (op_name, prefix, left_value, right_value)
+    if tensor_id:
+        line = '%s for %s.' % (line, tensor_id)
+        print_error_log(line)
+    else:
+        line = '%s.' % line
+        print_warn_log(line)
+    return line
+
+
+def print_cannot_compare_warning(op_name: str, left_shape: str, right_shape: str) -> str:
+    """
+    Print cannot compare warning
+    :param op_name: the op name
+    :param left_shape: the left_shape
+    :param right_shape: the right shape
+    :return message
+    """
+    prefix = '[%s] ' % op_name if op_name else ''
+    message = '%sDue to the different shapes on the left and right,the left dump data%s can not ' \
+              'be compared to the right dump data%s. Please check the batch of the dump data or ' \
+              'the shape may be changed due to optimization.' % (prefix, left_shape, right_shape)
+    print_warn_log(message)
+    return message
+
+
+def print_npu_path_valid_message(npu_dump_dir: str, dump_file_path_format: str) -> str:
+    """
+    Print npu path valid message
+    :param npu_dump_dir: the npu dump directory
+    :param dump_file_path_format : correct dump file path format
+    :return message
+    """
+    message = "The {0} does not match the path format," \
+              "please save dump files in the {1} path format".format(npu_dump_dir, dump_file_path_format)
+    print_error_log(message)
+    return message
+
+
+def print_out_of_range_error(op_name: str, index_type: str, index: int, range_str: str) -> None:
+    """
+    Print out of range error
+    :param op_name: the op name
+    :param index_type: the tensor type
+    :param index: the index
+    :param range_str: the count
+    """
+    prefix = ''
+    if op_name:
+        prefix = '[%s] ' % op_name
+    message = '%sThe %s index (%d) is out of range %s. Please check the index.' % \
+          (prefix, index_type, index, range_str)
+    print_error_log(message)
+
+
+def print_skip_inner_op_msg(op_name: str, is_error: bool) -> None:
+    """
+    Print warn or error log for skip inner operator
+    :param op_name: the op name
+    :param is_error: the log lever is error
+    :return message
+    """
+    message = '[%s] The op "%s" is inner node for multi to multi relation. Skip the op "%s".' \
+          % (op_name, op_name, op_name)
+    if is_error:
+        print_error_log(message)
+    else:
+        print_warn_log(message)
+
+
+def print_deprecated_warning(file_name: str) -> None:
+    """
+    Print deprecated warning
+    :param file_name: the file name
+    """
+    message = 'Note that "%s" will be deprecated in a future release. It'\
+              ' is recommended to use the next-generation "msaccucmp.py".' % file_name
+    print_warn_log(message)
+
+
+def print_skip_quant_info(op_name: str) -> None:
+    """
+    Print the op skipped info
+    :param op_name: the op name
+    """
+    message = '[%s] This op is in a quant/dequant op pair. Skip the op.' % op_name
+    print_info_log(message)

msprobe/msaccucmp/cmp_utils/multi_process/__init__.py

@@ -0,0 +1,16 @@
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+

msprobe/msaccucmp/cmp_utils/multi_process/multi_convert_process.py

@@ -0,0 +1,140 @@
+# coding=utf-8
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+"""
+Function:
+MultiConvertProcess class. This class mainly involves the process function.
+"""
+import os
+import multiprocessing
+import psutil
+from msprobe.msaccucmp.cmp_utils import log
+from msprobe.msaccucmp.cmp_utils.constant.const_manager import ConstManager
+from msprobe.msaccucmp.cmp_utils.file_utils import FileUtils
+
+from msprobe.msaccucmp.cmp_utils.multi_process.progress import Progress
+from msprobe.msaccucmp.cmp_utils.constant.compare_error import CompareError
+
+
+class MultiConvertProcess:
+    """
+    The class for multi process for convert
+    """
+    MAX_MULTI = 4
+    MULTI_THREAD_RESULT_COUNT = 2
+    MULTI_THREAD_RETURN_CODE_INDEX = 0
+    MULTI_THREAD_ERROR_FILE_INDEX = 1
+
+    def __init__(self: any, process_func: any, input_path: list, output_path: str) -> None:
+        self._process_func = process_func
+        self._output_path = output_path
+        self._input_path = input_path
+        self._progress = None
+
+    def process(self: any) -> int:
+        """
+        Process by multi process
+        """
+        return_code = CompareError.MSACCUCMP_NONE_ERROR
+
+        # split big file and common file
+        multi_process_file_list, big_file_list = self._split_big_file()
+
+        if len(multi_process_file_list) > 0:
+            ret = self._do_multi_process(multi_process_file_list)
+            if ret != CompareError.MSACCUCMP_NONE_ERROR:
+                return_code = ret
+        # big file do not multi process
+        for big_file in big_file_list:
+            ret, _ = self._process_func(big_file)
+            self._handle_result_callback([ret, big_file])
+            if ret != CompareError.MSACCUCMP_NONE_ERROR:
+                return_code = ret
+        if return_code != CompareError.MSACCUCMP_NONE_ERROR:
+            error_file_path = os.path.join(self._output_path, ConstManager.CONVERT_FAILED_FILE_LIST_NAME)
+            if os.path.exists(error_file_path):
+                log.print_info_log('The list of files that failed to be converted has been written to "%r".'
+                                   % error_file_path)
+
+        return return_code
+
+    def get_max_file_size(self: any) -> int:
+        """
+        Get max file size
+        :return int
+        """
+        mem = psutil.virtual_memory()
+        available = mem.available
+        cpu_count = int((multiprocessing.cpu_count() + 1) / 2)
+        if cpu_count != 0:
+            return available / cpu_count / self.MAX_MULTI
+        else:
+            return 0
+
+    def _handle_result_callback(self: any, result: list) -> None:
+        self._progress.update_progress()
+        self._progress.update_and_print_progress()
+        if len(result) == self.MULTI_THREAD_RESULT_COUNT:
+            cur_ret = result[self.MULTI_THREAD_RETURN_CODE_INDEX]
+            if cur_ret != CompareError.MSACCUCMP_NONE_ERROR:
+                error_file_path = os.path.join(self._output_path, ConstManager.CONVERT_FAILED_FILE_LIST_NAME)
+                FileUtils.save_data_to_file(error_file_path, "%s\n" % result[self.MULTI_THREAD_ERROR_FILE_INDEX],
+                                            'a+', delete=False)
+
+    def _do_multi_process(self: any, file_list: list) -> int:
+        cpu_count = int((multiprocessing.cpu_count() + 1) / 2)
+        pool = multiprocessing.Pool(cpu_count)
+        all_task = []
+        for cur_path in file_list:
+            task = pool.apply_async(self._process_func,
+                                    args=(cur_path,),
+                                    callback=self._handle_result_callback)
+            all_task.append(task)
+        pool.close()
+        pool.join()
+        for task in all_task:
+            result = task.get()
+            if len(result) != self.MULTI_THREAD_RESULT_COUNT:
+                continue
+            cur_ret = result[self.MULTI_THREAD_RETURN_CODE_INDEX]
+            if cur_ret != CompareError.MSACCUCMP_NONE_ERROR:
+                return cur_ret
+        return CompareError.MSACCUCMP_NONE_ERROR
+
+    def _split_big_file(self: any) -> (list, list):
+        if len(self._input_path) == 1:
+            files = os.listdir(self._input_path[0])
+        else:
+            files = self._input_path
+        self._progress = Progress(len(files))
+        multi_process_file_list = []
+        big_file_list = []
+        max_file_size = self.get_max_file_size()
+        for cur_file in files:
+            if cur_file == ConstManager.MAPPING_FILE_NAME:
+                continue
+            if len(self._input_path) == 1:
+                cur_path = os.path.join(self._input_path[0], cur_file)
+            else:
+                cur_path = cur_file
+            if os.path.isfile(cur_path):
+                # skip big file
+                if os.path.getsize(cur_path) > max_file_size:
+                    big_file_list.append(cur_path)
+                else:
+                    multi_process_file_list.append(cur_path)
+        return multi_process_file_list, big_file_list

msprobe/msaccucmp/cmp_utils/multi_process/progress.py

@@ -0,0 +1,78 @@
+# coding=utf-8
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
+#
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#
+#          http://license.coscl.org.cn/MulanPSL2
+#
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
+
+"""
+Function:
+Progress class. This class mainly involves the print_progress function.
+"""
+import time
+import math
+
+from msprobe.msaccucmp.cmp_utils import log
+from msprobe.msaccucmp.cmp_utils.constant.const_manager import ConstManager
+from msprobe.msaccucmp.cmp_utils.constant.compare_error import CompareError
+
+
+class Progress:
+    """
+    The class for progress
+    """
+    PROGRESS_GREATER_THAN_COUNT = 50
+    PROGRESS_GREATER_THAN = '>'
+    INTERVAL_TIME_SECOND = 1
+
+    def __init__(self: any, total_count: int) -> None:
+        self.total_count = total_count
+        self.last_progress_time = 0
+        self.current_count = 0
+
+    def update_progress(self: any, update_count: int = 1) -> None:
+        """
+        Update the progress
+        """
+        self.current_count += update_count
+
+    def is_done(self: any) -> bool:
+        """
+        check if the process is done
+        """
+        return self.current_count == self.total_count
+
+    def update_and_print_progress(self: any, progress: int = None) -> None:
+        """
+        Print the progress realtime
+        :param progress: the progress
+        """
+        if progress is None:
+            if self.total_count != 0:
+                progress = round(self.current_count * 100.0 / self.total_count, 2)
+            else:
+                progress = 0
+                log.print_error_log('Can not divide zero.')
+        current_time = time.time()
+        denominator = 1
+        greater_than_count = 0
+        if self.PROGRESS_GREATER_THAN_COUNT != 0:
+            denominator = ConstManager.MAX_PROGRESS // self.PROGRESS_GREATER_THAN_COUNT
+        if denominator != 0:
+            greater_than_count = math.floor(progress / denominator)
+        progress_info = '%s%s' % (self.PROGRESS_GREATER_THAN * greater_than_count,
+                                  ' ' * (self.PROGRESS_GREATER_THAN_COUNT - greater_than_count))
+        if current_time - self.last_progress_time >= self.INTERVAL_TIME_SECOND \
+                or progress == ConstManager.MAX_PROGRESS:
+            log.print_info_log('[ %s %d%%]' % (progress_info, progress))
+            self.last_progress_time = current_time