mindstudio-probe 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py

@@ -0,0 +1,212 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 标准库导入
+import multiprocessing
+from multiprocessing import Manager
+import os
+import signal
+import sys
+import time
+
+# 第三方库导入
+from mindspore import context
+import numpy as np
+from tqdm import tqdm
+
+# 本地应用/库特定导入
+from msprobe.core.common.const import Const, CompareConst, MsCompareConst
+from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker, BasicInfoAndStatus
+from msprobe.mindspore.api_accuracy_checker.multi_data_manager import MultiDataManager
+from msprobe.mindspore.common.log import logger
+
+
+class MultiApiAccuracyChecker(ApiAccuracyChecker):
+    def __init__(self, args):
+        # 可以添加 MultiApiAccuracyChecker 特有的属性或方法
+        self.api_infos = dict()
+
+        # 使用 Manager 创建共享变量，确保进程间的同步
+        self.manager = Manager()
+        self.is_first_write = self.manager.Value('b', True)  # 创建共享变量
+
+        # 初始化 DataManager 时传入共享的 is_first_write
+        self.multi_data_manager = MultiDataManager(args.out_path, args.result_csv_path, self.is_first_write)
+
+        self.args = args  # 将 args 保存为类的属性
+
+        # 初始化一个属性来存储当前的设备ID（用于日志中显示）
+        self.current_device_id = None
+
+    def process_on_device(self, device_id, api_infos, progress_queue):
+        """
+        在特定设备上处理一部分API。
+
+        参数:
+            device_id (int): 要使用的设备ID。
+            api_infos (list): 包含API名称和对应信息的元组列表。
+            progress_queue (multiprocessing.Queue): 用于通信进度更新的队列。
+        """
+
+        # 设置当前设备ID
+        self.current_device_id = device_id
+
+        # 设置 MindSpore context 的 device_id
+        context.set_context(device_id=device_id)
+
+        # 遍历当前进程分配的任务
+        for _, (api_name_str, api_info) in enumerate(api_infos):
+            logger.debug(f"Processing API: {api_name_str}, Device: {device_id}")
+
+            if not self.multi_data_manager.is_unique_api(api_name_str):
+                logger.debug(f"API {api_name_str} is not unique, skipping.")
+                progress_queue.put(1)
+                continue
+
+            # 处理前向
+            forward_output_list = self.process_forward(api_name_str, api_info)
+            if forward_output_list is not Const.EXCEPTION_NONE:
+                self.multi_data_manager.record(forward_output_list)
+
+            # 处理反向
+            backward_output_list = self.process_backward(api_name_str, api_info)
+            if backward_output_list is not Const.EXCEPTION_NONE:
+                self.multi_data_manager.record(backward_output_list)
+
+            # 保存结果
+            self.multi_data_manager.save_results(api_name_str)
+            progress_queue.put(1)  # 更新进度
+
+    def run_and_compare(self):
+        # 获取要使用的设备ID列表
+        device_ids = self.args.device_id
+
+        # 按设备数划分要处理的 API 项
+        partitioned_api_infos = list(self.api_infos.items())
+
+        # 在主进程中进行交叉任务切分（基于取模的方式）
+        partitioned_api_infos_split = [[] for _ in range(len(device_ids))]
+        for idx, api_info in enumerate(partitioned_api_infos):
+            device_index = idx % len(device_ids)  # 使用取模方法分配任务
+            partitioned_api_infos_split[device_index].append(api_info)
+
+        # 创建一个共享进度队列
+        progress_queue = multiprocessing.Queue()
+
+        # 进度条
+        total_tasks = len(partitioned_api_infos)  # 计算总任务数
+        with tqdm(total=total_tasks, desc="Total Progress", ncols=100) as pbar:
+            # 创建多进程
+            processes = []
+            for index, device_id in enumerate(device_ids):
+                process = multiprocessing.Process(target=self.process_on_device,
+                                                  args=(device_id, partitioned_api_infos_split[index], progress_queue))
+                processes.append(process)
+                process.start()
+
+            # 主进程更新进度条
+            completed_tasks = 0
+            while completed_tasks < total_tasks:
+                try:
+                    completed_tasks += progress_queue.get(timeout=Const.PROGRESS_TIMEOUT)  # 设置超时时间（秒）
+                    pbar.update(1)
+                except multiprocessing.queues.Empty:
+                    logger.error("Timeout while waiting for progress updates. Skipping remaining tasks.")
+                    break
+
+                # 检查子进程状态
+                for process in processes:
+                    if not process.is_alive():
+                        if process.exitcode != 0:
+                            logger.error(f"Process {process.pid} exited with code {process.exitcode}.")
+                            total_tasks -= len(partitioned_api_infos_split[processes.index(process)])
+                        processes.remove(process)
+
+            # 确保所有子进程完成或终止
+            for process in processes:
+                process.join(timeout=Const.PROGRESS_TIMEOUT)
+                if process.is_alive():
+                    logger.error(f"Process {process.pid} did not terminate. Forcing termination.")
+                    process.terminate()
+
+    def process_forward(self, api_name_str, api_info):
+        """
+        Overrides the parent class's process_forward method to log the device ID when exceptions occur.
+
+        Parameters:
+            api_name_str (str): The name of the API.
+            api_info (object): The API information object.
+
+        Returns:
+            list or None: The forward output list or None if an error occurs.
+        """
+        if not api_info.check_forward_info():
+            logger.debug(
+                f"[Device {self.current_device_id}] API: {api_name_str} lacks forward information, skipping "
+                f"forward check.")
+            return Const.EXCEPTION_NONE
+
+        try:
+            forward_inputs_aggregation = self.prepare_api_input_aggregation(api_info, Const.FORWARD)
+        except Exception as e:
+            logger.warning(
+                f"[Device {self.current_device_id}] Exception occurred while getting forward API inputs for "
+                f"{api_name_str}. Skipping forward check. Detailed exception information: {e}.")
+            return Const.EXCEPTION_NONE
+
+        forward_output_list = None
+        try:
+            forward_output_list = self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation,
+                                                              Const.FORWARD)
+        except Exception as e:
+            logger.warning(
+                f"[Device {self.current_device_id}] Exception occurred while running and comparing {api_name_str} "
+                f"forward API. Detailed exception information: {e}.")
+        return forward_output_list
+
+    def process_backward(self, api_name_str, api_info):
+        """
+        Overrides the parent class's process_backward method to log the device ID when exceptions occur.
+
+        Parameters:
+            api_name_str (str): The name of the API.
+            api_info (object): The API information object.
+
+        Returns:
+            list or None: The backward output list or None if an error occurs.
+        """
+        if not api_info.check_backward_info():
+            logger.debug(
+                f"[Device {self.current_device_id}] API: {api_name_str} lacks backward information, skipping "
+                f"backward check.")
+            return Const.EXCEPTION_NONE
+
+        try:
+            backward_inputs_aggregation = self.prepare_api_input_aggregation(api_info, Const.BACKWARD)
+        except Exception as e:
+            logger.warning(
+                f"[Device {self.current_device_id}] Exception occurred while getting backward API inputs for "
+                f"{api_name_str}. Skipping backward check. Detailed exception information: {e}.")
+            return Const.EXCEPTION_NONE
+
+        backward_output_list = None
+        try:
+            backward_output_list = self.run_and_compare_helper(api_info, api_name_str, backward_inputs_aggregation,
+                                                               Const.BACKWARD)
+        except Exception as e:
+            logger.warning(
+                f"[Device {self.current_device_id}] Exception occurred while running and comparing {api_name_str} "
+                f"backward API. Detailed exception information: {e}.")
+        return backward_output_list

msprobe/mindspore/api_accuracy_checker/multi_data_manager.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import multiprocessing
+import os
+
+from msprobe.mindspore.api_accuracy_checker.data_manager import (DataManager, ResultCsvEntry, write_csv_header,
+                                                                 get_result_csv_header, get_detail_csv_header,
+                                                                 check_csv_header)
+from msprobe.mindspore.common.log import logger
+
+
+class MultiDataManager(DataManager):
+    def __init__(self, csv_dir, result_csv_path, shared_is_first_write):
+        super().__init__(csv_dir, result_csv_path)
+
+        # 使用共享的 is_first_write 变量来控制表头写入
+        self.shared_is_first_write = shared_is_first_write
+        # 创建锁对象，确保线程安全
+        self.lock = multiprocessing.Lock()
+
+    def save_results(self, api_name_str):
+        """保存结果，线程安全操作"""
+
+        with self.lock:  # 确保保存操作不会被多个进程同时进行
+            if self.is_first_write and self.shared_is_first_write.value:
+                self.shared_is_first_write.value = False
+                self.is_first_write = False  # 写入后标记为 False，避免重复写入表头
+                # 直接写入表头
+                logger.info("Writing CSV headers for the first time.")
+                write_csv_header(self.detail_out_path, get_detail_csv_header)
+                write_csv_header(self.result_out_path, get_result_csv_header)
+
+            """写入详细输出和结果摘要并清理结果"""
+            self.to_detail_csv(self.detail_out_path)
+            logger.debug(f"Detailed output for {api_name_str} written to {self.detail_out_path}.")
+
+            self.to_result_csv(self.result_out_path)
+            logger.debug(f"Result summary for {api_name_str} written to {self.result_out_path}.")
+
+            # 清理记录，准备下一次调用
+            self.clear_results()
+
+    def clear_results(self):
+        """清空 self.results 数据，线程安全操作"""
+        logger.debug("Clearing results data.")
+        self.results.clear()

msprobe/mindspore/api_accuracy_checker/type_mapping.py

@@ -1,7 +1,23 @@
-from mindspore.common import dtype as mstype
-import numpy as np
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import mindspore
+import numpy as np
 import torch
+from mindspore._c_expression import typing
+from mindspore.common import dtype as mstype
 
 INT8 = "Int8"
 UINT8 = "UInt8"
@@ -18,7 +34,6 @@ BOOL = "Bool"
 BFLOAT16 = "BFloat16"
 INT4 = "Int4"
 
-
 dtype_str_to_ms_dtype = {
     INT8: mstype.int8,
     UINT8: mstype.uint8,
@@ -37,7 +52,6 @@ dtype_str_to_ms_dtype = {
 }
 ms_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_ms_dtype.items()}
 
-
 dtype_str_to_np_dtype = {
     INT8: np.int8,
     UINT8: np.uint8,
@@ -75,6 +89,8 @@ FLOAT_TYPE_STR = "float"
 SLICE_TYPE_STR = "slice"
 TUPLE_TYPE_STR = "tuple"
 STR_TYPE_STR = "str"
+MINDSPORE_DTYPE_TYPE_STR = "mindspore.dtype"
+TORCH_DTYPE_TYPE_STR = "torch.dtype"
 
 api_info_type_str_to_type = {
     MINDSPORE_TENSOR_TYPE_STR: mindspore.Tensor,
@@ -83,6 +99,7 @@ api_info_type_str_to_type = {
     FLOAT_TYPE_STR: float,
     SLICE_TYPE_STR: slice,
     STR_TYPE_STR: str,
+    MINDSPORE_DTYPE_TYPE_STR: typing.Type,
 }
 type_to_api_info_type_str = {value: key for key, value in api_info_type_str_to_type.items()}
 
@@ -111,4 +128,4 @@ uint_dtype_str_list = [
     UINT16,
     UINT32,
     UINT64,
-]
+]

msprobe/mindspore/api_accuracy_checker/utils.py

@@ -1,8 +1,24 @@
-from msprobe.core.common.exceptions import ApiAccuracyCheckerException
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import ApiAccuracyCheckerException
 from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list
 from msprobe.mindspore.common.log import logger
 
+
 def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None):
     '''
     Args:
@@ -22,30 +38,30 @@ def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_t
         3. value is not accepted type
         4. value is not accepted value
     '''
-    parse_failed_exception = ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed)
     if not isinstance(dict_instance, dict):
-        logger.error_log_with_exp("check_and_get_from_json_dict failed: input is not a dict", parse_failed_exception)
+        error_info = "check_and_get_from_json_dict failed: input is not a dict"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     value = dict_instance.get(key)
     if value is None:
-        logger.error_log_with_exp(f"check_and_get_from_json_dict failed: {key_description} is missing",
-                                  parse_failed_exception)
+        error_info = f"check_and_get_from_json_dict failed: {key_description} is missing"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     elif accepted_type is not None and not isinstance(value, accepted_type):
-        logger.error_log_with_exp(
-            f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}",
-            parse_failed_exception)
+        error_info = f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     elif accepted_value is not None and value not in accepted_value:
-        logger.error_log_with_exp(
-            f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}",
-            parse_failed_exception)
+        error_info = f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}"
+        raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info)
     return value
 
-def convert_to_tuple(input):
-    if isinstance(input, (tuple, list)):
-        return tuple(input)
+
+def convert_to_tuple(args):
+    if isinstance(args, (tuple, list)):
+        return tuple(args)
     else:
-        input_list = [input]
+        input_list = [args]
         return tuple(input_list)
 
+
 def trim_output_compute_element_list(compute_element_list, forward_or_backward):
     '''
     Args:
@@ -55,12 +71,13 @@ def trim_output_compute_element_list(compute_element_list, forward_or_backward):
     trimmed_list = []
     for compute_element in compute_element_list:
         if compute_element.get_parameter() is None or \
-            (forward_or_backward == Const.BACKWARD and compute_element.get_dtype() not in float_dtype_str_list):
+                (forward_or_backward == Const.BACKWARD and compute_element.get_dtype() not in float_dtype_str_list):
             # trim case: 1. parameter is None. 2. backward output has non float parameter
             continue
         trimmed_list.append(compute_element)
     return trimmed_list
 
+
 class GlobalContext:
     def __init__(self):
         self.is_constructed = True
@@ -77,4 +94,4 @@ class GlobalContext:
         return self.is_constructed
 
 
-global_context = GlobalContext()
+global_context = GlobalContext()

msprobe/mindspore/cell_processor.py

@@ -1,4 +1,19 @@
-from msprobe.core.data_dump.scope import ModuleRangeScope
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope
 from msprobe.core.common.const import Const
 
 
@@ -9,10 +24,7 @@ class CellProcessor:
     module_node = {}
 
     def __init__(self, scope):
-        if isinstance(scope, ModuleRangeScope):
-            self.scope = scope
-        else:
-            self.scope = None
+        self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None
 
     @staticmethod
     def set_cell_count(cell_name):
@@ -21,30 +33,29 @@ class CellProcessor:
         else:
             CellProcessor.cell_count[cell_name] += 1
         return CellProcessor.cell_count[cell_name]
-    
+
     @classmethod
     def reset_cell_stats(cls):
         cls.cell_count = {}
         cls.cell_stack = []
         cls.api_parent_node = ""
         cls.module_node = {}
-    
+
     def node_hook(self, name_prefix, start_or_stop, **kwargs):
-        def begin_hook(cell, input):
-            index = self.set_cell_count(name_prefix)
-            cell.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index)
+        def begin_hook(cell, input_data):
+            full_name = self.set_and_get_reserved_name(cell, name_prefix, is_called_by_pre_hook=True)
             if CellProcessor.cell_stack:
                 CellProcessor.module_node[full_name] = CellProcessor.cell_stack[-1]
             else:
                 CellProcessor.module_node[full_name] = None
-            
+
             CellProcessor.cell_stack.append(full_name)
             CellProcessor.api_parent_node = full_name
 
             if self.scope:
                 self.scope.begin_module(full_name)
 
-        def end_hook(cell, input, output):
+        def end_hook(cell, input_data, output_data):
             if CellProcessor.cell_stack:
                 CellProcessor.cell_stack.pop()
             if CellProcessor.cell_stack:
@@ -56,3 +67,13 @@ class CellProcessor:
                 self.scope.end_module(cell.mindstudio_reserved_name)
 
         return begin_hook if Const.START == start_or_stop else end_hook
+
+    def set_and_get_reserved_name(self, cell, cell_name, is_called_by_pre_hook=False):
+        if not is_called_by_pre_hook and hasattr(cell, 'has_pre_hook_called') and cell.has_pre_hook_called:
+            cell.has_pre_hook_called = False
+        else:
+            if is_called_by_pre_hook:
+                cell.has_pre_hook_called = True
+            index = self.set_cell_count(cell_name)
+            cell.mindstudio_reserved_name = cell_name + Const.SEP + str(index)
+        return cell.mindstudio_reserved_name