mindstudio-probe 1.2.2__py3-none-any.whl → 8.1.0__py3-none-any.whl

msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py

@@ -27,10 +27,14 @@ import numpy as np
 from tqdm import tqdm
 
 # 本地应用/库特定导入
-from msprobe.core.common.const import Const, CompareConst, MsCompareConst
+from msprobe.core.common.const import Const, CompareConst
 from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker, BasicInfoAndStatus
 from msprobe.mindspore.api_accuracy_checker.multi_data_manager import MultiDataManager
 from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.common.const import MsCompareConst
+
+from msprobe.core.data_dump.data_collector import build_data_collector
+from msprobe.core.common.utils import Const, print_tools_ends_info, DumpPathAggregation
 
 
 class MultiApiAccuracyChecker(ApiAccuracyChecker):
@@ -50,6 +54,12 @@ class MultiApiAccuracyChecker(ApiAccuracyChecker):
         # 初始化一个属性来存储当前的设备ID（用于日志中显示）
         self.current_device_id = None
 
+        self.save_error_data = args.save_error_data
+        if self.save_error_data:
+            config, dump_path_aggregation = self.init_save_error_data(args)
+            self.data_collector = build_data_collector(config)
+            self.data_collector.update_dump_paths(dump_path_aggregation)
+
     def process_on_device(self, device_id, api_infos, progress_queue):
         """
         在特定设备上处理一部分API。

msprobe/mindspore/api_accuracy_checker/torch_mindtorch_importer.py

@@ -19,7 +19,8 @@ import sys
 from pathlib import Path
 import mindspore
 from msprobe.mindspore.common.log import logger
-from msprobe.core.common.const import Const, CompareConst, MsCompareConst
+from msprobe.core.common.const import Const, CompareConst
+from msprobe.mindspore.common.const import MsCompareConst
 import torch as mindtorch
 from torch import Tensor as mindtorch_tensor
 import torch.nn.functional as mindtorch_func

msprobe/mindspore/cell_processor.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -13,21 +13,50 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope
+from collections import OrderedDict
+
+from mindspore import Tensor
+from mindspore.common.hook_handle import HookHandle
+from mindspore.ops.operations import _inner_ops as inner
+
 from msprobe.core.common.const import Const
+from msprobe.core.common.exceptions import MsprobeException
+from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope, BaseScope
+from msprobe.mindspore.common.const import Const as MsConst
+from msprobe.mindspore.common.log import logger
+from msprobe.mindspore.common.utils import (
+    is_mindtorch,
+    get_cells_and_names_with_index,
+    has_kwargs_in_forward_hook,
+    is_graph_mode_cell_dump_allowed
+)
+from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
+from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump
+from msprobe.core.common.runtime import Runtime
+
+
+def get_cell_construct(construct):
+    def _construct(self, *args, **kwargs):
+        if hasattr(self, 'msprobe_hook'):
+            setattr(self, 'msprobe_input_kwargs', kwargs)
+        return construct(self, *args, **kwargs)
+    return _construct
 
 
 class CellProcessor:
     cell_count = {}
     cell_stack = []
-    api_parent_node = ""
+    api_parent_node = None
     module_node = {}
+    cell_bw_hook_kernels = {}
+    cell_backward_pre_hook = []
+    cell_backward_hook = []
 
     def __init__(self, scope):
         self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None
 
     @staticmethod
-    def set_cell_count(cell_name):
+    def set_and_get_calls_number(cell_name):
         if cell_name not in CellProcessor.cell_count:
             CellProcessor.cell_count[cell_name] = 0
         else:
@@ -38,42 +67,184 @@ class CellProcessor:
     def reset_cell_stats(cls):
         cls.cell_count = {}
         cls.cell_stack = []
-        cls.api_parent_node = ""
+        cls.api_parent_node = None
         cls.module_node = {}
+        cls.cell_bw_hook_kernels = {}
+        cls.cell_backward_pre_hook = []
+        cls.cell_backward_hook = []
 
-    def node_hook(self, name_prefix, start_or_stop, **kwargs):
-        def begin_hook(cell, input_data):
-            full_name = self.set_and_get_reserved_name(cell, name_prefix, is_called_by_pre_hook=True)
-            if CellProcessor.cell_stack:
-                CellProcessor.module_node[full_name] = CellProcessor.cell_stack[-1]
-            else:
-                CellProcessor.module_node[full_name] = None
+    def register_cell_hook(self, models, build_hook, config: DebuggerConfig):
+        if not models:
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
+                                   'The model cannot be None, when level is "L0" or "mix"')
+
+        is_registered = False
+        model_type = Const.MODULE if is_mindtorch() else Const.CELL
+        cells_with_index_in_pynative_mode, cells_with_index_in_graph_mode = get_cells_and_names_with_index(models)
+        construct_name = '_call_impl' if is_mindtorch() else '_run_construct'
+
+        for index, cells_and_names in cells_with_index_in_pynative_mode.items():
+            model = models if index == "-1" else models[int(index)]
+            for name, cell in cells_and_names:
+                if cell == model:
+                    continue
+
+                if not has_kwargs_in_forward_hook():
+                    if not hasattr(cell.__class__, 'msprobe_construct'):
+                        setattr(cell.__class__, 'msprobe_construct', True)
+                        if hasattr(cell.__class__, construct_name):
+                            setattr(cell.__class__, construct_name,
+                                    get_cell_construct(getattr(cell.__class__, construct_name)))
+                setattr(cell, 'msprobe_hook', True)
+
+                cell_index = (index + Const.SEP) if index != "-1" else ""
+                prefix = f'{model_type}{Const.SEP}{cell_index}{name}{Const.SEP}{cell.__class__.__name__}{Const.SEP}'
+
+                forward_pre_hook = self.build_cell_hook(prefix, build_hook)
+                cell.register_forward_pre_hook(forward_pre_hook)
+
+                if not is_registered:
+                    logger.info("The cell hook function is successfully mounted to the model.")
+                is_registered = True
+
+        if is_graph_mode_cell_dump_allowed(config):
+            cells_and_names_in_graph_mode = []
+            for index, cells_and_names in cells_with_index_in_graph_mode.items():
+                model = models if index == "-1" else models[int(index)]
+                for name, cell in cells_and_names:
+                    if cell == model:
+                        continue
+                    cell_index = (index + Const.SEP) if index != "-1" else ""
+                    cells_and_names_in_graph_mode.append((f'{cell_index}{name}', cell))
+
+            if cells_and_names_in_graph_mode:
+                Runtime.run_mode = MsConst.PYNATIVE_GRAPH_MODE
+                GraphModeCellDump(config, cells_and_names_in_graph_mode, strict=False).handle()
 
-            CellProcessor.cell_stack.append(full_name)
-            CellProcessor.api_parent_node = full_name
+    def build_cell_hook(self, cell_name, build_data_hook):
+        def forward_pre_hook(cell, args):
+            index = CellProcessor.set_and_get_calls_number(cell_name)
+            full_forward_name = f'{cell_name}{Const.FORWARD}{Const.SEP}{index}'
+            full_backward_name = f'{cell_name}{Const.BACKWARD}{Const.SEP}{index}'
 
-            if self.scope:
-                self.scope.begin_module(full_name)
+            self.set_construct_info_in_pre_hook(full_forward_name)
 
-        def end_hook(cell, input_data, output_data):
-            if CellProcessor.cell_stack:
-                CellProcessor.cell_stack.pop()
-            if CellProcessor.cell_stack:
-                CellProcessor.api_parent_node = CellProcessor.cell_stack[-1]
+            if not hasattr(cell, 'msprobe_forward_hook'):
+                if is_mindtorch():
+                    cell.register_forward_hook(forward_hook, prepend=True, with_kwargs=True)
+                else:
+                    forward_hook_dict = getattr(cell, '_forward_hook', OrderedDict())
+                    if has_kwargs_in_forward_hook():
+                        forward_hook_with_kwargs_dict = getattr(cell, '_forward_hook_with_kwargs', OrderedDict())
+                        handle = HookHandle(forward_hook_dict, extra_dict=forward_hook_with_kwargs_dict)
+                        forward_hook_with_kwargs_dict[handle.handle_id] = True
+                    else:
+                        handle = HookHandle(forward_hook_dict)
+                    forward_hook_dict[handle.handle_id] = forward_hook
+                    forward_hook_dict.move_to_end(handle.handle_id, last=False)
+
+                setattr(cell, 'msprobe_forward_hook', True)
+
+            def get_backward_hook(backward_data_hook, full_backward_name):
+                def backward_hook_fn(cell, grad_input, grad_output):
+                    new_output = backward_data_hook(cell, grad_input, grad_output)
+                    self.set_construct_info_in_hook(full_backward_name)
+                    cell.has_pre_hook_called = False
+                    return new_output
+                return backward_hook_fn
+
+            enable_hooked = sum(
+                [isinstance(ele, Tensor) and ele.dtype not in MsConst.NonDifferentiableType for ele in args]
+            )
+            if enable_hooked:
+                backward_hook = OrderedDict()
+                hook_set = build_data_hook(BaseScope.Module_Type_Module, full_forward_name)
+                backward_hook[full_backward_name] = get_backward_hook(hook_set.backward_hook, full_backward_name)
+                CellProcessor.cell_backward_hook.append(backward_hook)
+                bw_hook = inner.CellBackwardHook(full_backward_name, cell,
+                                                 self.cell_backward_hook[-1])
+                bw_hook.register_backward_hook()
+                CellProcessor.cell_bw_hook_kernels[full_forward_name] = bw_hook
+
+                args = bw_hook(*args)
+
+            return args
+
+        def forward_hook(cell, args, kwargs_or_output, output_or_kwargs=None):
+            index = CellProcessor.cell_count.get(cell_name, 0)
+            full_forward_name = f'{cell_name}{Const.FORWARD}{Const.SEP}{index}'
+            full_backward_name = f'{cell_name}{Const.BACKWARD}{Const.SEP}{index}'
+
+            self.set_construct_info_in_hook(full_forward_name)
+            
+            hook_set = build_data_hook(BaseScope.Module_Type_Module, full_forward_name)
+            hook_result = hook_set.forward_hook(cell, args, kwargs_or_output, output_or_kwargs)
+            if hook_result is not None:
+                outputs = hook_result
             else:
-                CellProcessor.api_parent_node = None
+                outputs = output_or_kwargs if has_kwargs_in_forward_hook() else kwargs_or_output
+
+            bw_hook = CellProcessor.cell_bw_hook_kernels.get(full_forward_name)
+            if bw_hook:
+                if not isinstance(outputs, (Tensor, tuple)):
+                    logger.warning("For backward hooks to be called,"
+                                   " cell output should be a Tensor or a tuple of Tensors"
+                                   f" but received {type(outputs)}")
+                if isinstance(outputs, tuple):
+                    new_outputs = bw_hook(*outputs)
+                else:
+                    new_outputs = bw_hook(outputs)
+                if isinstance(outputs, tuple) and len(outputs) == 1:
+                    new_outputs = (new_outputs,)
+                outputs = new_outputs
+
+            def get_backward_pre_hook(full_backward_name, backward_data_hook):
+                def backward_pre_hook_fn(cell, grad_output):
+                    cell.has_pre_hook_called = True
+                    self.set_construct_info_in_pre_hook(full_backward_name)
+                    if backward_data_hook:
+                        backward_data_hook(cell, (), grad_output)
+                        self.set_construct_info_in_hook(full_backward_name)
+                        cell.has_pre_hook_called = False
+                return backward_pre_hook_fn
 
-            if self.scope:
-                self.scope.end_module(cell.mindstudio_reserved_name)
+            backward_pre_hook = OrderedDict()
+            backward_data_hook = None if bw_hook else hook_set.backward_hook
+            backward_pre_hook[full_backward_name] = get_backward_pre_hook(full_backward_name, backward_data_hook)
+            CellProcessor.cell_backward_pre_hook.append(backward_pre_hook)
+            bw_pre_hook = inner.CellBackwardHook(full_backward_name, cell,
+                                                 self.cell_backward_pre_hook[-1])
+            bw_pre_hook.register_backward_pre_hook()
 
-        return begin_hook if Const.START == start_or_stop else end_hook
+            if isinstance(outputs, tuple):
+                result = bw_pre_hook(*outputs)
+            else:
+                result = bw_pre_hook(outputs)
+            if isinstance(outputs, tuple):
+                if len(outputs) == 1:
+                    result = (result,)
+                if len(result) != len(outputs):
+                    raise TypeError(
+                        f"The backward pre hook return value size is {len(result)} "
+                        f"not equal to output size {len(outputs)}"
+                    )
+            return result
+
+        return forward_pre_hook
 
-    def set_and_get_reserved_name(self, cell, cell_name, is_called_by_pre_hook=False):
-        if not is_called_by_pre_hook and hasattr(cell, 'has_pre_hook_called') and cell.has_pre_hook_called:
-            cell.has_pre_hook_called = False
+    def set_construct_info_in_pre_hook(self, full_name):
+        if self.cell_stack:
+            CellProcessor.module_node[full_name] = self.cell_stack[-1]
         else:
-            if is_called_by_pre_hook:
-                cell.has_pre_hook_called = True
-            index = self.set_cell_count(cell_name)
-            cell.mindstudio_reserved_name = cell_name + Const.SEP + str(index)
-        return cell.mindstudio_reserved_name
+            CellProcessor.module_node[full_name] = None
+        CellProcessor.cell_stack.append(full_name)
+        CellProcessor.api_parent_node = full_name
+        if self.scope:
+            self.scope.begin_module(full_name)
+
+    def set_construct_info_in_hook(self, full_name):
+        if self.cell_stack:
+            CellProcessor.cell_stack.pop()
+        CellProcessor.api_parent_node = CellProcessor.cell_stack[-1] if self.cell_stack else None
+        if self.scope:
+            self.scope.end_module(full_name)

msprobe/mindspore/code_mapping/graph_parser.py

@@ -34,19 +34,6 @@ class Parser:
             if isinstance(subgraph_node.attrs, list):
                 subgraph_node.attrs.extend(attrs)
 
-    @staticmethod
-    def parse_graph_attributes(text: str, graph_node: GraphNode) -> None:
-        attr_pattern = re.compile(r'# Attrs:\s*(.*)', re.DOTALL)
-        match = attr_pattern.search(text, graph_node.pos)
-        if match:
-            attrs = match.group(1).strip().split('\n')
-            for attr in attrs:
-                if not attr:
-                    break
-                key, value = attr.split(':')
-                if isinstance(graph_node.attrs, dict):
-                    graph_node.attrs[key.strip()] = value.strip()
-
     @staticmethod
     def parse_code_info(text: str, start_pos: int, end_pos: int) -> List[str]:
         code_info = []
@@ -124,8 +111,9 @@ class Parser:
             scope_match = scope_pattern.search(text, end_pos)
             scope = scope_match.group(1) if scope_match else ""
 
-            id_pattern = re.compile(r'.*cnode_primal_attrs:'
-                                    r'\s*\{.*\b(?:forward_unique_id|unique_id):\s*\"(\d+)\".*', re.IGNORECASE)
+            id_pattern = re.compile(
+                r'cnode_primal_attrs:'r'\s*\{[\w+]{1, 10000}\b(?:forward_unique_id|unique_id):\s*\"(\d+)\"',
+                re.IGNORECASE)
             unique_id_match = id_pattern.search(text, end_pos, scope_match.start())
             unique_id = unique_id_match.group(1) if unique_id_match else None
 
@@ -186,7 +174,7 @@ class Parser:
                     node_info.var_inputs.append(callee_name)
 
     def parse_subgraphs(self, text: str) -> None:
-        subgraph_pattern = re.compile(r'subgraph\s+@(\S+)(\([^\)]*\))?\s+.*\{')
+        subgraph_pattern = re.compile(r'/subgraph\s+@([\w+]{1,1000)(\([^\)]{1,100}\))?\s+\S[^\{]\{/+')
         matches = list(subgraph_pattern.finditer(text))
         end_pos = 0
         for match in matches:
@@ -203,11 +191,6 @@ class Parser:
             subgraph_info.end = end_pos
             logging.info('Parsed subgraph: %s', subgraph_name)
 
-    def count_nodes(self) -> Tuple[int, int]:
-        total_nodes = len(self.nodes)
-        total_cnodes = sum(1 for node in self.nodes.values() if node.name.startswith('CNode'))
-        return total_nodes, total_cnodes
-
     def create_backward_map(self):
         for node in self.nodes.values():
             if node.scope and node.scope.startswith("Gradients"):

msprobe/mindspore/common/const.py

@@ -15,6 +15,7 @@
 
 import numpy as np
 import mindspore as ms
+from mindspore import dtype as mstype
 
 from msprobe.core.common.const import Const as CoreConst
 
@@ -23,14 +24,20 @@ class Const:
     CELL = "cell"
     API = "api"
     KERNEL = "kernel"
+    CELL_AND_API = 'cell_and_api'
     TOOL_LEVEL_DICT = {
         CoreConst.LEVEL_L0: CELL,
         CoreConst.LEVEL_L1: API,
-        CoreConst.LEVEL_L2: KERNEL
+        CoreConst.LEVEL_L2: KERNEL,
+        CoreConst.LEVEL_MIX: CELL_AND_API
     }
-    PYNATIVE_MODE = "pynative"
+
+    PYNATIVE_MODE = CoreConst.PYNATIVE_MODE
+    GRAPH_MODE = "graph"
     GRAPH_GE_MODE = "graph_ge"
     GRAPH_KBYK_MODE = "graph_kbyk"
+    PYNATIVE_GRAPH_MODE = CoreConst.PYNATIVE_GRAPH_MODE
+
     JIT_LEVEL = "jit_level"
     JIT_LEVEL_O0 = "O0"
     JIT_LEVEL_O1 = "O1"
@@ -61,6 +68,7 @@ class Const:
     DROPOUT_API_NAME_PREFIX = "dropout"
 
     GRAPH_DATA_MODE_LIST = [CoreConst.ALL, CoreConst.INPUT, CoreConst.OUTPUT]
+    GRAPH_CELL_DUMP_DATA_MODE_LIST = [CoreConst.ALL, CoreConst.FORWARD, CoreConst.BACKWARD]
 
     HOOK_MS_PREFIX_DICT = {
         OPS_DATA_PREFIX: OPS_PREFIX,
@@ -69,6 +77,69 @@ class Const:
         MINT_NN_FUNC_DATA_PREFIX: MINT_NN_FUNC_PREFIX
     }
 
+    NonDifferentiableType = (
+        mstype.bool_, mstype.int8, mstype.byte, mstype.uint8, mstype.ubyte,
+        mstype.int16, mstype.short, mstype.uint16, mstype.ushort,
+        mstype.int32, mstype.intc, mstype.uint32, mstype.uintc,
+        mstype.int64, mstype.intp, mstype.uint64, mstype.uintp
+    )
+
+
+class MsCompareConst:
+    # api_info field
+    MINT = "Mint"
+    MINT_FUNCTIONAL = "MintFunctional"
+    TENSOR_API = "Tensor"
+    FUNCTIONAL_API = "Functional"
+    FUSION_API = "FUSION"
+
+    API_NAME_STR_LENGTH = 4
+    MAX_RECURSION_DEPTH = 20
+
+    # Mindtorch api_info field
+    MINDTORCH_TENSOR = "Tensor"
+    MINDTORCH = "Torch"
+    MINDTORCH_FUNC = "Functional"
+    MINDTORCH_NPU = "NPU"
+    MINDTORCH_DIST = "Distributed"
+
+    MT_VALID_API_TYPES = [
+        MINDTORCH, MINDTORCH_FUNC, MINDTORCH_TENSOR
+    ]
+    SUPPORTED_FUSION_LIST = ["flash_attention_score"]
+
+    TASK_FIELD = "task"
+    STATISTICS_TASK = "statistics"
+    FRAMEWORK = "framework"
+    TENSOR_TASK = "tensor"
+    DUMP_DATA_DIR_FIELD = "dump_data_dir"
+    DATA_FIELD = "data"
+
+    # supported api yaml
+    SUPPORTED_API_LIST_FILE = "checker_support_api.yaml"
+    SUPPORTED_TENSOR_LIST_KEY = "tensor"
+
+    # detail_csv
+    DETAIL_CSV_API_NAME = "API Name"
+    DETAIL_CSV_BENCH_DTYPE = "Bench Dtype"
+    DETAIL_CSV_TESTED_DTYPE = "Tested Dtype"
+    DETAIL_CSV_SHAPE = "Shape"
+    DETAIL_CSV_PASS_STATUS = "Status"
+    DETAIL_CSV_MESSAGE = "Message"
+    DETAIL_CSV_FILE_NAME = "accuracy_checking_details"
+
+    # result_csv
+    RESULT_CSV_FORWARD_TEST_SUCCESS = "Forward Test Success"
+    RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success"
+    RESULT_CSV_FILE_NAME = "accuracy_checking_result"
+
+    EPSILON = 1e-8
+
+    class ProcessStatus:
+        SUCCESS = "success"
+        API_NOT_FOUND = "api_not_found"
+        EXCEPTION_SKIP = "exception_skip"
+
 
 class FreeBenchmarkConst:
     ADD_NOISE = "add_noise"