mindstudio-probe 8.1.1__py3-none-any.whl → 8.1.2__py3-none-any.whl

msprobe/docs/35.nan_analyze.md

@@ -23,10 +23,11 @@
 msprobe -f pytorch nan_analyze -i dump_step_path -o output_dir_path
 ```
 
-| 参数                 | 说明                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-|--------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 参数                 | 说明                                          |
+|--------------------|---------------------------------------------|
+| -f 或 --framework   | 指定训练框架。pytorch。必选。                          |
 | -i 或 --input_path  | dump数据的目录。需指定到step层级，如`-i /xxx/dump/step0/` |
-| -o 或 --output_path | 输出文件的目录，可选，不填时默认在当前目录下创建 \"./output/" 目录。                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| -o 或 --output_path | 输出文件的目录，可选，不填时默认在当前目录下创建 \"./output/" 目录。   |
 
 ### 输出文件介绍
 

msprobe/docs/FAQ.md

@@ -36,6 +36,9 @@
      该信息说明 module 挂载了被 PyTorch 框架废弃的 register_backward_hook，这与工具使用的 register_full_backward_hook 接口会产生冲突，故工具会跳过该 module 的反向数据采集。
    - 如果您希望所有 module 数据都能采集下来，可以将模型中使用的 register_backward_hook 接口改为 PyTorch 框架推荐的 register_full_backward_pre_hook 或 register_full_backward_hook 接口。
 
+
+5. 在使用 msprobe 进行 Pytorch 框架的数据采集功能时，请注意确认环境变量 NPU_ASD_ENABLE=0 ，即关闭特征值检测功能。 由于工具冲突， 在该功能开启的情况下可能导致某些 api 数据采集的缺失。
+
 # 2 精度预检(PyTorch)
 
 1. 预检工具在 dump 和 run_ut 的过程中，是否需要同时开启或关闭 jit 编译（jit_compile）？

msprobe/mindspore/api_accuracy_checker/api_runner.py

@@ -13,6 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import (
+    Any,
+    Dict,
+    List,
+    Tuple,
+    Union
+)
 import os
 import numpy as np
 import mindspore
@@ -39,6 +46,22 @@ if torch_mindtorch_importer.is_valid_pt_mt_env:
 else:
     import torch
 
+# 为了可读性，我们先给每种返回形态起个别名
+ForwardResult = Tuple[
+    List[ComputeElement],
+    Tuple[Any, ...],
+    Dict[str, Any],
+    Tuple[Any, ...],
+]
+
+BackwardResultMT = Tuple[
+    List[ComputeElement],
+    Union[Any, Tuple[Any, ...]],
+    Tuple[Any, ...],
+]
+
+PyTorchBackward = List[ComputeElement]
+
 
 class ApiInputAggregation:
     def __init__(self, inputs, kwargs, gradient_inputs) -> None:
@@ -179,7 +202,12 @@ class ApiRunner:
         return api_instance
 
     @staticmethod
-    def run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform):
+    def run_api(
+        api_instance,
+        api_input_aggregation,
+        forward_or_backward: str,
+        api_platform: str,
+    ) -> Union[ForwardResult, BackwardResultMT, PyTorchBackward]:
         inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform)
                        for compute_element in api_input_aggregation.inputs)
         kwargs = {key: value.get_parameter(get_origin=False, tensor_platform=api_platform)

msprobe/mindspore/cell_processor.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import threading
 from collections import OrderedDict
 
 from mindspore import Tensor
@@ -21,6 +22,8 @@ from mindspore.ops.operations import _inner_ops as inner
 
 from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import MsprobeException
+from msprobe.core.common.runtime import Runtime
+from msprobe.core.common.utils import ModuleQueue, ThreadSafe
 from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope, BaseScope
 from msprobe.mindspore.common.const import Const as MsConst
 from msprobe.mindspore.common.log import logger
@@ -32,7 +35,6 @@ from msprobe.mindspore.common.utils import (
 )
 from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
 from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump
-from msprobe.core.common.runtime import Runtime
 
 
 def get_cell_construct(construct):
@@ -40,13 +42,15 @@ def get_cell_construct(construct):
         if hasattr(self, 'msprobe_hook'):
             setattr(self, 'msprobe_input_kwargs', kwargs)
         return construct(self, *args, **kwargs)
+
     return _construct
 
 
 class CellProcessor:
+    cell_queue = ModuleQueue()
     cell_count = {}
-    cell_stack = []
-    api_parent_node = None
+    cell_stack = {}
+    api_parent_node = {}
     module_node = {}
     cell_bw_hook_kernels = {}
     cell_backward_pre_hook = []
@@ -65,9 +69,10 @@ class CellProcessor:
 
     @classmethod
     def reset_cell_stats(cls):
+        cls.cell_queue = ModuleQueue()
         cls.cell_count = {}
-        cls.cell_stack = []
-        cls.api_parent_node = None
+        cls.cell_stack = {}
+        cls.api_parent_node = {}
         cls.module_node = {}
         cls.cell_bw_hook_kernels = {}
         cls.cell_backward_pre_hook = []
@@ -122,6 +127,7 @@ class CellProcessor:
                 GraphModeCellDump(config, cells_and_names_in_graph_mode, strict=False).handle()
 
     def build_cell_hook(self, cell_name, build_data_hook):
+        @ThreadSafe.synchronized
         def forward_pre_hook(cell, args):
             index = CellProcessor.set_and_get_calls_number(cell_name)
             full_forward_name = f'{cell_name}{Const.FORWARD}{Const.SEP}{index}'
@@ -146,11 +152,13 @@ class CellProcessor:
                 setattr(cell, 'msprobe_forward_hook', True)
 
             def get_backward_hook(backward_data_hook, full_backward_name):
+                @ThreadSafe.synchronized
                 def backward_hook_fn(cell, grad_input, grad_output):
                     new_output = backward_data_hook(cell, grad_input, grad_output)
                     self.set_construct_info_in_hook(full_backward_name)
                     cell.has_pre_hook_called = False
                     return new_output
+
                 return backward_hook_fn
 
             enable_hooked = sum(
@@ -170,13 +178,14 @@ class CellProcessor:
 
             return args
 
+        @ThreadSafe.synchronized
         def forward_hook(cell, args, kwargs_or_output, output_or_kwargs=None):
             index = CellProcessor.cell_count.get(cell_name, 0)
             full_forward_name = f'{cell_name}{Const.FORWARD}{Const.SEP}{index}'
             full_backward_name = f'{cell_name}{Const.BACKWARD}{Const.SEP}{index}'
 
             self.set_construct_info_in_hook(full_forward_name)
-            
+
             hook_set = build_data_hook(BaseScope.Module_Type_Module, full_forward_name)
             hook_result = hook_set.forward_hook(cell, args, kwargs_or_output, output_or_kwargs)
             if hook_result is not None:
@@ -199,6 +208,7 @@ class CellProcessor:
                 outputs = new_outputs
 
             def get_backward_pre_hook(full_backward_name, backward_data_hook):
+                @ThreadSafe.synchronized
                 def backward_pre_hook_fn(cell, grad_output):
                     cell.has_pre_hook_called = True
                     self.set_construct_info_in_pre_hook(full_backward_name)
@@ -206,6 +216,7 @@ class CellProcessor:
                         backward_data_hook(cell, (), grad_output)
                         self.set_construct_info_in_hook(full_backward_name)
                         cell.has_pre_hook_called = False
+
                 return backward_pre_hook_fn
 
             backward_pre_hook = OrderedDict()
@@ -233,18 +244,28 @@ class CellProcessor:
         return forward_pre_hook
 
     def set_construct_info_in_pre_hook(self, full_name):
-        if self.cell_stack:
-            CellProcessor.module_node[full_name] = self.cell_stack[-1]
+        tid = threading.get_ident()
+        if tid not in self.cell_stack:
+            CellProcessor.cell_stack[tid] = []
+
+        if self.cell_stack[tid]:
+            CellProcessor.module_node[full_name] = self.cell_stack[tid][-1]
         else:
-            CellProcessor.module_node[full_name] = None
-        CellProcessor.cell_stack.append(full_name)
-        CellProcessor.api_parent_node = full_name
+            parent_name = CellProcessor.cell_queue.find_last(full_name)
+            CellProcessor.module_node[full_name] = parent_name
+
+        CellProcessor.cell_queue.add_name(full_name)
+        CellProcessor.cell_stack[tid].append(full_name)
+        CellProcessor.api_parent_node[tid] = full_name
         if self.scope:
             self.scope.begin_module(full_name)
 
     def set_construct_info_in_hook(self, full_name):
-        if self.cell_stack:
-            CellProcessor.cell_stack.pop()
-        CellProcessor.api_parent_node = CellProcessor.cell_stack[-1] if self.cell_stack else None
+        tid = threading.get_ident()
+        CellProcessor.api_parent_node[tid] = None
+        if self.cell_stack.get(tid):
+            CellProcessor.cell_stack[tid].pop()
+        if self.cell_stack.get(tid):
+            CellProcessor.api_parent_node[tid] = CellProcessor.cell_stack[tid][-1]
         if self.scope:
             self.scope.end_module(full_name)

msprobe/mindspore/code_mapping/bind.py

@@ -119,9 +119,17 @@ def find_npy_files(npy_path):
 
     # 如果是目录，使用Path.rglob查找所有.npy文件
     if npy_path_obj.is_dir():
-        for file in npy_path_obj.rglob(Const.NUMPY_PATTERN):
-            check_file_or_directory_path(file)
-            npy_files.append(file.resolve())
+        base_depth = len(npy_path_obj.resolve().parts)
+        for root, dirs, files in os.walk(npy_path_obj):
+            current_depth = len(Path(root).resolve().parts) - base_depth
+            if current_depth >= 10:
+                dirs[:] = []
+
+            for filename in files:
+                if filename.endswith(Const.NUMPY_SUFFIX):
+                    file_path = Path(root) / filename
+                    check_file_or_directory_path(file_path)
+                    npy_files.append(file_path.resolve())
     else:
         logger.info(f"The specified path is neither an .npy file nor a directory: {npy_path}")
 
@@ -254,7 +262,18 @@ def bind_code_info_for_data(input_dir: str, nodes: Dict[str, GraphNode]) -> Dict
                 corresponding_name = None
             name_without_ext = os.path.splitext(corresponding_name)[0]
         npy_path = os.path.realpath(npy_file)
-        node_scope = name_without_ext.split(".")[1]
+
+        parts = name_without_ext.split(".")
+        if len(parts) < 2:
+            logger.error(
+                f'File name "{file_name}" in "{directory}" '
+                f'does not conform to expected format (missing scope separator ".")!'
+            )
+            raise Exception(
+                f'File name "{file_name}" has incorrect format, cannot extract node scope!'
+            )
+        node_scope = parts[1]
+
         trie = Trie()
         for key, value in match_dict.items():
             trie.insert(key, value)

msprobe/mindspore/code_mapping/graph_parser.py

@@ -77,7 +77,7 @@ class Parser:
 
     @staticmethod
     def extract_constants(inputs_str: str) -> List[str]:
-        constant_pattern = re.compile(r'\b(\w+\(.*?\))')
+        constant_pattern = re.compile(r'\b([A-Za-z_][A-Za-z0-9_]{0,10000})\(([A-Za-z0-9_\s,.\-+/]{0,10000})\)')
         constants = constant_pattern.findall(inputs_str)
         return constants
 
@@ -90,7 +90,8 @@ class Parser:
             self.nodes[func_name] = func_graph_info
 
     def parse_nodes(self, text: str, subgraph_info: GraphNode) -> None:
-        node_pattern = re.compile(r'(%\d+)\((\S+)\)\s*=\s*(\S+)\(')
+        node_pattern = re.compile(
+            r'(%\d{1,10000})\(([A-Za-z0-9_\.]{1,10000})\)\s*=\s*([A-Za-z_][A-Za-z0-9_]{0,10000})\(')
         matches = list(node_pattern.finditer(text))
         for i, match in enumerate(matches):
             series_number = match.group(1)
@@ -106,8 +107,9 @@ class Parser:
 
             constants = self.__class__.extract_constants(args_str)
 
-            scope_pattern = re.compile(r'# .*scope.*:\s*\((.*?)\)', re.IGNORECASE | re.MULTILINE)
-
+            scope_pattern = re.compile(
+                r'^(?=.{0,300}$)[ \t]*\#[ \t]*[^\r\n]*?scope[^\r\n]*?:[ \t]*\(([^)\r\n]{1,200})\)[ \t]*$',
+                re.IGNORECASE | re.MULTILINE)
             scope_match = scope_pattern.search(text, end_pos)
             scope = scope_match.group(1) if scope_match else ""
 

msprobe/mindspore/common/utils.py

@@ -95,6 +95,9 @@ def save_tensor_as_npy(tensor, file_path):
 
 
 def convert_to_int(value):
+    if isinstance(value, bool):
+        logger.error('The value in rank_id or step should be int, please check!')
+        raise CompareException(CompareException.INVALID_OBJECT_TYPE_ERROR)
     try:
         return int(value)
     except Exception:

msprobe/mindspore/compare/common_dir_compare.py

@@ -30,9 +30,10 @@ from msprobe.core.common.utils import CompareException
 from msprobe.core.common.exceptions import FileCheckException
 from msprobe.core.common.file_utils import check_file_or_directory_path, write_df_to_csv, create_directory, \
                                            check_path_before_create, load_npy
-from msprobe.core.common.const import CompareConst, FileCheckConst
+from msprobe.core.common.const import CompareConst
 from msprobe.core.compare.npy_compare import compare_ops_apply
 from msprobe.core.compare.multiprocessing_compute import check_accuracy
+from msprobe.mindspore.compare.utils import check_name_map_dict
 
 
 def common_dir_compare(input_params: Dict, output_dir: str) -> Optional[pd.DataFrame]:
@@ -49,6 +50,7 @@ def common_dir_compare(input_params: Dict, output_dir: str) -> Optional[pd.DataF
     npu_root = Path(input_params.get('npu_path'))
     bench_root = Path(input_params.get('bench_path'))
     name_map_dict = input_params.get('map_dict', {})
+    check_name_map_dict(name_map_dict)
     file_tree = build_mirror_file_tree(npu_root, bench_root)
     
     # 处理文件比对
@@ -114,24 +116,42 @@ def build_mirror_file_tree(npu_root: Path, bench_root: Path) -> Dict[Path, Tuple
     file_tree = {}
     
     # 遍历NPU目录构建树结构
-    for npu_path in npu_root.rglob('*.npy'):
-        dir_path = npu_path.relative_to(npu_root).parent
-        npu_dir_pair = os.path.join(npu_root, dir_path)
-        bench_dir_pair = os.path.join(bench_root, dir_path)
-        try:
-            check_file_or_directory_path(bench_dir_pair, isdir=True)
-        except FileCheckException:
+    # 使用os.walk遍历目录,限制深度为10层
+    for root, dirs, files in os.walk(npu_root):
+        # 计算当前目录深度
+        depth = len(Path(root).relative_to(npu_root).parts)
+        if depth > 10:
+            dirs.clear()  # 清空dirs列表以阻止继续递归
             continue
-        # 添加到文件树
-        if dir_path not in file_tree:
-            file_tree[dir_path] = (npu_dir_pair, bench_dir_pair)
+            
+        # 检查当前目录下是否有npy文件
+        if any(f.endswith('.npy') for f in files):
+            # 获取相对路径
+            dir_path = Path(root).relative_to(npu_root)
+            npu_dir_pair = os.path.join(npu_root, dir_path)
+            bench_dir_pair = os.path.join(bench_root, dir_path)
+            
+            try:
+                check_file_or_directory_path(bench_dir_pair, isdir=True)
+            except FileCheckException:
+                continue
+                
+            # 添加到文件树
+            if dir_path not in file_tree:
+                file_tree[dir_path] = (npu_dir_pair, bench_dir_pair)
     
     return file_tree
 
 
 def find_npy_files(directory):
     npy_files_dict = {}
-    for root, _, files in os.walk(directory):
+    # 限制递归深度为1层,即只遍历当前目录和其直接子目录
+    for root, dirs, files in os.walk(directory, topdown=True):
+        # 计算当前目录深度
+        depth = root[len(directory):].count(os.sep)
+        # 如果深度超过10层则跳过
+        if depth > 10:
+            dirs.clear()
         for file in files:
             if file.endswith(".npy"):
                 # 分割文件名并去掉最后两个元素

msprobe/mindspore/compare/ms_graph_compare.py

@@ -168,8 +168,13 @@ class GraphMSComparator:
         self.output_path = output_path
         self.base_npu_path = input_param.get('npu_path', None)
         self.base_bench_path = input_param.get('bench_path', None)
-        self.rank_list = [convert_to_int(rank_id) for rank_id in input_param.get('rank_id', [])]
-        self.step_list = [convert_to_int(step_id) for step_id in input_param.get('step_id', [])]
+        rank_id_list = input_param.get('rank_id', [])
+        step_id_list = input_param.get('step_id', [])
+        if not isinstance(rank_id_list, list) or not isinstance(step_id_list, list):
+            logger.error("'rank_id' and 'step_id' should both be lists, please check!")
+            raise CompareException(CompareException.INVALID_OBJECT_TYPE_ERROR)
+        self.rank_list = [convert_to_int(rank_id) for rank_id in rank_id_list]
+        self.step_list = [convert_to_int(step_id) for step_id in step_id_list]
         # split by rank and step, generate rank step path
         self.npu_rank_step_dict = self.generate_rank_step_path(self.base_npu_path)
         self.bench_rank_step_dict = self.generate_rank_step_path(self.base_bench_path)

msprobe/mindspore/compare/utils.py

@@ -17,7 +17,8 @@ import os
 
 from msprobe.core.common.const import Const
 from msprobe.core.common.file_utils import load_npy, FileChecker, FileCheckConst
-from msprobe.core.common.utils import detect_framework_by_dump_json
+from msprobe.core.common.utils import detect_framework_by_dump_json, CompareException, check_op_str_pattern_valid
+from msprobe.core.common.log import logger
 
 
 def read_npy_data(dir_path, file_name):
@@ -35,3 +36,10 @@ def read_npy_data(dir_path, file_name):
 def check_cross_framework(bench_json_path):
     framework = detect_framework_by_dump_json(bench_json_path)
     return framework == Const.PT_FRAMEWORK
+
+
+def check_name_map_dict(name_map_dict):
+    if not isinstance(name_map_dict, dict):
+        logger.error("'map_dict' should be a dict, please check!")
+        raise CompareException(CompareException.INVALID_OBJECT_TYPE_ERROR)
+    check_op_str_pattern_valid(str(name_map_dict))

msprobe/mindspore/debugger/debugger_config.py

@@ -81,18 +81,22 @@ class DebuggerConfig:
         target_module_type = (torch.nn.Module, "torch.nn.Module") if is_mindtorch() else (nn.Cell, "mindspore.nn.Cell")
         if models is None or isinstance(models, target_module_type[0]):
             return models
-        error_model = None
         if isinstance(models, (list, tuple)):
+            error_model = None
             for model in models:
                 if not isinstance(model, target_module_type[0]):
                     error_model = model
                     break
-        else:
-            error_model = models
+            if error_model is not None:
+                error_info = (
+                    f"The 'model' parameter must be a {target_module_type[1]} or list[{target_module_type[1]}] "
+                    f"type, currently there is a {type(error_model)} type.")
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, error_info)
 
-        if error_model is not None:
+        else:
             error_info = (f"The 'model' parameter must be a {target_module_type[1]} or list[{target_module_type[1]}] "
-                          f"type, currently there is a {type(error_model)} type.")
+                          f"type, currently there is a {type(models)} type.")
             raise MsprobeException(
                 MsprobeException.INVALID_PARAM_ERROR, error_info)
         return models
@@ -125,16 +129,14 @@ class DebuggerConfig:
             self.level_ori = Const.LEVEL_MIX
         return True
 
-    def check_config_with_l2(self):
-        if self.level_ori != Const.LEVEL_L2:
-            return
-        if self.task != Const.TENSOR:
+    def check_config_with_l2(self, is_graph_config):
+        if not is_graph_config and self.task != Const.TENSOR:
             raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
                                    f"When level is set to L2, the task must be set to tensor.")
-        if self.scope:
+        if not is_graph_config and self.scope:
             raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
                                    f"When level is set to L2, the scope cannot be configured.")
-        if not self.list or len(self.list) != 1:
+        if not is_graph_config and (not self.list or len(self.list) != 1):
             raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR,
                                    f"When level is set to L2, the list must be configured as a list with one api name.")