mindstudio-probe 8.2.0__tar.gz → 8.3.0__tar.gz

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 8.2.0
+Version: 8.3.0
 Summary: Ascend Probe Utils
-Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
+Home-page: https://gitcode.com/Ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team
 Author-email: pmail_mindstudio@huawei.com
 License: Apache License 2.0

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/mindstudio_probe.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: mindstudio-probe
-Version: 8.2.0
+Version: 8.3.0
 Summary: Ascend Probe Utils
-Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe
+Home-page: https://gitcode.com/Ascend/mstt/tree/master/debug/accuracy_tools/msprobe
 Author: Ascend Team
 Author-email: pmail_mindstudio@huawei.com
 License: Apache License 2.0

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/mindstudio_probe.egg-info/SOURCES.txt

@@ -30,6 +30,7 @@ msprobe/core/common/global_lock.py
 msprobe/core/common/inplace_op_checker.py
 msprobe/core/common/inplace_ops.yaml
 msprobe/core/common/log.py
+msprobe/core/common/megatron_utils.py
 msprobe/core/common/parallel_state.py
 msprobe/core/common/runtime.py
 msprobe/core/common/utils.py
@@ -113,6 +114,7 @@ msprobe/docs/04.kernel_dump_PyTorch.md
 msprobe/docs/05.data_dump_PyTorch.md
 msprobe/docs/06.data_dump_MindSpore.md
 msprobe/docs/07.accuracy_checker_PyTorch.md
+msprobe/docs/08.accuracy_checker_online_PyTorch.md
 msprobe/docs/09.accuracy_checker_MindSpore.md
 msprobe/docs/10.accuracy_compare_PyTorch.md
 msprobe/docs/11.accuracy_compare_MindSpore.md
@@ -331,6 +333,7 @@ msprobe/nan_analyze/analyzer.py
 msprobe/nan_analyze/graph.py
 msprobe/nan_analyze/utils.py
 msprobe/pytorch/__init__.py
+msprobe/pytorch/attl_manager.py
 msprobe/pytorch/function_factory.py
 msprobe/pytorch/pt_config.py
 msprobe/pytorch/pytorch_service.py
@@ -370,6 +373,14 @@ msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py
 msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py
 msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py
 msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml
+msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py
 msprobe/pytorch/bench_functions/__init__.py
 msprobe/pytorch/bench_functions/apply_adam.py
 msprobe/pytorch/bench_functions/apply_adam_w.py

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/README.md

@@ -35,17 +35,17 @@ export MSPROBE_LOG_LEVEL={x}
 
 ## 环境和依赖
 
-- 硬件环境请参见《[昇腾产品形态说明](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F80RC22%2Fquickstart%2Fquickstart%2Fquickstart_18_0002.html)》。
-- 软件环境请参见《[CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F80RC22%2Fsoftwareinst%2Finstg%2Finstg_0000.html%3FMode%3DPmIns%26OS%3DUbuntu%26Software%3DcannToolKit)》安装昇腾设备开发或运行环境，即toolkit软件包。
+- 硬件环境请参见《[昇腾产品形态说明](https://www.hiascend.com/document/detail/zh/canncommercial/80RC22/quickstart/quickstart/quickstart_18_0002.html)》。
+- 软件环境请参见《[CANN 软件安装指南](https://www.hiascend.com/document/detail/zh/canncommercial/80RC22/softwareinst/instg/instg_0000.html?Mode=PmIns&OS=Ubuntu&Software=cannToolKit)》安装昇腾设备开发或运行环境，即toolkit软件包。
 
 以上环境依赖请根据实际环境选择适配的版本。
 
 ## 版本配套说明
 
-- msprobe支持AscendPyTorch 1.11.0或更高版本，支持的PyTorch和CANN以及PyTorch和python软件版本配套关系请参见《[Ascend Extension for PyTorch插件](https://gitee.com/ascend/pytorch)》。
+- msprobe支持AscendPyTorch 1.11.0或更高版本，支持的PyTorch和CANN以及PyTorch和python软件版本配套关系请参见《[Ascend Extension for PyTorch插件](https://gitcode.com/Ascend/pytorch)》。
 - msprobe支持MindSpore 2.4.0或更高版本，支持的MindSpore和CANN以及MindSpore和python软件版本配套关系请参见《[MindSpore版本发布列表](https://www.mindspore.cn/versions)》。
 - msprobe支持MSAdapter 2.1.0。
-- msprobe支持的固件驱动版本与配套CANN软件支持的固件驱动版本相同，开发者可通过“[昇腾社区-固件与驱动](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fhardware%2Ffirmware-drivers%2Fcommunity%3Fproduct%3D2%26model%3D28%26cann%3D8.0.RC3.alpha003%26driver%3D1.0.25.alpha)”页面根据产品型号与CANN软件版本获取配套的固件与驱动。
+- msprobe支持的固件驱动版本与配套CANN软件支持的固件驱动版本相同，开发者可通过“[昇腾社区-固件与驱动](https://www.hiascend.com/hardware/firmware-drivers/community?product=2&model=28&cann=8.0.RC3.alpha003&driver=1.0.25.alpha)”页面根据产品型号与CANN软件版本获取配套的固件与驱动。
 
 
 ## 🚨 工具限制与注意事项
@@ -84,7 +84,9 @@ msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API
 
 精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现，给出精度情况的诊断和分析。对应 config.json 中的 "run_ut" task。
 
-PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)
+PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)和[在线预检](./docs/08.accuracy_checker_online_PyTorch.md)
+
+在线预检计划在 Mindstudio 8.3.0 版本正式下线。
 
 MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.md)
 

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/core/common/const.py

@@ -274,6 +274,7 @@ class Const:
     TENSOR_TYPE = "torch.Tensor"
     DTENSOR_TYPE = "torch.distributed.tensor.DTensor"
     FAKE_TENSOR_TYPE = "torch._subclasses.fake_tensor.FakeTensor"
+    AC_TENSOR_TYPE = "torch.distributed._functional_collectives.AsyncCollectiveTensor"
 
     SUPPORT_API_FILE_NAME = "support_wrap_ops.yaml"
 
@@ -405,6 +406,8 @@ class Const:
 
     MIX_DUMP_NAMES = {'graph', 'pynative'}
 
+    MEGATRON_MICRO_STEP_NUMBER = 'megatron_micro_step_number'
+
 
 class CompareConst:
     """
@@ -425,6 +428,8 @@ class CompareConst:
     NPU_MIN = "NPU min"
     NPU_MEAN = "NPU mean"
     NPU_NORM = "NPU l2norm"
+    NPU_P2POP_PEER = "NPU P2POp peer"
+
     BENCH_MAX = "Bench max"
     BENCH_MIN = "Bench min"
     BENCH_MEAN = "Bench mean"
@@ -823,6 +828,7 @@ class MonitorConst:
     ACTV_OUT = "output"
     ACTVGRAD_IN = "input_grad"
     ACTVGRAD_OUT = "output_grad"
+    FSDP_FLAT_SEP = "_fsdp_wrapped_module."
     # used for tasks
     ACTV = "actv"
     ACTVGRAD = "actv_grad"

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/core/common/db_manager.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import re
 import sqlite3
 from typing import List, Tuple, Dict, Any
 from functools import wraps
@@ -20,6 +21,14 @@ from msprobe.pytorch.common.log import logger
 from msprobe.core.common.file_utils import check_path_before_create, change_mode
 from msprobe.core.common.const import FileCheckConst
 
+SAFE_SQL_PATTERN = re.compile(r'^[a-zA-Z0-9_]+$')
+
+
+def check_identifier_safety(name):
+    """验证标识符是否安全（防止SQL注入）"""
+    if not isinstance(name, str) or SAFE_SQL_PATTERN.match(name) is None:
+        raise ValueError(f"Invalid SQL identifier: {name}, potential SQL injection risk!")
+
 
 def _db_operation(func):
     """数据库操作装饰器，自动管理连接"""
@@ -68,6 +77,7 @@ class DBManager:
         where_values = []
         if where_list:
             for col, val in where_list.items():
+                check_identifier_safety(col)
                 where_clauses.append(f"{col} = ?")
                 where_values.append(val)
             if where_clauses:
@@ -84,13 +94,22 @@ class DBManager:
         :param batch_size: 每批插入的大小
         :return: 插入的行数
         """
+        check_identifier_safety(table_name)
+
         if not data:
             return 0
         columns = len(data[0])
-        if key_list and columns != len(key_list):
-            raise ValueError(
-                f"When inserting into table {table_name}, the length of key list ({key_list})"
-                f"does not match the data({columns}).")
+        if key_list:
+            if not isinstance(key_list, list):
+                raise TypeError(
+                    f"key_list must be a list, got {type(key_list)}"
+                )
+            if columns != len(key_list):
+                raise ValueError(
+                    f"When inserting into table {table_name}, the length of key list ({key_list})"
+                    f"does not match the data({columns}).")
+            for key in key_list:
+                check_identifier_safety(key)
 
         batch_size = self.DEFAULT_INSERT_SIZE
         placeholders = ", ".join(["?"] * columns)
@@ -121,12 +140,16 @@ class DBManager:
         :param where: WHERE条件
         :return: 查询结果列表(字典形式)
         """
+        check_identifier_safety(table_name)
 
         if not columns:
             raise ValueError("columns parameter cannot be empty, specify columns to select (e.g. ['id', 'name'])")
         if not isinstance(columns, list) or not all(isinstance(col, str) for col in columns):
             raise TypeError("columns must be a list of strings (e.g. ['id', 'name'])")
         
+        for col in columns:
+            check_identifier_safety(col)
+        
         cols = ", ".join(columns)
         sql = f"SELECT {cols} FROM {table_name}"
 
@@ -147,6 +170,14 @@ class DBManager:
         :param where_params: WHERE条件参数
         :return: 影响的行数
         """
+        check_identifier_safety(table_name)
+        if not updates:
+            raise ValueError("columns parameter cannot be empty, specify it to update (e.g. {'name': 'xxx'}")
+        if not isinstance(updates, dict):
+            raise TypeError(f"updates must be a dictionary, got: {type(updates)}")
+        for key in updates.keys():
+            check_identifier_safety(key)
+
         set_clause = ", ".join([f"{k} = ?" for k in updates.keys()])
         sql = f"UPDATE {table_name} SET {set_clause}"
 

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/core/common/file_utils.py

@@ -12,29 +12,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import atexit
 import csv
 import fcntl
 import io
+import json
+import multiprocessing
 import os
 import pickle
-from multiprocessing import shared_memory
-import stat
-import json
 import re
 import shutil
+import stat
 import sys
 import zipfile
-import multiprocessing
-import yaml
+from multiprocessing import shared_memory
+
 import numpy as np
 import pandas as pd
+import yaml
 
+from msprobe.core.common.const import FileCheckConst, CompareConst, Const
 from msprobe.core.common.decorator import recursion_depth_decorator
-from msprobe.core.common.log import logger
 from msprobe.core.common.exceptions import FileCheckException
-from msprobe.core.common.const import FileCheckConst, CompareConst
 from msprobe.core.common.global_lock import global_lock, is_main_process
+from msprobe.core.common.log import logger
 
 proc_lock = multiprocessing.Lock()
 
@@ -46,16 +48,15 @@ class FileChecker:
     Attributes:
         file_path: The file or dictionary path to be verified.
         path_type: file or dictionary
-        ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability
+        ability(str): one of [FileCheckConst.READ_ABLE, FileCheckConst.WRITE_ABLE, FileCheckConst.READ_WRITE_ABLE]
         file_type(str): The correct file type for file
     """
 
-    def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True):
+    def __init__(self, file_path, path_type, ability=None, file_type=None):
         self.file_path = file_path
         self.path_type = self._check_path_type(path_type)
-        self.ability = ability
+        self.ability = self._check_ability_type(ability)
         self.file_type = file_type
-        self.is_script = is_script
 
     @staticmethod
     def _check_path_type(path_type):
@@ -64,9 +65,17 @@ class FileChecker:
             raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
         return path_type
 
+    @staticmethod
+    def _check_ability_type(ability):
+        ability_list = [FileCheckConst.READ_ABLE, FileCheckConst.WRITE_ABLE, FileCheckConst.READ_WRITE_ABLE]
+        if ability and ability not in ability_list:
+            logger.error(f'The ability must be one of {ability_list}.')
+            raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR)
+        return ability
+
     def common_check(self):
         """
-        功能：用户校验基本文件权限：软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符
+        功能：基本文件权限校验，包括文件存在性、软连接、文件长度、文件类型、文件读写权限、文件属组、文件路径特殊字符、文件后缀等
         注意：文件后缀的合法性，非通用操作，可使用其他独立接口实现
         """
         check_path_exists(self.file_path)
@@ -75,13 +84,13 @@ class FileChecker:
         check_path_length(self.file_path)
         check_path_type(self.file_path, self.path_type)
         self.check_path_ability()
-        if self.is_script:
-            check_path_owner_consistent(self.file_path)
+        check_path_owner_consistent(self.file_path)
         check_path_pattern_valid(self.file_path)
         check_common_file_size(self.file_path)
         check_file_suffix(self.file_path, self.file_type)
+        check_path_no_others_write(self.file_path)
         if self.path_type == FileCheckConst.FILE:
-            check_dirpath_before_read(self.file_path)
+            check_dirpath_permission(self.file_path)
         return self.file_path
 
     def check_path_ability(self):
@@ -137,7 +146,8 @@ class FileOpen:
         check_path_pattern_valid(self.file_path)
         if os.path.exists(self.file_path):
             check_common_file_size(self.file_path)
-            check_dirpath_before_read(self.file_path)
+            check_path_no_others_write(self.file_path)
+            check_dirpath_permission(self.file_path)
 
     def check_ability_and_owner(self):
         if self.mode in self.SUPPORT_READ_MODE:
@@ -172,7 +182,7 @@ def check_path_exists(path):
     if not os.path.exists(path):
         logger.error('The file path %s does not exist.' % path)
         raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR)
-    
+
 
 def check_path_not_exists(path):
     if os.path.exists(path):
@@ -256,12 +266,15 @@ def check_path_type(file_path, file_type):
             raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
 
 
-def check_others_writable(directory):
-    dir_stat = os.stat(directory)
-    is_writable = (
-        bool(dir_stat.st_mode & stat.S_IWGRP) or  # 组可写
-        bool(dir_stat.st_mode & stat.S_IWOTH)     # 其他用户可写
-    )
+def check_group_writable(file_path):
+    path_stat = os.stat(file_path)
+    is_writable = bool(path_stat.st_mode & stat.S_IWGRP)
+    return is_writable
+
+
+def check_others_writable(file_path):
+    path_stat = os.stat(file_path)
+    is_writable = bool(path_stat.st_mode & stat.S_IWOTH)
     return is_writable
 
 
@@ -309,7 +322,7 @@ def check_path_before_create(path):
                                  'The file path {} contains special characters.'.format(path))
 
 
-def check_dirpath_before_read(path):
+def check_dirpath_permission(path):
     path = os.path.realpath(path)
     dirpath = os.path.dirname(path)
     if dedup_log('check_dirpath_before_read', dirpath):
@@ -319,15 +332,16 @@ def check_dirpath_before_read(path):
             check_path_owner_consistent(dirpath)
         except FileCheckException:
             logger.warning(f"The directory {dirpath} is not yours.")
-    
 
-def check_file_or_directory_path(path, isdir=False):
+
+def check_file_or_directory_path(path, isdir=False, is_strict=False):
     """
     Function Description:
         check whether the path is valid
     Parameter:
         path: the path to check
         isdir: the path is dir or file
+        is_strict: whether to perform stricter validation (e.g., verify group cannot write to path)
     Exception Description:
         when invalid data throw exception
     """
@@ -337,6 +351,33 @@ def check_file_or_directory_path(path, isdir=False):
         path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE)
     path_checker.common_check()
 
+    if is_strict:
+        if check_group_writable(path):
+            raise FileCheckException(
+                FileCheckException.FILE_PERMISSION_ERROR,
+                f"The directory/file must not allow write access to group. Directory/File path: {path}"
+            )
+
+
+def check_path_no_others_write(file_path):
+    if dedup_log('check_path_no_others_write', file_path):
+        if check_group_writable(file_path):
+            logger.warning(f"The directory/file path is writable by group: {file_path}.")
+
+    if check_others_writable(file_path):
+        raise FileCheckException(
+            FileCheckException.FILE_PERMISSION_ERROR,
+            f"The directory/file must not allow write access to others. Directory/File path: {file_path}"
+        )
+
+
+def check_path_no_group_others_write(file_path):
+    if check_group_writable(file_path) or check_others_writable(file_path):
+        raise FileCheckException(
+            FileCheckException.FILE_PERMISSION_ERROR,
+            f"The directory/file must not allow write access to group or others. Directory/File path: {file_path}"
+        )
+
 
 def change_mode(path, mode):
     if not os.path.exists(path) or os.path.islink(path):
@@ -388,6 +429,14 @@ def check_file_type(path):
         raise FileCheckException(FileCheckException.INVALID_FILE_ERROR)
 
 
+def root_privilege_warning():
+    if os.getuid() == 0:
+        logger.warning(
+            "msprobe is being run as root. "
+            "To avoid security risks, it is recommended to switch to a regular user to run it."
+        )
+
+
 def load_yaml(yaml_path):
     path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX)
     checked_path = path_checker.common_check()
@@ -422,6 +471,26 @@ def load_json(json_path):
     return data
 
 
+def load_construct_json(json_path):
+    construct_dict_o = load_json(json_path)
+    if Const.MEGATRON_MICRO_STEP_NUMBER in construct_dict_o:
+        construct_dict = {}
+        micro_step_dict = {Const.MEGATRON_MICRO_STEP_NUMBER: construct_dict_o.get(Const.MEGATRON_MICRO_STEP_NUMBER)}
+        del construct_dict_o[Const.MEGATRON_MICRO_STEP_NUMBER]
+        for key, value in construct_dict_o.items():
+            if isinstance(value, list):
+                if len(value) != 2:
+                    logger.error(f'Parse construct json file "{os.path.basename(json_path)}" failed.')
+                    raise RuntimeError()
+                construct_dict[key] = value[0]
+                micro_step_dict[key] = value[1]
+            else:
+                construct_dict[key] = value
+                micro_step_dict[key] = 0
+        return construct_dict, micro_step_dict
+    return construct_dict_o, {}
+
+
 def save_json(json_path, data, indent=None, mode="w"):
     check_path_before_create(json_path)
     json_path = os.path.realpath(json_path)
@@ -520,6 +589,9 @@ def move_directory(src_path, dst_path):
     check_file_or_directory_path(src_path, isdir=True)
     check_path_before_create(dst_path)
     try:
+        if os.path.exists(dst_path):
+            logger.warning(f"The destination directory {dst_path} already exists, it will be removed.")
+            shutil.rmtree(dst_path)
         shutil.move(src_path, dst_path)
     except Exception as e:
         logger.error(f"move directory {src_path} to {dst_path} failed")
@@ -945,7 +1017,13 @@ class SharedDict:
     def _safe_load(self):
         with io.BytesIO(self._shm.buf[:]) as buff:
             try:
-                self._dict = SafeUnpickler(buff).load()
+                data = SafeUnpickler(buff).load()
+                if not isinstance(data, dict):
+                    logger.debug(f"Data from shared memory is '{type(data)}' type, expected 'dict'.")
+                    self._dict = {}
+                    self._changed = True
+                else:
+                    self._dict = data
             except Exception as e:
                 logger.debug(f'shared dict is unreadable, reason: {e}, create new dict.')
                 self._dict = {}

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/core/common/framework_adapter.py

@@ -12,10 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.import functools
+
 import functools
+
 from msprobe.core.common.const import Const
-from msprobe.core.common.file_utils import check_file_or_directory_path
-from msprobe.core.common.file_utils import save_npy
+from msprobe.core.common.file_utils import check_file_or_directory_path, save_npy
 
 
 class FrameworkDescriptor:
@@ -103,7 +104,7 @@ class FmkAdp:
     @classmethod
     def tensor_norm(cls, tensor):
         return cls.process_tensor(tensor, lambda x: x.norm())
-    
+
     @classmethod
     def save_tensor(cls, tensor, filepath):
         if cls.fmk == Const.PT_FRAMEWORK:
@@ -151,7 +152,7 @@ class FmkAdp:
 
     @classmethod
     def load_checkpoint(cls, path, to_cpu=True, weights_only=True):
-        check_file_or_directory_path(path)
+        check_file_or_directory_path(path, is_strict=not weights_only)
         if cls.fmk == Const.PT_FRAMEWORK:
             try:
                 if to_cpu:
@@ -161,9 +162,9 @@ class FmkAdp:
             except Exception as e:
                 raise RuntimeError(f"load pt file {path} failed: {e}") from e
         return mindspore.load_checkpoint(path)
-    
+
     @classmethod
     def asnumpy(cls, tensor):
         if cls.fmk == Const.PT_FRAMEWORK:
             return tensor.float().numpy()
-        return tensor.float().asnumpy()
+        return tensor.float().asnumpy()

mindstudio_probe-8.3.0/msprobe/core/common/megatron_utils.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import wraps
+
+
+class MegatronStepInfo:
+    is_megatron = False
+    is_forward = False
+    is_backward = False
+    forward_micro_step = -1
+    backward_micro_step = -1
+
+    @classmethod
+    def reset(cls):
+        """重置所有类属性到初始状态"""
+        cls.is_megatron = False
+        cls.is_forward = False
+        cls.is_backward = False
+        cls.forward_micro_step = -1
+        cls.backward_micro_step = -1
+
+
+def wrap_megatron_step(func, is_forward=True):
+    @wraps(func)
+    def wrapped_func(*args, **kwargs):
+        if not MegatronStepInfo.is_megatron:
+            MegatronStepInfo.is_megatron = True
+        if is_forward:
+            MegatronStepInfo.is_forward = True
+            MegatronStepInfo.is_backward = False
+            MegatronStepInfo.forward_micro_step += 1
+        else:
+            MegatronStepInfo.is_forward = False
+            MegatronStepInfo.is_backward = True
+            MegatronStepInfo.backward_micro_step += 1
+        return func(*args, **kwargs)
+
+    return wrapped_func
+
+
+def get_micro_step():
+    return MegatronStepInfo.forward_micro_step if MegatronStepInfo.is_forward else MegatronStepInfo.backward_micro_step
+
+
+def is_megatron():
+    return MegatronStepInfo.is_megatron

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/core/common/utils.py

@@ -28,7 +28,7 @@ import numpy as np
 from msprobe.core.common.const import Const, CompareConst
 from msprobe.core.common.decorator import recursion_depth_decorator
 from msprobe.core.common.exceptions import MsprobeException
-from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path, load_json)
+from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path, load_json, load_construct_json)
 from msprobe.core.common.log import logger
 
 device = collections.namedtuple('device', ['type', 'index'])
@@ -83,7 +83,8 @@ class MsprobeBaseException(Exception):
     INVALID_API_NAME_ERROR = 36
     CROSS_FRAME_ERROR = 37
     MISSING_THRESHOLD_ERROR = 38
-    WRONG_THRESHOLD_ERROR = 38
+    WRONG_THRESHOLD_ERROR = 39
+    MULTIPROCESS_ERROR = 40
 
     def __init__(self, code, error_info: str = ""):
         super(MsprobeBaseException, self).__init__()
@@ -348,8 +349,18 @@ def get_stack_construct_by_dump_json_path(dump_json_path):
     stack_json = os.path.join(directory, "stack.json")
     construct_json = os.path.join(directory, "construct.json")
 
+    stack_json_exist = os.path.exists(stack_json)
+    construct_json_exist = os.path.exists(construct_json)
+
+    if not stack_json_exist and not construct_json_exist:
+        logger.info("stack.json and construct.json not found")
+        return {}, {}
+    if not stack_json_exist or not construct_json_exist:
+        logger.error("stack.json or construct.json not found, please check.")
+        raise CompareException(CompareException.INVALID_PATH_ERROR)
+
     stack = load_json(stack_json)
-    construct = load_json(construct_json)
+    construct, _ = load_construct_json(construct_json)
     return stack, construct
 
 

{mindstudio_probe-8.2.0 → mindstudio_probe-8.3.0}/msprobe/core/compare/find_first/analyzer.py

@@ -47,7 +47,6 @@ class DiffAnalyzer:
             analyze_func()
             if self._diff_nodes:
                 self._gen_analyze_info()
-                self._post_process()
                 return
         logger.info('Cannot find any diff node, no need to generate analyze file.')
 
@@ -56,12 +55,6 @@ class DiffAnalyzer:
         self._resolve_input_path(self._output_path)
         logger.info("Pre Process completed.")
 
-    def _post_process(self):
-        for rank_path in self._paths.values():
-            dump_path = rank_path.dump_path
-            logger.debug(f"Remove {dump_path} success")
-        logger.info("Post Process completed.")
-
     """
     这里需要生成stack，但是直接用dict中自带就行，在op_items.NPU_Stack_Info中
     """
@@ -105,6 +98,8 @@ class DiffAnalyzer:
                 logger.warning(f'Rank {path.rank} has no dump data!')
                 continue
             for op_name, op_data in dump_data.items():
+                if is_ignore_op(op_name):
+                    continue
                 if is_communication_op(op_name):
                     self._first_comm_nodes[path.rank] = op_name
                     break
@@ -131,10 +126,16 @@ class DiffAnalyzer:
         for rank, nodes in list(self._rank_comm_nodes_dict.items())[:-1]:
             searched_ranks.add(rank)
             seen_nodes = set()
+            last_node = None
             for cur_node in nodes.values():
+                is_overflow = last_node and hasattr(last_node, 'layer') and hasattr(cur_node, 'layer') and \
+                last_node.layer >= cur_node.layer
+                if is_overflow:
+                    cur_node.layer = last_node.layer + 1
                 conn_info = cur_node.find_connected_nodes()
                 if not conn_info.get('ranks'):
                     conn_info['ranks'] = self._rank_comm_nodes_dict.keys()
+                last_node = cur_node
                 if not self._find_connection(conn_info, cur_node, searched_ranks, seen_nodes):
                     logger.debug(f'Cannot find connected communication node for "{cur_node.node_id}".')