mindstudio-probe 8.3.3__py3-none-any.whl → 26.0.0a1__py3-none-any.whl

msprobe/mindspore/{debugger → dump/debugger}/precision_debugger.py

@@ -1,30 +1,30 @@
-# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import os
 from collections import defaultdict, namedtuple
 
-import mindspore as ms
 from mindspore.ops.operations import _inner_ops as inner
 from mindspore._c_expression import MSContext
 
 from msprobe.core.common.const import Const, MsgConst
-from msprobe.core.common.utils import check_token_range, ThreadSafe
+from msprobe.core.common.utils import check_token_range, ThreadSafe, check_rank_id
 from msprobe.core.common.runtime import Runtime
-from msprobe.core.debugger.precision_debugger import BasePrecisionDebugger
-from msprobe.mindspore.cell_processor import CellProcessor
+from msprobe.core.dump.debugger.precision_debugger import BasePrecisionDebugger
+from msprobe.mindspore.dump.cell_processor import CellProcessor
 from msprobe.mindspore.common.const import Const as MsConst
 from msprobe.mindspore.common.utils import (
     set_register_backward_hook_functions,
@@ -32,18 +32,17 @@ from msprobe.mindspore.common.utils import (
     is_graph_mode_cell_dump_allowed,
     wrap_backward_hook_call_func
 )
-from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
-from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump
-from msprobe.mindspore.dump.hook_cell.api_register import get_api_register
-from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell
-from msprobe.mindspore.grad_probe.grad_monitor import GradientMonitor
-from msprobe.mindspore.ms_config import parse_task_config
-from msprobe.mindspore.mindspore_service import MindsporeService
-from msprobe.mindspore.task_handler_factory import TaskHandlerFactory
+from msprobe.mindspore.dump.debugger.debugger_config import DebuggerConfig
+from msprobe.mindspore.dump.dump_processor.graph_mode_cell_dump import GraphModeCellDump
+from msprobe.mindspore.dump.dump_processor.hook_cell.api_register import get_api_register
+from msprobe.mindspore.dump.dump_processor.hook_cell.hook_cell import HOOKCell
+from msprobe.mindspore.dump.ms_config import parse_task_config
+from msprobe.mindspore.dump.mindspore_service import MindsporeService
+from msprobe.mindspore.dump.task_handler_factory import TaskHandlerFactory
 
 try:
-    from mindspore._c_expression import _dump_start, _dump_stop, _dump_step, _set_init_iter, _dump_set_dynamic
     import mindspore as ms
+    from mindspore._c_expression import _dump_start, _dump_stop, _dump_step, _set_init_iter, _dump_set_dynamic
 except ImportError:
     enable_dynamic_kbyk_dump = False
 else:
@@ -73,9 +72,6 @@ class PrecisionDebugger(BasePrecisionDebugger):
         set_register_backward_hook_functions()
         super().__init__(config_path, task, dump_path, level, step)
 
-        if self.task == Const.GRAD_PROBE:
-            self.gm = GradientMonitor(self.common_config, self.task_config)
-            return
         self.common_config.level = level if level else self.common_config.level
         self.common_config.dump_path = dump_path if dump_path else self.common_config.dump_path
         self.config = DebuggerConfig(self.common_config, self.task_config)
@@ -130,10 +126,23 @@ class PrecisionDebugger(BasePrecisionDebugger):
         return is_graph
 
     @classmethod
-    def start(cls, model=None, token_range=None):
+    def handle_graph_cell_dump(cls, instance):
+        """处理graph cell dump相关逻辑"""
+        is_graph_mode = instance.config.execution_mode in [MsConst.GRAPH_GE_MODE, MsConst.GRAPH_KBYK_MODE]
+
+        if is_graph_mode:
+            GraphModeCellDump.step(instance.config.dump_path, instance.config.step, instance.config.task)
+
+        if Runtime.run_mode == MsConst.PYNATIVE_GRAPH_MODE:
+            dump_path = os.path.join(instance.config.dump_path, MsConst.GRAPH_MODE)
+            GraphModeCellDump.step(dump_path, instance.config.step, instance.config.task)
+
+    @classmethod
+    def start(cls, model=None, token_range=None, rank_id=None):
         instance = cls._get_instance()
         if instance is None:
             return
+        check_rank_id(rank_id)
         if cls._is_kernel_dump():
             cls._start_kernel_dump()
         else:
@@ -144,7 +153,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
                     if not instance.service:
                         instance.service = MindsporeService(instance.config)
                     instance.config.check_model(model, token_range)
-                    instance.service.start(model, token_range)
+                    instance.service.start(model, token_range, rank_id)
             else:
                 if not instance.first_start:
                     get_api_register().restore_all_api()
@@ -159,8 +168,6 @@ class PrecisionDebugger(BasePrecisionDebugger):
         if instance is None:
             return
 
-        if instance.task == Const.GRAD_PROBE:
-            instance.gm.stop()
         if instance.service:
             with ThreadSafe():
                 instance.service.stop()
@@ -181,8 +188,10 @@ class PrecisionDebugger(BasePrecisionDebugger):
         if instance.service:
             with ThreadSafe():
                 instance.service.step()
-        if is_graph_mode_cell_dump_allowed(instance.config):
-            GraphModeCellDump.step(instance.config.dump_path, instance.config.step, instance.config.task)
+
+        if instance.config.level in [MsConst.CELL, MsConst.CELL_AND_API]:
+            cls.handle_graph_cell_dump(instance)
+
         if enable_dynamic_kbyk_dump and instance.config.level_ori == Const.LEVEL_L2:
             _dump_step(1)
         if cls._is_kernel_dump() and _msprobe_c:
@@ -192,16 +201,6 @@ class PrecisionDebugger(BasePrecisionDebugger):
         CellProcessor.reset_cell_stats()
         Runtime.step_count += 1
 
-    @classmethod
-    @ThreadSafe.synchronized
-    def monitor(cls, opt):
-        instance = cls._instance
-        if not instance:
-            raise Exception(MsgConst.NOT_CREATED_INSTANCE)
-        if instance.task != Const.GRAD_PROBE:
-            return
-        instance.gm.monitor(opt)
-
     @classmethod
     @ThreadSafe.synchronized
     def save(cls, variable, name, save_backward=True):
@@ -227,8 +226,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
             return not instance._is_graph_dump(instance.config)
         if instance.config.execution_mode != MsConst.PYNATIVE_MODE:
             return False
-        else:
-            return instance.config.task != Const.FREE_BENCHMARK
+        return True
 
     @classmethod
     def _is_kernel_dump(cls):

msprobe/mindspore/dump/{cell_dump_process.py → dump_processor/cell_dump_process.py}

@@ -1,19 +1,19 @@
-# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
-import atexit
 from multiprocessing import Pool
 import os
 import re
@@ -46,7 +46,10 @@ KEY_FORWARD = CoreConst.FORWARD
 KEY_BACKWARD = CoreConst.BACKWARD
 KEY_INPUT = CoreConst.INPUT
 KEY_OUTPUT = CoreConst.OUTPUT
-KEY_DUMP_TENSOR_DATA = "dump_tensor_data/"
+if (ms.__version__ > "2.7.0"):
+    KEY_DUMP_TENSOR_DATA = "dump_tensor_data/"
+else:
+    KEY_DUMP_TENSOR_DATA = "dump_tensor_data_"
 KEY_STATISTIC_CSV = "statistic.csv"
 KEY_TD_FLAG = "td_flag"
 # 设置落盘文件检测超时时间
@@ -520,7 +523,7 @@ def generate_dump_info(path):
         return
 
     if dump_task == CoreConst.TENSOR:
-        dump_data = {"task": "tensor", "level": "L0", "dump_data_dir": path, "data": {}}
+        dump_data = {"task": "tensor", "level": "L0", "framework": "mindspore", "dump_data_dir": path, "data": {}}
         with Pool(processes=10) as pool:
             file_paths = []
             for file in os.listdir(path):
@@ -590,7 +593,7 @@ def is_download_finished(directory, save_flag):
     if not os.path.exists(directory):
         logger.warning(f"The specified directory {directory} does not exist.")
         return False
-    
+
     # 遍历当前目录中的所有条目
     for entry_path in os.listdir(directory):
         if entry_path.startswith(save_flag):
@@ -600,8 +603,11 @@ def is_download_finished(directory, save_flag):
 
 
 def process_step(dump_path, flag_path, step, step_list):
-    if step not in step_list:
+    if step_list and step not in step_list:
         return
+    
+    global dump_task
+    dump_task = CoreConst.TENSOR
 
     if not os.path.exists(dump_path):
         logger.warning('No grap cell data is dumped.')
@@ -619,7 +625,7 @@ def process_step(dump_path, flag_path, step, step_list):
     rank_path = os.path.join(step_path, rank_dir)
     npy_path = os.path.join(rank_path, CoreConst.DUMP_TENSOR_DATA)
     save_finish_flag = f"step_{step}"
-    start_time = time.time() 
+    start_time = time.time()
     while True:
         is_finished = is_download_finished(flag_path, save_finish_flag)
         if not is_finished:
@@ -717,6 +723,10 @@ def process_statistics_step(dump_path, step, step_list):
         rank_dir_kbk = CoreConst.RANK + CoreConst.REPLACEMENT_CHARACTER + str(rank_id)
     rank_path_kbk = os.path.join(dump_path, rank_dir_kbk)
 
+    data_dir = os.path.join(rank_path_kbk, "Net")
+    if not os.path.isdir(data_dir):
+        logger.warning('No grap cell data is dumped.')
+        return
     # 按相同step数将csv文件名分组存入file_dict
     file_dict = {}
     depth_limit = 4
@@ -734,6 +744,10 @@ def process_statistics_step(dump_path, step, step_list):
                 else:
                     file_dict[step_dir] = [file_path]
 
+    if not file_dict:
+        logger.warning('Net exist but no grap cell data is dumped.')
+        return
+
     # 将相同step的csv文件合并，并加工后存入相应step目录下
     merge_file(dump_path, rank_dir_kbk, file_dict)
 

msprobe/mindspore/dump/{cell_dump_with_insert_gradient.py → dump_processor/cell_dump_with_insert_gradient.py}

@@ -1,17 +1,18 @@
-# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import atexit
 from multiprocessing import Pool
@@ -30,7 +31,7 @@ from msprobe.core.common.file_utils import (
     load_npy, save_json, remove_path, load_yaml,
     create_directory, read_csv, write_df_to_csv, write_csv, move_file, move_directory)
 from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.dump.cell_dump_process import CellDumpConfig
+from msprobe.mindspore.dump.dump_processor.cell_dump_process import CellDumpConfig
 
 
 CONSTRUCT_FILE_NAME = "construct.json"
@@ -44,7 +45,10 @@ KEY_FORWARD = CoreConst.FORWARD
 KEY_BACKWARD = CoreConst.BACKWARD
 KEY_INPUT = CoreConst.INPUT
 KEY_OUTPUT = CoreConst.OUTPUT
-KEY_DUMP_TENSOR_DATA = "dump_tensor_data_"
+if (ms.__version__ > "2.7.0"):
+    KEY_DUMP_TENSOR_DATA = "dump_tensor_data/"
+else:
+    KEY_DUMP_TENSOR_DATA = "dump_tensor_data_"
 KEY_STATISTIC_CSV = "statistic.csv"
 KEY_TD_FLAG = "td_flag"
 td = ops.TensorDump()
@@ -464,7 +468,7 @@ def generate_dump_info(path):
         return
 
     if dump_task == CoreConst.TENSOR:
-        dump_data = {"task": "tensor", "level": "L0", "dump_data_dir": path, "data": {}}
+        dump_data = {"task": "tensor", "level": "L0", "framework": "mindspore", "dump_data_dir": path, "data": {}}
         with Pool(processes=10) as pool:
             file_paths = []
             for file in os.listdir(path):

msprobe/mindspore/dump/{dump_tool_factory.py → dump_processor/dump_tool_factory.py}

@@ -1,25 +1,26 @@
-# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 from msprobe.mindspore.common.const import Const
 from msprobe.core.common.log import logger
 from msprobe.mindspore.common.utils import is_graph_mode_cell_dump_allowed
-from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
-from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump
-from msprobe.mindspore.dump.kernel_kbyk_dump import KernelKbykDump
-from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump
+from msprobe.mindspore.dump.debugger.debugger_config import DebuggerConfig
+from msprobe.mindspore.dump.dump_processor.kernel_graph_dump import KernelGraphDump
+from msprobe.mindspore.dump.dump_processor.kernel_kbyk_dump import KernelKbykDump
+from msprobe.mindspore.dump.dump_processor.graph_mode_cell_dump import GraphModeCellDump
 
 
 class DumpToolFactory:

msprobe/mindspore/dump/{graph_mode_cell_dump.py → dump_processor/graph_mode_cell_dump.py}

@@ -1,20 +1,20 @@
-# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd.
-# All rights reserved.
+# -------------------------------------------------------------------------
+#  This file is part of the MindStudio project.
+# Copyright (c) 2025 Huawei Technologies Co.,Ltd.
 #
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# MindStudio is licensed under Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#          http://license.coscl.org.cn/MulanPSL2
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+# See the Mulan PSL v2 for more details.
+# -------------------------------------------------------------------------
 
 import os
-import glob
 import tempfile
 import mindspore as ms
 from mindspore import hal, ops, Tensor
@@ -24,9 +24,9 @@ from msprobe.core.common.const import Const as CoreConst
 from msprobe.core.common.runtime import Runtime
 from msprobe.mindspore.common.const import Const
 from msprobe.mindspore.common.log import logger
-from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
-import msprobe.mindspore.dump.cell_dump_process as cellDumperWithDumpGradient
-import msprobe.mindspore.dump.cell_dump_with_insert_gradient as cellDumperWithInsertGradient
+from msprobe.mindspore.dump.debugger.debugger_config import DebuggerConfig
+import msprobe.mindspore.dump.dump_processor.cell_dump_process as cellDumperWithDumpGradient
+import msprobe.mindspore.dump.dump_processor.cell_dump_with_insert_gradient as cellDumperWithInsertGradient
 
 tensordump_flag = True
 DEFAULT_RANK_DIR = "rank0"
@@ -87,6 +87,7 @@ class GraphModeCellDump:
                     "Importing _dump_step failed, "
                     "please use the latest version package of MindSpore."
                 )
+            ms.runtime.synchronize()
             _dump_step(1)
             cellDumperWithDumpGradient.process_statistics_step(dump_path, Runtime.step_count, step_list)