mindspore 2.7.0rc1__cp311-cp311-win_amd64.whl → 2.7.1__cp311-cp311-win_amd64.whl

mindspore/profiler/dynamic_profiler.py

@@ -24,7 +24,9 @@ import multiprocessing
 
 from mindspore import log as logger
 from mindspore.train import Callback
-from mindspore.profiler import Profiler, tensorboard_trace_handler, schedule
+from mindspore.profiler import tensorboard_trace_handler, schedule
+from mindspore.profiler.profiler import Profile
+from mindspore.profiler.experimental_config import _ExperimentalConfig
 from mindspore.profiler.common.file_manager import FileManager
 from mindspore.profiler.common.path_manager import PathManager
 from mindspore.profiler.dynamic_profile.dynamic_profiler_config_context import DynamicProfilerConfigContext
@@ -41,7 +43,7 @@ def print_msg(msg):
 
 class DynamicProfilerMonitorBase(Callback):
     """
-    Dynamic profile callback base class implementing the dynamic profile functionality.
+    Dynamic profiler callback base class implementing the dynamic profiler functionality.
     """
 
     NPU_MONITOR_START = "NPU_MONITOR_START"
@@ -88,15 +90,23 @@ class DynamicProfilerMonitorBase(Callback):
     @no_exception_func()
     def step_begin(self, run_context):
         """
-        Start profile at the begin of step.
+        Start profiler at the begin of step.
 
         Args:
             run_context (RunContext): Context of the train running.
         """
         prof_json = self._get_prof_args()
+        if not prof_json:
+            return
+        if self._is_dyno:
+            # Dyno monitor process
+            if self.NPU_MONITOR_START in prof_json:
+                self._call_dyno_monitor(prof_json)
+                return
+
         prof_args = DynamicProfilerConfigContext(prof_json)
         if not prof_args.is_valid:
-            logger.error("Dynamic profile json is not valid, please check the json file.")
+            logger.error("Dynamic profiler json is not valid, please check the json file.")
             return
 
         if prof_args.start_step in (-1, self._last_start_step):
@@ -109,16 +119,22 @@ class DynamicProfilerMonitorBase(Callback):
         # Prevent repeated calls of the start function within a complete interval
         if step_num == start_step:
             if self._is_started:
-                logger.error("Dynamic profile is already started at step %d, "
-                             "please wait the first profile finished at step %d.",
+                logger.error("Dynamic profiler is already started at step %d, "
+                             "please wait the first profiler finished at step %d.",
                              self._last_start_step, self._last_stop_step)
                 return
 
             if self._profiler is None:
-                prof_path = os.path.join(self._output_path, f"rank{self._rank_id}_start{start_step}_stop{stop_step}")
+                output_path = prof_args.prof_path if prof_args.prof_path != "./" else self._output_path
+                prof_path = os.path.join(
+                    output_path,
+                    f"rank{self._rank_id}_start{start_step}_stop{stop_step}"
+                )
                 PathManager.check_input_directory_path(prof_path)
-                self._profiler = Profiler(on_trace_ready=tensorboard_trace_handler(dir_name=prof_path),
-                                          start_profile=False, **prof_args.args)
+                profiler_config = self._get_prof_config(prof_args, prof_path, start_step, stop_step,
+                                                        start_profile=False,
+                                                        skip_first=0)
+                self._profiler = Profile(**profiler_config)
                 print_msg(f"Rank {self._rank_id} create output path {prof_path}")
 
             self._profiler.start()
@@ -128,10 +144,58 @@ class DynamicProfilerMonitorBase(Callback):
             print_msg(f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
                       f"will stop at step {stop_step}")
 
+    @staticmethod
+    def _get_prof_config(prof_args, prof_path, start_step, stop_step, start_profile, skip_first):
+        """
+        Get profiler config.
+
+        Args:
+            prof_args: Profiler config.
+            prof_path: Profiler output path.
+            start_step: Start step.
+            stop_step: Stop step.
+            start_profile: enable start_profile.
+            skip_first: skip first step.
+        """
+        profiler_config = {
+            "activities": prof_args.args.get("activities"),
+            "with_stack": prof_args.args.get("with_stack"),
+            "profile_memory": prof_args.args.get("profile_memory"),
+            "parallel_strategy": prof_args.args.get("parallel_strategy"),
+            "start_profile": start_profile,
+            "record_shapes": prof_args.args.get("record_shapes"),
+            "schedule": schedule(
+                wait=0,
+                warmup=0,
+                active=stop_step - start_step + 1,
+                repeat=1,
+                skip_first=skip_first
+            ),
+            "on_trace_ready": tensorboard_trace_handler(
+                dir_name=prof_path,
+                analyse_flag=prof_args.analyse,
+                async_mode=prof_args.analyse_mode == "async",
+            ),
+            "experimental_config": _ExperimentalConfig(
+                profiler_level=prof_args.args.get("profiler_level"),
+                aic_metrics=prof_args.args.get("aic_metrics"),
+                l2_cache=prof_args.args.get("l2_cache"),
+                mstx=prof_args.args.get("mstx"),
+                data_simplification=prof_args.args.get("data_simplification"),
+                export_type=prof_args.args.get("export_type"),
+                mstx_domain_include=prof_args.args.get("mstx_domain_include"),
+                mstx_domain_exclude=prof_args.args.get("mstx_domain_exclude"),
+                sys_io=prof_args.args.get("sys_io"),
+                sys_interconnection=prof_args.args.get("sys_interconnection"),
+                host_sys=prof_args.args.get("host_sys")
+            )
+        }
+        return profiler_config
+
     @no_exception_func()
     def step_end(self, run_context):
         """
-        Stop profile at the end of step.
+        Stop profiler at the end of step.
 
         Args:
             run_context (RunContext): Context of the train running.
@@ -140,26 +204,22 @@ class DynamicProfilerMonitorBase(Callback):
         prof_args = DynamicProfilerConfigContext(prof_json)
 
         if not prof_args.is_valid:
-            logger.error("Dynamic profile json is not valid, please check the json file.")
+            logger.error("Dynamic profiler json is not valid, please check the json file.")
             return
 
         if prof_args.stop_step == -1:
             return
 
+        if self._profiler:
+            self._profiler.step()
+
         cb_params = run_context.original_args()
         step_num = cb_params.cur_step_num
 
         if step_num == self._last_stop_step and self._is_started:
-            if self._profiler:
-                self._profiler.stop()
-                if prof_args.analyse_mode:
-                    self._profiler.analyse(mode=prof_args.analyse_mode)
-                else:
-                    ProfilerInterface.finalize()
-                    ProfilerInterface.clear()
-                self._profiler = None
-                self._is_started = False
-                print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
+            self._profiler = None
+            self._is_started = False
+            print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
 
     @no_exception_func()
     def step(self):
@@ -271,7 +331,7 @@ class DynamicProfilerMonitorBase(Callback):
 
         prof_args = DynamicProfilerConfigContext(prof_json)
         if not prof_args.is_valid:
-            logger.error("Dynamic profile config is not valid, please check the json or dyno config.")
+            logger.error("Dynamic profiler config is not valid, please check the json or dyno config.")
             return
         self._handle_profiler_setup(prof_args)
 
@@ -293,14 +353,13 @@ class DynamicProfilerMonitorBase(Callback):
             if not (start_step >= 0 and 0 <= start_step <= stop_step):
                 self._profiler = None
                 logger.error(
-                    "Rank %d Dynamic profile start at step %d and stop at step %d must be "
+                    "Rank %d Dynamic profiler start at step %d and stop at step %d must be "
                     "greater than or equal to 0, and stop step should not be less than start step",
                     self._rank_id, start_step, stop_step
                 )
                 return
 
             # Setup profiler configuration
-            active_steps = stop_step - start_step + 1
             output_path = args.prof_path if args.prof_path != "./" else self._output_path
             prof_path = os.path.join(
                 output_path,
@@ -308,26 +367,12 @@ class DynamicProfilerMonitorBase(Callback):
             )
             print_msg(f"Rank {self._rank_id} create output path {prof_path}")
             print_msg(
-                f"Rank {self._rank_id} Dynamic profile start at step {start_step}, "
+                f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
                 f"will stop at step {stop_step}"
             )
-            profiler_config = {
-                "schedule": schedule(
-                    wait=0,
-                    warmup=0,
-                    active=active_steps,
-                    repeat=1,
-                    skip_first=1
-                ),
-                "on_trace_ready": tensorboard_trace_handler(
-                    dir_name=prof_path,
-                    analyse_flag=args.analyse,
-                    async_mode=args.analyse_mode == "async",
-                ),
-                **args.args
-            }
-
-            self._profiler = Profiler(**profiler_config)
+            profiler_config = self._get_prof_config(args, prof_path, start_step, stop_step, start_profile=True,
+                                                    skip_first=1)
+            self._profiler = Profile(**profiler_config)
 
     def _is_valid_start_stop_step(self, step_num, start_step, stop_step):
         """Verify whether start_step and stop_step are valid parameters."""
@@ -546,10 +591,10 @@ if sys.version_info >= (3, 8):
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
-        This class to enable the dynamic profile monitoring of MindSpore neural networks.
+        This class to enable the dynamic profiler monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): (Ascend only) Dynamic profile json config file directory. The requirement is a shared path
+            cfg_path (str): (Ascend only) Dynamic profiler json config file directory. The requirement is a shared path
                 that can be accessed by all nodes. The parameters of the json configuration file are as follows:
 
                 - start_step (int, required) - Sets the step number at which the Profiler starts collecting data.
@@ -795,10 +840,10 @@ else:
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
-        This class to enable the dynamic profile monitoring of MindSpore neural networks.
+        This class to enable the dynamic profiler monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): Dynamic profile json config file directory. The requirement is a shared path
+            cfg_path (str): Dynamic profiler json config file directory. The requirement is a shared path
                 that can be accessed by all nodes.
             output_path (str, optional): Output data path. Default: ``"./dyn_profile_data"`` .
             poll_interval (int, optional): The polling period of the monitoring process, in seconds.

mindspore/profiler/envprofiler.py

@@ -16,7 +16,9 @@
 import os
 import json
 from mindspore import log as logger
-from mindspore.profiler.profiler import Profiler
+from mindspore.profiler.profiler import Profile
+from mindspore.profiler.experimental_config import _ExperimentalConfig
+from mindspore.profiler import tensorboard_trace_handler
 from mindspore.profiler.common.constant import (
     ProfilerLevel,
     AicoreMetrics,
@@ -47,8 +49,31 @@ class EnvProfiler:
 
         params = cls._convert_options_to_profiler_params(options)
         logger.info(f"params: {params}")
-        if params["start_profile"]:
-            cls.profiler = Profiler(**params)
+        if params["start"]:
+            experimental_config = _ExperimentalConfig(profiler_level=params.get("profiler_level"),
+                                                      aic_metrics=params.get("aic_metrics"),
+                                                      l2_cache=params.get("l2_cache"),
+                                                      mstx=params.get("mstx"),
+                                                      data_simplification=params.get("data_simplification"),
+                                                      export_type=params.get("export_type"),
+                                                      mstx_domain_include=params.get("mstx_domain_include"),
+                                                      mstx_domain_exclude=params.get("mstx_domain_exclude"),
+                                                      sys_io=params.get("sys_io"),
+                                                      sys_interconnection=params.get("sys_interconnection"),
+                                                      host_sys=params.get("host_sys"))
+            cls.profiler = Profile(activities=params.get("activities"),
+                                   with_stack=params.get("with_stack"),
+                                   profile_memory=params.get("profile_memory"),
+                                   data_process=params.get("data_process"),
+                                   parallel_strategy=params.get("parallel_strategy"),
+                                   start_profile=params.get("start_profile"),
+                                   hbm_ddr=params.get("hbm_ddr"),
+                                   pcie=params.get("pcie"),
+                                   sync_enable=params.get("sync_enable"),
+                                   record_shapes=params.get("record_shapes"),
+                                   on_trace_ready=tensorboard_trace_handler(params.get("output_path")),
+                                   experimental_config=experimental_config)
+            cls.profiler.start()
             logger.info("Profiler init success.")
 
     def analyse(self):
@@ -59,7 +84,7 @@ class EnvProfiler:
         if not self.profiler:
             logger.info("Profiler is not initialized, skip analyse.")
             return
-        self.profiler.analyse()
+        self.profiler.stop()
         logger.info("analyse end")
 
     @classmethod
@@ -101,7 +126,7 @@ class EnvProfiler:
             params["output_path"] = options["output_path"]
 
         # if start is not set, default is False
-        params["start_profile"] = options.get("start", False)
+        params["start"] = options.get("start", False)
 
         for param, (_, default_value) in ProfilerParameters.PARAMS.items():
             if param in options and param not in cls.NOT_SUPPORTED_PARAMS:

mindspore/profiler/experimental_config.py

@@ -59,7 +59,7 @@ class _ExperimentalConfig:
               storage and l2 cache etc.
         l2_cache (bool, optional): (Ascend only) Whether to collect l2 cache data, collect when True.
             Default: ``False`` . The l2_cache.csv file is generated in the ASCEND_PROFILER_OUTPUT folder. In GE backend,
-            only wait and skip_first parameters in schedule configuration can be set to 0.
+            only support :class:`mindspore.profiler.schedule` configuration wait and skip_first parameter is 0.
         mstx (bool, optional): (Ascend only) Whether to collect light weight profiling data, collect when True.
             Default: ``False`` .
         data_simplification (bool, optional): (Ascend only) Whether to remove FRAMEWORK data and other redundant data.
@@ -84,7 +84,8 @@ class _ExperimentalConfig:
             HCCS data, PCIe data, and Stars Chip Trans. Default: ``False``.
         host_sys (list, optional): Collect the data of system call classes on the host side.
             Default: ``[]``, indicating that system class data on the host side is not collected.
-            You need to set `start_profile` of :class:`mindspore.profiler.profile` to ``False``.When collecting DISK or
+            You need to set `start_profile` of :class:`mindspore.profiler.profile` to ``False``.Currently, only
+            the **root user** supports collecting DISK or OSRT data, when collecting DISK or
             OSRT data, it is necessary to install the iotop, perf, and ltrace third-party tools in advance.
             For detailed steps, please refer to `Installing Third-party Tools
             <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/atlasprofiling_16_0136.
@@ -98,6 +99,21 @@ class _ExperimentalConfig:
             <https://gitee.com/mindspore/mindspore/blob/master/docs/api/api_python/mindspore/script/
             msprof_data_collection.sh>`_.
 
+            Final deliverables by `MindStudio Insight
+            <https://www.hiascend.com/developer/download/community/result?module=pt+sto+cann>`_
+            tool visualizes the analysis results.
+            For detailed analysis, please refer to `host-side CPU data analysis
+            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
+            atlasprofiling_16_0106.html>`_, `host-side MEM data analysis
+            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
+            atlasprofiling_16_0107.html>`_, `host-side DISK data analysis
+            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
+            atlasprofiling_16_0108.html>`_, `host-side NETWORK data analysis
+            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
+            atlasprofiling_16_0109.html>`_, `host-side OSRT data analysis
+            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
+            atlasprofiling_16_0110.html>`_.
+
             - HostSystem.CPU: Collect the CPU utilization at the process level.
             - HostSystem.MEM: Collect the memory utilization at the process level.
             - HostSystem.DISK: Collect the disk I/O utilization at the process level.

mindspore/profiler/platform/cpu_profiler.py

@@ -16,10 +16,11 @@
 import mindspore._c_expression as c_expression
 
 from mindspore.profiler.common.registry import PROFILERS
-from mindspore.profiler.common.constant import DeviceTarget, ProfilerActivity
+from mindspore.profiler.common.constant import DeviceTarget, ProfilerActivity, AnalysisMode
 from mindspore.profiler.common.util import print_msg_with_pid
 from mindspore.profiler.common.profiler_context import ProfilerContext
 from mindspore.profiler.common.profiler_path_manager import ProfilerPathManager
+from mindspore.profiler.common.process_pool import MultiProcessPool
 from mindspore.profiler.platform.base_profiler import BaseProfiler
 from mindspore.profiler.analysis.time_converter import TimeConverter
 from mindspore.profiler.analysis.task_manager import TaskManager
@@ -68,7 +69,7 @@ class CpuProfiler(BaseProfiler):
         if ProfilerContext().device_target_set != {DeviceTarget.CPU.value}:
             return
         self._logger.info("CpuProfiler analyse.")
-        CPUProfilerAnalysis.online_analyse()
+        CPUProfilerAnalysis.online_analyse(**kwargs)
 
     def finalize(self) -> None:
         """Finalize profiling data."""
@@ -81,12 +82,17 @@ class CPUProfilerAnalysis:
     """
 
     @classmethod
-    def online_analyse(cls):
+    def online_analyse(cls, async_mode: bool = False):
         """
         Online analysis for CPU
         """
         cls._pre_analyse_online()
-        cls._run_tasks(**ProfilerContext().to_dict())
+        if async_mode:
+            ProfilerContext().mode = AnalysisMode.ASYNC_MODE.value
+            MultiProcessPool().add_async_job(cls._run_tasks, **ProfilerContext().to_dict())
+        else:
+            ProfilerContext().mode = AnalysisMode.SYNC_MODE.value
+            cls._run_tasks(**ProfilerContext().to_dict())
 
     @classmethod
     def _pre_analyse_online(cls):

mindspore/profiler/platform/npu_profiler.py

@@ -36,6 +36,8 @@ from mindspore.profiler.platform.base_profiler import BaseProfiler
 from mindspore.profiler.common.profiler_path_manager import ProfilerPathManager
 from mindspore.profiler.common.profiler_info import ProfilerInfo
 from mindspore.profiler.common.process_pool import MultiProcessPool
+from mindspore.profiler.common.constant import MsprofModeName
+from mindspore.profiler.common.util import no_exception_func
 from mindspore.profiler.analysis.task_manager import TaskManager
 from mindspore.profiler.analysis.time_converter import TimeConverter
 from mindspore.profiler.analysis.parser.ascend_cann_parser import AscendMsprofParser
@@ -58,6 +60,7 @@ from mindspore.profiler.analysis.viewer.ms_operator_details_viewer import MsOper
 from mindspore.profiler.common.util import print_msg_with_pid
 from mindspore.profiler.common.log import ProfilerLogger
 from mindspore.profiler.mstx import Mstx
+from mindspore.profiler.common.util import get_device_id
 
 
 @PROFILERS.register_module(DeviceTarget.NPU.value)
@@ -68,6 +71,7 @@ class NpuProfiler(BaseProfiler):
 
     def __init__(self) -> None:
         super().__init__()
+        self._is_env_not_valid = self._is_environment_not_valid()
         self._prof_ctx = ProfilerContext()
         self._prof_info = ProfilerInfo()
         self._prof_path_mgr = ProfilerPathManager()
@@ -78,7 +82,7 @@ class NpuProfiler(BaseProfiler):
         # initialize profiler backend
         self._profiler.init(
             self._prof_ctx.ascend_ms_dir,
-            int(self._prof_ctx.device_id),
+            int(get_device_id()),
             json.dumps(self._prof_ctx.npu_profiler_params),
         )
         self._logger.info("NpuProfiler init profiler backend params %s",
@@ -97,6 +101,8 @@ class NpuProfiler(BaseProfiler):
 
     def start(self) -> None:
         """Start profiling."""
+        if self._is_env_not_valid:
+            return
         self._logger.info("NpuProfiler start.")
 
         Mstx.enable = self._prof_ctx.npu_profiler_params.get("mstx", False)
@@ -115,6 +121,8 @@ class NpuProfiler(BaseProfiler):
 
     def stop(self) -> None:
         """Stop profiling."""
+        if self._is_env_not_valid:
+            return
         self._logger.info("NpuProfiler stop.")
 
         Mstx.enable = False
@@ -149,16 +157,29 @@ class NpuProfiler(BaseProfiler):
 
     def analyse(self, **kwargs) -> None:
         """Analyse the profiling data."""
+        if self._is_env_not_valid:
+            return
         self._logger.info("NpuProfiler analyse.")
 
-        NPUProfilerAnalysis.online_analyse()
+        NPUProfilerAnalysis.online_analyse(async_mode=kwargs.get('async_mode'))
 
     def finalize(self) -> None:
         """Finalize profiling data."""
+        if self._is_env_not_valid:
+            return
         self._logger.info("NpuProfiler finalize.")
         if self._profiler:
             self._profiler.finalize()
 
+    @staticmethod
+    def _is_environment_not_valid() -> bool:
+        # check msprof dynamic environment variable
+        if os.getenv(MsprofModeName.MSPROF_DYNAMIC_ENV) is not None:
+            logger.error(f"The environment variable '{MsprofModeName.MSPROF_DYNAMIC_ENV}' has been set."
+                         f"Please execute 'unset {MsprofModeName.MSPROF_DYNAMIC_ENV}'.")
+            return True
+        return False
+
 
 class NPUProfilerAnalysis:
     """
@@ -166,6 +187,7 @@ class NPUProfilerAnalysis:
     """
 
     @classmethod
+    @no_exception_func()
     def online_analyse(cls, async_mode: bool = False):
         """
         Online analysis for NPU
@@ -179,6 +201,7 @@ class NPUProfilerAnalysis:
             cls._run_tasks(**ProfilerContext().to_dict())
 
     @classmethod
+    @no_exception_func()
     def offline_analyse(
             cls,
             path: str,
@@ -262,11 +285,15 @@ class NPUProfilerAnalysis:
         task_mgr = cls._construct_task_mgr(**kwargs)
         task_mgr.run()
         ProfilerLogger.get_instance().info(json.dumps(task_mgr.cost_time, indent=4))
-        activities = kwargs.get("activities")
-        if activities and ProfilerActivity.NPU.value in activities:
-            ProfilerPathManager().move_db_file()
-        if kwargs.get("data_simplification") and ProfilerActivity.NPU.value in kwargs.get("activities"):
-            ProfilerPathManager().simplify_data()
+        activities = kwargs.get("activities", [])
+        export_type = kwargs.get("export_type", [])
+        if ProfilerActivity.NPU.value in activities:
+            if ExportType.Db.value in export_type:
+                ProfilerPathManager().move_db_file()
+            else:
+                ProfilerPathManager().remove_db_file()
+            if kwargs.get("data_simplification"):
+                ProfilerPathManager().simplify_data()
 
     @classmethod
     def _construct_task_mgr(cls, **kwargs) -> TaskManager: