mindspore 2.7.0__cp310-cp310-win_amd64.whl → 2.7.0rc1__cp310-cp310-win_amd64.whl

mindspore/profiler/common/profiler_op_analyse.py

@@ -16,7 +16,7 @@
 import json
 import os
 from collections import defaultdict
-from mindspore.profiler.common.path_manager import PathManager
+from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
 
 
 class OpAnalyser:
@@ -34,7 +34,7 @@ class OpAnalyser:
     def __init__(self, output_path, dev_id, op_names=None):
         """The parser for parsing framework files."""
         self._dev_id = dev_id
-        self._output_path = PathManager.get_real_path(output_path)
+        self._output_path = output_path
         self.op_names = op_names
         self.op_name = ''
         self.framework_list = []
@@ -66,8 +66,7 @@ class OpAnalyser:
         for filename in self.framework_info_dir:
             op_side = filename.split('_')[0]
             framework_file_path = os.path.join(self._output_path, filename)
-            PathManager.check_input_file_path(framework_file_path)
-            PathManager.check_directory_path_readable(framework_file_path)
+            framework_file_path = validate_and_normalize_path(framework_file_path)
             with open(framework_file_path, 'r') as f_obj:
                 framework_info = f_obj.readlines()
             for line_info in framework_info:
@@ -86,8 +85,7 @@ class OpAnalyser:
         for filename in self.cpu_detail_info_dir:
             op_side = filename.split('_')[0]
             op_detail_file_path = os.path.join(self._output_path, filename)
-            PathManager.check_input_file_path(op_detail_file_path)
-            PathManager.check_directory_path_readable(op_detail_file_path)
+            op_detail_file_path = validate_and_normalize_path(op_detail_file_path)
             with open(op_detail_file_path, 'r') as f_obj:
                 op_detail_info = f_obj.readlines()
             for line_info in op_detail_info[1:]:
@@ -101,8 +99,7 @@ class OpAnalyser:
         """Get gpu operators execute times."""
         if self.gpu_op_type_info_dir:
             gpu_op_type_file_path = os.path.join(self._output_path, self.gpu_op_type_info_dir[0])
-            PathManager.check_input_file_path(gpu_op_type_file_path)
-            PathManager.check_directory_path_readable(gpu_op_type_file_path)
+            gpu_op_type_file_path = validate_and_normalize_path(gpu_op_type_file_path)
             with open(gpu_op_type_file_path, 'r') as fp:
                 op_type_info = fp.readlines()
                 for line_info in op_type_info[1:]:
@@ -121,8 +118,7 @@ class OpAnalyser:
         for filename in self.activity_info_dir:
             op_side = filename.split('_')[0]
             activity_file_path = os.path.join(self._output_path, filename)
-            PathManager.check_input_file_path(activity_file_path)
-            PathManager.check_directory_path_readable(activity_file_path)
+            activity_file_path = validate_and_normalize_path(activity_file_path)
             with open(activity_file_path, 'r') as file:
                 activity_info = file.readlines()
             for line_info in activity_info[1:]:

mindspore/{ops/_op_impl/cpu/joinedstr_op.py → profiler/common/validator/__init__.py}

@@ -1,4 +1,4 @@
-# Copyright 2025 Huawei Technologies Co., Ltd
+# Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,17 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
-"""JoinedStr op"""
-from mindspore.ops.op_info_register import op_info_register, CpuRegOp, DataType
-
-joinedstr_op_info = CpuRegOp("JoinedStr") \
-    .input(0, "x", "dynamic") \
-    .output(0, "y", "dynamic") \
-    .dtype_format(DataType.I32_Default, DataType.I32_Default) \
-    .get_op_info()
-
-
-@op_info_register(joinedstr_op_info)
-def _joinedstr_cpu():
-    """JoinedStr cpu register"""
-    return

mindspore/profiler/common/validator/validate_path.py

@@ -0,0 +1,84 @@
+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Validate the input path."""
+import os
+import re
+
+
+def check_valid_character_of_path(file_path):
+    """
+    Validates path.
+
+    The output path of profiler only supports alphabets(a-zA-Z), digit(0-9) or {'-', '_', '.', '/'}.
+
+    Note:
+        Chinese and other paths are not supported at present.
+
+    Args:
+        path (str):  Normalized Path.
+
+    Returns:
+        bool, whether valid.
+    """
+    re_path = r'^[/\\_a-zA-Z0-9-_.@]+$'
+    path_valid = re.fullmatch(re_path, file_path)
+    if not path_valid:
+        msg = "The output path of profiler only supports alphabets(a-zA-Z), " \
+              "digit(0-9) or {'-', '_', '.', '/', '@'}, but got the absolute path= " + file_path
+        raise RuntimeError(msg)
+
+
+def validate_and_normalize_path(
+        path,
+        check_absolute_path=False,
+        allow_parent_dir=True,
+):
+    """
+    Validates path and returns its normalized form.
+
+    If path has a valid scheme, treat path as url, otherwise consider path a
+    unix local path.
+
+    Note:
+        File scheme (rfc8089) is currently not supported.
+
+    Args:
+        path (str): Path to be normalized.
+        check_absolute_path (bool): Whether check path scheme is supported.
+        allow_parent_dir (bool): Whether allow parent dir in path.
+
+    Returns:
+        str, normalized path.
+    """
+    if not path:
+        raise RuntimeError("The path is invalid!")
+
+    path_str = str(path)
+    if not allow_parent_dir:
+        path_components = path_str.split("/")
+        if ".." in path_components:
+            raise RuntimeError("The parent path is not allowed!")
+
+    # path does not have valid schema, treat it as unix local path.
+    if check_absolute_path:
+        if not path_str.startswith("/"):
+            raise RuntimeError("The path is invalid!")
+    try:
+        # most unix systems allow
+        normalized_path = os.path.realpath(path)
+    except ValueError as err:
+        raise RuntimeError("The path is invalid!") from err
+    check_valid_character_of_path(normalized_path)
+    return normalized_path

mindspore/profiler/dynamic_profiler.py

@@ -24,9 +24,7 @@ import multiprocessing
 
 from mindspore import log as logger
 from mindspore.train import Callback
-from mindspore.profiler import tensorboard_trace_handler, schedule
-from mindspore.profiler.profiler import Profile
-from mindspore.profiler.experimental_config import _ExperimentalConfig
+from mindspore.profiler import Profiler, tensorboard_trace_handler, schedule
 from mindspore.profiler.common.file_manager import FileManager
 from mindspore.profiler.common.path_manager import PathManager
 from mindspore.profiler.dynamic_profile.dynamic_profiler_config_context import DynamicProfilerConfigContext
@@ -43,7 +41,7 @@ def print_msg(msg):
 
 class DynamicProfilerMonitorBase(Callback):
     """
-    Dynamic profiler callback base class implementing the dynamic profiler functionality.
+    Dynamic profile callback base class implementing the dynamic profile functionality.
     """
 
     NPU_MONITOR_START = "NPU_MONITOR_START"
@@ -90,23 +88,15 @@ class DynamicProfilerMonitorBase(Callback):
     @no_exception_func()
     def step_begin(self, run_context):
         """
-        Start profiler at the begin of step.
+        Start profile at the begin of step.
 
         Args:
             run_context (RunContext): Context of the train running.
         """
         prof_json = self._get_prof_args()
-        if not prof_json:
-            return
-        if self._is_dyno:
-            # Dyno monitor process
-            if self.NPU_MONITOR_START in prof_json:
-                self._call_dyno_monitor(prof_json)
-                return
-
         prof_args = DynamicProfilerConfigContext(prof_json)
         if not prof_args.is_valid:
-            logger.error("Dynamic profiler json is not valid, please check the json file.")
+            logger.error("Dynamic profile json is not valid, please check the json file.")
             return
 
         if prof_args.start_step in (-1, self._last_start_step):
@@ -119,22 +109,16 @@ class DynamicProfilerMonitorBase(Callback):
         # Prevent repeated calls of the start function within a complete interval
         if step_num == start_step:
             if self._is_started:
-                logger.error("Dynamic profiler is already started at step %d, "
-                             "please wait the first profiler finished at step %d.",
+                logger.error("Dynamic profile is already started at step %d, "
+                             "please wait the first profile finished at step %d.",
                              self._last_start_step, self._last_stop_step)
                 return
 
             if self._profiler is None:
-                output_path = prof_args.prof_path if prof_args.prof_path != "./" else self._output_path
-                prof_path = os.path.join(
-                    output_path,
-                    f"rank{self._rank_id}_start{start_step}_stop{stop_step}"
-                )
+                prof_path = os.path.join(self._output_path, f"rank{self._rank_id}_start{start_step}_stop{stop_step}")
                 PathManager.check_input_directory_path(prof_path)
-                profiler_config = self._get_prof_config(prof_args, prof_path, start_step, stop_step,
-                                                        start_profile=False,
-                                                        skip_first=0)
-                self._profiler = Profile(**profiler_config)
+                self._profiler = Profiler(on_trace_ready=tensorboard_trace_handler(dir_name=prof_path),
+                                          start_profile=False, **prof_args.args)
                 print_msg(f"Rank {self._rank_id} create output path {prof_path}")
 
             self._profiler.start()
@@ -144,58 +128,10 @@ class DynamicProfilerMonitorBase(Callback):
             print_msg(f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
                       f"will stop at step {stop_step}")
 
-    @staticmethod
-    def _get_prof_config(prof_args, prof_path, start_step, stop_step, start_profile, skip_first):
-        """
-        Get profiler config.
-
-        Args:
-            prof_args: Profiler config.
-            prof_path: Profiler output path.
-            start_step: Start step.
-            stop_step: Stop step.
-            start_profile: enable start_profile.
-            skip_first: skip first step.
-        """
-        profiler_config = {
-            "activities": prof_args.args.get("activities"),
-            "with_stack": prof_args.args.get("with_stack"),
-            "profile_memory": prof_args.args.get("profile_memory"),
-            "parallel_strategy": prof_args.args.get("parallel_strategy"),
-            "start_profile": start_profile,
-            "record_shapes": prof_args.args.get("record_shapes"),
-            "schedule": schedule(
-                wait=0,
-                warmup=0,
-                active=stop_step - start_step + 1,
-                repeat=1,
-                skip_first=skip_first
-            ),
-            "on_trace_ready": tensorboard_trace_handler(
-                dir_name=prof_path,
-                analyse_flag=prof_args.analyse,
-                async_mode=prof_args.analyse_mode == "async",
-            ),
-            "experimental_config": _ExperimentalConfig(
-                profiler_level=prof_args.args.get("profiler_level"),
-                aic_metrics=prof_args.args.get("aic_metrics"),
-                l2_cache=prof_args.args.get("l2_cache"),
-                mstx=prof_args.args.get("mstx"),
-                data_simplification=prof_args.args.get("data_simplification"),
-                export_type=prof_args.args.get("export_type"),
-                mstx_domain_include=prof_args.args.get("mstx_domain_include"),
-                mstx_domain_exclude=prof_args.args.get("mstx_domain_exclude"),
-                sys_io=prof_args.args.get("sys_io"),
-                sys_interconnection=prof_args.args.get("sys_interconnection"),
-                host_sys=prof_args.args.get("host_sys")
-            )
-        }
-        return profiler_config
-
     @no_exception_func()
     def step_end(self, run_context):
         """
-        Stop profiler at the end of step.
+        Stop profile at the end of step.
 
         Args:
             run_context (RunContext): Context of the train running.
@@ -204,22 +140,26 @@ class DynamicProfilerMonitorBase(Callback):
         prof_args = DynamicProfilerConfigContext(prof_json)
 
         if not prof_args.is_valid:
-            logger.error("Dynamic profiler json is not valid, please check the json file.")
+            logger.error("Dynamic profile json is not valid, please check the json file.")
             return
 
         if prof_args.stop_step == -1:
             return
 
-        if self._profiler:
-            self._profiler.step()
-
         cb_params = run_context.original_args()
         step_num = cb_params.cur_step_num
 
         if step_num == self._last_stop_step and self._is_started:
-            self._profiler = None
-            self._is_started = False
-            print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
+            if self._profiler:
+                self._profiler.stop()
+                if prof_args.analyse_mode:
+                    self._profiler.analyse(mode=prof_args.analyse_mode)
+                else:
+                    ProfilerInterface.finalize()
+                    ProfilerInterface.clear()
+                self._profiler = None
+                self._is_started = False
+                print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
 
     @no_exception_func()
     def step(self):
@@ -331,7 +271,7 @@ class DynamicProfilerMonitorBase(Callback):
 
         prof_args = DynamicProfilerConfigContext(prof_json)
         if not prof_args.is_valid:
-            logger.error("Dynamic profiler config is not valid, please check the json or dyno config.")
+            logger.error("Dynamic profile config is not valid, please check the json or dyno config.")
             return
         self._handle_profiler_setup(prof_args)
 
@@ -353,13 +293,14 @@ class DynamicProfilerMonitorBase(Callback):
             if not (start_step >= 0 and 0 <= start_step <= stop_step):
                 self._profiler = None
                 logger.error(
-                    "Rank %d Dynamic profiler start at step %d and stop at step %d must be "
+                    "Rank %d Dynamic profile start at step %d and stop at step %d must be "
                     "greater than or equal to 0, and stop step should not be less than start step",
                     self._rank_id, start_step, stop_step
                 )
                 return
 
             # Setup profiler configuration
+            active_steps = stop_step - start_step + 1
             output_path = args.prof_path if args.prof_path != "./" else self._output_path
             prof_path = os.path.join(
                 output_path,
@@ -367,12 +308,26 @@ class DynamicProfilerMonitorBase(Callback):
             )
             print_msg(f"Rank {self._rank_id} create output path {prof_path}")
             print_msg(
-                f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
+                f"Rank {self._rank_id} Dynamic profile start at step {start_step}, "
                 f"will stop at step {stop_step}"
             )
-            profiler_config = self._get_prof_config(args, prof_path, start_step, stop_step, start_profile=True,
-                                                    skip_first=1)
-            self._profiler = Profile(**profiler_config)
+            profiler_config = {
+                "schedule": schedule(
+                    wait=0,
+                    warmup=0,
+                    active=active_steps,
+                    repeat=1,
+                    skip_first=1
+                ),
+                "on_trace_ready": tensorboard_trace_handler(
+                    dir_name=prof_path,
+                    analyse_flag=args.analyse,
+                    async_mode=args.analyse_mode == "async",
+                ),
+                **args.args
+            }
+
+            self._profiler = Profiler(**profiler_config)
 
     def _is_valid_start_stop_step(self, step_num, start_step, stop_step):
         """Verify whether start_step and stop_step are valid parameters."""
@@ -591,10 +546,10 @@ if sys.version_info >= (3, 8):
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
-        This class to enable the dynamic profiler monitoring of MindSpore neural networks.
+        This class to enable the dynamic profile monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): (Ascend only) Dynamic profiler json config file directory. The requirement is a shared path
+            cfg_path (str): (Ascend only) Dynamic profile json config file directory. The requirement is a shared path
                 that can be accessed by all nodes. The parameters of the json configuration file are as follows:
 
                 - start_step (int, required) - Sets the step number at which the Profiler starts collecting data.
@@ -840,10 +795,10 @@ else:
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
-        This class to enable the dynamic profiler monitoring of MindSpore neural networks.
+        This class to enable the dynamic profile monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): Dynamic profiler json config file directory. The requirement is a shared path
+            cfg_path (str): Dynamic profile json config file directory. The requirement is a shared path
                 that can be accessed by all nodes.
             output_path (str, optional): Output data path. Default: ``"./dyn_profile_data"`` .
             poll_interval (int, optional): The polling period of the monitoring process, in seconds.

mindspore/profiler/envprofiler.py

@@ -16,9 +16,7 @@
 import os
 import json
 from mindspore import log as logger
-from mindspore.profiler.profiler import Profile
-from mindspore.profiler.experimental_config import _ExperimentalConfig
-from mindspore.profiler import tensorboard_trace_handler
+from mindspore.profiler.profiler import Profiler
 from mindspore.profiler.common.constant import (
     ProfilerLevel,
     AicoreMetrics,
@@ -49,31 +47,8 @@ class EnvProfiler:
 
         params = cls._convert_options_to_profiler_params(options)
         logger.info(f"params: {params}")
-        if params["start"]:
-            experimental_config = _ExperimentalConfig(profiler_level=params.get("profiler_level"),
-                                                      aic_metrics=params.get("aic_metrics"),
-                                                      l2_cache=params.get("l2_cache"),
-                                                      mstx=params.get("mstx"),
-                                                      data_simplification=params.get("data_simplification"),
-                                                      export_type=params.get("export_type"),
-                                                      mstx_domain_include=params.get("mstx_domain_include"),
-                                                      mstx_domain_exclude=params.get("mstx_domain_exclude"),
-                                                      sys_io=params.get("sys_io"),
-                                                      sys_interconnection=params.get("sys_interconnection"),
-                                                      host_sys=params.get("host_sys"))
-            cls.profiler = Profile(activities=params.get("activities"),
-                                   with_stack=params.get("with_stack"),
-                                   profile_memory=params.get("profile_memory"),
-                                   data_process=params.get("data_process"),
-                                   parallel_strategy=params.get("parallel_strategy"),
-                                   start_profile=params.get("start_profile"),
-                                   hbm_ddr=params.get("hbm_ddr"),
-                                   pcie=params.get("pcie"),
-                                   sync_enable=params.get("sync_enable"),
-                                   record_shapes=params.get("record_shapes"),
-                                   on_trace_ready=tensorboard_trace_handler(params.get("output_path")),
-                                   experimental_config=experimental_config)
-            cls.profiler.start()
+        if params["start_profile"]:
+            cls.profiler = Profiler(**params)
             logger.info("Profiler init success.")
 
     def analyse(self):
@@ -84,7 +59,7 @@ class EnvProfiler:
         if not self.profiler:
             logger.info("Profiler is not initialized, skip analyse.")
             return
-        self.profiler.stop()
+        self.profiler.analyse()
         logger.info("analyse end")
 
     @classmethod
@@ -126,7 +101,7 @@ class EnvProfiler:
             params["output_path"] = options["output_path"]
 
         # if start is not set, default is False
-        params["start"] = options.get("start", False)
+        params["start_profile"] = options.get("start", False)
 
         for param, (_, default_value) in ProfilerParameters.PARAMS.items():
             if param in options and param not in cls.NOT_SUPPORTED_PARAMS:

mindspore/profiler/experimental_config.py

@@ -59,7 +59,7 @@ class _ExperimentalConfig:
               storage and l2 cache etc.
         l2_cache (bool, optional): (Ascend only) Whether to collect l2 cache data, collect when True.
             Default: ``False`` . The l2_cache.csv file is generated in the ASCEND_PROFILER_OUTPUT folder. In GE backend,
-            only support :class:`mindspore.profiler.schedule` configuration wait and skip_first parameter is 0.
+            only wait and skip_first parameters in schedule configuration can be set to 0.
         mstx (bool, optional): (Ascend only) Whether to collect light weight profiling data, collect when True.
             Default: ``False`` .
         data_simplification (bool, optional): (Ascend only) Whether to remove FRAMEWORK data and other redundant data.
@@ -98,21 +98,6 @@ class _ExperimentalConfig:
             <https://gitee.com/mindspore/mindspore/blob/master/docs/api/api_python/mindspore/script/
             msprof_data_collection.sh>`_.
 
-            Final deliverables by `MindStudio Insight
-            <https://www.hiascend.com/developer/download/community/result?module=pt+sto+cann>`_
-            tool visualizes the analysis results.
-            For detailed analysis, please refer to `host-side CPU data analysis
-            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
-            atlasprofiling_16_0106.html>`_, `host-side MEM data analysis
-            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
-            atlasprofiling_16_0107.html>`_, `host-side DISK data analysis
-            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
-            atlasprofiling_16_0108.html>`_, `host-side NETWORK data analysis
-            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
-            atlasprofiling_16_0109.html>`_, `host-side OSRT data analysis
-            <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/Profiling/
-            atlasprofiling_16_0110.html>`_.
-
             - HostSystem.CPU: Collect the CPU utilization at the process level.
             - HostSystem.MEM: Collect the memory utilization at the process level.
             - HostSystem.DISK: Collect the disk I/O utilization at the process level.

mindspore/profiler/platform/cpu_profiler.py

@@ -16,11 +16,10 @@
 import mindspore._c_expression as c_expression
 
 from mindspore.profiler.common.registry import PROFILERS
-from mindspore.profiler.common.constant import DeviceTarget, ProfilerActivity, AnalysisMode
+from mindspore.profiler.common.constant import DeviceTarget, ProfilerActivity
 from mindspore.profiler.common.util import print_msg_with_pid
 from mindspore.profiler.common.profiler_context import ProfilerContext
 from mindspore.profiler.common.profiler_path_manager import ProfilerPathManager
-from mindspore.profiler.common.process_pool import MultiProcessPool
 from mindspore.profiler.platform.base_profiler import BaseProfiler
 from mindspore.profiler.analysis.time_converter import TimeConverter
 from mindspore.profiler.analysis.task_manager import TaskManager
@@ -69,7 +68,7 @@ class CpuProfiler(BaseProfiler):
         if ProfilerContext().device_target_set != {DeviceTarget.CPU.value}:
             return
         self._logger.info("CpuProfiler analyse.")
-        CPUProfilerAnalysis.online_analyse(**kwargs)
+        CPUProfilerAnalysis.online_analyse()
 
     def finalize(self) -> None:
         """Finalize profiling data."""
@@ -82,17 +81,12 @@ class CPUProfilerAnalysis:
     """
 
     @classmethod
-    def online_analyse(cls, async_mode: bool = False):
+    def online_analyse(cls):
         """
         Online analysis for CPU
         """
         cls._pre_analyse_online()
-        if async_mode:
-            ProfilerContext().mode = AnalysisMode.ASYNC_MODE.value
-            MultiProcessPool().add_async_job(cls._run_tasks, **ProfilerContext().to_dict())
-        else:
-            ProfilerContext().mode = AnalysisMode.SYNC_MODE.value
-            cls._run_tasks(**ProfilerContext().to_dict())
+        cls._run_tasks(**ProfilerContext().to_dict())
 
     @classmethod
     def _pre_analyse_online(cls):

mindspore/profiler/platform/npu_profiler.py

@@ -151,7 +151,7 @@ class NpuProfiler(BaseProfiler):
         """Analyse the profiling data."""
         self._logger.info("NpuProfiler analyse.")
 
-        NPUProfilerAnalysis.online_analyse(async_mode=kwargs.get('async_mode'))
+        NPUProfilerAnalysis.online_analyse()
 
     def finalize(self) -> None:
         """Finalize profiling data."""