mindspore 2.5.0__cp310-cp310-win_amd64.whl → 2.6.0rc1__cp310-cp310-win_amd64.whl

mindspore/profiler/common/msprof_cmd_tool.py

@@ -21,6 +21,7 @@ from typing import Dict, List, Optional
 
 from mindspore import log as logger
 from mindspore.profiler.common.command_executor import CommandExecutor
+from mindspore.profiler.common.constant import ExportType
 
 
 class MsprofCmdTool:
@@ -37,22 +38,30 @@ class MsprofCmdTool:
         self._msprof_profile_path = msprof_profile_path
         self._check_environment()
 
-    def run_ms_export_cmd(self) -> None:
-        """Run msprof export command."""
-        CommandExecutor.execute(
-            [
-                self._MSPROF_CMD,
-                "--export=on",
-                f"--output={self._msprof_profile_path}"
-            ]
-        )
-
-    def run_ms_py_export_cmd(self, model_id: int, iter_list: List[int]) -> None:
+    def run_ms_export_cmd(self, analyse_export_type: List[str]) -> None:
+        """Run msprof export command.
+
+        Args:
+            analyse_export_type (List[str]): The type of data to export.
+        """
+        export_cmd = [
+            self._MSPROF_CMD,
+            "--export=on",
+            f"--output={self._msprof_profile_path}",
+        ]
+        if ExportType.Text.value in analyse_export_type:
+            CommandExecutor.execute(export_cmd)
+        if ExportType.Db.value in analyse_export_type:
+            export_cmd.append("--type=db")
+            CommandExecutor.execute(export_cmd)
+
+    def run_ms_py_export_cmd(self, model_id: int, iter_list: List[int], analyse_export_type: List[str]) -> None:
         """Export timeline and summary data for the specified model and iterations.
 
         Args:
             model_id (int): The ID of the model to export data for.
             iter_list (List[int]): A list of iteration IDs to export data for.
+            analyse_export_type (List[str]): The type of data to export.
 
         Raises:
             FileNotFoundError: If msprof.py path cannot be found.
@@ -66,24 +75,34 @@ class MsprofCmdTool:
         export_cmd = ["python3", script_path]
         iter_param = self._get_iteration_params(model_id, iter_list)
 
-        for export_type in ("timeline", "summary"):
+        if ExportType.Text.value in analyse_export_type:
+            for export_type in ("timeline", "summary"):
+                cmd = (
+                    export_cmd
+                    + ["export", export_type, "-dir", self._msprof_profile_path]
+                    + iter_param
+                )
+                CommandExecutor.execute(cmd)
+        if ExportType.Db.value in analyse_export_type:
             cmd = (
                 export_cmd
-                + ["export", export_type, "-dir", self._msprof_profile_path]
-                + iter_param
+                + ["export", "db", "-dir", self._msprof_profile_path]
             )
             CommandExecutor.execute(cmd)
 
-    def run_ms_analyze_cmd(self) -> None:
+    def run_ms_analyze_cmd(self, analyse_export_type: List[str]) -> None:
         """Run msprof analyze command."""
-        CommandExecutor.execute(
-            [
-                self._MSPROF_CMD,
-                "--analyze=on",
-                "--rule=communication,communication_matrix",
-                f"--output={self._msprof_profile_path}",
-            ]
-        )
+        analyze_cmd = [
+            self._MSPROF_CMD,
+            "--analyze=on",
+            "--rule=communication,communication_matrix",
+            f"--output={self._msprof_profile_path}",
+        ]
+        if ExportType.Text.value in analyse_export_type:
+            CommandExecutor.execute(analyze_cmd)
+        if ExportType.Db.value in analyse_export_type:
+            analyze_cmd.append("--type=db")
+            CommandExecutor.execute(analyze_cmd)
 
     @lru_cache(maxsize=1)
     def get_msprof_info(self):

mindspore/profiler/common/path_manager.py

@@ -17,6 +17,7 @@ import os
 import re
 import shutil
 import glob
+import stat
 
 from mindspore import log as logger
 from mindspore.profiler.common.constant import FileConstant
@@ -369,3 +370,26 @@ class PathManager:
             msg = f"Invalid input path is a soft link: {path}"
             raise ProfilerPathErrorException(msg)
         return os.path.realpath(expanded_path)
+
+    @classmethod
+    def check_cann_lib_valid(cls, path: str) -> bool:
+        """
+        Function Description:
+           check if cann lib path is valid
+        Parameter:
+            path: the cann lib path to check
+        Return:
+            bool: True if the path is valid, False otherwise
+        """
+        lib_path = os.path.realpath(path)
+        if not os.path.exists(lib_path):
+            return False
+        if os.path.isdir(lib_path) or os.path.islink(lib_path):
+            return False
+        if bool(os.stat(lib_path).st_mode & stat.S_IWOTH):
+            return False
+        if os.name == 'nt':
+            return False
+        if os.stat(path).st_uid == 0 or os.stat(path).st_uid == os.getuid():
+            return True
+        return False

mindspore/profiler/common/profiler_context.py

@@ -30,7 +30,8 @@ from mindspore.profiler.common.constant import (
     DeviceTarget,
     ProfilerLevel,
     ProfilerActivity,
-    AicoreMetrics
+    AicoreMetrics,
+    ExportType
 )
 from mindspore.profiler.common.profiler_output_path import ProfilerOutputPath
 from mindspore.profiler.common.profiler_parameters import ProfilerParameters
@@ -59,6 +60,7 @@ class ProfilerContext:
         self._mode: str = AnalysisMode.SYNC_MODE.value
         self._pretty: bool = False
         self._profiler_path_mgr: ProfilerOutputPath = None
+        self._on_trace_ready_output_path = None
         self._jit_level: Optional[str] = ""
         self._context_mode: Optional[int] = -1
 
@@ -71,12 +73,32 @@ class ProfilerContext:
         """
         Set profiler parameters and paths
         """
+        # output_path and on_trace_ready cannot be set at the same time. If both are set,
+        # only paths in on_trace_ready take effect
+        if self._on_trace_ready_output_path:
+            final_path = self._on_trace_ready_output_path
+            if "output_path" in kwargs:
+                logger.warning(f"Both on_trace_ready path and output_path are provided. "
+                               f"The on_trace_ready path takes effect. Final path is {final_path}")
+            kwargs["output_path"] = final_path
+
         self._profiler_params_mgr: ProfilerParameters = ProfilerParameters(**kwargs)
         self._profiler_path_mgr: ProfilerOutputPath = ProfilerOutputPath(
             device_id=int(self._device_id), rank_id=int(self._rank_id)
         )
+
         self._profiler_path_mgr.output_path = self._profiler_params_mgr.output_path
 
+    @property
+    def on_trace_ready_output_path(self) -> str:
+        """Get the on trace ready output path."""
+        return self._on_trace_ready_output_path
+
+    @on_trace_ready_output_path.setter
+    def on_trace_ready_output_path(self, value: str):
+        """Set the tensorboard profile path to on trace ready output path."""
+        self._on_trace_ready_output_path = value
+
     def to_dict(self) -> Dict[str, Any]:
         """
         Convert the profiler context to a dictionary for multiprocessing.
@@ -105,10 +127,12 @@ class ProfilerContext:
             if param in profiler_parameters:
                 if param == "profiler_level":
                     value = ProfilerLevel(profiler_parameters[param])
-                elif param == "aicore_metrics":
+                elif param == "aic_metrics":
                     value = AicoreMetrics(profiler_parameters[param])
                 elif param == "activities":
                     value = [ProfilerActivity(activity) for activity in profiler_parameters[param]]
+                elif param == "export_type":
+                    value = [ExportType(export_type) for export_type in profiler_parameters[param]]
                 elif param == "schedule":
                     continue
                 else:

mindspore/profiler/common/profiler_meta_data.py

@@ -0,0 +1,74 @@
+# Copyright 2024-2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler Meta Data"""
+import os
+from typing import Dict
+
+import mindspore.communication as comm
+from mindspore.profiler.common.constant import DeviceTarget
+from mindspore.profiler.common.profiler_context import ProfilerContext
+import mindspore.communication._comm_helper as comm_helper
+from mindspore.profiler.common.file_manager import FileManager
+from mindspore import log as logger
+
+
+class ProfilerMetaData:
+    """
+        Profiler MetaData
+        This class is used to handle metadata.
+    """
+    metadata: Dict[str, str] = {}
+
+    @classmethod
+    def get_metadata(cls) -> Dict[str, str]:
+        """Get metadata"""
+        return cls.metadata
+
+    @classmethod
+    def set_metadata(cls, value: Dict[str, str]):
+        """Set metadata"""
+        cls.metadata = value
+
+    @classmethod
+    def dump_metadata(cls):
+        """Dump metadata to file."""
+        cls.add_group_info_to_metadata()
+        if not cls.metadata:
+            return
+        save_path = os.path.join(ProfilerContext().ascend_ms_dir, "profiler_metadata.json")
+        FileManager.create_json_file(save_path, cls.metadata)
+        cls.metadata.clear()
+
+    @classmethod
+    def add_group_info_to_metadata(cls):
+        """Add parallel group info to metadata"""
+        try:
+            if ProfilerContext().device_target == DeviceTarget.NPU.value and comm.GlobalComm.INITED \
+                    and comm.GlobalComm.BACKEND == comm_helper.Backend.HCCL:
+                group_info = {}
+                # pylint: disable=protected-access
+                for group_name in comm_helper._get_group_map().keys():
+                    comm_name = comm.get_comm_name(group_name)
+                    if not comm_name:
+                        continue
+                    group_info[comm_name] = {
+                        "group_name": group_name,
+                        "group_rank": comm.get_local_rank(group_name),
+                        "global_ranks": comm.get_process_group_ranks(group_name)
+                    }
+                if group_info:
+                    cls.metadata.update({"parallel_group_info": group_info})
+        except Exception as err:    # pylint: disable=broad-except
+            logger.error(f"Failed to get parallel group info, Exception: {str(err)}")

mindspore/profiler/common/profiler_parameters.py

@@ -21,6 +21,7 @@ from mindspore.profiler.common.constant import (
     ProfilerLevel,
     ProfilerActivity,
     AicoreMetrics,
+    ExportType
 )
 from mindspore.profiler.schedule import Schedule
 
@@ -35,7 +36,7 @@ class ProfilerParameters:
         "output_path": (str, "./data"),
         "profiler_level": (ProfilerLevel, ProfilerLevel.Level0),
         "activities": (list, [ProfilerActivity.CPU, ProfilerActivity.NPU]),
-        "aicore_metrics": (AicoreMetrics, AicoreMetrics.AiCoreNone),
+        "aic_metrics": (AicoreMetrics, AicoreMetrics.AiCoreNone),
         "with_stack": (bool, False),
         "profile_memory": (bool, False),
         "data_process": (bool, False),
@@ -46,6 +47,7 @@ class ProfilerParameters:
         "pcie": (bool, False),
         "sync_enable": (bool, True),
         "data_simplification": (bool, True),
+        "export_type": (list, [ExportType.Text]),
         "mstx": (bool, False),
         "schedule": (Schedule, None),
         "on_trace_ready": (Optional[Callable[..., Any]], None)
@@ -58,10 +60,7 @@ class ProfilerParameters:
         self.is_set_schedule: bool = False
         self._set_schedule(**kwargs)
         self._check_deprecated_params(**kwargs)
-        # Initialize parameters with kwargs
-        for param, (_, default_value) in self.PARAMS.items():
-            setattr(self, param, kwargs.get(param, default_value))
-
+        self._init_params(kwargs)
         self._check_params_type()
         self._handle_compatibility()
 
@@ -75,10 +74,12 @@ class ProfilerParameters:
         for param, (_, _) in self.PARAMS.items():
             if param == "profiler_level":
                 params[param] = getattr(self, param).value
-            elif param == "aicore_metrics":
+            elif param == "aic_metrics":
                 params[param] = getattr(self, param).value
             elif param == "activities":
                 params[param] = [item.value for item in getattr(self, param)]
+            elif param == "export_type":
+                params[param] = [item.value for item in getattr(self, param)]
             elif param == "schedule":
                 params[param] = getattr(self, param).to_dict()
             elif param == "on_trace_ready":
@@ -97,19 +98,27 @@ class ProfilerParameters:
         """
         return {
             "profile_memory": self.profile_memory,
-            "aic_metrics": self.aicore_metrics.value,
+            "aicore_metrics": self.aic_metrics.value,
             "l2_cache": self.l2_cache,
             "hbm_ddr": self.hbm_ddr,
             "pcie": self.pcie,
             "parallel_strategy": self.parallel_strategy,
             "profiler_level": self.profiler_level.value,
-            "aicore_metrics": self.aicore_metrics.value,
             "with_stack": self.with_stack,
             "mstx": self.mstx,
             "cpu_trace": ProfilerActivity.CPU in self.activities,
             "npu_trace": ProfilerActivity.NPU in self.activities,
         }
 
+    def _init_params(self, kwargs):
+        """
+        Initialize parameters with kwargs
+        """
+        for param, (_, default_value) in self.PARAMS.items():
+            if param == "schedule" and kwargs.get(param) is None:
+                kwargs["schedule"] = Schedule(wait=0, active=1)
+            setattr(self, param, kwargs.get(param) if kwargs.get(param) is not None else default_value)
+
     def _check_params_type(self) -> None:
         """
         Check profiler input params type, if type is invalid reset to default value.
@@ -121,11 +130,19 @@ class ProfilerParameters:
 
                 # Callable特殊处理
                 if key == "on_trace_ready":
-                    if not callable(value):
+                    if value is not None and not callable(value):
                         setattr(self, key, default_value)
+                        logger.warning(
+                            f"For Profiler, on_trace_ready value is Invalid, reset to {default_value}."
+                        )
                 elif key == "schedule":
                     if not isinstance(value, Schedule):
                         setattr(self, key, Schedule(wait=0, active=1))
+                        logger.warning(
+                            f"For Profiler, schedule value is Invalid, reset to {Schedule(wait=0, active=1)}"
+                        )
+                elif key == "export_type":
+                    setattr(self, key, self._check_and_get_export_type(value))
                 # 检查可迭代类型
                 elif isinstance(expected_type, type) and issubclass(expected_type, (list, tuple, set)):
                     if not (isinstance(value, expected_type) and
@@ -190,16 +207,40 @@ class ProfilerParameters:
             warnings.warn("when 'mstx' is disabled, 'profiler_level' cannot be set to 'ProfilerLevel.LevelNone', "
                           "reset to 'ProfilerLevel.Level0'.")
 
-        if self.__dict__.get('profiler_level') == ProfilerLevel.Level0 and \
-            self.__dict__.get('aicore_metrics') != AicoreMetrics.AiCoreNone:
-            self.aicore_metrics = AicoreMetrics.AiCoreNone
-            warnings.warn("when 'profiler_level' is set to 'ProfilerLevel.Level0', "
-                          "'aicore_metrics' cannot be set to other value except 'AicoreMetrics.AiCoreNone', "
-                          "reset to 'AicoreMetrics.AiCoreNone'.")
+        if self.__dict__.get('profiler_level') in (ProfilerLevel.LevelNone, ProfilerLevel.Level0) and \
+            self.__dict__.get('aic_metrics') != AicoreMetrics.AiCoreNone:
+            self.aic_metrics = AicoreMetrics.AiCoreNone
+            warnings.warn(f"when 'profiler_level' is set to '{self.__dict__.get('profiler_level')}', "
+                          f"'aic_metrics' cannot be set to other value except 'AicoreMetrics.AiCoreNone', "
+                          f"reset to 'AicoreMetrics.AiCoreNone'.")
+
+        if self.__dict__.get('profiler_level') in (ProfilerLevel.Level1, ProfilerLevel.Level2) and \
+            self.__dict__.get('aic_metrics') == AicoreMetrics.AiCoreNone:
+            self.aic_metrics = AicoreMetrics.PipeUtilization
+            warnings.warn(f"when 'profiler_level' is set to '{self.__dict__.get('profiler_level')}', "
+                          f"'aic_metrics' cannot be set to 'AicoreMetrics.AiCoreNone', "
+                          f"reset to 'AicoreMetrics.PipeUtilization'.")
+
+    def _check_and_get_export_type(self, export_type) -> list:
+        """
+        Check export type.
+        """
+        if not export_type:
+            return [ExportType.Text]
+
+        if isinstance(export_type, str):
+            if export_type in [ExportType.Text.value, ExportType.Db.value]:
+                return [ExportType(export_type)]
+
+        if isinstance(export_type, list):
+            if all(isinstance(type, ExportType) for type in export_type):
+                return list(set(export_type))
+
+        if isinstance(export_type, ExportType):
+            return [export_type]
 
-        if self.__dict__.get('profiler_level') != ProfilerLevel.Level0 and \
-            self.__dict__.get('aicore_metrics') == AicoreMetrics.AiCoreNone:
-            self.aicore_metrics = AicoreMetrics.PipeUtilization
+        logger.warning("Invalid parameter export_type, reset it to text.")
+        return [ExportType.Text]
 
     def __getattr__(self, name):
         """

mindspore/profiler/common/profiler_path_manager.py

@@ -15,6 +15,8 @@
 """Profiler Path Manager"""
 import os
 import socket
+import glob
+import shutil
 from datetime import datetime, timezone
 
 from mindspore import log as logger
@@ -29,10 +31,39 @@ class ProfilerPathManager:
     ProfilerPathManager is responsible for creating and managing all paths used by profiler.
     """
 
-    _ASCEND_MS_DIR = "{}_{}_{}_ascend_ms"
+    _ASCEND_MS_DIR = "{}_{}_ascend_ms"
+    MAX_WORKER_NAME_LENGTH = 226
 
     def __init__(self):
         self._prof_ctx = ProfilerContext()
+        self._worker_name = None
+        self._dir_path = None
+
+    def init(self, worker_name: str = None, dir_name: str = None) -> None:
+        """
+        Init the profiler path.
+        """
+        valid_wk_name = worker_name and isinstance(worker_name, str)
+        valid_wk_len = isinstance(worker_name, str) and len(worker_name) < self.MAX_WORKER_NAME_LENGTH
+        if (valid_wk_name and valid_wk_len) or worker_name is None:
+            self._worker_name = worker_name
+        else:
+            logger.warning("Invalid parameter worker_name, reset it to default.")
+            self._worker_name = None
+
+        valid_dir_name = dir_name and isinstance(dir_name, str)
+        if valid_dir_name:
+            dir_path = PathManager.get_real_path(dir_name)
+            PathManager.check_input_directory_path(dir_path)
+            self._dir_path = dir_path
+        elif dir_name is None:
+            self._dir_path = dir_name
+        else:
+            logger.warning(f"Invalid parameter dir_name, reset it to default.")
+            self._dir_path = None
+
+        if self._dir_path:
+            self._prof_ctx.on_trace_ready_output_path = self._dir_path
 
     def clean_analysis_cache(self):
         """
@@ -86,6 +117,33 @@ class ProfilerPathManager:
             elif os.path.isdir(cache_path):
                 PathManager.remove_path_safety(cache_path)
 
+    def move_db_file(self):
+        """
+        Copy the db file to the output path.
+        """
+        if not self._prof_ctx.msprof_profile_output_path:
+            return
+        db_files = glob.glob(os.path.join(
+            os.path.dirname(self._prof_ctx.msprof_profile_output_path),
+            'msprof*.db'
+        )) + glob.glob(os.path.join(
+            os.path.dirname(self._prof_ctx.msprof_profile_output_path),
+            "analyze",
+            "communication_analyzer.db"
+        ))
+        for db_file in db_files:
+            if os.path.isfile(db_file):
+                db_file_name = os.path.basename(db_file)
+                if db_file_name == "communication_analyzer.db":
+                    new_file_name = os.path.join(self._prof_ctx.ascend_profiler_output_path, db_file_name)
+                    shutil.copy(db_file, new_file_name)
+                else:
+                    new_file_name = f"ascend_mindspore_profiler_{self._prof_ctx.rank_id}.db" if self._prof_ctx.rank_id \
+                        else f"ascend_mindspore_profiler.db"
+                    new_file_path = os.path.join(self._prof_ctx.ascend_profiler_output_path, new_file_name)
+                    shutil.move(db_file, new_file_path)
+
+
     def create_output_path(self):
         """
         Create ASCEND_PROFILER_OUTPUT dir, this method should call before analysis
@@ -112,9 +170,10 @@ class ProfilerPathManager:
         """
         Generate xxx_ascend_ms name
         """
-        return self._ASCEND_MS_DIR.format(
-            socket.gethostname(),
-            os.getpid(),
-            # save time with microseconds
-            datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f")[:-3],
-        )
+        timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f")[:-3]
+        if not self._worker_name:
+            worker_name = f"{socket.gethostname()}_{os.getpid()}"
+        else:
+            worker_name = f"{self._worker_name}_{os.getpid()}"
+
+        return self._ASCEND_MS_DIR.format(worker_name, timestamp)