mindspore 2.4.10__cp310-cp310-win_amd64.whl → 2.6.0__cp310-cp310-win_amd64.whl

mindspore/profiler/__init__.py

@@ -16,13 +16,30 @@
 This module provides Python APIs to enable the profiling of MindSpore neural networks.
 Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
 and use Profiler.analyse() to stop profiling and analyse the results.
-Users can visualize the results using the MindInsight tool.
 Now, Profiler supports AICORE operator, AICPU operator, HostCPU operator, memory,
 correspondence, cluster, etc data analysis.
 """
-__all__ = ["Profiler", "EnvProfiler", "ProfilerLevel", "DynamicProfilerMonitor"]
+__all__ = [
+    "tensorboard_trace_handler",
+    "schedule",
+    "Profiler",
+    "EnvProfiler",
+    "ProfilerLevel",
+    "ProfilerActivity",
+    "AicoreMetrics",
+    "ExportType",
+    "DynamicProfilerMonitor",
+    "mstx",
+    "_ExperimentalConfig",
+    "profile"
+]
 
-from mindspore.profiler.profiling import Profiler
-from mindspore.profiler.envprofiling import EnvProfiler
-from mindspore.profiler.profiling import ProfilerLevel
+from mindspore.profiler.mstx import Mstx as mstx
+from mindspore.profiler.profiler import Profiler
+from mindspore.profiler.profiler import tensorboard_trace_handler
+from mindspore.profiler.schedule import Schedule as schedule
+from mindspore.profiler.envprofiler import EnvProfiler
+from mindspore.profiler.common.constant import ProfilerLevel, ProfilerActivity, AicoreMetrics, ExportType
 from mindspore.profiler.dynamic_profiler import DynamicProfilerMonitor
+from mindspore.profiler.experimental_config import _ExperimentalConfig
+from mindspore.profiler.profiler import Profile as profile

mindspore/profiler/analysis/parser/ascend_cann_parser.py

@@ -0,0 +1,170 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Parser for Ascend CANN profiling data."""
+import os
+import glob
+from typing import Dict, Optional
+
+import numpy as np
+
+from mindspore import log as logger
+from mindspore.profiler.analysis.parser.base_parser import BaseParser
+from mindspore.profiler.common.file_manager import FileManager
+from mindspore.profiler.common.path_manager import PathManager
+from mindspore.profiler.common.constant import ProfilerLevel, ExportType
+from mindspore.profiler.common.ascend_msprof_exporter import AscendMsprofExporter
+from mindspore.profiler.common.log import ProfilerLogger
+
+
+class AscendMsprofParser(BaseParser):
+    """Parser for MindSpore profiling data on Ascend platform."""
+
+    _MSPROF_TIMELINE_FILE_PATTERN = ["msprof_[0-9]*.json", "msprof_slice_*.json"]
+    _OP_SUMMARY_FILE_PATTERN = "op_summary_*.csv"
+    OVERSIZE_MB = 1024
+
+    def __init__(self, next_parser: Optional[BaseParser] = None, **kwargs):
+        """Initialize AscendMsprofParser."""
+        super().__init__(next_parser)
+        self._kwargs = kwargs
+        self._msprof_profile_output_path = self._kwargs.get(
+            "msprof_profile_output_path"
+        )
+        self._msprof_profile_host_path = self._kwargs.get(
+            "msprof_profile_host_path"
+        )
+        self._msprof_profile_device_path = self._kwargs.get(
+            "msprof_profile_device_path"
+        )
+        self._ascend_ms_dir = self._kwargs.get("ascend_ms_dir")
+        self._profiler_level = kwargs.get("profiler_level")
+        self._export_type = kwargs.get("export_type")
+        ProfilerLogger.init(self._ascend_ms_dir)
+        self._logger = ProfilerLogger.get_instance()
+        self.op_summary = None
+        self.op_summary_headers = None
+        self.msprof_timeline = []
+
+    def _parse(self, data=None) -> Dict:
+        """Parse profiling data and update the input dictionary.
+
+        Args:
+            data (Dict, optional): Input data dictionary. Defaults to None.
+
+        Returns:
+            Dict: Updated data with op_summary, headers and timeline information.
+        """
+        if data is None:
+            data = {}
+        self._check_msprof_data_size()
+        AscendMsprofExporter(**self._kwargs).export()
+        self._logger.info("AscendMsprofExporter export done.")
+        self._parse_op_summary()
+        self._logger.info("AscendMsprofParser parse op summary done.")
+        self._parse_msprof_timeline()
+        self._logger.info("AscendMsprofParser parse msprof timeline done.")
+        data.update(
+            {
+                "op_summary": self.op_summary,
+                "op_summary_headers": self.op_summary_headers,
+                "msprof_timeline": self.msprof_timeline,
+            }
+        )
+        return data
+
+    def _parse_op_summary(self):
+        """Parse operation summary data from CSV files.
+
+        Raises:
+            RuntimeError: If no op summary files are found or read file failed.
+        """
+        if (self._profiler_level == ProfilerLevel.LevelNone.value or
+                self._export_type == [ExportType.Db.value]):
+            return
+        file_path_list = glob.glob(
+            os.path.join(
+                self._msprof_profile_output_path, self._OP_SUMMARY_FILE_PATTERN
+            )
+        )
+        if not file_path_list:
+            logger.error(
+                f"Failed to find op_summary_*.csv in directory: {self._msprof_profile_output_path}"
+            )
+            return
+
+        for file_path in file_path_list:
+            csv_data_np, self.op_summary_headers = FileManager.read_csv_file_as_numpy(
+                file_path=file_path, extern_headers=["Step ID"]
+            )
+            if self.op_summary is None:
+                self.op_summary = csv_data_np
+            else:
+                self.op_summary = np.concatenate([self.op_summary, csv_data_np], axis=0)
+
+    def _parse_msprof_timeline(self) -> None:
+        """Parse msprof timeline data from JSON files.
+
+        Raises:
+            RuntimeError: If no msprof JSON files are found in the specified directory.
+        """
+        if self._export_type == [ExportType.Db.value]:
+            return
+        file_path_list = []
+        for pattern in self._MSPROF_TIMELINE_FILE_PATTERN:
+            file_path_list.extend(glob.glob(os.path.join(self._msprof_profile_output_path, pattern)))
+
+        if not file_path_list:
+            raise RuntimeError(
+                f"Failed to find msprof JSON files in directory: {self._msprof_profile_output_path}"
+            )
+        for file_path in file_path_list:
+            self._process_timeline_file(file_path)
+        if not self.msprof_timeline:
+            logger.error(
+                "Failed to collect msprof timeline data"
+            )
+
+    def _process_timeline_file(self, file_path: str) -> None:
+        """Process a single timeline JSON file.
+
+        Args:
+            file_path (str): Path to the JSON file to process.
+        """
+        try:
+            timeline_data = FileManager.read_json_file(file_path)
+            if timeline_data:
+                self.msprof_timeline.extend(timeline_data)
+            else:
+                logger.warning(
+                    f"Msporf timeline data in {file_path} is empty."
+                )
+        except RuntimeError as err:
+            logger.error(
+                f"Failed to read file: {file_path}. Error: {str(err)}"
+            )
+
+    def _check_msprof_data_size(self) -> None:
+        """Check the size of profiling data."""
+        host_data_size = PathManager.get_directory_size(self._msprof_profile_host_path)
+        device_data_size = PathManager.get_directory_size(self._msprof_profile_device_path)
+        total_size = host_data_size + device_data_size
+
+        if total_size >= self.OVERSIZE_MB:
+            msg = (
+                f"The size of profiling data is too large: {total_size} MB, "
+                "which may spend a lot of time on analysing, you can consider "
+                "reducing the number of collection steps"
+            )
+            logger.warning(msg)

mindspore/profiler/analysis/parser/base_parser.py

@@ -0,0 +1,158 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Base parser"""
+from typing import (
+    Any,
+    List,
+    Callable,
+    Optional,
+)
+from multiprocessing import Process
+from mindspore import log as logger
+from mindspore.profiler.common.log import ProfilerLogger
+
+
+class BaseParser:
+    """
+    Base class for all parsers in the workflow.
+    """
+    EXEC_HOOK_TIMEOUT = 30 * 60
+
+    def __init__(self, next_parser: Optional["BaseParser"] = None):
+        """
+        Initialize the BaseParser.
+
+        Args:
+            next_parser (Optional[BaseParser]): The next parser in the chain.
+        """
+        self.next_parser: Optional["BaseParser"] = next_parser
+        self._post_hooks: List[Callable[[Any], None]] = []
+        self._logger = ProfilerLogger.get_instance()
+
+    def set_next(self, next_parser: "BaseParser") -> "BaseParser":
+        """
+        Set the next parser in the chain.
+
+        Args:
+            next_parser (BaseParser): The next parser to be set.
+
+        Returns:
+            BaseParser: The next parser that was set.
+        """
+        if not isinstance(next_parser, BaseParser):
+            raise ValueError(
+                f"next_parser {next_parser.__class__.__name__} must be a BaseParser"
+            )
+        self.next_parser = next_parser
+        return self
+
+    def parse(self, data: Any) -> Any:
+        """
+        Parse the input data and execute post-hooks.
+
+        Args:
+            data (Any): The input data to be parsed.
+
+        Returns:
+            Any: The parsed result.
+        """
+        try:
+            result = self._parse(data)
+            self._execute_post_hooks(result)
+        except Exception as e: # pylint: disable=W0703
+            logger.error("Parser [%s] error: %s", self.__class__.__name__, str(e))
+            self._logger.error("Parser [%s] error: %s", self.__class__.__name__, str(e), exc_info=True)
+            return data
+        return result
+
+    def register_post_hook(self, hook: Callable[[Any], None]) -> "BaseParser":
+        """
+        Register a post-hook to be executed after parsing.
+
+        Args:
+            hook (Callable[[Any], None]): The hook function to be registered.
+
+        Returns:
+            BaseParser: The current parser instance.
+
+        Raises:
+            ValueError: If the hook is not callable.
+        """
+        if callable(hook):
+            self._post_hooks.append(hook)
+        else:
+            raise ValueError("Hook must be callable")
+        return self
+
+    def _execute_post_hooks(self, res: Any) -> None:
+        """
+        Execute all registered post-hooks asynchronously.
+
+        Args:
+            res (Any): The result to be passed to the post-hooks.
+        """
+        if not self._post_hooks:
+            return
+
+        processes = []
+        for hook in self._post_hooks:
+            p = Process(target=hook, args=(res,))
+            p.start()
+            hook_class = hook.__self__.__class__.__name__ if hasattr(hook, '__self__') else 'Unknown'
+            hook_name = f"{hook_class}.{hook.__name__}"
+            processes.append((p, hook_name))
+            self._logger.info("Parser [%s] post hook [%s] start", self.__class__.__name__, hook_name)
+
+        for p, hook_name in processes:
+            try:
+                p.join(timeout=self.EXEC_HOOK_TIMEOUT)
+                if p.is_alive():
+                    logger.error(
+                        "Parser [%s] post hook [%s] timeout after %s seconds, terminating",
+                        self.__class__.__name__, hook_name, self.EXEC_HOOK_TIMEOUT
+                    )
+                    p.terminate()
+                    p.join()
+                else:
+                    self._logger.info(
+                        "Parser [%s] post hook [%s] completed",
+                        self.__class__.__name__, hook_name
+                    )
+            except Exception as e:  # pylint: disable=W0703
+                self._logger.error(
+                    "Parser [%s] post hook [%s] failed: %s",
+                    self.__class__.__name__, hook_name, str(e), exc_info=True
+                )
+                if p.is_alive():
+                    p.terminate()
+
+    def _parse(self, data: Any) -> Any:
+        """
+        Abstract method to be implemented by subclasses for parsing logic.
+
+        Args:
+            data (Any): The input data to be parsed.
+
+        Raises:
+            NotImplementedError: If not implemented by subclass.
+        """
+        raise NotImplementedError("Subclasses should implement this!")
+
+
+class DummyParser(BaseParser):
+    """Dummy parser"""
+    def _parse(self, data: Any) -> Any:
+        """Dummy parse"""
+        return data

mindspore/profiler/analysis/parser/framework_cann_relation_parser.py

@@ -0,0 +1,45 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Parser for analyzing the relationship between framework and CANN data."""
+from typing import Dict, Any
+
+from mindspore.profiler.analysis.parser.base_parser import BaseParser
+from mindspore.profiler.analysis.parser.timeline_assembly_factory.ascend_timeline_assembler import (
+    AscendTimelineAssembler
+)
+from mindspore.profiler.common.log import ProfilerLogger
+
+
+class FrameworkCannRelationParser(BaseParser):
+    """FrameworkCannRelationParser"""
+
+    def __init__(self, **kwargs):
+        """Initialize the AscendTraceAnalyser."""
+        super().__init__()
+        self.assembler = AscendTimelineAssembler(**kwargs)
+        self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
+        ProfilerLogger.init(kwargs.get("ascend_ms_dir"))
+        self._logger = ProfilerLogger.get_instance()
+
+    def _parse(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Parse the relation of framework and cann data."""
+        self.assembler.assemble(data)
+        self._logger.info("FrameworkCannRelationParser assemble done")
+        trace_view_container = self.assembler.get_trace_view_container()
+        self._logger.info("FrameworkCannRelationParser get trace view container done")
+        data.update({
+            "trace_view_container": trace_view_container,
+        })
+        return data

mindspore/profiler/analysis/parser/ms_framework_parser.py

@@ -0,0 +1,142 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Parser for MindSpore framework profiling data."""
+import os
+import struct
+from typing import List, Dict, Any, Optional
+
+from mindspore import log as logger
+from mindspore.profiler.common.tlv_decoder import TLVDecoder
+from mindspore.profiler.common.file_manager import FileManager
+from mindspore.profiler.common.constant import ProfilerActivity, FileConstant, DeviceTarget
+from mindspore.profiler.analysis.parser.base_parser import BaseParser
+from mindspore.profiler.analysis.parser.timeline_event.fwk_event import FwkFixSizeFormat, OpRangeStructField
+from mindspore.profiler.common.log import ProfilerLogger
+
+
+class FrameworkParser(BaseParser):
+    """Parser for MindSpore framework profiling data."""
+
+    _OP_RANGE_FILE_NAME = "mindspore.op_range"
+    _CPU_OP_TIMESTAMP_FILE_NAME = "cpu_op_execute_timestamp_{}.txt"
+
+    def __init__(self, next_parser: Optional[BaseParser] = None, **kwargs):
+        """Initialize FrameworkParser."""
+        super().__init__(next_parser)
+        self._rank_id = kwargs.get("rank_id")
+        self._activities = kwargs.get("activities")
+        self._step_list = kwargs.get("step_list")
+        self._framework_path = kwargs.get("framework_path")
+        self._device_target = kwargs.get("device_target")
+        self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
+        self._op_range_path = os.path.join(
+            self._framework_path,
+            self._OP_RANGE_FILE_NAME
+        )
+        self._cpu_op_path = os.path.join(
+            self._framework_path,
+            self._CPU_OP_TIMESTAMP_FILE_NAME.format(self._rank_id),
+        )
+        ProfilerLogger.init(self._ascend_ms_dir)
+        self._logger = ProfilerLogger.get_instance()
+
+    def _parse(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Parse MindSpore framework profiling data.
+
+        Args:
+            data (Dict[str, Any]): Input data dictionary.
+
+        Returns:
+            Dict[str, Any]: Updated data with:
+                - mindspore_op_list: List of parsed operation events
+                - cpu_op_lines: List of CPU operation timestamp data
+        """
+        if ProfilerActivity.CPU.value not in self._activities:
+            return data
+        mindspore_op_list = self._parse_op_range_data()
+        self._logger.info("FrameworkParser parse op range done.")
+        cpu_op_lines = self._parse_cpu_op_data()
+        self._logger.info("FrameworkParser parse cpu op done.")
+        data.update(
+            {
+                "mindspore_op_list": mindspore_op_list,
+                "cpu_op_lines": cpu_op_lines,
+            }
+        )
+        return data
+
+    def _parse_op_range_data(self) -> List[Dict]:
+        """Read and decode MindSpore op_range data.
+
+        Returns:
+            List[Dict]: List of parsed MindSpore operation events.
+        """
+        if self._device_target == DeviceTarget.CPU.value:
+            return []
+
+        if not os.path.exists(self._op_range_path):
+            logger.error("Failed to find op_range data. Skipping parse host profiler data.")
+            return []
+
+        try:
+            op_range_bytes = FileManager.read_file_content(self._op_range_path, "rb")
+            op_range_list = TLVDecoder.decode(
+                op_range_bytes, FwkFixSizeFormat.OpRangeStruct, struct.calcsize(FwkFixSizeFormat.OpRangeStruct)
+            )
+            self._logger.info("FrameworkParser parse op range done, op_range_list length: %d", len(op_range_list))
+            if not op_range_list:
+                logger.error(
+                    f"Failed to decode op_range data: empty result from file {self._op_range_path}"
+                )
+            return self._filter_op_range_list(op_range_list)
+        except RuntimeError as err:
+            logger.error(f"Failed to read file: {self._op_range_path}. Error: {str(err)}")
+            return []
+
+    def _filter_op_range_list(self, op_range_list: List[Dict]) -> List[Dict]:
+        """Filter op_range_list based on step_list.
+
+        Args:
+            op_range_list (List[Dict]): List of operation events to filter.
+
+        Returns:
+            List[Dict]: Filtered list of operation events.
+        """
+        if not self._step_list or not isinstance(self._step_list, list):
+            return op_range_list
+
+        first_step = min(
+            op[FileConstant.FIX_SIZE_DATA][OpRangeStructField.STEP.value]
+            for op in op_range_list
+        )
+        adjusted_step_list = [step - 1 + first_step for step in self._step_list]
+        return [
+            op for op in op_range_list
+            if op[FileConstant.FIX_SIZE_DATA][OpRangeStructField.STEP.value] in adjusted_step_list
+        ]
+
+    def _parse_cpu_op_data(self) -> List[str]:
+        """Parse CPU op timestamp data.
+
+        Returns:
+            List[str]: List of CPU operation timestamp data lines.
+        """
+        if not os.path.exists(self._cpu_op_path):
+            return []
+        try:
+            return FileManager.read_txt_file(self._cpu_op_path)
+        except RuntimeError as err:
+            logger.warning(f"Failed to read file: {self._cpu_op_path}. Error: {str(err)}")
+            return []