mindspore 2.4.10__cp310-cp310-win_amd64.whl → 2.6.0rc1__cp310-cp310-win_amd64.whl

mindspore/profiler/common/ascend_msprof_exporter.py

@@ -0,0 +1,139 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Exporter for Ascend MSPROF data."""
+import os
+import glob
+from collections import defaultdict
+from typing import Dict, List, Optional
+
+from mindspore import log as logger
+from mindspore.profiler.common.path_manager import PathManager
+from mindspore.profiler.common.file_manager import FileManager
+from mindspore.profiler.common.msprof_cmd_tool import MsprofCmdTool
+
+
+class AscendMsprofExporter:
+    """Exporter for Ascend MSPROF data."""
+
+    _DRV_VERSION = 467473
+    _DEFAULT_MODEL_ID = 4294967295
+    _STEP_TRACE_FILE_PATTERN = "step_trace_*.csv"
+    _STEP_TRACE_MODEL_ID_INDEX = 9
+    _STEP_TRACE_ITERATION_ID_INDEX = 1
+
+    def __init__(self, **kwargs):
+        """Initialize AscendMsprofExporter."""
+        self._msprof_profile_output_path = kwargs.get("msprof_profile_output_path")
+        self._msprof_profile_path = kwargs.get("msprof_profile_path")
+        self._step_list = kwargs.get("step_list")
+        self._export_type = kwargs.get("export_type")
+        self._msprof_tool = MsprofCmdTool(self._msprof_profile_path)
+
+    def export(self) -> None:
+        """Determine the export strategy and execute the appropriate export method."""
+        support_all_export = self._check_drv_version()
+        if self._step_list or not support_all_export:
+            self._single_export(self._step_list)
+        else:
+            self._all_export()
+        self._msprof_tool.run_ms_analyze_cmd(self._export_type)
+
+    def _single_export(self, step_list: Optional[List[int]]) -> None:
+        """Perform single export for each model and iteration.
+
+        Args:
+            step_list (Optional[List[int]]): iteration IDs.
+        """
+        if not step_list:
+            model_iteration_dict = self._generate_step_trace()
+            if not model_iteration_dict:
+                return
+        else:
+            model_iteration_dict = {self._DEFAULT_MODEL_ID: step_list}
+
+        for model_id, iter_list in model_iteration_dict.items():
+            self._msprof_tool.run_ms_py_export_cmd(model_id, iter_list, self._export_type)
+
+    def _all_export(self) -> None:
+        """Perform all-export for all data."""
+        self._msprof_tool.run_ms_export_cmd(self._export_type)
+
+    def _check_drv_version(self) -> bool:
+        """Check if the driver version supports all-export.
+
+        Returns:
+            bool: True if driver version supports all-export, False otherwise.
+        """
+        msprof_info = self._msprof_tool.get_msprof_info()
+        status = msprof_info.get("status", 1)
+        drv_version = (
+            msprof_info.get("data", {}).get("version_info", {}).get("drv_version", 0)
+        )
+
+        if status != 1 and drv_version and drv_version >= self._DRV_VERSION:
+            return True
+
+        logger.warning(
+            "Current driver package does not support all-export mode. "
+            "Using single export mode, which may affect performance. "
+            "Consider upgrading the driver package."
+        )
+        return False
+
+    def _generate_step_trace(self) -> Optional[Dict[int, List[int]]]:
+        """Generate step trace data.
+
+        Returns:
+            Optional[Dict[int, List[int]]]: Model IDs to iteration IDs mapping, or None if failed.
+        """
+        try:
+            # step1: run msprof command
+            self._msprof_tool.run_ms_export_cmd(self._export_type)
+            step_trace_file = glob.glob(
+                os.path.join(
+                    self._msprof_profile_output_path, self._STEP_TRACE_FILE_PATTERN
+                )
+            )
+            if not step_trace_file:
+                logger.warning(
+                    f"No step trace csv file found in {self._msprof_profile_output_path}."
+                )
+                return None
+            # step2: parse the step trace file to get model id and iteration id
+            model_iteration_dict = self._parse_step_trace_file(step_trace_file[0])
+            # step3: remove the unused files
+            PathManager.remove_path_safety(self._msprof_profile_output_path)
+            return model_iteration_dict
+
+        except RuntimeError as err:
+            logger.warning(f"Failed to get step trace data. Error: {str(err)}")
+            return None
+
+    def _parse_step_trace_file(self, file_path: str) -> Dict[int, List[int]]:
+        """Parse the step trace file to extract model and iteration information.
+
+        Args:
+            file_path (str): Path to the step trace file.
+
+        Returns:
+            Dict[int, List[int]]: Model IDs to iteration IDs mapping.
+        """
+        model_iteration_dict = defaultdict(list)
+        step_trace_data = FileManager.read_csv_file(file_path)
+        for row in step_trace_data:
+            model_id = int(row[self._STEP_TRACE_MODEL_ID_INDEX])
+            iteration_id = int(row[self._STEP_TRACE_ITERATION_ID_INDEX])
+            model_iteration_dict[model_id].append(iteration_id)
+        return model_iteration_dict

mindspore/profiler/common/command_executor.py

@@ -0,0 +1,90 @@
+# Copyright 2020-2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler command executor"""
+import shlex
+import subprocess
+from typing import Union, Tuple
+
+from mindspore import log as logger
+
+
+class CommandExecutor:
+    """
+    Execute command in subprocess.
+    """
+    @staticmethod
+    def execute(
+            cmd: Union[str, list],
+            timeout: int = 10 * 60 * 60,
+            check: bool = True,
+            encoding: str = "utf-8",
+    ) -> Tuple[str, str, int]:
+        """
+        Execute a shell command with improved error handling and logging.
+
+        Args:
+            cmd (Union[str, list]): The command to execute, either as a string or list.
+            timeout (int): Maximum execution time in seconds. Defaults to 10 hours.
+            check (bool): If True, raise an exception for non-zero exit codes. Defaults to True.
+            encoding (str): Specify the encoding for output. Defaults to 'utf-8'.
+
+        Returns:
+            Tuple[str, str, int]: A tuple containing (stdout, stderr, return_code).
+
+        Raises:
+            RuntimeError: If any error occurs during command execution.
+        """
+        if isinstance(cmd, str):
+            cmd = shlex.split(cmd)
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+                check=check,
+                shell=False,
+                encoding=encoding,
+            )
+            logger.info(
+                f"Command '{cmd}' executed successfully. Return code: {result.returncode}"
+            )
+            return result.stdout, result.stderr, result.returncode
+
+        except subprocess.TimeoutExpired as e:
+            error_msg = f"Command '{cmd}' timed out after {timeout} seconds"
+            logger.error(error_msg)
+            raise RuntimeError(error_msg) from e
+
+        except subprocess.CalledProcessError as e:
+            error_msg = (
+                f"Command '{cmd}' failed with return code {e.returncode}, "
+                f"Standard Error: {e.stderr}"
+            )
+            logger.error(error_msg)
+            raise RuntimeError(error_msg) from e
+
+        except (FileNotFoundError, PermissionError) as e:
+            error_msg = f"Error executing command '{cmd}': {str(e)}"
+            logger.error(error_msg)
+            raise RuntimeError(error_msg) from e
+
+        except Exception as e: # pylint: disable=broad-exception-caught
+            error_msg = (
+                f"Unexpected error occurred while executing command '{cmd}': {str(e)}"
+            )
+            logger.error(error_msg)
+            raise RuntimeError(error_msg) from e

mindspore/profiler/common/constant.py

@@ -18,12 +18,195 @@ from enum import Enum
 
 class DeviceTarget(Enum):
     """The device target enum."""
-    CPU = 'cpu'
-    GPU = 'gpu'
-    ASCEND = 'ascend'
+
+    CPU = "CPU"
+    GPU = "GPU"
+    NPU = "Ascend"
 
 
 class ProfilerLevel(Enum):
+    """The profiler level enum."""
+
+    LevelNone = "LevelNone"
     Level0 = "Level0"
     Level1 = "Level1"
     Level2 = "Level2"
+
+
+class ProfilerActivity(Enum):
+    """The profiler activity enum."""
+
+    NPU = "NPU"
+    GPU = "GPU"
+    CPU = "CPU"
+
+
+class AicoreMetrics(Enum):
+    """The aicore metrics enum."""
+
+    PipeUtilization = "PipeUtilization"
+    ArithmeticUtilization = "ArithmeticUtilization"
+    Memory = "Memory"
+    MemoryL0 = "MemoryL0"
+    MemoryUB = "MemoryUB"
+    ResourceConflictRatio = "ResourceConflictRatio"
+    L2Cache = "L2Cache"
+    MemoryAccess = "MemoryAccess"
+    AiCoreNone = "None"
+
+
+class OverlapAnalysisTidName(Enum):
+    """The overlap analysis tidName."""
+    COMPUTING = "Computing"
+    COMMUNICATION = "Communication"
+    COMMUNICATION_NOT_OVERLAP = "Communication(Not Overlapped)"
+    FREE = "Free"
+
+
+class OpSummaryHeaders(Enum):
+    """The op summary headers."""
+    STEP_ID = "Step ID"
+    DEVICE_ID = "Device_id"
+    MODEL_NAME = "Model Name"
+    MODEL_ID = "Model ID"
+    TASK_ID = "Task ID"
+    STREAM_ID = "Stream ID"
+    OP_NAME = "Op Name"
+    OP_TYPE = "OP Type"
+    TASK_TYPE = "Task Type"
+    TASK_START_TIME = "Task Start Time(us)"
+    TASK_DURATION = "Task Duration(us)"
+    TASK_WAIT_TIME = "Task Wait Time(us)"
+    BLOCK_DIM = "Block Dim"
+    MIX_BLOCK_DIM = "Mix Block Dim"
+    HF32_ELIGIBLE = "HF32 Eligible"
+    INPUT_SHAPES = "Input Shapes"
+    INPUT_DATA_TYPES = "Input Data Types"
+    INPUT_FORMATS = "Input Formats"
+    OUTPUT_SHAPES = "Output Shapes"
+    OUTPUT_DATA_TYPES = "Output Data Types"
+    OUTPUT_FORMATS = "Output Formats"
+    CONTEXT_ID = "Context ID"
+
+
+class EventConstant:
+    """Timeline event constant values"""
+
+    START_FLOW = "s"
+    END_FLOW = "f"
+    META_EVENT = 'M'
+    COMPLETE_EVENT = 'X'
+    INSTANT_EVENT = 'i'
+    COUNTER_EVENT = 'C'
+
+    PROCESS_NAME = "process_name"
+    PROCESS_LABEL = "process_labels"
+    PROCESS_SORT = "process_sort_index"
+    THREAD_NAME = "thread_name"
+    THREAD_SORT = "thread_sort_index"
+
+    CPU_LABEL = "CPU"
+    ASCEND_LABEL = "NPU"
+
+    HOST_TO_DEVICE_FLOW_CAT = "HostToDevice"
+    MINDSPORE_NPU_FLOW_CAT = "async_npu"
+    MINDSPORE_SELF_FLOW_CAT = "async_mindspore"
+    MSTX_FLOW_CAT = "MsTx"
+    MINDSPORE_SELF_FLOW_NAME = "mindspore_to_self"
+    MINDSPORE_NPU_FLOW_NAME = "mindspore_to_npu"
+    MSTX_FLOW_NAME = "mindspore_to_mstx"
+
+    MINDSPORE_PID = 1
+    CPU_OP_PID = 2
+    SCOPE_LAYER_PID = 3
+
+    MINDSPORE_SORT_IDX = 1
+    CPU_OP_SORT_IDX = 2
+    SCOPE_LAYER_SORT_IDX = 12
+
+    # field name
+    SEQUENCE_NUMBER = "Sequence number"
+    FORWARD_THREAD_ID = "Fwd thread id"
+    OP_NAME = "op_name"
+    INPUT_SHAPES = "Input Dims"
+    INPUT_DTYPES = "Input type"
+    CALL_STACK = "Call stack"
+    MODULE_HIERARCHY = "Module Hierarchy"
+    FLOPS = "flops"
+    NAME = "name"
+    CUSTOM_INFO = "custom_info"
+    TOP_SCOPE_NAMES = ('Default', 'Gradients', 'recompute_Default')
+    KERNEL_LAUNCH_KEYWORDS = ("KernelLaunch", "LaunchTask")
+    MSTX_KEYWORD = "Mstx"
+    STACK_EVENT_CAT = "python_function"
+
+    FLOW_OP = "flow"
+    INVALID_FLOW_ID = 18446744073709551615
+
+
+class TimeConstant:
+    """Time constant values"""
+
+    NS_TO_US = 0.001
+    MS_TO_US = 1000
+
+
+class FileConstant:
+    """File constant values"""
+
+    # tlv constant struct
+    FIX_SIZE_DATA = "fix_size_data"
+    CANN_FILE_REGEX = r"^PROF_\d+_\d+_[0-9a-zA-Z]+"
+    FRAMEWORK_DIR = "FRAMEWORK"
+
+
+class ProfilerStepNameConstant:
+    """Profiler step name."""
+
+    PROFILER_STEP = "ProfilerStep#"
+
+
+class TimelineLayerName(Enum):
+    """Timeline layer types."""
+    MINDSPORE = "MindSpore"
+    CPU_OP = "CPU OP"
+    MSTX = ["python", "python3"]
+    CANN = "CANN"
+    SCOPER_LAYER = "Scope Layer"
+    ASCEND_HARDWARE = "Ascend Hardware"
+    HCCL = "HCCL"
+    AI_CORE_FREQ = "AI Core Freq"
+    HBM = "HBM"
+    PCLE = "PCle"
+    HCCS = "HCCS"
+    LLC = "LLC"
+    NPU_MEM = "NPU MEM"
+    STARS_SOC_INFO = "Stars Soc Info"
+    STARS_Chip_Trans = "Stars Chip Trans"
+    ACC_PMU = "Acc PMU"
+    SIO = "SIO"
+    QOS = "QoS"
+    OVERLAP_ANALYSIS = "Overlap Analysis"
+
+
+class AnalysisMode(Enum):
+    """analysis mode"""
+    SYNC_MODE = "sync"
+    ASYNC_MODE = "async"
+
+
+class JitLevel:
+    """jit level"""
+    KBK_LEVEL = "O0"
+    KBK_DVM_LEVEL = "O1"
+    GRAPH_LEVEL = "O2"
+
+
+class ExportType(Enum):
+    Db = "db"
+    Text = "text"
+
+
+class CannLibName:
+    """CANN lib name"""
+    CANN_MSPTI = "libmspti.so"

mindspore/profiler/common/file_manager.py

@@ -0,0 +1,208 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Profiler file manager"""
+import csv
+import json
+import os
+import shutil
+from typing import List, Dict, Optional, Tuple
+import numpy as np
+
+from mindspore import log as logger
+from mindspore.profiler.common.path_manager import PathManager
+
+
+class FileManager:
+    """Profiler file manager"""
+
+    MAX_PATH_LENGTH = 4096
+    MAX_FILE_NAME_LENGTH = 255
+    DATA_FILE_AUTHORITY = 0o640
+    DATA_DIR_AUTHORITY = 0o700
+    FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+
+    @classmethod
+    def read_file_content(cls, path: str, mode: str = "r"):
+        """Read the content in the input file."""
+        PathManager.check_input_file_path(path)
+        PathManager.check_directory_path_readable(path)
+        try:
+            with open(path, mode) as file:
+                return file.read()
+        except Exception as err:
+            raise RuntimeError(f"Failed read file: {path}, error: {err}") from err
+
+    @classmethod
+    def read_json_file(cls, file_path: str) -> Optional[Dict]:
+        """Read json file and return dict data"""
+        PathManager.check_input_file_path(file_path)
+        PathManager.check_directory_path_readable(file_path)
+        try:
+            with open(file_path, "r", encoding="utf-8") as json_file:
+                data = json.load(json_file)
+                return data
+        except Exception as err:
+            raise RuntimeError(
+                f"Failed read json file: {file_path}, error: {err}"
+            ) from err
+
+    @classmethod
+    def create_json_file(
+            cls, output_file_path: str, json_data: List, indent: int = None
+    ) -> None:
+        """Create json file with least authority"""
+        PathManager.check_directory_path_writeable(os.path.dirname(output_file_path))
+        if not json_data:
+            logger.warning("Json data is empty, file path: %s", output_file_path)
+            return
+
+        try:
+            with os.fdopen(
+                    os.open(output_file_path, cls.FLAGS, cls.DATA_FILE_AUTHORITY), "w"
+            ) as fp:
+                data = json.dumps(json_data, indent=indent, ensure_ascii=False)
+                fp.write(data)
+        except Exception as err:
+            raise RuntimeError(
+                f"Failed create json file: {output_file_path}, error: {err}"
+            ) from err
+
+    @classmethod
+    def read_csv_file(cls, file_path: str) -> list:
+        """Read csv file and return list"""
+        PathManager.check_input_file_path(file_path)
+        PathManager.check_directory_path_readable(file_path)
+        result_data = []
+        try:
+            with open(file_path, newline="", encoding="utf-8") as csv_file:
+                reader = csv.reader(csv_file)
+                for row in reader:
+                    result_data.append(row)
+            return result_data
+        except Exception as err:
+            raise RuntimeError(f"Failed read csv file: {file_path}, error: {err}") from err
+
+    @classmethod
+    def read_csv_file_as_numpy(
+            cls, file_path: str, extern_headers: list = None
+    ) -> Tuple[np.ndarray, List[str]]:
+        """Read csv file and return numpy array"""
+        PathManager.check_input_file_path(file_path)
+        PathManager.check_directory_path_readable(file_path)
+        try:
+            with open(file_path, newline="", encoding="utf-8") as csv_file:
+                reader = csv.DictReader(csv_file, delimiter=",", quotechar='"')
+                headers = reader.fieldnames
+                if extern_headers:
+                    headers = extern_headers + headers
+
+                csv_data = [
+                    tuple([row.get(field) for field in headers]) for row in reader
+                ]
+                csv_data_np = np.array(
+                    csv_data, dtype=np.dtype([(field, object) for field in headers])
+                )
+
+            return csv_data_np, headers
+        except Exception as err:
+            raise RuntimeError(f"Failed read csv file: {file_path}, error: {err}") from err
+
+    @classmethod
+    def create_csv_file(cls, file_path: str, data: list, headers: list = None) -> None:
+        """Create csv file and write the data"""
+        if not data:
+            logger.error(
+                "Create csv file failed, data is empty, file path: %s", file_path
+            )
+            return
+        PathManager.check_directory_path_writeable(os.path.dirname(file_path))
+        try:
+            with os.fdopen(
+                    os.open(file_path, cls.FLAGS, cls.DATA_FILE_AUTHORITY), "w"
+            ) as fp:
+                writer = csv.writer(fp)
+                if headers:
+                    writer.writerow(headers)
+                writer.writerows(data)
+        except Exception as err:
+            raise RuntimeError(
+                f"Failed create csv file: {file_path}, error: {err}"
+            ) from err
+
+    @classmethod
+    def combine_csv_file(
+            cls, source_file_list: list, target_file_path: str, header_map: dict = None
+    ):
+        """Merge multiple CSV files into one"""
+        headers, all_data = [], []
+        for source_file in source_file_list:
+            data = cls.read_csv_file(source_file)
+            if len(data) > 1:
+                headers = data[0]
+                all_data.extend(data[1:])
+        if all_data:
+            if isinstance(header_map, dict):
+                headers = [header_map.get(header, header) for header in headers]
+            FileManager.create_csv_file(target_file_path, all_data, headers)
+
+    @classmethod
+    def read_txt_file(cls, file_path: str) -> list:
+        """Read txt file and return list"""
+        PathManager.check_input_file_path(file_path)
+        PathManager.check_directory_path_readable(file_path)
+        result_data = []
+        try:
+            with open(file_path, "r", encoding="utf-8") as file:
+                for line in file.readlines():
+                    result_data.append(line.strip())
+        except Exception as err:
+            raise RuntimeError(f"Failed read txt file: {file_path}, error: {err}") from err
+        return result_data
+
+    @classmethod
+    def copy_file(cls, src_path: str, dst_path: str):
+        """
+        Function Description:
+            copy file safety
+        Parameter:
+            src_path: file source path
+            dst_path: file destination path
+        Exception Description:
+            when src_path is link throw exception
+        """
+        if not os.path.exists(src_path):
+            logger.warning("The source file does not exist: %s", src_path)
+            return
+
+        PathManager.check_input_file_path(src_path)
+        src_dir = os.path.dirname(src_path)
+        PathManager.check_directory_path_readable(src_dir)
+        dst_dir = os.path.dirname(dst_path)
+        PathManager.check_directory_path_writeable(dst_dir)
+
+        try:
+            shutil.copy2(src_path, dst_path)
+        except (shutil.Error, IOError) as err:
+            msg = f"Failed to copy from '{src_path}' to '{dst_path}': {err}"
+            raise RuntimeError(msg) from err
+
+    @classmethod
+    def get_csv_file_list_by_start_name(cls, source_path: str, start_name: str):
+        """Get all the csv files that match the name"""
+        file_list = []
+        for file_name in os.listdir(source_path):
+            if file_name.startswith(start_name) and file_name.endswith(".csv"):
+                file_list.append(os.path.join(source_path, file_name))
+        return file_list