mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.5.0__cp311-cp311-win_amd64.whl

mindspore/profiler/analysis/viewer/ascend_kernel_details_viewer.py

@@ -0,0 +1,252 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Ascend kernel details viewer"""
+import os
+import csv
+from decimal import Decimal
+
+from mindspore import log as logger
+from mindspore.profiler.analysis.viewer.base_viewer import BaseViewer
+from mindspore.profiler.analysis.parser.timeline_assembly_factory.trace_view_container import TraceViewContainer
+from mindspore.profiler.common.constant import OpSummaryHeaders
+from mindspore.profiler.common.path_manager import PathManager
+from mindspore.profiler.common.constant import TimelineLayerName
+from mindspore.profiler.common.constant import ProfilerStepNameConstant, JitLevel, ProfilerLevel
+from mindspore.profiler.common.log import ProfilerLogger
+
+
+class AscendKernelDetailsViewer(BaseViewer):
+    """
+    Ascend kernel details viewer
+    """
+    KERNEL_DETAILS_FILE_NAME = "kernel_details.csv"
+    EXCLUDE_HEADERS = [OpSummaryHeaders.DEVICE_ID.value]
+    LEVEL0_EXCLUDE_HEADERS = [
+        OpSummaryHeaders.MIX_BLOCK_DIM.value,
+        OpSummaryHeaders.HF32_ELIGIBLE.value,
+        OpSummaryHeaders.INPUT_SHAPES.value,
+        OpSummaryHeaders.INPUT_DATA_TYPES.value,
+        OpSummaryHeaders.INPUT_FORMATS.value,
+        OpSummaryHeaders.OUTPUT_SHAPES.value,
+        OpSummaryHeaders.OUTPUT_DATA_TYPES.value,
+        OpSummaryHeaders.OUTPUT_FORMATS.value,
+        OpSummaryHeaders.CONTEXT_ID.value,
+    ]
+    RENAME_HEADERS = {
+        OpSummaryHeaders.OP_NAME.value: "Name",
+        OpSummaryHeaders.OP_TYPE.value: "Type",
+        OpSummaryHeaders.TASK_TYPE.value: "Accelerator Core",
+        OpSummaryHeaders.TASK_START_TIME.value: "Start Time(us)",
+        OpSummaryHeaders.TASK_DURATION.value: "Duration(us)",
+        OpSummaryHeaders.TASK_WAIT_TIME.value: "Wait Time(us)",
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self._save_path = os.path.join(
+            kwargs.get("ascend_profiler_output_path"),
+            self.KERNEL_DETAILS_FILE_NAME
+        )
+        self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
+        self._is_set_schedule = kwargs.get("is_set_schedule")
+        self._jit_level = kwargs.get("jit_level")
+        self._profiler_level = kwargs.get("profiler_level")
+        self.op_summary_headers = None
+        self.op_summary = None
+        self.trace_container = None
+        self.kernel_details_headers = None
+        ProfilerLogger.init(self._ascend_ms_dir)
+        self._logger = ProfilerLogger.get_instance()
+
+    def save(self, data):
+        """
+        Save kernel details to csv file.
+        """
+        self._logger.info("AscendKernelDetailsViewer start")
+        try:
+            if self._profiler_level == ProfilerLevel.LevelNone.value:
+                return
+            self._check_input_data(data)
+            self._update_kernel_name()
+            self._update_headers()
+            self._write_data()
+            self._logger.info("Kernel details saved done")
+        except Exception as e:  # pylint: disable=W0703
+            self._logger.error("Failed to save kernel details: %s", str(e), exc_info=True)
+        self._logger.info("AscendKernelDetailsViewer end")
+
+    def _check_input_data(self, data):
+        """
+        Check input data.
+        """
+        self.trace_container = data.get("trace_view_container", None)
+        self.op_summary = data.get("op_summary", None)
+        self.op_summary_headers = data.get("op_summary_headers", None)
+
+        if self.op_summary is None or self.op_summary.size == 0:
+            raise ValueError("op summary is empty")
+
+        if self.trace_container is None:
+            raise ValueError("trace view container is None")
+
+    def _write_data(self):
+        """
+        Write data to csv file.
+        """
+        self._logger.info("Kernel details saved start")
+        PathManager.check_directory_path_writeable(os.path.dirname(self._save_path))
+        with open(self._save_path, "w", newline="", encoding="utf-8") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(self.kernel_details_headers)
+            for row in self.op_summary:
+                writer.writerow([row[field] for field in self.op_summary_headers])
+        self._logger.info("Kernel details saved done")
+
+    def _update_headers(self):
+        """
+        Update kernel details headers.
+        """
+        # filter exclude headers
+        self.op_summary_headers = [
+            header
+            for header in self.op_summary_headers
+            if header not in self.EXCLUDE_HEADERS
+        ]
+
+        if self._profiler_level == ProfilerLevel.Level0.value:
+            self.op_summary_headers = [
+                header
+                for header in self.op_summary_headers
+                if header not in self.LEVEL0_EXCLUDE_HEADERS
+            ]
+
+        if not self._is_set_schedule or self._jit_level == JitLevel.GRAPH_LEVEL:
+            self.op_summary_headers.remove(OpSummaryHeaders.STEP_ID.value)
+        # rename headers
+        self.kernel_details_headers = [
+            self.RENAME_HEADERS.get(header, header)
+            for header in self.op_summary_headers
+        ]
+
+    def _update_kernel_name(self):
+        """
+        Update op summary op name to framework launch op name.
+        """
+        self._logger.info("Update kernel name start")
+        step_id_to_time_dict = _generate_step_id_dict_by_trace_container(self.trace_container)
+        dev_kernels = self.trace_container.hardware_op_event
+        _generate_hardware_op_event_step_id(dev_kernels, step_id_to_time_dict)
+
+        if not dev_kernels:
+            logger.warning(
+                "Cannot find the device kernels with MindSpore framework launch op, "
+                "Please verify if it's in graph mode."
+            )
+            return
+
+        # build device kernel to framework launch op map
+        dev_kernel_to_fwk_op = {}
+        for _, per_tid_kernels in dev_kernels.items():
+            for kernel in per_tid_kernels:
+                dev_kernel_name = kernel.name
+                dev_kerel_ts = str(kernel.ts)
+                dev_kernel_to_fwk_op[(dev_kernel_name, dev_kerel_ts)] = kernel
+
+        launch_ops = [None] * len(self.op_summary)
+        step_ids = [None] * len(self.op_summary)
+        for index, summary in enumerate(self.op_summary):
+            dev_kernel_name = summary[OpSummaryHeaders.OP_NAME.value]
+            dev_kernel_ts = str(summary[OpSummaryHeaders.TASK_START_TIME.value]).strip("\t")
+            fwk_langch_op_name = None
+            step_id = None
+            if dev_kernel_to_fwk_op.get((dev_kernel_name, dev_kernel_ts)):
+                kernel = dev_kernel_to_fwk_op.get((dev_kernel_name, dev_kernel_ts))
+                if kernel.parent:
+                    fwk_langch_op_name = kernel.parent.name
+                step_id = kernel.step_id
+
+            if step_id is None:
+                step_id = _get_step_id_by_ts(Decimal(dev_kernel_ts), step_id_to_time_dict)
+
+            if fwk_langch_op_name is None:
+                self._logger.warning(
+                    "Can not find fwk launch op for dev kernel %s, ts %s",
+                    dev_kernel_name,
+                    dev_kernel_ts,
+                )
+                launch_ops[index] = dev_kernel_name
+            else:
+                launch_ops[index] = f"{fwk_langch_op_name}/{dev_kernel_name}"
+
+            if step_id is None and self._is_set_schedule:
+                self._logger.warning(
+                    "Can not find step id for dev kernel %s, ts %s",
+                    dev_kernel_name,
+                    dev_kernel_ts,
+                )
+            else:
+                step_ids[index] = step_id
+
+        # update op summary op name
+        self.op_summary[OpSummaryHeaders.OP_NAME.value] = launch_ops
+
+        # update op summary step id
+        if self._is_set_schedule:
+            if not self._jit_level == JitLevel.GRAPH_LEVEL:
+                self.op_summary[OpSummaryHeaders.STEP_ID.value] = step_ids
+        self._logger.info("Update kernel name done")
+
+
+def _generate_step_id_dict_by_trace_container(trace_container: TraceViewContainer):
+    """
+    Generate the step id to time dict.
+    """
+    # Retrieve all events from the trace container for the Mindspore timeline layer
+    events = trace_container.get_pool_by_name(TimelineLayerName.MINDSPORE.value).get_all_events()
+
+    # Filter events that contain "ProfilerStep" and create a dictionary mapping (start_ts, end_ts) to step ID
+    step_id_to_time_dict = {}
+    for event in events:
+        event_name = event.name
+        if ProfilerStepNameConstant.PROFILER_STEP in event_name:
+            step_id_to_time_dict[event.name.split("#")[-1]] = (event.ts, event.dur + event.ts)
+
+    return step_id_to_time_dict
+
+
+def _generate_hardware_op_event_step_id(hardware_op_events_dict: dict, step_id_to_time_dict: dict):
+    """
+    Generate the hardware op event step id.
+    """
+    for hardware_op_events_list in hardware_op_events_dict.values():
+        # Associate each hardware operation event with its step ID
+        for hardware_op_event in hardware_op_events_list:
+            kernel_event = hardware_op_event.parent
+            if not kernel_event:
+                continue
+
+            hardware_op_event.step_id = _get_step_id_by_ts(kernel_event.ts, step_id_to_time_dict)
+
+
+def _get_step_id_by_ts(ts: Decimal, step_events_dict: dict):
+    """
+    Retrieves the step ID for a given timestamp from the step events dictionary.
+    """
+    # Iterate through the step events dictionary to find the step ID for the given timestamp
+    for step_id, (st, et) in step_events_dict.items():
+        if st <= ts <= et:
+            return step_id
+
+    return None

mindspore/profiler/analysis/viewer/ascend_memory_viewer.py

@@ -0,0 +1,313 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""ascend memory viewer"""
+import os
+from decimal import Decimal
+
+from mindspore.profiler.analysis.viewer.base_viewer import BaseViewer
+from mindspore.profiler.common.file_manager import FileManager
+from mindspore.profiler.common.log import ProfilerLogger
+
+
+class MemoryRecordBean:
+    """Memory Record Struct"""
+
+    KB_TO_MB = 1000
+    HEADERS = [
+        "Component",
+        "Timestamp(us)",
+        "Total Allocated(KB)",
+        "Total Reserved(KB)",
+        "Total Active(KB)",
+        "Device Type",
+    ]
+
+    def __init__(self, data: list):
+        self._data = dict(zip(self.HEADERS, data))
+
+    @property
+    def row(self) -> list:
+        """
+        Get row data
+        """
+        return [
+            self.component,
+            self.time_us_str,
+            self.total_allocated_mb,
+            self.total_reserved_mb,
+            self.total_active_mb,
+            self.device_type,
+        ]
+
+    @property
+    def component(self) -> str:
+        """
+        Get component
+        """
+        return self._data.get("Component", "")
+
+    @property
+    def time_us_str(self) -> str:
+        """
+        Get time in us
+        """
+        ts_us = self._data.get("Timestamp(us)", 0)
+        return str(ts_us)
+
+    @property
+    def total_allocated_mb(self) -> float:
+        """
+        Get total allocated memory in MB
+        """
+        return float(self._data.get("Total Allocated(KB)", 0)) / self.KB_TO_MB
+
+    @property
+    def total_reserved_mb(self) -> float:
+        """
+        Get total reserved memory in MB
+        """
+        return float(self._data.get("Total Reserved(KB)", 0)) / self.KB_TO_MB
+
+    @property
+    def total_active_mb(self) -> float:
+        """
+        Get total active memory in MB
+        """
+        return float(self._data.get("Total Active(KB)", 0)) / self.KB_TO_MB
+
+    @property
+    def device_type(self) -> float:
+        """
+        Get device type
+        """
+        return self._data.get("Device Type", "")
+
+    @property
+    def total_allocated_kb(self) -> float:
+        """
+        Get total allocated memory in KB
+        """
+        return float(self._data.get("Total Allocated(KB)", 0))
+
+    @property
+    def total_reserved_kb(self) -> float:
+        """
+        Get total reserved memory in KB
+        """
+        return float(self._data.get("Total Reserved(KB)", 0))
+
+    @property
+    def total_active_kb(self) -> float:
+        """
+        Get total active memory in KB
+        """
+        return float(self._data.get("Total Active(KB)", 0))
+
+    @property
+    def time_us(self) -> Decimal:
+        """
+        Get time in us
+        """
+        return Decimal(self._data.get("Timestamp(us)", 0))
+
+    def is_ge_component(self):
+        """
+        Determine if it is GE
+        """
+        return self.component == "GE"
+
+
+class AscendMemoryViewer(BaseViewer):
+    """
+    Ascend Memory Viewer
+
+    generate memory_record.csv and npu_module_mem.csv
+    """
+
+    GE_MEMORY_RECORD_HEADERS = [
+        "Device id",
+        "Component",
+        "Timestamp(us)",
+        "Total Allocated(KB)",
+        "Total Reserved(KB)",
+        "Device",
+    ]
+    MS_MEMORY_RECORD_HEADERS = [
+        "Timestamp(ns)",
+        "Total Allocated(Byte)",
+        "Total Reserved(Byte)",
+        "Total Active(Byte)",
+    ]
+    TARGET_MEMORY_RECORD_HEADERS = [
+        "Component",
+        "Timestamp(us)",
+        "Total Allocated(MB)",
+        "Total Reserved(MB)",
+        "Total Active(MB)",
+        "Device Type",
+    ]
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self._enable_profile_memory = kwargs.get("profile_memory", False)
+        self._rank_id = kwargs.get("rank_id", 0)
+        self._output_path = kwargs.get("ascend_profiler_output_path")
+        self._framework_path = kwargs.get("framework_path")
+        self._msprof_profiler_output_path = kwargs.get("msprof_profile_output_path")
+        self._ascend_ms_dir = kwargs.get("ascend_ms_dir")
+        ProfilerLogger.init(self._ascend_ms_dir)
+        self._logger = ProfilerLogger.get_instance()
+        self._ge_memory_record = []
+        self._ms_memory_record = []
+
+    def save(self, data=None):
+        """
+        Save memory data
+        """
+        self._logger.info("AscendMemoryViewer start")
+        if not self._enable_profile_memory:
+            return
+        try:
+            self._copy_npu_module_mem_csv()
+            self._parse_memory_record()
+        except Exception as e: # pylint: disable=W0703
+            self._logger.error("Failed to save memory data: %s", e, exc_info=True)
+        self._logger.info("AscendMemoryViewer end")
+
+    def _copy_npu_module_mem_csv(self):
+        """Generate npu_module_mem.csv"""
+        npu_module_mem_file_list = FileManager.get_csv_file_list_by_start_name(
+            self._msprof_profiler_output_path, "npu_module_mem"
+        )
+        target_file_path = os.path.join(
+            self._output_path, "npu_module_mem.csv"
+        )
+        FileManager.combine_csv_file(npu_module_mem_file_list, target_file_path)
+        self._logger.info("npu_module_mem.csv saved to %s", target_file_path)
+
+    def _parse_memory_record(self):
+        """Generate memory_record.csv"""
+        self._parse_ge_memory_record()
+        self._parse_ms_memory_record()
+        combined_memory_data = self._combine_ge_ms_memory_record()
+        target_file_path = os.path.join(
+            self._output_path, "memory_record.csv"
+        )
+        FileManager.create_csv_file(
+            target_file_path, combined_memory_data, self.TARGET_MEMORY_RECORD_HEADERS
+        )
+        self._logger.info("memory_record.csv saved to %s", target_file_path)
+
+    def _parse_ge_memory_record(self):
+        """Parse ge memory record data"""
+        memory_record_file_list = FileManager.get_csv_file_list_by_start_name(
+            self._msprof_profiler_output_path, "memory_record"
+        )
+        for file in memory_record_file_list:
+            data = FileManager.read_csv_file(file)
+            if len(data) > 1:
+                self._ge_memory_record.extend(data[1:])
+
+    def _parse_ms_memory_record(self):
+        """Parse mindspore memory record data"""
+        memory_record_file = os.path.join(
+            self._framework_path,
+            f"cpu_ms_memory_record_{self._rank_id}.txt",
+        )
+        data = FileManager.read_csv_file(memory_record_file)
+        if len(data) > 1:
+            self._ms_memory_record.extend(data[1:])
+
+    def _get_app_reserved_memory(self) -> list:
+        """Get the reserved memory of the application from npu_mem.csv"""
+        npu_module_mem_file_list = FileManager.get_csv_file_list_by_start_name(
+            self._msprof_profiler_output_path, "npu_mem"
+        )
+        app_mems = []
+        for file in npu_module_mem_file_list:
+            md_mems = FileManager.read_csv_file(file)
+            for mem in md_mems:
+                if mem[1] == "APP":
+                    app_mems.append(
+                        MemoryRecordBean(
+                            [
+                                mem[1],
+                                mem[-1].rstrip("\t"),
+                                0.0,
+                                float(mem[4]),
+                                0.0,
+                                f"NPU:{self._rank_id}",
+                            ]
+                        ).row
+                    )
+        return app_mems
+
+    def _combine_ge_ms_memory_record(self) -> list:
+        """Combine ge and mindspore memory record data"""
+        memory_records = []
+        for ge_memory in self._ge_memory_record:
+            memory_record = dict(zip(self.GE_MEMORY_RECORD_HEADERS, ge_memory))
+            memory_records.append(
+                MemoryRecordBean(
+                    [
+                        memory_record.get("Component", "GE"),
+                        memory_record.get("Timestamp(us)"),
+                        memory_record.get("Total Allocated(KB)", 0),
+                        memory_record.get("Total Reserved(KB)", 0),
+                        0,
+                        memory_record.get("Device"),
+                    ]
+                )
+            )
+        for ms_memory in self._ms_memory_record:
+            memory_record = dict(zip(self.MS_MEMORY_RECORD_HEADERS, ms_memory))
+            memory_records.append(
+                MemoryRecordBean(
+                    [
+                        "MindSpore",
+                        Decimal(memory_record.get("Timestamp(ns)", 0)) / 1000,
+                        float(memory_record.get("Total Allocated(Byte)", 0)) / 1024,
+                        float(memory_record.get("Total Reserved(Byte)", 0)) / 1024,
+                        float(memory_record.get("Total Active(Byte)", 0)) / 1024,
+                        f"NPU:{self._rank_id}",
+                    ]
+                )
+            )
+        memory_records.sort(key=lambda x: x.time_us)
+        last_ge_memory, last_ms_memory = MemoryRecordBean([0] * 6), MemoryRecordBean(
+            [0] * 6
+        )
+        result_data = []
+        for memory_record in memory_records:
+            result_data.append(memory_record.row)
+            last_memory = (
+                last_ms_memory if memory_record.is_ge_component() else last_ge_memory
+            )
+            combined_mem = MemoryRecordBean(
+                [
+                    "MindSpore+GE",
+                    memory_record.time_us,
+                    memory_record.total_allocated_kb + last_memory.total_allocated_kb,
+                    memory_record.total_reserved_kb + last_memory.total_reserved_kb,
+                    memory_record.total_active_kb + last_memory.total_active_kb,
+                    f"NPU:{self._rank_id}",
+                ]
+            )
+            result_data.append(combined_mem.row)
+            if memory_record.is_ge_component():
+                last_ge_memory = memory_record
+            else:
+                last_ms_memory = memory_record
+        return result_data + self._get_app_reserved_memory()