mindspore 2.4.0__cp310-cp310-macosx_10_15_x86_64.whl

mindspore/profiler/parser/optime_parser.py

@@ -0,0 +1,250 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Op compute time files parser."""
+import os
+import stat
+from mindspore.profiler.common.util import fwrite_format
+from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
+    ProfilerIOException
+from mindspore import log as logger
+from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
+from mindspore.profiler.parser.container import HWTSContainer
+
+TIMELINE_FILE_COLUMN_TITLE = 'op_name, stream_id, start_time(ms), duration(ms)'
+
+
+class OPComputeTimeParser:
+    """
+    Join hwts info and framework info, get op time info, and output to the result file.
+
+    Args:
+         hwts_output_file (str): The file path of hwts_output_file. Such as: './output_format_data_hwts_0.txt".
+         output_filename (str): The output data file path and name. Such as: './output_op_compute_time_0.txt'.
+         op_task_info (dict): The task and op relation info. The format: {task_id, [opname, stream_id, block dim]}.
+    """
+
+    _dst_file_title = 'title:op compute time'
+    _dst_file_column_title = 'op_name       compute_time(ms) stream_id execution_times'
+    _dst_file_column_title += '\n------------  ---------------  ---------'
+
+    def __init__(self, hwts_output_file, output_filename, op_task_info,
+                 output_path, device_id):
+        hwts_output_file = validate_and_normalize_path(hwts_output_file)
+        self._hwts_output_file = hwts_output_file
+        self._output_filename = output_filename
+        self._op_task_info = op_task_info
+        self._output_path = output_path
+        self._device_id = device_id
+        self._min_cycle_counter = float("inf")
+
+    @property
+    def min_cycle_counter(self):
+        """Get minimum cycle counter."""
+        return self._min_cycle_counter
+
+    @staticmethod
+    def _convert_op_time_unit(op_data_list, op_name_time_dict, op_name_stream_dict,
+                              op_name_count_dict, op_name_task_dict, op_name_start_time):
+        """
+        Calculate the execution time of operator and convert it into millisecond.
+
+        Args:
+            op_data_list (list): The list of operator metadata.
+            op_name_time_dict (dict): The mapping relation of operator name and its execution time.
+            op_name_stream_dict (dict): The mapping relation of operator name and its stream id.
+            op_name_count_dict (dict): The mapping relation of operator name and its count.
+            op_name_task_dict (dict): The mapping relation of operator name and its task id.
+            op_name_start_time (dict): The mapping relation of operator name and its start time.
+        """
+        factor = 1e5
+        for item in op_data_list:
+            op_name = item.op_name
+            # Unit conversion: converting the cycle counter into ms.
+            op_start_time_str = str(item.cycle_counter / factor)
+            op_duration = item.duration / factor
+            op_duration_str = str(item.duration / factor)
+            if op_name in op_name_time_dict.keys():
+                op_name_time_dict[op_name] += op_duration
+                op_name_count_dict[op_name] += 1
+                op_name_start_time[op_name].append(
+                    (op_start_time_str, op_duration_str)
+                )
+
+            else:
+                op_name_time_dict[op_name] = op_duration
+                op_name_stream_dict[op_name] = item.stream_id
+                op_name_task_dict[op_name] = item.task_id
+                op_name_count_dict[op_name] = 1
+                op_name_start_time[op_name] = []
+                op_name_start_time[op_name].append(
+                    (op_start_time_str, op_duration_str)
+                )
+
+    def execute(self):
+        """Execute the parser, compute all op, get op time, and write it to the output file."""
+        # Calculate the execution time of operators,
+        # and update the minimum cycle counter.
+        tmp_result_data = self._calculate_op_execution_time()
+
+        # Convert time units from nanoseconds to milliseconds.
+        # The unit of the cycle counter is 10 nanoseconds.
+        op_name_time_dict = {}
+        op_name_stream_dict = {}
+        op_name_count_dict = {}
+        op_name_task_dict = {}
+        op_name_start_time = {}
+        self._convert_op_time_unit(
+            tmp_result_data, op_name_time_dict, op_name_stream_dict,
+            op_name_count_dict, op_name_task_dict, op_name_start_time
+        )
+
+        result_data = ""
+        total_time = 0
+        for op_name, time in op_name_time_dict.items():
+            if op_name in op_name_stream_dict.keys():
+                stream_id = op_name_stream_dict.get(op_name)
+                if op_name_count_dict.get(op_name) == 0:
+                    raise ValueError("The number of operations can not be 0.")
+                avg_time = time / op_name_count_dict.get(op_name)
+                total_time += avg_time
+                result_data += ("%s %s %s %s\n" % (op_name, str(avg_time), stream_id, op_name_count_dict.get(op_name)))
+        result_data += ("total op  %s 0" % (str(total_time)))
+
+        timeline_data = []
+        for op_name, _ in op_name_time_dict.items():
+            if op_name in op_name_stream_dict.keys():
+                stream_id = op_name_stream_dict[op_name]
+                start_time_list = op_name_start_time.get(op_name)
+                for (start_time, duration) in start_time_list:
+                    timeline_data.append([op_name, stream_id, start_time, duration])
+
+        # Write the metadata of operators into the file,
+        # including operator name, average time, and stream id.
+        self._write_op_time_into_file(result_data)
+        # Write the timeline data into file,
+        # including operator name, stream id, start time, and duration.
+        self._write_timeline_data_into_file(timeline_data)
+
+    def _get_op_task_id_map(self):
+        """
+        Read hwts data file, get the task time info.
+
+        Returns:
+           list: all hwts task time info.
+        """
+
+        op_map_result = []
+        hwts_list = []
+
+        if not os.path.exists(self._hwts_output_file):
+            logger.critical('The hwts output file does not exist.')
+            raise ProfilerFileNotFoundException('hwts output file')
+
+        with open(self._hwts_output_file, 'r') as data_file:
+            lines = data_file.readlines()
+            for line in lines:
+                if line.startswith("Start of task") or line.startswith("End of task"):
+                    line_split = line.split()
+                    container = HWTSContainer(line_split)
+                    hwts_list.append(container)
+
+        # hwts op map by taskId
+        for hwts in hwts_list:
+            if hwts.task_id in self._op_task_info.keys():
+                hwts.op_name = self._op_task_info[hwts.task_id]
+                op_map_result.append(hwts)
+
+        return op_map_result
+
+    def _write_op_time_into_file(self, result_data):
+        """
+        Write the metadata of operators into the file, including
+            op name, average time, and stream id.
+
+        Args:
+            result_data (str): The metadata to be written into the file.
+                    'op_name_1', 'avg_time_1', 'stream_id_1',
+                    'op_name_2', 'avg_time_2', 'stream_id_2',
+                    ...
+        """
+
+        fwrite_format(self._output_filename, data_source=self._dst_file_title, is_start=True)
+        fwrite_format(self._output_filename, data_source=self._dst_file_column_title)
+        fwrite_format(self._output_filename, data_source=result_data)
+
+    def _write_timeline_data_into_file(self, timeline_data):
+        """
+        Write the timeline information into the file, including
+            operator name, stream id, start time and duration.
+
+        Args:
+            timeline_data (list): The metadata to be written into the file.
+                [
+                    ['op_name_1', 'stream_id_1', 'start_time_1', 'duration_1'],
+                    ['op_name_2', 'stream_id_2', 'start_time_2', 'duration_2'],
+                    [...]
+                ]
+        """
+        # sorted by start times
+        timeline_data.sort(key=lambda x: float(x[2]))
+        filename = 'output_timeline_data_{}.txt'.format(self._device_id)
+        file_path = os.path.join(self._output_path, filename)
+        file_path = validate_and_normalize_path(file_path)
+
+        # write to file
+        try:
+            with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f_obj:
+                f_obj.write(TIMELINE_FILE_COLUMN_TITLE + '\n')
+                for timeline in timeline_data:
+                    timeline = [str(item) for item in timeline]
+                    f_obj.write(','.join(timeline) + '\n')
+            os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE)
+        except (IOError, OSError) as err:
+            logger.critical('Error occurred when writing intermediate timeline file: %s', err)
+            raise ProfilerIOException from err
+
+    def _calculate_op_execution_time(self):
+        """
+        Calculate the execution time of each operator.
+
+        Returns:
+            list, including the intermediate data of op execution time.
+        """
+        tmp_result_data = []
+        op_map_list = self._get_op_task_id_map()
+        tmp_op_dict = dict()
+
+        cur_index = 0
+        length = len(op_map_list)
+        min_cycle_counter = float("inf")
+        while cur_index < length:
+            op_time = op_map_list[cur_index]
+            if op_time.status == "Start":
+                tmp_op_dict[op_time.op_name] = op_time
+            elif op_time.status == "End" and op_time.op_name in tmp_op_dict:
+                op_start = tmp_op_dict.get(op_time.op_name, None)
+                if op_start:
+                    op_start.duration = op_time.cycle_counter - op_start.cycle_counter
+                    tmp_result_data.append(op_start)
+                    del tmp_op_dict[op_time.op_name]
+            if not op_time.op_name.startswith("assign"):
+                min_cycle_counter = min(min_cycle_counter, op_time.cycle_counter)
+
+            cur_index += 1
+
+        # Update the value of minimum cycle counter.
+        self._min_cycle_counter = min_cycle_counter / 1e5  # Convert the time unit from 10ns to 1ms
+
+        return tmp_result_data

mindspore/profiler/parser/profiler_info.py

@@ -0,0 +1,213 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Record profiler information"""
+import glob
+import os
+import stat
+
+from mindspore.version import __version__ as ms_version
+from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
+from mindspore import log as logger
+
+
+class ProfilerInfo:
+    """
+    This class is used to record profiler information.
+    it contains context_mode, rank_id, rank_size, parallel_mode, pipeline_stage_num, pipeline_stage_id,
+    profiling_start_time, profiling_stop_time, analyse_start_time, analyse_end_time
+    """
+
+    _file_name = "profiler_info_{}.json"
+    _file_path = ""
+    _profiler_info_dict = dict()
+    JIT_LEVEL = "jit_level"
+
+    @staticmethod
+    def init_info(context_mode, rank_id):
+        """Profiler info initialization must include context_mode, rank_id and output_path."""
+        ProfilerInfo._profiler_info_dict["context_mode"] = context_mode
+        ProfilerInfo._profiler_info_dict["rank_id"] = rank_id
+        ProfilerInfo._profiler_info_dict["ms_version"] = ms_version
+        ProfilerInfo._file_name = ProfilerInfo._file_name.format(rank_id)
+
+    @staticmethod
+    def set_parallel_info(parallel_mode="", stage_num=1):
+        """Set parallel info include parallel_mode, pipeline_stage_num and pipeline_stage_id."""
+        info = dict()
+        info["parallel_mode"] = parallel_mode
+        info["stage_num"] = stage_num
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_profiling_start_time(start_time):
+        """Set the profiling start time."""
+        info = dict()
+        info["profiling_start_time"] = start_time
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_profiling_stop_time(stop_time):
+        """Set the profiling stop time."""
+        info = dict()
+        info["profiling_stop_time"] = stop_time
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_analyse_start_time(start_time):
+        """Set the analyse start time."""
+        info = dict()
+        info["analyse_start_time"] = start_time
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_analyse_end_time(end_time):
+        """Set the analyse end time."""
+        info = dict()
+        info["analyse_end_time"] = end_time
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_export_start_time(start_time):
+        """Set the export start time."""
+        info = dict()
+        info["export_start_time"] = start_time
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_export_end_time(end_time):
+        """Set the export end time."""
+        info = dict()
+        info["export_end_time"] = end_time
+        ProfilerInfo._profiler_info_dict.update(info)
+
+    @staticmethod
+    def set_export_flag(flag):
+        """Set whether all-export or not."""
+        ProfilerInfo._profiler_info_dict["all_export"] = flag
+
+    @staticmethod
+    def set_system_time(sys_time):
+        """Set system time."""
+        ProfilerInfo._profiler_info_dict["system_time"] = sys_time
+
+    @staticmethod
+    def set_system_cnt(sys_cnt):
+        """Set system cnt."""
+        ProfilerInfo._profiler_info_dict["system_cnt"] = sys_cnt
+
+    @staticmethod
+    def set_diff_time(diff_time):
+        """synchronize timestamps between different devices"""
+        ProfilerInfo._profiler_info_dict["diff_time"] = diff_time
+
+    @staticmethod
+    def set_graph_ids(graph_ids):
+        """Set the graph id list."""
+        ProfilerInfo._profiler_info_dict["graph_ids"] = graph_ids
+
+    @staticmethod
+    def set_rank_size(rank_size):
+        """Set the rank size."""
+        ProfilerInfo._profiler_info_dict["rank_size"] = rank_size
+
+    @staticmethod
+    def set_heterogeneous(is_heterogeneous):
+        """Set is it heterogeneous."""
+        ProfilerInfo._profiler_info_dict["is_heterogeneous"] = is_heterogeneous
+
+    @staticmethod
+    def get_profiler_info():
+        """Get the profiler info."""
+        return ProfilerInfo._profiler_info_dict
+
+    @staticmethod
+    def set_profiling_options(profiling_options):
+        """Set profiling options to profiler info dict"""
+        ProfilerInfo._profiler_info_dict["profiling_options"] = profiling_options
+
+    @staticmethod
+    def set_jit_level(jit_level):
+        """Set jit_level to profiler info dict"""
+        ProfilerInfo._profiler_info_dict[ProfilerInfo.JIT_LEVEL] = jit_level
+
+    @staticmethod
+    def set_data_simplification(data_simplification):
+        """
+        Function Description:
+            Set the data simplification to profiler info dict
+        Parameter:
+            data_simplification: Whether data simplification is enabled
+        """
+        ProfilerInfo._profiler_info_dict["data_simplification"] = data_simplification
+
+    @staticmethod
+    def save(output_path):
+        """Save the profiler info to file."""
+        ProfilerInfo._file_path = os.path.join(output_path, ProfilerInfo._file_name)
+        FileManager.create_json_file(output_path, ProfilerInfo._profiler_info_dict, ProfilerInfo._file_name, indent=4)
+        os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)
+
+    @staticmethod
+    def load_profiler_info_dict(input_path):
+        """Load the profiler info from input path."""
+        ProfilerInfo._file_path = os.path.join(input_path, ProfilerInfo._file_name)
+        try:
+            load_info_dict = FileManager.read_json_file(ProfilerInfo._file_path)
+        except RuntimeError as err:
+            logger.warning(f"Cannot read file: {ProfilerInfo._file_path}, Error: {err}")
+            return
+        if not load_info_dict:
+            msg = f"Offline analysis failed load the ProfilerInfo._profiler_info_dict from: {ProfilerInfo._file_path}"
+            logger.warning(msg)
+            return
+        ProfilerInfo._profiler_info_dict = load_info_dict
+        os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)
+
+    @staticmethod
+    def get_rank_id(profiler_dir: str):
+        """
+        Function Description:
+            Get rank id from profiler_info_*.json
+        Parameter:
+            profiler_dir: the directory path of profiler data, eg: rank_0/profiler
+        Return:
+            str type rank id
+        """
+        prof_info_path = os.path.join(profiler_dir, "profiler_info_*.json")
+        prof_info_path = glob.glob(prof_info_path)
+        if not prof_info_path:
+            logger.warning("Cannot find profiler_info.json in the profiler directory.")
+            return "-1"
+
+        info_data = FileManager.read_json_file(prof_info_path[0])
+        return info_data.get("rank_id", "-1")
+
+    @staticmethod
+    def get_device_id(prof_dir: str):
+        """
+        Function Description:
+            Get device id from PROF_XXX dir
+        Parameter:
+            prof_dir: the directory path of PROF_XXX
+        Return:
+            str type device id
+        """
+        device_dir = os.path.join(prof_dir, "device_*")
+        device_dir = glob.glob(device_dir)
+        if not device_dir:
+            logger.warning("Cannot find device_XXX in the %s.", prof_dir)
+            return "-1"
+
+        return device_dir[0].split("device_")[-1]