mindspore 2.6.0rc1__cp39-cp39-win_amd64.whl → 2.7.0rc1__cp39-cp39-win_amd64.whl

mindspore/profiler/parser/optime_parser.py

@@ -1,250 +0,0 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Op compute time files parser."""
-import os
-import stat
-from mindspore.profiler.common.util import fwrite_format
-from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
-    ProfilerIOException
-from mindspore import log as logger
-from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
-from mindspore.profiler.parser.container import HWTSContainer
-
-TIMELINE_FILE_COLUMN_TITLE = 'op_name, stream_id, start_time(ms), duration(ms)'
-
-
-class OPComputeTimeParser:
-    """
-    Join hwts info and framework info, get op time info, and output to the result file.
-
-    Args:
-         hwts_output_file (str): The file path of hwts_output_file. Such as: './output_format_data_hwts_0.txt".
-         output_filename (str): The output data file path and name. Such as: './output_op_compute_time_0.txt'.
-         op_task_info (dict): The task and op relation info. The format: {task_id, [opname, stream_id, block dim]}.
-    """
-
-    _dst_file_title = 'title:op compute time'
-    _dst_file_column_title = 'op_name       compute_time(ms) stream_id execution_times'
-    _dst_file_column_title += '\n------------  ---------------  ---------'
-
-    def __init__(self, hwts_output_file, output_filename, op_task_info,
-                 output_path, device_id):
-        hwts_output_file = validate_and_normalize_path(hwts_output_file)
-        self._hwts_output_file = hwts_output_file
-        self._output_filename = output_filename
-        self._op_task_info = op_task_info
-        self._output_path = output_path
-        self._device_id = device_id
-        self._min_cycle_counter = float("inf")
-
-    @property
-    def min_cycle_counter(self):
-        """Get minimum cycle counter."""
-        return self._min_cycle_counter
-
-    @staticmethod
-    def _convert_op_time_unit(op_data_list, op_name_time_dict, op_name_stream_dict,
-                              op_name_count_dict, op_name_task_dict, op_name_start_time):
-        """
-        Calculate the execution time of operator and convert it into millisecond.
-
-        Args:
-            op_data_list (list): The list of operator metadata.
-            op_name_time_dict (dict): The mapping relation of operator name and its execution time.
-            op_name_stream_dict (dict): The mapping relation of operator name and its stream id.
-            op_name_count_dict (dict): The mapping relation of operator name and its count.
-            op_name_task_dict (dict): The mapping relation of operator name and its task id.
-            op_name_start_time (dict): The mapping relation of operator name and its start time.
-        """
-        factor = 1e5
-        for item in op_data_list:
-            op_name = item.op_name
-            # Unit conversion: converting the cycle counter into ms.
-            op_start_time_str = str(item.cycle_counter / factor)
-            op_duration = item.duration / factor
-            op_duration_str = str(item.duration / factor)
-            if op_name in op_name_time_dict.keys():
-                op_name_time_dict[op_name] += op_duration
-                op_name_count_dict[op_name] += 1
-                op_name_start_time[op_name].append(
-                    (op_start_time_str, op_duration_str)
-                )
-
-            else:
-                op_name_time_dict[op_name] = op_duration
-                op_name_stream_dict[op_name] = item.stream_id
-                op_name_task_dict[op_name] = item.task_id
-                op_name_count_dict[op_name] = 1
-                op_name_start_time[op_name] = []
-                op_name_start_time[op_name].append(
-                    (op_start_time_str, op_duration_str)
-                )
-
-    def execute(self):
-        """Execute the parser, compute all op, get op time, and write it to the output file."""
-        # Calculate the execution time of operators,
-        # and update the minimum cycle counter.
-        tmp_result_data = self._calculate_op_execution_time()
-
-        # Convert time units from nanoseconds to milliseconds.
-        # The unit of the cycle counter is 10 nanoseconds.
-        op_name_time_dict = {}
-        op_name_stream_dict = {}
-        op_name_count_dict = {}
-        op_name_task_dict = {}
-        op_name_start_time = {}
-        self._convert_op_time_unit(
-            tmp_result_data, op_name_time_dict, op_name_stream_dict,
-            op_name_count_dict, op_name_task_dict, op_name_start_time
-        )
-
-        result_data = ""
-        total_time = 0
-        for op_name, time in op_name_time_dict.items():
-            if op_name in op_name_stream_dict.keys():
-                stream_id = op_name_stream_dict.get(op_name)
-                if op_name_count_dict.get(op_name) == 0:
-                    raise ValueError("The number of operations can not be 0.")
-                avg_time = time / op_name_count_dict.get(op_name)
-                total_time += avg_time
-                result_data += ("%s %s %s %s\n" % (op_name, str(avg_time), stream_id, op_name_count_dict.get(op_name)))
-        result_data += ("total op  %s 0" % (str(total_time)))
-
-        timeline_data = []
-        for op_name, _ in op_name_time_dict.items():
-            if op_name in op_name_stream_dict.keys():
-                stream_id = op_name_stream_dict[op_name]
-                start_time_list = op_name_start_time.get(op_name)
-                for (start_time, duration) in start_time_list:
-                    timeline_data.append([op_name, stream_id, start_time, duration])
-
-        # Write the metadata of operators into the file,
-        # including operator name, average time, and stream id.
-        self._write_op_time_into_file(result_data)
-        # Write the timeline data into file,
-        # including operator name, stream id, start time, and duration.
-        self._write_timeline_data_into_file(timeline_data)
-
-    def _get_op_task_id_map(self):
-        """
-        Read hwts data file, get the task time info.
-
-        Returns:
-           list: all hwts task time info.
-        """
-
-        op_map_result = []
-        hwts_list = []
-
-        if not os.path.exists(self._hwts_output_file):
-            logger.critical('The hwts output file does not exist.')
-            raise ProfilerFileNotFoundException('hwts output file')
-
-        with open(self._hwts_output_file, 'r') as data_file:
-            lines = data_file.readlines()
-            for line in lines:
-                if line.startswith("Start of task") or line.startswith("End of task"):
-                    line_split = line.split()
-                    container = HWTSContainer(line_split)
-                    hwts_list.append(container)
-
-        # hwts op map by taskId
-        for hwts in hwts_list:
-            if hwts.task_id in self._op_task_info.keys():
-                hwts.op_name = self._op_task_info[hwts.task_id]
-                op_map_result.append(hwts)
-
-        return op_map_result
-
-    def _write_op_time_into_file(self, result_data):
-        """
-        Write the metadata of operators into the file, including
-            op name, average time, and stream id.
-
-        Args:
-            result_data (str): The metadata to be written into the file.
-                    'op_name_1', 'avg_time_1', 'stream_id_1',
-                    'op_name_2', 'avg_time_2', 'stream_id_2',
-                    ...
-        """
-
-        fwrite_format(self._output_filename, data_source=self._dst_file_title, is_start=True)
-        fwrite_format(self._output_filename, data_source=self._dst_file_column_title)
-        fwrite_format(self._output_filename, data_source=result_data)
-
-    def _write_timeline_data_into_file(self, timeline_data):
-        """
-        Write the timeline information into the file, including
-            operator name, stream id, start time and duration.
-
-        Args:
-            timeline_data (list): The metadata to be written into the file.
-                [
-                    ['op_name_1', 'stream_id_1', 'start_time_1', 'duration_1'],
-                    ['op_name_2', 'stream_id_2', 'start_time_2', 'duration_2'],
-                    [...]
-                ]
-        """
-        # sorted by start times
-        timeline_data.sort(key=lambda x: float(x[2]))
-        filename = 'output_timeline_data_{}.txt'.format(self._device_id)
-        file_path = os.path.join(self._output_path, filename)
-        file_path = validate_and_normalize_path(file_path)
-
-        # write to file
-        try:
-            with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f_obj:
-                f_obj.write(TIMELINE_FILE_COLUMN_TITLE + '\n')
-                for timeline in timeline_data:
-                    timeline = [str(item) for item in timeline]
-                    f_obj.write(','.join(timeline) + '\n')
-            os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE)
-        except (IOError, OSError) as err:
-            logger.critical('Error occurred when writing intermediate timeline file: %s', err)
-            raise ProfilerIOException from err
-
-    def _calculate_op_execution_time(self):
-        """
-        Calculate the execution time of each operator.
-
-        Returns:
-            list, including the intermediate data of op execution time.
-        """
-        tmp_result_data = []
-        op_map_list = self._get_op_task_id_map()
-        tmp_op_dict = dict()
-
-        cur_index = 0
-        length = len(op_map_list)
-        min_cycle_counter = float("inf")
-        while cur_index < length:
-            op_time = op_map_list[cur_index]
-            if op_time.status == "Start":
-                tmp_op_dict[op_time.op_name] = op_time
-            elif op_time.status == "End" and op_time.op_name in tmp_op_dict:
-                op_start = tmp_op_dict.get(op_time.op_name, None)
-                if op_start:
-                    op_start.duration = op_time.cycle_counter - op_start.cycle_counter
-                    tmp_result_data.append(op_start)
-                    del tmp_op_dict[op_time.op_name]
-            if not op_time.op_name.startswith("assign"):
-                min_cycle_counter = min(min_cycle_counter, op_time.cycle_counter)
-
-            cur_index += 1
-
-        # Update the value of minimum cycle counter.
-        self._min_cycle_counter = min_cycle_counter / 1e5  # Convert the time unit from 10ns to 1ms
-
-        return tmp_result_data

mindspore/profiler/parser/profiler_info.py

@@ -1,213 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Record profiler information"""
-import glob
-import os
-import stat
-
-from mindspore.version import __version__ as ms_version
-from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
-from mindspore import log as logger
-
-
-class ProfilerInfo:
-    """
-    This class is used to record profiler information.
-    it contains context_mode, rank_id, rank_size, parallel_mode, pipeline_stage_num, pipeline_stage_id,
-    profiling_start_time, profiling_stop_time, analyse_start_time, analyse_end_time
-    """
-
-    _file_name = "profiler_info_{}.json"
-    _file_path = ""
-    _profiler_info_dict = dict()
-    JIT_LEVEL = "jit_level"
-
-    @staticmethod
-    def init_info(context_mode, rank_id):
-        """Profiler info initialization must include context_mode, rank_id and output_path."""
-        ProfilerInfo._profiler_info_dict["context_mode"] = context_mode
-        ProfilerInfo._profiler_info_dict["rank_id"] = rank_id
-        ProfilerInfo._profiler_info_dict["ms_version"] = ms_version
-        ProfilerInfo._file_name = ProfilerInfo._file_name.format(rank_id)
-
-    @staticmethod
-    def set_parallel_info(parallel_mode="", stage_num=1):
-        """Set parallel info include parallel_mode, pipeline_stage_num and pipeline_stage_id."""
-        info = dict()
-        info["parallel_mode"] = parallel_mode
-        info["stage_num"] = stage_num
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_profiling_start_time(start_time):
-        """Set the profiling start time."""
-        info = dict()
-        info["profiling_start_time"] = start_time
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_profiling_stop_time(stop_time):
-        """Set the profiling stop time."""
-        info = dict()
-        info["profiling_stop_time"] = stop_time
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_analyse_start_time(start_time):
-        """Set the analyse start time."""
-        info = dict()
-        info["analyse_start_time"] = start_time
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_analyse_end_time(end_time):
-        """Set the analyse end time."""
-        info = dict()
-        info["analyse_end_time"] = end_time
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_export_start_time(start_time):
-        """Set the export start time."""
-        info = dict()
-        info["export_start_time"] = start_time
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_export_end_time(end_time):
-        """Set the export end time."""
-        info = dict()
-        info["export_end_time"] = end_time
-        ProfilerInfo._profiler_info_dict.update(info)
-
-    @staticmethod
-    def set_export_flag(flag):
-        """Set whether all-export or not."""
-        ProfilerInfo._profiler_info_dict["all_export"] = flag
-
-    @staticmethod
-    def set_system_time(sys_time):
-        """Set system time."""
-        ProfilerInfo._profiler_info_dict["system_time"] = sys_time
-
-    @staticmethod
-    def set_system_cnt(sys_cnt):
-        """Set system cnt."""
-        ProfilerInfo._profiler_info_dict["system_cnt"] = sys_cnt
-
-    @staticmethod
-    def set_diff_time(diff_time):
-        """synchronize timestamps between different devices"""
-        ProfilerInfo._profiler_info_dict["diff_time"] = diff_time
-
-    @staticmethod
-    def set_graph_ids(graph_ids):
-        """Set the graph id list."""
-        ProfilerInfo._profiler_info_dict["graph_ids"] = graph_ids
-
-    @staticmethod
-    def set_rank_size(rank_size):
-        """Set the rank size."""
-        ProfilerInfo._profiler_info_dict["rank_size"] = rank_size
-
-    @staticmethod
-    def set_heterogeneous(is_heterogeneous):
-        """Set is it heterogeneous."""
-        ProfilerInfo._profiler_info_dict["is_heterogeneous"] = is_heterogeneous
-
-    @staticmethod
-    def get_profiler_info():
-        """Get the profiler info."""
-        return ProfilerInfo._profiler_info_dict
-
-    @staticmethod
-    def set_profiling_options(profiling_options):
-        """Set profiling options to profiler info dict"""
-        ProfilerInfo._profiler_info_dict["profiling_options"] = profiling_options
-
-    @staticmethod
-    def set_jit_level(jit_level):
-        """Set jit_level to profiler info dict"""
-        ProfilerInfo._profiler_info_dict[ProfilerInfo.JIT_LEVEL] = jit_level
-
-    @staticmethod
-    def set_data_simplification(data_simplification):
-        """
-        Function Description:
-            Set the data simplification to profiler info dict
-        Parameter:
-            data_simplification: Whether data simplification is enabled
-        """
-        ProfilerInfo._profiler_info_dict["data_simplification"] = data_simplification
-
-    @staticmethod
-    def save(output_path):
-        """Save the profiler info to file."""
-        ProfilerInfo._file_path = os.path.join(output_path, ProfilerInfo._file_name)
-        FileManager.create_json_file(output_path, ProfilerInfo._profiler_info_dict, ProfilerInfo._file_name, indent=4)
-        os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)
-
-    @staticmethod
-    def load_profiler_info_dict(input_path):
-        """Load the profiler info from input path."""
-        ProfilerInfo._file_path = os.path.join(input_path, ProfilerInfo._file_name)
-        try:
-            load_info_dict = FileManager.read_json_file(ProfilerInfo._file_path)
-        except RuntimeError as err:
-            logger.warning(f"Cannot read file: {ProfilerInfo._file_path}, Error: {err}")
-            return
-        if not load_info_dict:
-            msg = f"Offline analysis failed load the ProfilerInfo._profiler_info_dict from: {ProfilerInfo._file_path}"
-            logger.warning(msg)
-            return
-        ProfilerInfo._profiler_info_dict = load_info_dict
-        os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)
-
-    @staticmethod
-    def get_rank_id(profiler_dir: str):
-        """
-        Function Description:
-            Get rank id from profiler_info_*.json
-        Parameter:
-            profiler_dir: the directory path of profiler data, eg: rank_0/profiler
-        Return:
-            str type rank id
-        """
-        prof_info_path = os.path.join(profiler_dir, "profiler_info_*.json")
-        prof_info_path = glob.glob(prof_info_path)
-        if not prof_info_path:
-            logger.warning("Cannot find profiler_info.json in the profiler directory.")
-            return "-1"
-
-        info_data = FileManager.read_json_file(prof_info_path[0])
-        return info_data.get("rank_id", "-1")
-
-    @staticmethod
-    def get_device_id(prof_dir: str):
-        """
-        Function Description:
-            Get device id from PROF_XXX dir
-        Parameter:
-            prof_dir: the directory path of PROF_XXX
-        Return:
-            str type device id
-        """
-        device_dir = os.path.join(prof_dir, "device_*")
-        device_dir = glob.glob(device_dir)
-        if not device_dir:
-            logger.warning("Cannot find device_XXX in the %s.", prof_dir)
-            return "-1"
-
-        return device_dir[0].split("device_")[-1]