mindspore 2.6.0__cp310-cp310-win_amd64.whl → 2.7.0__cp310-cp310-win_amd64.whl

mindspore/profiler/parser/ascend_timeline_generator.py

@@ -1,545 +0,0 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""The integrator for integrating parsed profiling files."""
-
-import os.path
-import glob
-import json
-import stat
-from decimal import Decimal
-import numpy as np
-from mindspore import log as logger
-from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
-from mindspore.profiler.parser.base_timeline_generator import BaseTimelineGenerator
-from mindspore.profiler.parser.integrator import DeviceTarget
-from mindspore.profiler.parser.ascend_analysis.fwk_cann_parser import FwkCANNParser
-from mindspore.profiler.common.util import get_newest_file
-from mindspore.profiler.parser.ascend_analysis.constant import Constant
-
-
-class AscendTimelineGenerator(BaseTimelineGenerator):
-    """Generate ascend Timeline data from file."""
-    _timeline_display_filename = 'ascend_timeline_display_{}.json'
-    _timeline_summary_filename = 'ascend_timeline_summary_{}.json'
-    _cluster_analyse_filename = 'ascend_cluster_analyse_{}_{}_{}_{}.csv'
-
-    def __init__(self, profiling_dir, source_path, mindstudio_profiler_output, rank_id, rank_size, mode,
-                 step_list=None):
-        super().__init__(DeviceTarget.ASCEND.value, mode)
-        self._profiling_dir = profiling_dir
-        self._source_path = source_path
-        self._mindstudio_profiler_output = mindstudio_profiler_output
-        self._rank_id = rank_id
-        self._rank_size = rank_size
-        self._timeline_display_filename = self._timeline_display_filename.format(rank_id)
-        self._timeline_summary_filename = self._timeline_summary_filename.format(rank_id)
-        self._timeline_data = []
-        self._step_list = step_list
-
-        self.step_time_list_df = np.dtype(
-            [('Iteration ID', object), ('Steps', object), ('Iteration Start', float), ('Iteration Time', float)])
-
-        self.aicpu_time_list_dt = np.dtype(
-            [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float)])
-
-    def parse_cluster_data(self, op_summary, steptrace):
-        """
-        Parse cluster data and timeline summary data.
-
-        Args:
-            op_summary: op data
-            steptrace: step data
-        """
-
-        logger.info('parse cluster data...')
-        if isinstance(op_summary, np.ndarray) and op_summary.shape[0] == 0 or \
-                not isinstance(op_summary, np.ndarray) and not op_summary:
-            return
-        timeline_list = op_summary[~np.isin(op_summary['Task Type'], ['AI_CPU', 'HCCL'])][
-            ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
-
-        timeline_list = timeline_list.tolist()
-
-        if not isinstance(steptrace, np.ndarray) or steptrace.shape[0] == 0 or not steptrace.tolist():
-            iteration_time = op_summary[-1]['Task Start Time'] - op_summary[0]['Task Start Time'] + op_summary[-1][
-                'Task Duration'] + op_summary[-1]['Task Wait Time']
-            step_time_list = [['1', 'Steps', op_summary[0]['Task Start Time'], iteration_time]]
-        else:
-            step_time_list = np.empty((len(steptrace),), dtype=self.step_time_list_df)
-            step_time_list['Iteration ID'] = \
-                np.char.add("Model ID: ",
-                            np.char.add(steptrace['Model ID'].astype(str),
-                                        np.char.add(" Iteration ID: ",
-                                                    steptrace['Iteration ID'].astype(str))))
-            step_time_list['Steps'] = 'Steps'
-            step_time_list['Iteration Start'] = steptrace['Iteration End'] - steptrace['Iteration Time']
-            step_time_list['Iteration Time'] = steptrace['Iteration Time']
-            step_time_list = step_time_list.tolist()
-
-        # Add AI CPU data into timeline temp list and sort by start time.
-        aicpu_op = op_summary[op_summary['Task Type'] == 'AI_CPU']
-        if aicpu_op.size:
-            aicpu_time_list = np.empty((len(aicpu_op),), dtype=self.aicpu_time_list_dt)
-            aicpu_time_list['Op Name'] = aicpu_op['Op Name']
-            aicpu_time_list['Stream ID'] = aicpu_op['Stream ID']
-            aicpu_time_list['Task Start Time'] = aicpu_op['Task Start Time']
-            aicpu_time_list['Task Duration'] = aicpu_op['Task Duration'] + aicpu_op['Task Wait Time']
-            aicpu_time_list = aicpu_time_list.tolist()
-            timeline_list.extend(aicpu_time_list)
-        timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
-
-        # Add AllReduce info to timeline temp list and sort by start time.
-        communication_info = op_summary[op_summary['Task Type'] == 'HCCL'][
-            ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
-        if communication_info.size:
-            communication_info = communication_info.tolist()
-            communication_info.sort(key=lambda x: float(x[self._start_time_idx]))
-            logger.debug('AllReduce info found. Start adding info into timeline...')
-            self._get_cluster_timeline(timeline_list, communication_info, step_time_list)
-
-        # Update timeline summary info
-        timeline_summary = op_summary[['Op Name', 'Stream ID', 'Task Duration']]
-        self._timeline_summary['total_time'] = np.sum(timeline_summary['Task Duration'])
-        self._timeline_summary['num_of_streams'] = int(
-            len(np.unique(timeline_summary['Stream ID'], return_counts=True)[0]))
-        self._timeline_summary['num_of_ops'] = int(len(np.unique(timeline_summary['Op Name'], return_counts=True)[0]))
-        self._timeline_summary['op_exe_times'] = int(len(timeline_summary))
-        if self._timeline_summary['op_exe_times'] != 0:
-            self._timeline_summary['max_scope_name_num'] = int(np.max(
-                [len(x) for x in np.char.split(timeline_summary['Op Name'].astype(str), sep='/')]))
-        else:
-            self._timeline_summary['max_scope_name_num'] = 0
-        logger.info('Finished parse cluster data...')
-
-    def write_timeline_display(self):
-        """Write timeline display"""
-        logger.info('Writing timeline file...')
-        display_file_path = os.path.join(
-            self._profiling_dir,
-            self._timeline_display_filename
-        )
-        try:
-            with os.fdopen(os.open(display_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as fw:
-                json.dump(self._timeline_data, fw, indent=self.indent)
-            os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE)
-            logger.info('Finished file writing!')
-        except (IOError, OSError) as err:
-            logger.critical('Error occurred when write timeline display file: %s', err)
-            raise ProfilerIOException() from err
-
-    def parse_timeline_data(self, pretty=False):
-        """
-        Get detail timeline
-        Returns:
-            json, the content of timeline data.
-        """
-        logger.info("Start parse timeline data...")
-        self._pretty = pretty
-        timeline_data = []
-        all_scope_data = []
-
-        # get msprof data
-        msprof_file_name = fr'{self._mindstudio_profiler_output}/msprof_*.json'
-        file_list_msprof = glob.glob(msprof_file_name)
-        msprof_timeline = []
-        if not file_list_msprof:
-            logger.error('Could not find msprof_*.json file in %s', self._mindstudio_profiler_output)
-        else:
-            msprof_timeline = self._parse_msprof_data(get_newest_file(file_list_msprof))
-
-        # get cpu op
-        cpu_op_file_name = fr'{self._profiling_dir}/cpu_op_execute_timestamp_{self._rank_id}.txt'
-        file_list = glob.glob(cpu_op_file_name)
-        if not file_list:
-            logger.warning('Could not find cpu op file in %s', self._profiling_dir)
-        else:
-            cpu_timeline, scope_data = self.parse_cpu_timeline(file_list)
-            timeline_data.extend(cpu_timeline)
-            all_scope_data.extend(scope_data)
-
-        oprange_name = self._op_range_name.format(self._rank_id)
-        fwk_file_path = fr'{self._profiling_dir}/{self._framework_dir}/{oprange_name}'
-        if os.path.exists(fwk_file_path):
-            # It is faster not to submit to the pool
-            result = self._parse_fwk_device_data(msprof_timeline)
-            all_scope_data.extend(result.get('scope_data', []))
-            timeline_data.extend(result.get("trace_data", []))
-            self._kernel_events = result.get("kernels", [])
-        else:
-            # get Ascend Hardware for scope
-            scope_data = self._parse_ascend_hardware_scope(msprof_timeline)
-            all_scope_data.extend(scope_data)
-            timeline_data.extend(msprof_timeline)
-
-        # parse scope info
-        scope_timeline = self._parse_scope_info(all_scope_data)
-        timeline_data.extend(scope_timeline)
-
-        logger.info("All timeline data parse complete.")
-        self._timeline_data = timeline_data
-        return timeline_data
-
-    def parse_cpu_timeline(self, file_list):
-        """Load cpu operator data from file"""
-        ms_to_us = 1e3
-        ns_to_us = 1e-3
-        new_pid = Constant.CPU_OP
-        process_list = [{"name": "process_name",
-                         "pid": new_pid,
-                         "args": {
-                             "name": f"CPU OP"
-                         },
-                         "ph": "M"
-                         }, {"name": "process_sort_index", "pid": new_pid,
-                             "args": {"sort_index": new_pid}, "ph": "M"}
-                        ]
-        tid_set = set()
-        thread_list = []
-        new_timeline = []
-        scope_data = []
-        try:
-            flags = os.O_RDONLY
-            for file_path in file_list:
-                with os.fdopen(os.open(file_path, flags, 0o400), 'r') as fr:
-                    for line in fr:
-                        op_list = line.strip().split(';')
-                        op_full_name = op_list[0]
-                        time_arr = op_list[-1]
-                        time_arr = time_arr.split(" ")
-                        for time in time_arr:
-                            ts, dur, tid = time.split(",")   # origin unit of ts is ns and dur is ms.
-                            ts = Decimal(ts).quantize(Decimal('0.000')) * Decimal(ns_to_us).quantize(
-                                Decimal('0.000'))  # cast to us
-                            dur = Decimal(dur).quantize(Decimal('0.000')) * Decimal(ms_to_us).quantize(
-                                Decimal('0.000'))  # cast to us
-
-                            if op_full_name and op_full_name.startswith(Constant.TOP_SCOPE_NAMES):
-                                te = ts + dur
-                                scope_data.append((op_full_name.split('/')[:-1], ts, te))
-
-                            if int(tid) not in tid_set:
-                                tid_set.add(int(tid))
-                                thread_list.append({"name": "thread_name",
-                                                    "pid": new_pid,
-                                                    "tid": int(tid),
-                                                    "ph": "M",
-                                                    'args': {'name': f'thread {tid}'}
-                                                    })
-
-                            new_timeline.append({'name': op_list[0],
-                                                 'pid': new_pid,
-                                                 'tid': int(tid),
-                                                 'ph': 'X',
-                                                 'ts': str(ts),
-                                                 'dur': float(dur) * ms_to_us,
-                                                 'args':
-                                                     {'type': op_list[1]}
-                                                 })
-                break
-
-            return process_list + thread_list + new_timeline, scope_data
-
-        except (IOError, OSError, json.JSONDecodeError) as err:
-            logger.error('parse_cann_data failed! please theck. detail: %s', err)
-            return []
-
-    def _parse_fwk_device_data(self, cann_kernel_data):
-        """
-        Get framework op range trace data, flow events and hardware kernel events
-        """
-        fwkcann_parser = FwkCANNParser(self._source_path, cann_kernel_data, self._rank_id, self._step_list)
-        fwk_link_data = fwkcann_parser.generate_trace_data()
-        kernels = fwkcann_parser.kernels
-        scope_data = fwkcann_parser.scope_data_with_flow + fwkcann_parser.scope_data_without_flow
-        result = {"trace_data": fwk_link_data, "kernels": kernels, "scope_data": scope_data}
-        return result
-
-    def _parse_msprof_data(self, file_list):
-        """
-        parse msprof.json file
-        :param file_list:
-        :return:
-        """
-        flags = os.O_RDONLY
-        raw_data = []
-        try:
-            for file_path in file_list:
-                with os.fdopen(os.open(file_path, flags, 0o400), 'r') as fr:
-                    raw_data.extend(json.load(fr))
-
-            if not raw_data:
-                logger.error('Could not found msprof data in file list: %s .', file_list)
-
-            return raw_data
-
-        except (IOError, OSError, json.JSONDecodeError) as err:
-            logger.error('_parse_msprof_data failed! please theck. detail: %s', err)
-            return []
-
-    def _parse_ascend_hardware_scope(self, msprof_timeline):
-        """
-        parse ascend hardware scope
-        """
-        scope_data = []
-        try:
-
-            for event in msprof_timeline:
-                if event.get("ph") == "M":
-                    continue
-
-                op_full_name = event.get('name')
-                if op_full_name and op_full_name.startswith(Constant.TOP_SCOPE_NAMES):
-                    ts = Decimal(event.get('ts')).quantize(Decimal('0.000'))
-                    te = ts + Decimal(event.get('dur')).quantize(Decimal('0.000'))
-                    scope_data.append((op_full_name.split('/')[:-1], ts, te))
-
-            return scope_data
-
-        except (IOError, OSError, json.JSONDecodeError) as err:
-            logger.error('_parse_ascend_hardware_scope failed! please theck. detail: %s', err)
-            return []
-
-    def _parse_scope_info(self, scope_data):
-        """Parse scope info of op"""
-        if not scope_data:
-            return []
-        new_pid = Constant.SCOPE_LAYLER
-        scope_data.sort(key=lambda x: x[1])
-        process_list = [
-            {"name": "process_name",
-             "pid": new_pid,
-             "args": {
-                 "name": f"Scope Layer"
-             },
-             "ph": "M"},
-            {"name": "process_sort_index",
-             "pid": new_pid,
-             "args": {"sort_index": new_pid},
-             "ph": "M"}
-        ]
-
-        new_events = []
-        layer_stack = []
-        for layer_name in scope_data[0][0]:
-            layer_stack.append([layer_name, scope_data[0][1], scope_data[0][2]])
-
-        for op in scope_data[1:]:
-            if op[1] < layer_stack[0][2]:
-                # 并行算子只保留前面的
-                continue
-            flag = True  # 判断上层是否合并， 上层不合并下层也不合并
-            for layer_depth, layer_name in enumerate(op[0]):
-                if layer_depth >= len(layer_stack):
-                    layer_stack.append([layer_name, op[1], op[2]])
-                else:
-                    if layer_stack[layer_depth][0] == layer_name and flag:
-                        layer_stack[layer_depth][2] = op[2]  # 合并
-                    else:
-                        ts = layer_stack[layer_depth][1]
-                        new_events.append({
-                            "name": layer_stack[layer_depth][0],
-                            "pid": new_pid,
-                            "tid": layer_depth,
-                            "ph": "X",
-                            "ts": str(ts),
-                            "dur": float(layer_stack[layer_depth][2] - layer_stack[layer_depth][1])
-                        })
-                        layer_stack[layer_depth] = [layer_name, op[1], op[2]]
-                        flag = False
-
-        thread_list = []
-        for index, layer in enumerate(layer_stack):
-            thread_list.extend([{
-                "name": "thread_name",
-                "pid": new_pid,
-                "tid": index,
-                "args": {
-                    "name": f"layer{index}"
-                },
-                "ph": "M"
-            }, {
-                "name": "thread_sort_index",
-                "pid": new_pid,
-                "tid": index,
-                "args": {"sort_index": index},
-                "ph": "M"
-            }])
-            if layer:
-                ts = layer[1]
-                new_events.append({
-                    "name": layer[0],
-                    "pid": new_pid,
-                    "tid": index,
-                    "ph": "X",
-                    "ts": str(ts),
-                    "dur": float(layer[2] - layer[1])
-                })
-
-        return process_list + thread_list + new_events
-
-    def _produce_two_separated_timeline(self, timeline, op_name):
-        """Produce two separated timeline based on op_name."""
-        timeline_include_op_name = []
-        timeline_exclude_op_name = []
-        for time_item in timeline:
-            if op_name in time_item[self._op_name_idx]:
-                timeline_include_op_name.append(time_item)
-            else:
-                timeline_exclude_op_name.append(time_item)
-        return timeline_include_op_name, timeline_exclude_op_name
-
-    def _get_cluster_timeline(self, aicore_info, comm_info, step_info):
-        """
-        Analyse the cluster communication and computation data, and write result to file.
-
-        To analyse the cluster performance bottleneck based on timeline, define the time of a training
-        step as "t_total", propose five metrics as follows:
-            1) The time that "receive" operators not overlapped by others(t1)
-            2) The time that is consumed inside the stage(t_total - t1)
-            3) The time that "communication" operators not overlapped by others(t2)
-            4) The time that consumed by computation(t_total - t2)
-            5) The time that "collective communication" operators not overlapped by others(t3)
-        In pipeline parallel mode, we can locate slow stage based on t_total - t1. Inside each stage,
-        we can locate slow card based on t_total - t2. The value of t1 indicates the degree that
-        communication time between stages slow down the training. The value of t3 indicates the degree
-        that communication inside each stage slow down the training.
-        """
-        is_pipeline_parallel = False
-        comm_timeline = self._get_merged_time_list(
-            comm_info, display_name="communication"
-        )
-        aicore_timeline = self._get_merged_time_list(
-            aicore_info, get_interval_time=True
-        )
-        # Consider if the overlap will be 0 or not.
-        comm_not_overlapped_timeline = self._get_intersection_time(
-            aicore_timeline[0], comm_timeline[0]
-        )
-
-        # Process receive part.
-        all_timeline = aicore_info + comm_info
-        all_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
-        receive_timeline = self._produce_two_separated_timeline(
-            all_timeline, "Receive-op"
-        )
-        if receive_timeline[0]:
-            is_pipeline_parallel = True
-        receive_op_merged_timeline = self._get_merged_time_list(receive_timeline[0])[0]
-        timeline_exclude_receive_op_interval = self._get_merged_time_list(
-            receive_timeline[1], get_interval_time=True
-        )[0]
-        receive_op_not_overlapped_timeline = self._get_intersection_time(
-            timeline_exclude_receive_op_interval, receive_op_merged_timeline
-        )
-
-        # Process collective communication part.
-        collective_comm_timeline = self._produce_two_separated_timeline(
-            comm_info, "Receive-op"
-        )[-1]
-
-        collective_comm_not_overlapped_timeline = self._get_intersection_time(
-            aicore_timeline[0], self._get_merged_time_list(collective_comm_timeline)[0]
-        )
-
-        self._parse_cluster_metrices(step_info, receive_op_not_overlapped_timeline, comm_not_overlapped_timeline,
-                                     collective_comm_not_overlapped_timeline, is_pipeline_parallel)
-
-    def _parse_cluster_metrices(self, step_info, receive_op_not_overlapped_timeline, comm_not_overlapped_timeline,
-                                collective_comm_not_overlapped_timeline, is_pipeline_parallel):
-        """Write the cluster metrices"""
-        # Compute these five metrics mentioned above per step.
-        recieve_alone_time = self._compute_time_inside_step(receive_op_not_overlapped_timeline, step_info)
-        time_info = {"stage_time": [], "computation_time": []}
-        comm_alone_time = self._compute_time_inside_step(comm_not_overlapped_timeline, step_info)
-        collective_comm_alone_time = self._compute_time_inside_step(
-            collective_comm_not_overlapped_timeline, step_info
-        )
-        step_num = len(step_info)
-        for step in range(step_num):
-            try:
-                if is_pipeline_parallel:
-                    time_info.get("stage_time").append(step_info[step][self._duration_idx] - recieve_alone_time[step])
-            except IndexError as err:
-                logger.error(err)
-
-            try:
-                time_info.get("computation_time").append(step_info[step][self._duration_idx] - comm_alone_time[step])
-            except IndexError as err:
-                logger.error(err)
-
-        metrices_per_step_list = [
-            time_info.get("computation_time"), comm_alone_time, time_info.get("stage_time"),
-            recieve_alone_time, collective_comm_alone_time
-        ]
-        if step_num > 1:
-            for metric in metrices_per_step_list:
-                metric.append(sum(metric[1:]) / (step_num - 1))
-
-        try:
-            self._write_cluster_metrices(metrices_per_step_list, is_pipeline_parallel, "Ascend", self._rank_id)
-        except (IOError, OSError) as err:
-            logger.warning(err)
-            raise ProfilerIOException from err
-
-    def _compute_time_inside_step(self, metric_timeline, step_time_list):
-        """Compute per step time of metric_timeline."""
-        per_step_time_list = [0 for _ in range(len(step_time_list))]
-        step = 0
-        step_end_time = step_time_list[step][self._start_time_idx] + step_time_list[step][self._duration_idx]
-        for time_item in metric_timeline:
-            start_time = time_item[self._start_time_idx]
-            if start_time > step_end_time:
-                step += 1
-                if step >= len(step_time_list):
-                    logger.warning("Compute profiler compute_time_inside_step time, "
-                                   "find the data length is more than step count, "
-                                   "maybe current graph has multi sub graph, skip the last data.")
-                    break
-                step_end_time = step_time_list[step][self._start_time_idx] + step_time_list[step][self._duration_idx]
-            per_step_time_list[step] += time_item[self._duration_idx]
-
-        return per_step_time_list
-
-    def _get_intersection_time(self, first_time_list, second_time_list,
-                               display_name="communication_not_overlapped"):
-        """Get intersection time of two time list."""
-        first_list_idx, second_list_idx = 0, 0
-        first_list_len = len(first_time_list)
-        second_list_len = len(second_time_list)
-        intersection_segment_display_list = []
-
-        while first_list_idx < first_list_len and \
-                second_list_idx < second_list_len:
-            intersection_start = max(
-                first_time_list[first_list_idx][self._start_time_idx],
-                second_time_list[second_list_idx][self._start_time_idx]
-            )
-            intersection_end = min(
-                first_time_list[first_list_idx][self._duration_idx],
-                second_time_list[second_list_idx][self._duration_idx]
-            )
-            if intersection_start < intersection_end:
-                tid = self._tid_dict.get(display_name, [0, 0])
-                intersection_segment_display_list.append(
-                    [display_name, tid[0],
-                     intersection_start, intersection_end - intersection_start, tid[1]]
-                )
-            if first_time_list[first_list_idx][self._duration_idx] >= \
-                    second_time_list[second_list_idx][self._duration_idx]:
-                second_list_idx += 1
-            else:
-                first_list_idx += 1
-
-        return intersection_segment_display_list