mindspore 1.10.0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/profiler/parser/cpu_gpu_timeline_generator.py

@@ -0,0 +1,684 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""The integrator for integrating parsed profiling files."""
+import os
+import csv
+
+from mindspore import log as logger
+from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException, ProfilerFileNotFoundException, \
+    ProfilerParamValueErrorException
+from mindspore.profiler.parser.container import TimelineContainer
+from mindspore.profiler.parser.base_timeline_generator import BaseTimelineGenerator
+from mindspore.profiler.parser.integrator import DeviceTarget
+from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
+
+
+class GpuTimelineGenerator(BaseTimelineGenerator):
+    """Generate gpu Timeline data from file."""
+    _display_filename = 'gpu_timeline_display_{}.json'
+    _timeline_summary_filename = 'gpu_timeline_summary_{}.json'
+    _output_op_execute_time_file_path = "gpu_op_execute_timestamp_{}.txt"
+    _output_activity_execute_time_file_path = "activity_execute_timestamp_{}.txt"
+    _output_gpu_activity_info_file_path = "gpu_activity_data_{}.csv"
+    _step_trace_original_filename = 'step_trace_profiling_{}.txt'
+    _cluster_analyse_filename = 'gpu_cluster_analyse_{}_{}_{}_{}.csv'
+    _activity_keys_list = []
+
+    def __init__(self, profiling_dir, device_id, rank_size, model):
+        super().__init__(DeviceTarget.GPU.value, model)
+        self._device_id = device_id
+        self._rank_size = rank_size
+        self._profiling_dir = profiling_dir
+        self._timeline_meta = []
+        self._display_filename = self._display_filename.format(device_id)
+        self._timeline_summary_filename = self._timeline_summary_filename.format(device_id)
+        self._tid_dict = {
+            "receive_op_not_overlapped": (self._RECEIVE_ALONE, self._OP_OVERLAP_PID),
+            "exclude_receive_op": (self._ALLREDUCE_ALONE, self._OP_OVERLAP_PID),
+            "computation_op": (self._MERGED_COMPUTATION_TID, self._OP_OVERLAP_PID),
+            "communication_not_overlapped": (self._PURE_COMMUNICATION_TID, self._OP_OVERLAP_PID),
+            "communication": (self._MERGED_COMMUNICATION_TID, self._OP_OVERLAP_PID),
+            "free_time": (self._FREE_TIME_TID, self._OP_OVERLAP_PID)
+        }
+
+    def init_timeline(self, reduce_op_type):
+        """Init timeline metadata, adding all collected info."""
+        timeline_list = self._load_timeline_data(reduce_op_type)
+
+        # Init a dict for counting the num of streams.
+        stream_count_dict = {}
+        for timeline in timeline_list:
+            self._parse_timeline_data(timeline, 0)
+            # Updating the collection of streams.
+            if len(timeline) == 4:
+                self._update_num_of_streams(timeline, stream_count_dict)
+
+        # Add format thread meta data.
+        self._format_meta_data_list.extend(self._timeline_meta)
+        self._timeline_meta = self._format_meta_data_list
+
+        # Update timeline summary info
+        self._timeline_summary['num_of_streams'] += len(stream_count_dict)
+
+    def check_op_name(self, op_name):
+        """
+        Check whether the operator name exists.
+
+        Args:
+            op_name (str): The operator name or operator name prefix.
+
+        Returns:
+            bool, `True` if the operator name does exist, else `False`.
+        """
+        if not op_name:
+            raise ProfilerParamValueErrorException('The op_name should exist.')
+        for op_time_info in self._timeline_meta:
+            full_op_name = op_time_info['name']
+            if full_op_name and full_op_name.startswith(op_name):
+                return True
+        return False
+
+    def is_gpu_kernel_async_launch(self):
+        """Recognize the solution that launch the gpu kernel async."""
+        step_trace_profiling_path = self._get_and_validate_path(
+            self._step_trace_original_filename
+        )
+        try:
+            with open(step_trace_profiling_path, 'r') as f_obj:
+                line = next(f_obj)
+                first_string = line.strip().split()[0]
+                # the data format of launch the gpu kernel async is "Default/op1,160123 op-name"
+                # otherwise, the data format is "Default/op1 160123,12 "
+                return bool(len(first_string.split(',')) == 2)
+        except (IOError, OSError) as err:
+            logger.critical(f'Error occurred when read {step_trace_profiling_path}: {err}')
+            raise ProfilerIOException() from err
+        except StopIteration:
+            logger.warning('No step trace data exists.')
+            return False
+
+    def _get_and_validate_path(self, file_name):
+        """Generate op or activity file path from file name, and validate this path."""
+        file_path = os.path.join(
+            self._profiling_dir,
+            file_name.format(self._device_id)
+        )
+        file_path = validate_and_normalize_path(file_path)
+        if not os.path.exists(file_path):
+            logger.critical(f"Failed to find parsed timeline file {file_path}.")
+            raise ProfilerFileNotFoundException('parsed timeline file')
+
+        return file_path
+
+    def _parse_timeline_data(self, timeline, min_cycle_counter):
+        """Parse timeline data."""
+        # factor to convert the time unit of start_time(ts) from 1ns to 1us for timeline display
+        factor = 1000
+        op_meta = TimelineContainer(timeline)
+        timeline_dict = {}
+        timeline_dict['name'] = op_meta.op_name.split('/')[-1]
+        timeline_dict['ph'] = 'X'
+        timeline_dict['tid'] = op_meta.stream_id
+        timeline_dict['ts'] = (op_meta.start_time - min_cycle_counter) / factor
+        dur = op_meta.duration
+        timeline_dict['dur'] = dur
+        if op_meta.pid is None:
+            timeline_dict['pid'] = int(self._device_id)
+        else:
+            timeline_dict['pid'] = op_meta.pid
+        if op_meta.stream_id == "Scope Name":
+            # remove the level of scope name which has a format like "0-conv2-Conv2d".
+            timeline_dict['name'] = "-".join(op_meta.op_name.split('-')[1:])
+            timeline_dict['scope_level'] = int(op_meta.op_name.split('-')[0])
+        elif op_meta.stream_id[:len(self._host_cpu_op_label)] == self._host_cpu_op_label:
+            timeline_dict['pid'] = self._HOST_CPU_PID
+
+        if len(timeline) > 4:
+            # len(timeline) > 4 refers to activity data, else op data.
+            # Add args for activity data
+            args_dict = {}
+            for ix, value in enumerate(timeline[4:]):
+                args_dict[self._activity_keys_list[ix]] = value
+            timeline_dict['args'] = args_dict
+            timeline_dict['tid'] = f"Stream #{timeline_dict.get('tid', '0')}"
+        elif op_meta.stream_id not in ["Scope Name", "Steps"]:
+            # Update total time of operator execution.
+            self._timeline_summary['total_time'] += dur / factor
+            self._timeline_summary['op_exe_times'] += 1
+
+        self._update_format_meta_data(timeline_dict)
+        self._timeline_meta.append(timeline_dict)
+
+    def _load_timeline_data(self, reduce_op_type):
+        """Load timeline data from file."""
+        op_file_path = self._get_and_validate_path(
+            self._output_op_execute_time_file_path)
+
+        timeline_list, communication_info = self._load_op_data(op_file_path, reduce_op_type)
+        communication_info.sort(key=lambda x: float(x[2]))
+        # Add host cpu op timeline.
+        cpu_timeline_generator = CpuTimelineGenerator(self._profiling_dir, self._device_id, self._model)
+        cpu_timeline_list = cpu_timeline_generator.load_cpu_op_data()
+        if cpu_timeline_list:
+            self._clock_synchronize_to_gpu(cpu_timeline_list)
+            timeline_list.extend(cpu_timeline_list)
+        timeline_list.sort(key=lambda x: float(x[2]))
+        self._max_scope_name_num = self._get_max_scope_name_num(timeline_list)
+        self._timeline_summary['max_scope_name_num'] = self._max_scope_name_num
+
+        # Generate step time.
+        factor_start_time_uint_to_duration = 1e-3
+        self._set_step_start_and_end_op_name(timeline_list)
+        # Fit gpu kernel async launch solution.
+        if self.is_gpu_kernel_async_launch():
+            step_time_list = self._get_step_time_list_from_step_trace()
+        else:
+            step_time_list = self._get_step_time_list(timeline_list, factor_start_time_uint_to_duration)
+
+        # Add Scope Name.
+        default_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Default",
+                                                                      factor_start_time_uint_to_duration)
+        gradient_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Gradients",
+                                                                       factor_start_time_uint_to_duration)
+        recompute_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "recompute_Default",
+                                                                        factor_start_time_uint_to_duration)
+        cuda_op_timeline = self._load_activity_data()
+
+        # Add AllReduce info to timeline temp list and sort by start time.
+        if communication_info:
+            logger.debug('Allreduce info found, Start adding info to timeline...')
+            cluster_related_timeline = self._get_cluster_timeline(
+                timeline_list, cuda_op_timeline[1], communication_info, step_time_list)
+            timeline_list.extend(cluster_related_timeline)
+            timeline_list.extend(communication_info)
+            timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
+
+        timeline_list.extend(default_scope_name_time_list)
+        timeline_list.extend(gradient_scope_name_time_list)
+        timeline_list.extend(recompute_scope_name_time_list)
+        timeline_list.extend(step_time_list)
+
+        timeline_list.sort(key=lambda x: (float(x[self._start_time_idx])))
+
+        # Add cuda activity timeline.
+        timeline_list.extend(cuda_op_timeline[0])
+        timeline_list.sort(key=lambda x: float(x[2]))
+
+        return timeline_list
+
+    def _clock_synchronize_to_gpu(self, timeline_list):
+        """Synchronize the timestamp from device to host."""
+        start_time_file_path = os.path.join(self._profiling_dir, f"start_time_{self._device_id}.txt")
+
+        try:
+            with open(start_time_file_path) as f_obj:
+                lines = f_obj.readlines()
+                # lines[0] stores the host monotonic time of start training.
+                host_monotonic_start_time = int(lines[0].strip().split(':')[-1])
+                # lines[1] stores the gpu time of start training.
+                gpu_start_time = int(lines[1].strip().split(':')[-1])
+        except (IOError, OSError) as err:
+            logger.critical(f'Error occurred when read {start_time_file_path}: {err}')
+            raise ProfilerIOException()
+
+        time_diff = gpu_start_time - host_monotonic_start_time
+        for idx, time_item in enumerate(timeline_list):
+            timeline_list[idx][self._start_time_idx] = int(time_item[self._start_time_idx]) + time_diff
+
+    def _load_op_data(self, op_file_path, reduce_op_type):
+        """Load operator data from file"""
+        op_timeline_list = []
+        communication_info = []
+        try:
+            with open(op_file_path, 'r') as f_obj:
+                for line in f_obj:
+                    self._timeline_summary['num_of_ops'] += 1
+                    op_list = line.strip('\n').strip().split(';')
+                    time_arr = op_list[-1]
+                    time_arr = time_arr.split(" ")
+                    for time in time_arr:
+                        time = time.split(",")
+                        line_list = op_list[:2] + time
+                        communication_op_name = line_list[0].strip().split('/')[-1]
+                        if communication_op_name not in reduce_op_type:
+                            op_timeline_list.append(line_list)
+                        else:
+                            communication_info.append(line_list)
+        except (IOError, OSError) as err:
+            logger.critical('Error occurred when load operator timeline data intermediate file: %s', err)
+            raise ProfilerIOException()
+
+        return op_timeline_list, communication_info
+
+    def _load_activity_data(self):
+        """Load activity data from file"""
+        activity_timeline_list = []
+        cuda_compute_ops_timeline_list = []
+        args_dict = {}
+        activity_file_path = self._get_and_validate_path(
+            self._output_activity_execute_time_file_path)
+        activity_args_file_path = self._get_and_validate_path(
+            self._output_gpu_activity_info_file_path)
+
+        if not os.path.exists(activity_args_file_path):
+            logger.error(f'The file {activity_args_file_path} does not exist.')
+            raise ProfilerFileNotFoundException(activity_args_file_path)
+        with open(activity_args_file_path, 'r') as args_file:
+            csv_reader = csv.reader(args_file)
+            keys_list = next(csv_reader)
+            # keys_list format is: name, type, op_full_name, stream_id, block_dim, grid_dim, ...
+            self._activity_keys_list = keys_list[1:3] + keys_list[4:6]
+            for info in csv_reader:
+                args_dict[info[0]] = info[1:3] + info[4:6]
+
+        if not os.path.exists(activity_file_path):
+            logger.error(f'The file {activity_file_path} does not exist.')
+            raise ProfilerFileNotFoundException(activity_file_path)
+        with open(activity_file_path, 'r') as f_obj:
+            for line in f_obj:
+                line_list = line.strip('\n').split(';')
+                # concat activity args info.
+                line_list += args_dict.get(line_list[0])
+                if not line_list[0].startswith('nccl'):
+                    cuda_compute_ops_timeline_list.append(line_list)
+                activity_timeline_list.append(line_list)
+
+        return activity_timeline_list, cuda_compute_ops_timeline_list
+
+    def _get_step_time_list_from_step_trace(self):
+        """Produce the time of each step based on step_trace_profiling file."""
+        # Record the time of each step.
+        step_time_list = []
+        step_start_op_name = []
+        step_end_op_name = []
+        step_num = 1
+        tid = "Steps"
+        step_trace_profiling_path = self._get_and_validate_path(
+            self._step_trace_original_filename
+        )
+
+        try:
+            with open(step_trace_profiling_path, 'r') as f_obj:
+                for line in f_obj:
+                    line = line.strip().split()
+                    step_start_op_name.append(line[0].split(',')[0])
+                    step_end_op_name.append(line[3].split(',')[0])
+                    cur_step_start_time = float(line[0].split(',')[1])
+                    cur_step_end_time = float(line[3].split(',')[1])
+                    # convert duration time unit from ns to us.
+                    cur_step_duration_time = (cur_step_end_time - cur_step_start_time) / 1e3
+                    step_time_item = [str(step_num), tid, cur_step_start_time, cur_step_duration_time]
+                    step_time_list.append(step_time_item)
+                    step_num += 1
+        except (IOError, OSError) as err:
+            logger.critical(f'Error occurred when read {step_trace_profiling_path}: {err}')
+            raise ProfilerIOException()
+
+        return step_time_list
+
+    def _get_cluster_timeline(self, timeline, activity_info, comm_info, step_info):
+        """
+        Analyse the cluster communication and computation data, and write result to file.
+
+        To analyse the cluster performance bottleneck based on timeline, define the time of a training
+        step as "t_total", propose five metrics as follows:
+            1) The time that "receive" operators not overlapped by others(t1)
+            2) The time that is consumed inside the stage(t_total - t1)
+            3) The time that "communication" operators not overlapped by others(t2)
+            4) The time that consumed by computation(t_total - t2)
+            5) The time that "collective communication" operators not overlapped by others(t3)
+        In pipeline parallel mode, we can locate slow stage based on t_total - t1. Inside each stage,
+        we can locate slow card based on t_total - t2. The value of t1 indicates the degree that
+        communication time between stages slow down the training. The value of t3 indicates the degree
+        that communication inside each stage slow down the training.
+        """
+        time_info = {"stage_time": [], "computation_time": [], "recieve_alone_time": [], "comm_alone_time": [],
+                     "collective_comm_alone_time": [], "comm_not_overlapped_timeline": [], "free_timeline": [],
+                     "collective_comm_not_overlapped_timeline": [], "comm_timeline": [], "compute_timeline": [],
+                     "receive_op_not_overlapped_timeline": [], "receive_op_timeline": [],
+                     "collective_comm_timeline": [], "receive_op_merged_timeline": [], "is_pipeline_parallel": False}
+        time_info["comm_timeline"] = self._get_merged_time_list(
+            comm_info,
+            display_name="communication",
+            factor=1e-3
+        )
+        compute_op_timeline = timeline + activity_info
+        compute_op_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
+        time_info["compute_timeline"] = self._get_merged_time_list(
+            compute_op_timeline,
+            get_interval_time=True,
+            factor=1e-3
+        )
+        # Consider if the overlap will be 0 or not.
+        time_info["comm_not_overlapped_timeline"] = self._get_intersection_time(
+            time_info.get("compute_timeline")[0],
+            time_info.get("comm_timeline")[0]
+        )
+
+        # Process receive part.
+        all_timeline = timeline + comm_info
+        all_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
+        time_info["receive_op_timeline"] = self._produce_two_separated_timeline(
+            all_timeline,
+            "Receive-op"
+        )[0]
+        if time_info.get("receive_op_timeline"):
+            time_info["is_pipeline_parallel"] = True
+        time_info["receive_op_merged_timeline"] = self._get_merged_time_list(time_info.get("receive_op_timeline"),
+                                                                             factor=1e-3)[0]
+
+        time_info["receive_op_not_overlapped_timeline"] = self._get_intersection_time(
+            time_info.get("compute_timeline")[0],
+            time_info.get("receive_op_merged_timeline"),
+            display_name="receive_op_not_overlapped"
+        )
+
+        # Process collective communication part.
+        time_info["collective_comm_timeline"] = self._produce_two_separated_timeline(
+            comm_info,
+            "Receive-op"
+        )[-1]
+        collective_comm_merged_timeline = self._get_merged_time_list(time_info.get("collective_comm_timeline"),
+                                                                     factor=1e-3)[0]
+        time_info["collective_comm_not_overlapped_timeline"] = self._get_intersection_time(
+            time_info.get("compute_timeline")[0],
+            collective_comm_merged_timeline,
+            display_name="exclude_receive_op"
+        )
+
+        # Generate free time that exclude computation and communication time.
+        all_timeline = compute_op_timeline + comm_info
+        all_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
+        time_info["free_timeline"] = self._get_merged_time_list(
+            all_timeline,
+            get_interval_time=True,
+            display_name="free_time",
+            factor=1e-3
+        )[1]
+
+        # Compute these five metrics mentioned above per step.
+        time_info["recieve_alone_time"] = self._compute_time_inside_step(
+            time_info.get("receive_op_not_overlapped_timeline"), step_info)
+        time_info["comm_alone_time"] = self._compute_time_inside_step(time_info.get("comm_not_overlapped_timeline"),
+                                                                      step_info)
+        time_info["collective_comm_alone_time"] = self._compute_time_inside_step(
+            time_info.get("collective_comm_not_overlapped_timeline"),
+            step_info
+        )
+        step_num = len(step_info)
+        for step in range(step_num):
+            try:
+                if time_info.get("is_pipeline_parallel"):
+                    time_info.get("stage_time").append(
+                        step_info[step][self._duration_idx] - time_info.get("recieve_alone_time")[step]
+                    )
+            except IndexError as e:
+                logger.error(e)
+            try:
+                time_info.get("computation_time").append(
+                    step_info[step][self._duration_idx] - time_info.get("comm_alone_time")[step]
+                )
+            except IndexError as e:
+                logger.error(e)
+
+        metrices_per_step_list = [time_info.get("computation_time"), time_info.get("comm_alone_time"),
+                                  time_info.get("stage_time"), time_info.get("recieve_alone_time"),
+                                  time_info.get("collective_comm_alone_time")]
+        if step_num > 1:
+            for metric in metrices_per_step_list:
+                metric.append(sum(metric[1:]) / (step_num - 1))
+        try:
+            self._write_cluster_metrices(metrices_per_step_list, time_info.get("is_pipeline_parallel"), "Gpu",
+                                         self._device_id)
+        except (IOError, OSError) as e:
+            logger.warning(e)
+            raise ProfilerIOException
+
+        res_timeline = []
+        res_timeline.extend(time_info.get("comm_not_overlapped_timeline"))
+        res_timeline.extend(time_info.get("compute_timeline")[2])
+        res_timeline.extend(time_info.get("comm_timeline")[2])
+        res_timeline.extend(time_info.get("free_timeline"))
+        return res_timeline
+
+    def _compute_time_inside_step(self, metric_timeline, step_time_list):
+        """Compute per step time of metric_timeline."""
+        per_step_time_list = []
+        step = 0
+        cur_step_metric_time = 0
+        factor_us_to_ns = 1e3
+        step_end_time = step_time_list[step][self._start_time_idx] + \
+                        step_time_list[step][self._duration_idx] * factor_us_to_ns
+        for time_item in metric_timeline:
+            start_time = time_item[self._start_time_idx]
+            if start_time > step_end_time:
+                per_step_time_list.append(cur_step_metric_time)
+                step += 1
+                if step >= len(step_time_list):
+                    logger.warning("Compute profiler compute_time_inside_step time, "
+                                   "find the data length is more than step count, "
+                                   "maybe current graph has multi sub graph, skip the last data.")
+                    break
+                step_end_time = step_time_list[step][self._start_time_idx] + \
+                                step_time_list[step][self._duration_idx] * factor_us_to_ns
+                cur_step_metric_time = 0
+            cur_step_metric_time += time_item[self._duration_idx]
+        per_step_time_list.append(cur_step_metric_time)
+
+        return per_step_time_list
+
+    def _get_intersection_time(self, first_time_list, second_time_list,
+                               display_name="communication_not_overlapped"):
+        """Get intersection time of two time list."""
+        first_list_idx, second_list_idx = 0, 0
+        first_list_len = len(first_time_list)
+        second_list_len = len(second_time_list)
+        intersection_segment_display_list = []
+        factor_ns_to_us = 1e-3
+        while first_list_idx < first_list_len and second_list_idx < second_list_len:
+            intersection_start = max(
+                first_time_list[first_list_idx][self._start_time_idx],
+                second_time_list[second_list_idx][self._start_time_idx]
+            )
+            intersection_end = min(
+                first_time_list[first_list_idx][self._duration_idx],
+                second_time_list[second_list_idx][self._duration_idx]
+            )
+            if intersection_start < intersection_end:
+                intersection_segment_display_list.append(
+                    [display_name, self._tid_dict[display_name][0],
+                     intersection_start, (intersection_end - intersection_start) * factor_ns_to_us,
+                     self._tid_dict[display_name][1]]
+                )
+            if first_time_list[first_list_idx][self._duration_idx] >= \
+                    second_time_list[second_list_idx][self._duration_idx]:
+                second_list_idx += 1
+            else:
+                first_list_idx += 1
+
+        return intersection_segment_display_list
+
+    def _produce_two_separated_timeline(self, timeline, op_name):
+        """Produce two separated timeline based on op_name."""
+        timeline_include_op_name = []
+        timeline_exclude_op_name = []
+        for time_item in timeline:
+            if op_name in time_item[self._op_name_idx]:
+                timeline_include_op_name.append(time_item)
+            else:
+                timeline_exclude_op_name.append(time_item)
+        return timeline_include_op_name, timeline_exclude_op_name
+
+
+class CpuTimelineGenerator(GpuTimelineGenerator):
+    """Generate cpu Timeline data from file."""
+    _output_op_execute_time_file_path = "cpu_op_execute_timestamp_{}.txt"
+    _display_filename = 'cpu_timeline_display_{}.json'
+    _timeline_summary_filename = 'cpu_timeline_summary_{}.json'
+
+    def __init__(self, profiling_dir, device_id, model):
+        super().__init__(profiling_dir, device_id, 0, model)
+        self._device_target = DeviceTarget.CPU.value
+
+    def get_timeline_data(self):
+        """Get timeline data from file."""
+        timeline_list = self.load_cpu_op_data()
+        factor_ns_to_ms = 1e6
+        factor_us_to_ms = 1e3
+        for time_item in timeline_list:
+            time_item[self._start_time_idx] = float(time_item[self._start_time_idx]) / factor_ns_to_ms
+            time_item[self._duration_idx] = float(time_item[self._duration_idx]) / factor_us_to_ms
+
+        return timeline_list
+
+    def init_timeline(self):
+        """Init timeline metadata, adding all collected info."""
+        timeline_list = self._load_timeline_data()
+
+        # Init a dict for counting the num of streams.
+        stream_count_dict = {}
+        for timeline in timeline_list:
+            self._parse_timeline_data(timeline, 0)
+            # Updating the collection of streams.
+            if len(timeline) == 4:
+                self._update_num_of_streams(timeline, stream_count_dict)
+
+        # Add format thread meta data.
+        self._format_meta_data_list.extend(self._timeline_meta)
+        self._timeline_meta = self._format_meta_data_list
+
+        # Update timeline summary info
+        self._timeline_summary['num_of_streams'] += len(stream_count_dict.keys())
+
+    def load_cpu_op_data(self):
+        """Load cpu operator data from file"""
+        op_file_path = self._get_and_validate_path(
+            self._output_op_execute_time_file_path)
+        timeline_list = []
+        if not os.path.exists(op_file_path):
+            logger.info("No cpu operator info.")
+            return timeline_list
+        timeline_list = self._load_op_data(op_file_path)
+        factor_ms_to_us = 1e-3
+        for time_item in timeline_list:
+            time_item[self._duration_idx] = float(time_item[self._duration_idx]) / factor_ms_to_us
+
+        return timeline_list
+
+    def _get_and_validate_path(self, file_name):
+        """Generate op or activity file path from file name, and validate this path."""
+        file_path = os.path.join(
+            self._profiling_dir,
+            file_name.format(self._device_id)
+        )
+        file_path = validate_and_normalize_path(file_path)
+
+        return file_path
+
+    def _load_op_data(self, op_file_path):
+        """Load operator data from file"""
+        op_timeline_list = []
+        try:
+            with open(op_file_path, 'r') as f_obj:
+                for line in f_obj:
+                    self._timeline_summary['num_of_ops'] += 1
+                    op_list = line.strip('\n').strip().split(';')
+                    time_arr = op_list[-1]
+                    time_arr = time_arr.split(" ")
+                    for time in time_arr:
+                        time = time.split(",")
+                        if len(time) == 3:
+                            # for time value is [start_timestamp, duration, tid]
+                            # line_list[1] would be like "HostCpuOps" + str(tid)
+                            line_list = op_list[:1] + [op_list[1] + str(time[-1])] + time[:-1]
+                        else:
+                            # for time value is [start_timestamp, duration]
+                            line_list = op_list[:2] + time
+                        op_timeline_list.append(line_list)
+        except (IOError, OSError) as err:
+            logger.critical('Error occurred when load operator timeline data intermediate file: %s', err)
+            raise ProfilerIOException()
+
+        return op_timeline_list
+
+    def _load_timeline_data(self):
+        """Load timeline data from file."""
+        timeline_list = self.load_cpu_op_data()
+
+        timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
+        self._max_scope_name_num = self._get_max_scope_name_num(timeline_list)
+        self._timeline_summary['max_scope_name_num'] = self._max_scope_name_num
+
+        # Generate step time.
+        factor_start_time_uint_to_duration = 1e-3
+        self._set_step_start_and_end_op_name(timeline_list)
+
+        step_time_list = self._get_step_time_list(timeline_list, factor_start_time_uint_to_duration)
+
+        # Add merge compute time and free time
+        merge_compute_timeline = self._get_merged_time_list(
+            timeline_list, False, "computation_op", factor_start_time_uint_to_duration)[2]
+        free_time_timeline = self._get_merged_time_list(
+            timeline_list, True, "free_time", factor_start_time_uint_to_duration)[1]
+
+        # Add Scope Name.
+        default_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Default",
+                                                                      factor_start_time_uint_to_duration)
+        gradient_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Gradients",
+                                                                       factor_start_time_uint_to_duration)
+        recompute_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "recompute_Default",
+                                                                        factor_start_time_uint_to_duration)
+        timeline_list.extend(default_scope_name_time_list)
+        timeline_list.extend(gradient_scope_name_time_list)
+        timeline_list.extend(recompute_scope_name_time_list)
+        timeline_list.extend(step_time_list)
+
+        timeline_list.sort(key=lambda x: (float(x[self._start_time_idx]), x[self._tid_idx]))
+        timeline_list.sort(key=lambda x: float(x[2]))
+        timeline_list.extend(merge_compute_timeline)
+        timeline_list.extend(free_time_timeline)
+
+        return timeline_list
+
+    def _parse_timeline_data(self, timeline, min_cycle_counter):
+        """Parse timeline data."""
+        # factor to convert the time unit of start_time(ts) from 1ns to 1us for timeline display
+        factor = 1000
+        op_meta = TimelineContainer(timeline)
+        timeline_info = {}
+        timeline_info['name'] = op_meta.op_name.split('/')[-1]
+        timeline_info['ph'] = 'X'
+        timeline_info['tid'] = op_meta.stream_id
+        timeline_info['ts'] = (op_meta.start_time - min_cycle_counter) / factor
+        dur = op_meta.duration
+        timeline_info['dur'] = dur
+        timeline_info['pid'] = int(self._device_id)
+        if op_meta.stream_id == "Scope Name":
+            # remove the level of scope name which has a format like "0-conv2-Conv2d".
+            timeline_info['name'] = "-".join(op_meta.op_name.split('-')[1:])
+            timeline_info['scope_level'] = int(op_meta.op_name.split('-')[0])
+        elif self._host_cpu_op_label == op_meta.stream_id[:len(self._host_cpu_op_label)]:
+            timeline_info['pid'] = self._HOST_CPU_PID
+
+        if len(timeline) == 5:
+            # len(timeline) == 5 refers to analyse data.
+            timeline_info["pid"] = op_meta.pid
+        elif op_meta.stream_id not in ["Scope Name", "Steps"]:
+            # Update total time of operator execution.
+            self._timeline_summary['total_time'] += dur / factor
+            self._timeline_summary['op_exe_times'] += 1
+
+        self._update_format_meta_data(timeline_info)
+        self._timeline_meta.append(timeline_info)