mindspore 2.2.14__cp37-cp37m-manylinux1_x86_64.whl → 2.3.0rc1__cp37-cp37m-manylinux1_x86_64.whl

mindspore/profiler/parser/ascend_timeline_generator.py

@@ -14,70 +14,60 @@
 # ============================================================================
 """The integrator for integrating parsed profiling files."""
 
+import os.path
+import glob
+import json
+import stat
+from decimal import Decimal
 import numpy as np
 from mindspore import log as logger
 from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
 from mindspore.profiler.parser.base_timeline_generator import BaseTimelineGenerator
-from mindspore.profiler.parser.container import TimelineContainer
-from mindspore.profiler.parser.cpu_gpu_timeline_generator import CpuTimelineGenerator
 from mindspore.profiler.parser.integrator import DeviceTarget
+from mindspore.profiler.parser.ascend_analysis.fwk_cann_parser import FwkCANNParser
+from mindspore.profiler.common.util import get_newest_file
 
 
 class AscendTimelineGenerator(BaseTimelineGenerator):
     """Generate ascend Timeline data from file."""
-    _display_filename = 'ascend_timeline_display_{}.json'
+    _timeline_display_filename = 'ascend_timeline_display_{}.json'
     _timeline_summary_filename = 'ascend_timeline_summary_{}.json'
     _cluster_analyse_filename = 'ascend_cluster_analyse_{}_{}_{}_{}.csv'
+    top_scope_name = ('Default', 'Gradients', 'recompute_Default')
+    scope_index = 1
+    cpu_index = 2
 
-    def __init__(self, profiling_dir, device_id, rank_id, rank_size, model):
-        super().__init__(DeviceTarget.ASCEND.value, model)
+    def __init__(self, profiling_dir, source_path, mindstudio_profiler_output, rank_id, mode):
+        super().__init__(DeviceTarget.ASCEND.value, mode)
         self._profiling_dir = profiling_dir
-        self._device_id = device_id
+        self._source_path = source_path
+        self._mindstudio_profiler_output = mindstudio_profiler_output
         self._rank_id = rank_id
-        self._rank_size = rank_size
-        self._display_filename = self._display_filename.format(rank_id)
+        self._timeline_display_filename = self._timeline_display_filename.format(rank_id)
         self._timeline_summary_filename = self._timeline_summary_filename.format(rank_id)
+        self._timeline_data = []
 
         self.step_time_list_df = np.dtype(
             [('Iteration ID', object), ('Steps', object), ('Iteration Start', float), ('Iteration Time', float)])
 
         self.aicpu_time_list_dt = np.dtype(
-            [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float),
-             ('pid', int)])
+            [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float)])
 
-        self.communication_info_dt = np.dtype(
-            [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float),
-             ('pid', int)])
-
-    def init_timeline(self, op_summary, steptrace, pretty=False):
+    def parse_cluster_data(self, op_summary, steptrace):
         """
-        Init timeline metadata, adding all collected info.
+        Parse cluster data and timeline summary data.
 
         Args:
             op_summary: op data
             steptrace: step data
-            pretty: whether to format json file
         """
 
-        logger.info('Initiating timeline...')
-        self._pretty = pretty
-        if op_summary.size < 1:
-            return
+        logger.info('parse cluster data...')
+
         timeline_list = op_summary[~np.isin(op_summary['Task Type'], ['AI_CPU', 'HCCL'])][
             ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
 
         timeline_list = timeline_list.tolist()
-        cpu_timeline_generator = CpuTimelineGenerator(self._profiling_dir, self._rank_id, self._model)
-        cpu_timeline_list = cpu_timeline_generator.get_timeline_data()
-        if cpu_timeline_list:
-            timeline_list.extend(cpu_timeline_list)
-            timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
-        min_cycle_counter = 0
-        if timeline_list:
-            min_cycle_counter = timeline_list[0][2]
-
-        # Generate step time.
-        self._set_step_start_and_end_op_name(timeline_list)
 
         if not isinstance(steptrace, np.ndarray) or steptrace.shape[0] == 0 or not steptrace.tolist():
             iteration_time = op_summary[-1]['Task Start Time'] - op_summary[0]['Task Start Time'] + op_summary[-1][
@@ -95,13 +85,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             step_time_list['Iteration Time'] = steptrace['Iteration Time']
             step_time_list = step_time_list.tolist()
 
-        # Add Scope Name.
-        default_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Default")
-        gradient_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Gradients")
-        recompute_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "recompute_Default")
-
         # Add AI CPU data into timeline temp list and sort by start time.
-
         aicpu_op = op_summary[op_summary['Task Type'] == 'AI_CPU']
         if aicpu_op.size:
             aicpu_time_list = np.empty((len(aicpu_op),), dtype=self.aicpu_time_list_dt)
@@ -109,46 +93,18 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             aicpu_time_list['Stream ID'] = aicpu_op['Stream ID']
             aicpu_time_list['Task Start Time'] = aicpu_op['Task Start Time']
             aicpu_time_list['Task Duration'] = aicpu_op['Task Duration'] + aicpu_op['Task Wait Time']
-            aicpu_time_list['pid'] = 9000
             aicpu_time_list = aicpu_time_list.tolist()
             timeline_list.extend(aicpu_time_list)
         timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
 
         # Add AllReduce info to timeline temp list and sort by start time.
-        hccl_op = op_summary[op_summary['Task Type'] == 'HCCL']
-        if hccl_op.size:
-            communication_info = np.empty((len(hccl_op,)), dtype=self.communication_info_dt)
-            communication_info['Op Name'] = hccl_op['Op Name']
-            communication_info['Stream ID'] = hccl_op['Stream ID']
-            communication_info['Task Start Time'] = hccl_op['Task Start Time']
-            communication_info['Task Duration'] = hccl_op['Task Duration']
-            communication_info['pid'] = 10000
+        communication_info = op_summary[op_summary['Task Type'] == 'HCCL'][
+            ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
+        if communication_info.size:
             communication_info = communication_info.tolist()
             communication_info.sort(key=lambda x: float(x[self._start_time_idx]))
-        else:
-            communication_info = []
-        if communication_info:
             logger.debug('AllReduce info found. Start adding info into timeline...')
-            cluster_related_timeline = self._get_cluster_timeline(
-                timeline_list, communication_info, step_time_list)
-            timeline_list.extend(cluster_related_timeline)
-            timeline_list.extend(communication_info)
-            timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
-
-        # Add step time and scope name info.
-        timeline_list.extend(step_time_list)
-        timeline_list.extend(default_scope_name_time_list)
-        timeline_list.extend(recompute_scope_name_time_list)
-        timeline_list.extend(gradient_scope_name_time_list)
-        timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
-
-        # Init a dict for counting the num of streams.
-        for timeline in timeline_list:
-            self._parse_timeline_data(timeline, min_cycle_counter)
-
-        # Add format thread meta data.
-        self._format_meta_data_list.extend(self._timeline_meta)
-        self._timeline_meta = self._format_meta_data_list
+            self._get_cluster_timeline(timeline_list, communication_info, step_time_list)
 
         # Update timeline summary info
         timeline_summary = op_summary[['Op Name', 'Stream ID', 'Task Duration']]
@@ -162,37 +118,269 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
                 [len(x) for x in np.char.split(timeline_summary['Op Name'].astype(str), sep='/')]))
         else:
             self._timeline_summary['max_scope_name_num'] = 0
-        logger.info('Finished adding info into timeline...')
-
-    def _parse_timeline_data(self, timeline, min_cycle_counter):
-        """Parse timeline data."""
-        # factor to convert the time unit from 1ms to 1us for timeline display
-        factor = 1000
-        op_meta = TimelineContainer(timeline)
-        timeline_dict = {}
-        timeline_dict['name'] = op_meta.op_name.split('/')[-1]
-        timeline_dict['ph'] = 'X'
-        timeline_dict['tid'] = op_meta.stream_id
-        timeline_dict['ts'] = (op_meta.start_time - min_cycle_counter) * factor
-        dur = op_meta.duration * factor
-        timeline_dict['dur'] = dur
-        if op_meta.pid is None:
-            timeline_dict['pid'] = int(self._device_id)
-            # Update total time of operator execution.
-            if op_meta.stream_id not in ["Steps", "Scope Name"]:
-                self._timeline_summary['total_time'] += op_meta.duration
-        else:  # AllReduce and AI CPU pid
-            timeline_dict['pid'] = op_meta.pid
-
-        if op_meta.stream_id == "Scope Name":
-            # remove the level of scope name which has a format like "0-conv2-Conv2d".
-            timeline_dict['name'] = "-".join(op_meta.op_name.split('-')[1:])
-            timeline_dict['scope_level'] = int(op_meta.op_name.split('-')[0])
-        elif op_meta.stream_id[:len(self._host_cpu_op_label)] == self._host_cpu_op_label:
-            timeline_dict['pid'] = self._HOST_CPU_PID
-
-        self._update_format_meta_data(timeline_dict)
-        self._timeline_meta.append(timeline_dict)
+        logger.info('Finished parse cluster data...')
+
+    def write_timeline_display(self):
+        """Write timeline display"""
+        logger.info('Writing timeline file...')
+        display_file_path = os.path.join(
+            self._profiling_dir,
+            self._timeline_display_filename
+        )
+        try:
+            with os.fdopen(os.open(display_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as fw:
+                json.dump(self._timeline_data, fw, indent=self.indent)
+            os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE)
+            logger.info('Finished file writing!')
+        except (IOError, OSError) as err:
+            logger.critical('Error occurred when write timeline display file: %s', err)
+            raise ProfilerIOException() from err
+
+    def parse_timeline_data(self, pretty=False):
+        """
+        Get detail timeline
+        Returns:
+            json, the content of timeline data.
+        """
+        logger.info("Start parse timeline data...")
+        self._pretty = pretty
+        timeline_data = []
+
+        all_scope_data = []  # 所有带scope的算子
+
+        # get msprof data
+        msprof_file_name = fr'{self._mindstudio_profiler_output}/msprof_*.json'
+        file_list_msprof = glob.glob(msprof_file_name)
+        if not file_list_msprof:
+            logger.error('Could not find msprof_*.json file in %s', self._mindstudio_profiler_output)
+        else:
+            msprof_timeline = self._parse_msprof_data(get_newest_file(file_list_msprof))
+            timeline_data.extend(msprof_timeline)
+
+        # get Ascend Hardware for scope
+        scope_data = self._parse_ascend_hardware_scope(msprof_timeline)
+        all_scope_data.extend(scope_data)
+
+        # get cpu op
+        cpu_op_file_name = fr'{self._profiling_dir}/cpu_op_execute_timestamp_{self._rank_id}.txt'
+        file_list = glob.glob(cpu_op_file_name)
+        if not file_list:
+            logger.warning('Could not find cpu op file in %s', self._profiling_dir)
+        else:
+            cpu_timeline, scope_data = self.parse_cpu_timeline(file_list)
+            timeline_data.extend(cpu_timeline)
+            all_scope_data.extend(scope_data)
+
+        # parse scope info
+        scope_timeline = self._parse_scope_info(all_scope_data)
+        timeline_data.extend(scope_timeline)
+
+        oprange_name = self._op_range_name.format(self._rank_id)
+        fwk_file_path = fr'{self._profiling_dir}/{self._framework_dir}/{oprange_name}'
+        if os.path.exists(fwk_file_path):
+            # It is faster not to submit to the pool
+            msprof_side_data = msprof_timeline
+            result = self._parse_fwk_device_data(msprof_side_data)
+            timeline_data.extend(result.get("trace_data", []))
+            self._kernel_events = result.get("kernels", [])
+
+        logger.info("All timeline data parse complete.")
+        self._timeline_data = timeline_data
+        return timeline_data
+
+    def parse_cpu_timeline(self, file_list):
+        """Load cpu operator data from file"""
+        ms_to_us = 1e3
+        ps_to_ns = 1e-3
+        new_pid = int(f'{self.cpu_index}{self._rank_id}')
+        process_list = [{"name": "process_name",
+                         "pid": new_pid,
+                         "args": {
+                             "name": f"CPU OP Rank{self._rank_id}"
+                         },
+                         "ph": "M"
+                         }, {"name": "process_sort_index", "pid": new_pid,
+                             "args": {"sort_index": self.cpu_index}, "ph": "M"}
+                        ]
+        tid_set = set()
+        thread_list = []
+        new_timeline = []
+        scope_data = []
+        try:
+            flags = os.O_RDONLY
+            for file_path in file_list:
+                with os.fdopen(os.open(file_path, flags, 0o400), 'r') as fr:
+                    for line in fr:
+                        op_list = line.strip().split(';')
+                        op_full_name = op_list[0]
+                        time_arr = op_list[-1]
+                        time_arr = time_arr.split(" ")
+                        for time in time_arr:
+                            ts, dur, tid = time.split(",")
+                            ts = Decimal(ts).quantize(Decimal('0.000')) * Decimal(ps_to_ns).quantize(Decimal('0.000'))
+
+                            if op_full_name and op_full_name.startswith(self.top_scope_name):
+                                te = ts + Decimal(dur).quantize(Decimal('0.000'))
+                                scope_data.append((op_full_name.split('/')[:-1], ts, te))
+
+                            if int(tid) not in tid_set:
+                                tid_set.add(int(tid))
+                                thread_list.append({"name": "thread_name",
+                                                    "pid": new_pid,
+                                                    "tid": int(tid),
+                                                    "ph": "M",
+                                                    'args': {'name': f'thread {tid}'}
+                                                    })
+
+                            new_timeline.append({'name': op_list[0],
+                                                 'pid': new_pid,
+                                                 'tid': int(tid),
+                                                 'ph': 'X',
+                                                 'ts': str(ts),
+                                                 'dur': float(dur) * ms_to_us,
+                                                 'args':
+                                                     {'type': op_list[1]}
+                                                 })
+                break
+
+            return process_list + thread_list + new_timeline, scope_data
+
+        except (IOError, OSError, json.JSONDecodeError) as err:
+            logger.error('parse_cann_data failed! please theck. detail: %s', err)
+            return []
+
+    def _parse_fwk_device_data(self, cann_kernel_data):
+        """
+        Get framework op range trace data, flow events and hardware kernel events
+        """
+        fwkcann_parser = FwkCANNParser(self._source_path, cann_kernel_data, self._rank_id)
+        fwk_link_data = fwkcann_parser.generate_trace_data()
+        kernels = fwkcann_parser.kernels
+        result = {"trace_data": fwk_link_data, "kernels": kernels}
+        return result
+
+    def _parse_msprof_data(self, file_list):
+        """
+        parse msprof.json file
+        :param file_list:
+        :return:
+        """
+        flags = os.O_RDONLY
+        raw_data = []
+        try:
+            for file_path in file_list:
+                with os.fdopen(os.open(file_path, flags, 0o400), 'r') as fr:
+                    raw_data.extend(json.load(fr))
+
+            if not raw_data:
+                logger.error('Could not found msprof data in file list: %s .', file_list)
+
+            return raw_data
+
+        except (IOError, OSError, json.JSONDecodeError) as err:
+            logger.error('_parse_msprof_data failed! please theck. detail: %s', err)
+            return []
+
+    def _parse_ascend_hardware_scope(self, msprof_timeline):
+        """
+        parse ascend hardware scope
+        """
+        scope_data = []
+        try:
+
+            for event in msprof_timeline:
+                if event.get("ph") == "M":
+                    continue
+
+                op_full_name = event.get('name')
+                if op_full_name and op_full_name.startswith(self.top_scope_name):
+                    ts = Decimal(event.get('ts')).quantize(Decimal('0.000'))
+                    te = ts + Decimal(event.get('dur')).quantize(Decimal('0.000'))
+                    scope_data.append((op_full_name.split('/')[:-1], ts, te))
+
+            return scope_data
+
+        except (IOError, OSError, json.JSONDecodeError) as err:
+            logger.error('_parse_ascend_hardware_scope failed! please theck. detail: %s', err)
+            return []
+
+    def _parse_scope_info(self, scope_data):
+        """Parse scope info of op"""
+        if not scope_data:
+            return []
+        new_pid = int(f'{self.scope_index}{self._rank_id}')
+        scope_data.sort(key=lambda x: x[1])
+        process_list = [
+            {"name": "process_name",
+             "pid": new_pid,
+             "args": {
+                 "name": f"Scope Layer Rank{self._rank_id}"
+             },
+             "ph": "M"},
+            {"name": "process_sort_index",
+             "pid": new_pid,
+             "args": {"sort_index": self.scope_index},
+             "ph": "M"}
+        ]
+
+        new_events = []
+        layer_stack = []
+        for layer_name in scope_data[0][0]:
+            layer_stack.append([layer_name, scope_data[0][1], scope_data[0][2]])
+
+        for op in scope_data[1:]:
+            if op[1] < layer_stack[0][2]:
+                # 并行算子只保留前面的
+                continue
+            flag = True  # 判断上层是否合并， 上层不合并下层也不合并
+            for layer_depth, layer_name in enumerate(op[0]):
+                if layer_depth >= len(layer_stack):
+                    layer_stack.append([layer_name, op[1], op[2]])
+                else:
+                    if layer_stack[layer_depth][0] == layer_name and flag:
+                        layer_stack[layer_depth][2] = op[2]  # 合并
+                    else:
+                        ts = layer_stack[layer_depth][1]
+                        new_events.append({
+                            "name": layer_stack[layer_depth][0],
+                            "pid": new_pid,
+                            "tid": layer_depth,
+                            "ph": "X",
+                            "ts": str(ts),
+                            "dur": float(layer_stack[layer_depth][2] - layer_stack[layer_depth][1])
+                        })
+                        layer_stack[layer_depth] = [layer_name, op[1], op[2]]
+                        flag = False
+
+        thread_list = []
+        for index, layer in enumerate(layer_stack):
+            thread_list.extend([{
+                "name": "thread_name",
+                "pid": new_pid,
+                "tid": index,
+                "args": {
+                    "name": f"layer{index}"
+                },
+                "ph": "M"
+            }, {
+                "name": "thread_sort_index",
+                "pid": new_pid,
+                "tid": index,
+                "args": {"sort_index": index},
+                "ph": "M"
+            }])
+            if layer:
+                ts = layer[1]
+                new_events.append({
+                    "name": layer[0],
+                    "pid": new_pid,
+                    "tid": index,
+                    "ph": "X",
+                    "ts": str(ts),
+                    "dur": float(layer[2] - layer[1])
+                })
+
+        return process_list + thread_list + new_events
 
     def _produce_two_separated_timeline(self, timeline, op_name):
         """Produce two separated timeline based on op_name."""
@@ -258,22 +446,9 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             aicore_timeline[0], self._get_merged_time_list(collective_comm_timeline)[0]
         )
 
-        # Generate free time that exclude computation and communication time.
-        free_timeline = self._get_merged_time_list(
-            all_timeline, get_interval_time=True, display_name="free_time"
-        )[1]
-
         self._parse_cluster_metrices(step_info, receive_op_not_overlapped_timeline, comm_not_overlapped_timeline,
                                      collective_comm_not_overlapped_timeline, is_pipeline_parallel)
 
-        res_timeline = []
-        res_timeline.extend(comm_not_overlapped_timeline)
-        res_timeline.extend(aicore_timeline[2])
-        res_timeline.extend(comm_timeline[2])
-        res_timeline.extend(free_timeline)
-
-        return res_timeline
-
     def _parse_cluster_metrices(self, step_info, receive_op_not_overlapped_timeline, comm_not_overlapped_timeline,
                                 collective_comm_not_overlapped_timeline, is_pipeline_parallel):
         """Write the cluster metrices"""
@@ -304,6 +479,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         if step_num > 1:
             for metric in metrices_per_step_list:
                 metric.append(sum(metric[1:]) / (step_num - 1))
+
         try:
             self._write_cluster_metrices(metrices_per_step_list, is_pipeline_parallel, "Ascend", self._rank_id)
         except (IOError, OSError) as err:
@@ -312,10 +488,9 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
 
     def _compute_time_inside_step(self, metric_timeline, step_time_list):
         """Compute per step time of metric_timeline."""
-        per_step_time_list = [0 for i in range(len(step_time_list))]
+        per_step_time_list = [0 for _ in range(len(step_time_list))]
         step = 0
-        step_end_time = step_time_list[step][self._start_time_idx] + \
-                        step_time_list[step][self._duration_idx]
+        step_end_time = step_time_list[step][self._start_time_idx] + step_time_list[step][self._duration_idx]
         for time_item in metric_timeline:
             start_time = time_item[self._start_time_idx]
             if start_time > step_end_time:
@@ -325,8 +500,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
                                    "find the data length is more than step count, "
                                    "maybe current graph has multi sub graph, skip the last data.")
                     break
-                step_end_time = step_time_list[step][self._start_time_idx] + \
-                                step_time_list[step][self._duration_idx]
+                step_end_time = step_time_list[step][self._start_time_idx] + step_time_list[step][self._duration_idx]
             per_step_time_list[step] += time_item[self._duration_idx]
 
         return per_step_time_list
@@ -362,19 +536,3 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
                 first_list_idx += 1
 
         return intersection_segment_display_list
-
-    def _set_step_start_and_end_op_name(self, timeline_list):
-        """Set the start and end operator full name of each step."""
-        if not timeline_list or len(timeline_list) < 2:
-            return
-
-        start_op_idx = 0
-        self._step_end_op_name = timeline_list[-1][self._op_name_idx]
-        for i, timeline in enumerate(timeline_list):
-            if timeline[self._op_name_idx] == self._step_end_op_name:
-                start_op_idx = i + 1
-                break
-
-        if start_op_idx >= len(timeline_list):
-            start_op_idx = 0
-        self._step_start_op_name = timeline_list[start_op_idx][self._op_name_idx]

mindspore/profiler/parser/base_timeline_generator.py

@@ -88,6 +88,8 @@ class BaseTimelineGenerator:
     _op_name_list = []
     _device_target = DeviceTarget.ASCEND.value
     _model = context.GRAPH_MODE
+    _framework_dir = "FRAMEWORK"
+    _op_range_name = "op_range_{}"
 
     __col_names__ = ['op_name', 'stream_id', 'start_time', 'duration']
 
@@ -102,8 +104,12 @@ class BaseTimelineGenerator:
         self._model = model
         self._step_start_op_name = ""
         self._step_end_op_name = ""
+        self._kernel_events = []
         self._pretty = False
 
+    def get_kernel_event_list(self):
+        return self._kernel_events
+
     @property
     def indent(self):
         indent = 1 if self._pretty else None

mindspore/profiler/parser/framework_parser.py

@@ -512,7 +512,8 @@ class GpuFrameWorkParser:
                 line_info = line_info.strip(' ').strip('\n').split(',')
                 if not self.op_detail.get(line_info[2]):
                     # line_info[4]: op_occurrences, line_info[5]: op_detail_time(us), line_info[6]: op_avg_time(us);
-                    self.op_detail[line_info[2]] = [line_info[4], line_info[5], line_info[6], op_side]
+                    self.op_detail[line_info[2]] = [float(line_info[4]), float(line_info[5]),
+                                                    float(line_info[6]), op_side]
 
     def get_execute_times(self):
         """Get gpu operators execute times."""
@@ -833,7 +834,7 @@ class DynamicFrameWorkParser:
             for line_info in framework_info:
                 line_info = line_info.strip('\n').split(',')
                 op_name = line_info[3].split('/')[-1]
-                shape_info = ','.join(line_info[7:]).replace('"', '')
+                shape_info = ','.join(line_info[8:]).replace('"', '')
                 self._op_shape_info[op_name].append(shape_info)
 
     def _get_total_step_num(self, op_summary):

mindspore/profiler/parser/integrator.py

@@ -282,7 +282,7 @@ class Integrator:
             _ = next(csv_reader)
             for info in csv_reader:
                 framework_infos[info[3]] = [
-                    info[3], info[4], info[5], info[6], json.loads(info[7]) if info[7] else None
+                    info[3], info[5], info[6], info[7], json.loads(info[8]) if info[8] else None
                 ]
 
         with open(op_detail_file_path, 'r') as file:
@@ -290,6 +290,8 @@ class Integrator:
             _ = next(csv_reader)
             for info in csv_reader:
                 framework_info = framework_infos.get(info[0])
+                if not framework_info:
+                    continue
                 self._aicore_detail_data.append(
                     [
                         framework_info[1], framework_info[2], float(info[1]),

mindspore/profiler/parser/msadvisor_analyzer.py

@@ -31,7 +31,7 @@ class Msadvisor:
     """
     def __init__(self, job_id, rank_id, output_path, pretty=False):
         self._job_id, self._device_id = job_id.split("/")
-        self._rank_id = rank_id
+        self._rank_id = str(rank_id)
         self._output_path = output_path
         self._pretty = pretty
 

mindspore/profiler/parser/msadvisor_parser.py

@@ -39,7 +39,7 @@ class MsadvisorParser:
     def __init__(self, job_id, device_id, rank_id, output_path, pretty=False):
         self._job_id = job_id
         self._device_id = device_id
-        self._rank_id = rank_id
+        self._rank_id = str(rank_id)
         self._output_path = output_path
         self._aicore_path = ""
         self._aicpu_path = ""

mindspore/profiler/parser/profiler_info.py

@@ -95,6 +95,11 @@ class ProfilerInfo:
         """Set the graph id list."""
         ProfilerInfo._profiler_info_dict["all_export"] = flag
 
+    @staticmethod
+    def set_diff_time(diff_time):
+        """synchronize timestamps between different devices"""
+        ProfilerInfo._profiler_info_dict["diff_time"] = diff_time
+
     @staticmethod
     def set_graph_ids(graph_ids):
         """Set the graph id list."""