mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/profiler/parser/ascend_timeline_generator.py

@@ -15,13 +15,14 @@
 """The integrator for integrating parsed profiling files."""
 import os
 
+import numpy as np
 from mindspore import log as logger
 from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
-from mindspore.profiler.parser.container import TimelineContainer
-from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParser
 from mindspore.profiler.parser.base_timeline_generator import BaseTimelineGenerator
-from mindspore.profiler.parser.integrator import DeviceTarget
+from mindspore.profiler.parser.container import TimelineContainer
 from mindspore.profiler.parser.cpu_gpu_timeline_generator import CpuTimelineGenerator
+from mindspore.profiler.parser.integrator import DeviceTarget
+from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParser
 
 
 class AscendTimelineGenerator(BaseTimelineGenerator):
@@ -39,6 +40,17 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         self._display_filename = self._display_filename.format(rank_id)
         self._timeline_summary_filename = self._timeline_summary_filename.format(rank_id)
 
+        self.step_time_list_df = np.dtype(
+            [('Iteration ID', object), ('Steps', object), ('Iteration Start', float), ('Iteration Time', float)])
+
+        self.aicpu_time_list_dt = np.dtype(
+            [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float),
+             ('pid', int)])
+
+        self.communication_info_dt = np.dtype(
+            [('Op Name', object), ('Stream ID', int), ('Task Start Time', float), ('Task Duration', float),
+             ('pid', int)])
+
     @staticmethod
     def _get_all_reduce_names(communication_info):
         names = []
@@ -49,27 +61,48 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
                 names.append(all_reduce_name)
         return names
 
-    def init_timeline(self, communication_info, framework_info, aicpu_info, min_cycle_counter, source_path):
+    def init_timeline(self, op_summary, steptrace):
         """
         Init timeline metadata, adding all collected info.
 
         Args:
-            communication_info (list[list]): The metadata of communication operator.
-            framework_info (dict): The framework metadata.
-            aicpu_info (dict): The metadata of AI CPU operator.
-            min_cycle_counter (float): The minimum cycle counter of the timeline.
-            source_path (str): The source of file.
+            op_summary: op data
+            steptrace: step data
         """
-        if min_cycle_counter == float('inf'):
-            min_cycle_counter = 0
 
         logger.info('Initiating timeline...')
-        timeline_list = []
-        timeline_list.extend(self._get_op_timeline(communication_info, source_path))
+
+        timeline_list = op_summary[~np.isin(op_summary['Task Type'], ['AI_CPU', 'HCCL'])][
+            ['Op Name', 'Stream ID', 'Task Start Time', 'Task Duration']]
+
+        timeline_list = timeline_list.tolist()
+        cpu_timeline_generator = CpuTimelineGenerator(self._profiling_dir, self._rank_id, self._model)
+        cpu_timeline_list = cpu_timeline_generator.get_timeline_data()
+        if cpu_timeline_list:
+            timeline_list.extend(cpu_timeline_list)
+            timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
+        min_cycle_counter = 0
+        if timeline_list:
+            min_cycle_counter = timeline_list[0][2]
 
         # Generate step time.
         self._set_step_start_and_end_op_name(timeline_list)
-        step_time_list = self._get_step_time_list(timeline_list)
+
+        if not isinstance(steptrace, np.ndarray) or steptrace.shape[0] == 0 or not steptrace.tolist():
+            iteration_time = op_summary[-1]['Task Start Time'] - op_summary[0]['Task Start Time'] + op_summary[-1][
+                'Task Duration'] + op_summary[-1]['Task Wait Time']
+            step_time_list = [['1', 'Steps', op_summary[0]['Task Start Time'], iteration_time]]
+        else:
+            step_time_list = np.empty((len(steptrace),), dtype=self.step_time_list_df)
+            step_time_list['Iteration ID'] = \
+                np.char.add("Model ID: ",
+                            np.char.add(steptrace['Model ID'].astype(str),
+                                        np.char.add(" Iteration ID: ",
+                                                    steptrace['Iteration ID'].astype(str))))
+            step_time_list['Steps'] = 'Steps'
+            step_time_list['Iteration Start'] = steptrace['Iteration End'] - steptrace['Iteration Time']
+            step_time_list['Iteration Time'] = steptrace['Iteration Time']
+            step_time_list = step_time_list.tolist()
 
         # Add Scope Name.
         default_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "Default")
@@ -77,17 +110,32 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         recompute_scope_name_time_list = self._get_scope_name_time_list(timeline_list, "recompute_Default")
 
         # Add AI CPU data into timeline temp list and sort by start time.
-        aicpu_data = aicpu_info.get('info')
-        if aicpu_data:
-            timeline_list.extend(aicpu_data)
-            self._timeline_summary['op_exe_times'] += aicpu_info.get('op_exe_times', 0)
-            self._timeline_summary['num_of_streams'] += aicpu_info.get('num_of_streams', 0)
-            self._timeline_summary['num_of_ops'] += aicpu_info.get('num_of_ops', 0)
-            self._timeline_summary['total_time'] += aicpu_info.get('total_time', 0)
 
+        aicpu_op = op_summary[op_summary['Task Type'] == 'AI_CPU']
+        if aicpu_op.size:
+            aicpu_time_list = np.empty((len(aicpu_op),), dtype=self.aicpu_time_list_dt)
+            aicpu_time_list['Op Name'] = aicpu_op['Op Name']
+            aicpu_time_list['Stream ID'] = aicpu_op['Stream ID']
+            aicpu_time_list['Task Start Time'] = aicpu_op['Task Start Time']
+            aicpu_time_list['Task Duration'] = aicpu_op['Task Duration'] + aicpu_op['Task Wait Time']
+            aicpu_time_list['pid'] = 9000
+            aicpu_time_list = aicpu_time_list.tolist()
+            timeline_list.extend(aicpu_time_list)
         timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
 
         # Add AllReduce info to timeline temp list and sort by start time.
+        hccl_op = op_summary[op_summary['Task Type'] == 'HCCL']
+        if hccl_op.size:
+            communication_info = np.empty((len(hccl_op,)), dtype=self.communication_info_dt)
+            communication_info['Op Name'] = hccl_op['Op Name']
+            communication_info['Stream ID'] = hccl_op['Stream ID']
+            communication_info['Task Start Time'] = hccl_op['Task Start Time']
+            communication_info['Task Duration'] = hccl_op['Task Duration']
+            communication_info['pid'] = 10000
+            communication_info = communication_info.tolist()
+            communication_info.sort(key=lambda x: float(x[self._start_time_idx]))
+        else:
+            communication_info = []
         if communication_info:
             logger.debug('AllReduce info found. Start adding info into timeline...')
             cluster_related_timeline = self._get_cluster_timeline(
@@ -104,24 +152,24 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         timeline_list.sort(key=lambda x: float(x[self._start_time_idx]))
 
         # Init a dict for counting the num of streams.
-        stream_count_dict = {}
         for timeline in timeline_list:
             self._parse_timeline_data(timeline, min_cycle_counter)
-            # Updating the collection of streams.
-            if len(timeline) == 4:
-                self._update_num_of_streams(timeline, stream_count_dict)
 
         # Add format thread meta data.
         self._format_meta_data_list.extend(self._timeline_meta)
         self._timeline_meta = self._format_meta_data_list
-        # Get framework metadata.
-        # The length of list is the number of operators.
-        self._timeline_summary['num_of_ops'] += len(framework_info.get('object'))
-        self._add_framework_info(framework_info.get('object'))
-        logger.info('Finished adding info into timeline...')
 
         # Update timeline summary info
-        self._timeline_summary['num_of_streams'] += len(stream_count_dict.keys())
+        timeline_summary = op_summary[np.isin(op_summary['Task Type'], ['AI_CORE', 'AI_CPU', 'HCCL'])][[
+            'Op Name', 'Stream ID', 'Task Duration']]
+        self._timeline_summary['total_time'] = np.sum(timeline_summary['Task Duration'])
+        self._timeline_summary['num_of_streams'] = int(
+            len(np.unique(timeline_summary['Stream ID'], return_counts=True)[0]))
+        self._timeline_summary['num_of_ops'] = int(len(np.unique(timeline_summary['Op Name'], return_counts=True)[0]))
+        self._timeline_summary['op_exe_times'] = int(len(timeline_summary))
+        self._timeline_summary['max_scope_name_num'] = int(np.max(
+            [len(x) for x in np.char.split(timeline_summary['Op Name'].astype(str), sep='/')]))
+        logger.info('Finished adding info into timeline...')
 
     def init_pynative_timeline(self):
         """Init timeline for pynative model."""
@@ -207,21 +255,31 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         """Synchronize the timestamp from host to device."""
         host_start_file_path = os.path.join(source_path, f"host_start.log.{self._device_id}")
         dev_start_file_path = os.path.join(source_path, f"dev_start.log.{self._device_id}")
-
+        host_monotonic = 0
+        dev_cntvct = 0
         try:
             with open(host_start_file_path) as f_obj:
-                # f_obj.readlines()[2] stores host monotonic_raw time of start training.
-                host_monotonic = int(f_obj.readlines()[2].strip().split(':')[1])
-        except (IOError, OSError) as err:
-            logger.critical('Error occurred when read host_start.log: %s', err)
-            raise ProfilerIOException()
-        try:
+                lines = f_obj.readlines()
+            for line in lines:
+                info = line.strip().split(':')
+                if len(info) < 2 or info[0] != "clock_monotonic_raw":
+                    continue
+                host_monotonic = int(info[1])
+                break
+
             with open(dev_start_file_path) as f_obj:
-                # f_obj.readlines()[2] stores device cycle counter of start training.
-                dev_cntvct = int(f_obj.readlines()[2].strip().split(':')[1])
+                lines = f_obj.readlines()
+            for line in lines:
+                info = line.strip().split(':')
+                if len(info) < 2 or info[0] != "cntvct":
+                    continue
+                dev_cntvct = int(info[1])
+                break
         except (IOError, OSError) as err:
             logger.critical('Error occurred when read dev_start.log: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
+        if host_monotonic == 0 or dev_cntvct == 0:
+            logger.error('Error occurred when read host_monotonic or dev_cntvct time')
 
         factor = {"factor_ns_to_ms": 1e-6, "factor_ten_ns_to_ns": 10, "factor_ms_to_ns": 1e6}
         for idx, time_item in enumerate(timeline_list):
@@ -291,8 +349,6 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         that communication inside each stage slow down the training.
         """
         is_pipeline_parallel = False
-        time_info = {"receive_op_not_overlapped_timeline": [], "collective_comm_not_overlapped_timeline": [],
-                     "comm_not_overlapped_timeline": []}
         comm_timeline = self._get_merged_time_list(
             comm_info, display_name="communication"
         )
@@ -300,7 +356,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             aicore_info, get_interval_time=True
         )
         # Consider if the overlap will be 0 or not.
-        time_info["comm_not_overlapped_timeline"] = self._get_intersection_time(
+        comm_not_overlapped_timeline = self._get_intersection_time(
             aicore_timeline[0], comm_timeline[0]
         )
 
@@ -316,8 +372,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
         timeline_exclude_receive_op_interval = self._get_merged_time_list(
             receive_timeline[1], get_interval_time=True
         )[0]
-
-        time_info["receive_op_not_overlapped_timeline"] = self._get_intersection_time(
+        receive_op_not_overlapped_timeline = self._get_intersection_time(
             timeline_exclude_receive_op_interval, receive_op_merged_timeline
         )
 
@@ -326,7 +381,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             comm_info, "Receive-op"
         )[-1]
 
-        time_info["collective_comm_not_overlapped_timeline"] = self._get_intersection_time(
+        collective_comm_not_overlapped_timeline = self._get_intersection_time(
             aicore_timeline[0], self._get_merged_time_list(collective_comm_timeline)[0]
         )
 
@@ -335,12 +390,11 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             all_timeline, get_interval_time=True, display_name="free_time"
         )[1]
 
-        self._parse_cluster_metrices(step_info, time_info.get("receive_op_not_overlapped_timeline"),
-                                     time_info.get("comm_not_overlapped_timeline"),
-                                     time_info.get("collective_comm_not_overlapped_timeline"), is_pipeline_parallel)
+        self._parse_cluster_metrices(step_info, receive_op_not_overlapped_timeline, comm_not_overlapped_timeline,
+                                     collective_comm_not_overlapped_timeline, is_pipeline_parallel)
 
         res_timeline = []
-        res_timeline.extend(time_info.get("comm_not_overlapped_timeline"))
+        res_timeline.extend(comm_not_overlapped_timeline)
         res_timeline.extend(aicore_timeline[2])
         res_timeline.extend(comm_timeline[2])
         res_timeline.extend(free_timeline)
@@ -370,8 +424,10 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             except IndexError as err:
                 logger.error(err)
 
-        metrices_per_step_list = [time_info.get("computation_time"), comm_alone_time, time_info.get("stage_time"),
-                                  recieve_alone_time, collective_comm_alone_time]
+        metrices_per_step_list = [
+            time_info.get("computation_time"), comm_alone_time, time_info.get("stage_time"),
+            recieve_alone_time, collective_comm_alone_time
+        ]
         if step_num > 1:
             for metric in metrices_per_step_list:
                 metric.append(sum(metric[1:]) / (step_num - 1))
@@ -379,7 +435,7 @@ class AscendTimelineGenerator(BaseTimelineGenerator):
             self._write_cluster_metrices(metrices_per_step_list, is_pipeline_parallel, "Ascend", self._rank_id)
         except (IOError, OSError) as err:
             logger.warning(err)
-            raise ProfilerIOException
+            raise ProfilerIOException from err
 
     def _compute_time_inside_step(self, metric_timeline, step_time_list):
         """Compute per step time of metric_timeline."""

mindspore/profiler/parser/base_timeline_generator.py

@@ -198,7 +198,7 @@ class BaseTimelineGenerator:
         display_file_path = validate_and_normalize_path(display_file_path)
 
         try:
-            with os.fdopen(os.open(display_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660), 'w') as json_file:
+            with os.fdopen(os.open(display_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
                 json_file.write('[')
                 for _, item in enumerate(self._timeline_meta):
                     json.dump(item, json_file)
@@ -216,7 +216,7 @@ class BaseTimelineGenerator:
             return self._timeline_meta
         except (IOError, OSError) as err:
             logger.critical('Error occurred when write timeline display file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
     def write_timeline_summary(self):
         """Write timeline summary to json."""
@@ -229,11 +229,11 @@ class BaseTimelineGenerator:
 
         try:
             with os.fdopen(os.open(timeline_summary_file_path,
-                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660), 'w') as json_file:
+                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
                 json.dump(self._timeline_summary, json_file)
         except (IOError, OSError) as err:
             logger.critical('Error occurred when write timeline summary file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(timeline_summary_file_path):
             os.chmod(timeline_summary_file_path, stat.S_IREAD | stat.S_IWRITE)
 
@@ -438,6 +438,8 @@ class BaseTimelineGenerator:
         """Write cluster metric."""
         # Note that the feature of cluster bottleneck analyse is not supported in offline parse mode,
         # due to that parallel context is not set.
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            return
         parallel_mode, stage_num = BaseTimelineGenerator.get_parallel_context()
 
         unit = 1 if device_target == "Ascend" else 1e3
@@ -448,12 +450,14 @@ class BaseTimelineGenerator:
         )
         cluster_analyse_file_path = validate_and_normalize_path(cluster_analyse_file_path)
 
-        with os.fdopen(os.open(cluster_analyse_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660),
+        with os.fdopen(os.open(cluster_analyse_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
                        'w') as file_handle:
             csv_writer = csv.writer(file_handle)
             if is_pipeline_parallel:
-                header = ['computation_time', 'communication_alone_time', 'stage_time',
-                          'receive_alone_time', 'collective_communication_alone_time']
+                header = [
+                    'computation_time', 'communication_alone_time', 'stage_time',
+                    'receive_alone_time', 'collective_communication_alone_time'
+                ]
                 zip_metrices = zip(metrices[0], metrices[1], metrices[2], metrices[3], metrices[4])
             else:
                 header = ['computation_time', 'communication_alone_time']

mindspore/profiler/parser/cpu_gpu_timeline_generator.py

@@ -231,7 +231,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
                 gpu_start_time = int(lines[1].strip().split(':')[-1])
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when read {start_time_file_path}: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         time_diff = gpu_start_time - host_monotonic_start_time
         for idx, time_item in enumerate(timeline_list):
@@ -258,7 +258,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
                             communication_info.append(line_list)
         except (IOError, OSError) as err:
             logger.critical('Error occurred when load operator timeline data intermediate file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         return op_timeline_list, communication_info
 
@@ -324,7 +324,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
                     step_num += 1
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when read {step_trace_profiling_path}: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         return step_time_list
 
@@ -344,56 +344,56 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
         communication time between stages slow down the training. The value of t3 indicates the degree
         that communication inside each stage slow down the training.
         """
-        time_info = {"stage_time": [], "computation_time": [], "recieve_alone_time": [], "comm_alone_time": [],
-                     "collective_comm_alone_time": [], "comm_not_overlapped_timeline": [], "free_timeline": [],
-                     "collective_comm_not_overlapped_timeline": [], "comm_timeline": [], "compute_timeline": [],
-                     "receive_op_not_overlapped_timeline": [], "receive_op_timeline": [],
-                     "collective_comm_timeline": [], "receive_op_merged_timeline": [], "is_pipeline_parallel": False}
-        time_info["comm_timeline"] = self._get_merged_time_list(
+        time_info = {
+            "stage_time": [], "computation_time": [], "recieve_alone_time": [], "comm_alone_time": [],
+            "collective_comm_alone_time": []
+        }
+        is_pipeline_parallel = False
+        comm_timeline = self._get_merged_time_list(
             comm_info,
             display_name="communication",
             factor=1e-3
         )
         compute_op_timeline = timeline + activity_info
         compute_op_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
-        time_info["compute_timeline"] = self._get_merged_time_list(
+        compute_timeline = self._get_merged_time_list(
             compute_op_timeline,
             get_interval_time=True,
             factor=1e-3
         )
         # Consider if the overlap will be 0 or not.
-        time_info["comm_not_overlapped_timeline"] = self._get_intersection_time(
-            time_info.get("compute_timeline")[0],
-            time_info.get("comm_timeline")[0]
+        comm_not_overlapped_timeline = self._get_intersection_time(
+            compute_timeline[0],
+            comm_timeline[0]
         )
 
         # Process receive part.
         all_timeline = timeline + comm_info
         all_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
-        time_info["receive_op_timeline"] = self._produce_two_separated_timeline(
+        receive_op_timeline = self._produce_two_separated_timeline(
             all_timeline,
             "Receive-op"
         )[0]
-        if time_info.get("receive_op_timeline"):
-            time_info["is_pipeline_parallel"] = True
-        time_info["receive_op_merged_timeline"] = self._get_merged_time_list(time_info.get("receive_op_timeline"),
-                                                                             factor=1e-3)[0]
-
-        time_info["receive_op_not_overlapped_timeline"] = self._get_intersection_time(
-            time_info.get("compute_timeline")[0],
-            time_info.get("receive_op_merged_timeline"),
+        if receive_op_timeline:
+            is_pipeline_parallel = True
+        receive_op_merged_timeline = self._get_merged_time_list(receive_op_timeline,
+                                                                factor=1e-3)[0]
+
+        receive_op_not_overlapped_timeline = self._get_intersection_time(
+            compute_timeline[0],
+            receive_op_merged_timeline,
             display_name="receive_op_not_overlapped"
         )
 
         # Process collective communication part.
-        time_info["collective_comm_timeline"] = self._produce_two_separated_timeline(
+        collective_comm_timeline = self._produce_two_separated_timeline(
             comm_info,
             "Receive-op"
         )[-1]
-        collective_comm_merged_timeline = self._get_merged_time_list(time_info.get("collective_comm_timeline"),
+        collective_comm_merged_timeline = self._get_merged_time_list(collective_comm_timeline,
                                                                      factor=1e-3)[0]
-        time_info["collective_comm_not_overlapped_timeline"] = self._get_intersection_time(
-            time_info.get("compute_timeline")[0],
+        collective_comm_not_overlapped_timeline = self._get_intersection_time(
+            compute_timeline[0],
             collective_comm_merged_timeline,
             display_name="exclude_receive_op"
         )
@@ -401,7 +401,7 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
         # Generate free time that exclude computation and communication time.
         all_timeline = compute_op_timeline + comm_info
         all_timeline.sort(key=lambda x: float(x[self._start_time_idx]))
-        time_info["free_timeline"] = self._get_merged_time_list(
+        free_timeline = self._get_merged_time_list(
             all_timeline,
             get_interval_time=True,
             display_name="free_time",
@@ -409,18 +409,16 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
         )[1]
 
         # Compute these five metrics mentioned above per step.
-        time_info["recieve_alone_time"] = self._compute_time_inside_step(
-            time_info.get("receive_op_not_overlapped_timeline"), step_info)
-        time_info["comm_alone_time"] = self._compute_time_inside_step(time_info.get("comm_not_overlapped_timeline"),
-                                                                      step_info)
+        time_info["recieve_alone_time"] = self._compute_time_inside_step(receive_op_not_overlapped_timeline, step_info)
+        time_info["comm_alone_time"] = self._compute_time_inside_step(comm_not_overlapped_timeline, step_info)
         time_info["collective_comm_alone_time"] = self._compute_time_inside_step(
-            time_info.get("collective_comm_not_overlapped_timeline"),
+            collective_comm_not_overlapped_timeline,
             step_info
         )
         step_num = len(step_info)
         for step in range(step_num):
             try:
-                if time_info.get("is_pipeline_parallel"):
+                if is_pipeline_parallel:
                     time_info.get("stage_time").append(
                         step_info[step][self._duration_idx] - time_info.get("recieve_alone_time")[step]
                     )
@@ -433,24 +431,25 @@ class GpuTimelineGenerator(BaseTimelineGenerator):
             except IndexError as e:
                 logger.error(e)
 
-        metrices_per_step_list = [time_info.get("computation_time"), time_info.get("comm_alone_time"),
-                                  time_info.get("stage_time"), time_info.get("recieve_alone_time"),
-                                  time_info.get("collective_comm_alone_time")]
+        metrices_per_step_list = [
+            time_info.get("computation_time"), time_info.get("comm_alone_time"),
+            time_info.get("stage_time"), time_info.get("recieve_alone_time"),
+            time_info.get("collective_comm_alone_time")
+        ]
         if step_num > 1:
             for metric in metrices_per_step_list:
                 metric.append(sum(metric[1:]) / (step_num - 1))
         try:
-            self._write_cluster_metrices(metrices_per_step_list, time_info.get("is_pipeline_parallel"), "Gpu",
-                                         self._device_id)
-        except (IOError, OSError) as e:
-            logger.warning(e)
-            raise ProfilerIOException
+            self._write_cluster_metrices(metrices_per_step_list, is_pipeline_parallel, "Gpu", self._device_id)
+        except (IOError, OSError) as err:
+            logger.warning(err)
+            raise ProfilerIOException from err
 
         res_timeline = []
-        res_timeline.extend(time_info.get("comm_not_overlapped_timeline"))
-        res_timeline.extend(time_info.get("compute_timeline")[2])
-        res_timeline.extend(time_info.get("comm_timeline")[2])
-        res_timeline.extend(time_info.get("free_timeline"))
+        res_timeline.extend(comm_not_overlapped_timeline)
+        res_timeline.extend(compute_timeline[2])
+        res_timeline.extend(comm_timeline[2])
+        res_timeline.extend(free_timeline)
         return res_timeline
 
     def _compute_time_inside_step(self, metric_timeline, step_time_list):
@@ -609,7 +608,7 @@ class CpuTimelineGenerator(GpuTimelineGenerator):
                         op_timeline_list.append(line_list)
         except (IOError, OSError) as err:
             logger.critical('Error occurred when load operator timeline data intermediate file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         return op_timeline_list
 

mindspore/profiler/parser/flops_parser.py

@@ -181,7 +181,7 @@ class FlopsParser:
         sum_flops_utilization = 0.0
         # calculate the every step FLOPS utilization and the average values.
         utilization_save_filename = os.path.join(self._output_dir, self._flops_utilization_step_filename)
-        with open(utilization_save_filename, 'w') as f:
+        with os.fdopen(os.open(utilization_save_filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
             f.write("steps, FLOPS_Utilization %\n")
             for i, x in enumerate(op_all_step_comp):
                 current_utilization = x[0] / x[1] * 1e9 / peak_flops * 100
@@ -190,6 +190,8 @@ class FlopsParser:
                 f.write(",")
                 f.write(str(current_utilization))
                 f.write("\n")
+        os.chmod(utilization_save_filename, stat.S_IREAD | stat.S_IWRITE)
+
         if len(op_all_step_comp) >= 1:
             self._flops_summary['FLOPS_Utilization'] = sum_flops_utilization / len(op_all_step_comp)
         else:
@@ -222,7 +224,7 @@ class FlopsParser:
                 all_log_struct = aicore_file.read(self.AICORE_LOG_SIZE * read_count)
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when read {aicore_file_path} file: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         return read_count, all_log_struct
 
@@ -246,7 +248,7 @@ class FlopsParser:
                 peak_flops = device_frequency * 1e6 * ai_core_num * 4096 * 2
         except (IOError, OSError, json.JSONDecodeError) as err:
             logger.critical(f'Error occurred when read {info_json_file_path} file: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         return peak_flops
 
@@ -304,7 +306,7 @@ class FlopsParser:
                     op_avg_time_dict[op_name] = avg_time
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when read {optime_file_path} file: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         return op_avg_time_dict
 
@@ -317,7 +319,7 @@ class FlopsParser:
         For op_name like "Default/network", the "network" will be renamed as "network(Default)".
         For op_name like "recompute_Default/network", "network" --> "network(recompute_Default)".
         For op_name like "Gradients/network", "network" --> "network(Gradients)".
-        For op_name like "Gradients/recompute_Default/network"，"network" --> "network(recompute_Gradients)".
+        For op_name like "Gradients/recompute_Default/network", "network" --> "network(recompute_Gradients)".
         """
         # Only extracts the scope name, remove the operator name.
         scope_list = op_name.split('/')[:-1]
@@ -387,7 +389,7 @@ class FlopsParser:
         output_flops_scope_file_path = join_file_path(self._flops_scope_filename)
 
         try:
-            with open(output_file_path, 'w') as f:
+            with os.fdopen(os.open(output_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
                 header = "op_full_name, MFLOPs(10^6), GFLOPS(10^9), FLOPS utilization(%) \n"
                 f.writelines(header)
                 for op_flops in op_flops_list:
@@ -396,25 +398,27 @@ class FlopsParser:
             os.chmod(output_file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when writing {output_file_path} file: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         for key in self._flops_summary:
             self._flops_summary[key] = round(self._flops_summary[key], 3)
         try:
-            with open(output_summary_file_path, 'w') as json_file:
+            with os.fdopen(os.open(output_summary_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
+                           'w') as json_file:
                 json.dump(self._flops_summary, json_file)
             os.chmod(output_summary_file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when write {output_summary_file_path} file: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
         try:
-            with open(output_flops_scope_file_path, 'w') as json_file:
+            with os.fdopen(os.open(output_flops_scope_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
+                           'w') as json_file:
                 json.dump(self._flops_sankey_diagram, json_file)
             os.chmod(output_flops_scope_file_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             logger.critical(f'Error occurred when write {output_flops_scope_file_path} file: {err}')
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
 
     def _get_aicore_files(self, profiler_dir):
         """Get aicore files."""