mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/profiler/parser/msadvisor_parser.py

@@ -92,8 +92,10 @@ class MsadvisorParser:
         finally:
             pass
 
-        op = {"name": name, "pid": pid, "ts": ts, "dur": dur,
-              "args": {"Task Type": task_type, "Stream Id": sid, "Task Id": tid}, "ph": "X"}
+        op = {
+            "name": name, "pid": pid, "ts": ts, "dur": dur,
+            "args": {"Task Type": task_type, "Stream Id": sid, "Task Id": tid}, "ph": "X"
+        }
         return op
 
     @staticmethod
@@ -127,8 +129,10 @@ class MsadvisorParser:
         finally:
             pass
 
-        op = {"name": name, "pid": pid, "ts": ts, "dur": dur,
-              "args": {"Task Type": task_type, "Stream Id": sid, "Task Id": tid}, "ph": "X"}
+        op = {
+            "name": name, "pid": pid, "ts": ts, "dur": dur,
+            "args": {"Task Type": task_type, "Stream Id": sid, "Task Id": tid}, "ph": "X"
+        }
         return op
 
     def get_input_file(self):

mindspore/profiler/parser/op_intermediate_parser.py

@@ -15,6 +15,7 @@
 """Op intermediate files parser."""
 import csv
 import os
+import stat
 from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
     ProfilerIOException
 from mindspore import log as logger
@@ -97,7 +98,7 @@ class OPIntermediateParser:
 
         op_op_file_path = os.path.join(self._profiling_dir,
                                        self._file_name_op_intermediate_detail.format(self._rank_id))
-        with os.fdopen(os.open(op_op_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660), 'w') as op_file:
+        with os.fdopen(os.open(op_op_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as op_file:
             csv_writer = csv.writer(op_file)
             csv_writer.writerow(self._op_intermediate_op_header)
 
@@ -106,6 +107,7 @@ class OPIntermediateParser:
                     op_name, round(op_name_time_info[1] / op_name_time_info[0], self._ms_decimal_digits)
                 ]
                 csv_writer.writerow(op_info)
+        os.chmod(op_op_file_path, stat.S_IREAD | stat.S_IWRITE)
 
     def parser_pynative_op_type(self):
         """Parse pynative op intermediate type."""
@@ -134,7 +136,7 @@ class OPIntermediateParser:
 
         op_type_file_path = os.path.join(self._profiling_dir,
                                          self._file_name_op_intermediate_type.format(self._rank_id))
-        with os.fdopen(os.open(op_type_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660), 'w') as type_file:
+        with os.fdopen(os.open(op_type_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as type_file:
             csv_writer = csv.writer(type_file)
             csv_writer.writerow(self._op_intermediate_type_header)
 
@@ -144,3 +146,4 @@ class OPIntermediateParser:
                     round((op_type_time_info[1] / sum_avg_time) * 100, self._percent_decimal_digits)
                 ]
                 csv_writer.writerow(type_info)
+        os.chmod(op_type_file_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/optime_parser.py

@@ -205,7 +205,7 @@ class OPComputeTimeParser:
 
         # write to file
         try:
-            with open(file_path, 'w') as f_obj:
+            with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f_obj:
                 f_obj.write(TIMELINE_FILE_COLUMN_TITLE + '\n')
                 for timeline in timeline_data:
                     timeline = [str(item) for item in timeline]

mindspore/profiler/parser/profiler_info.py

@@ -41,12 +41,11 @@ class ProfilerInfo:
         ProfilerInfo._file_name = ProfilerInfo._file_name.format(rank_id)
 
     @staticmethod
-    def set_parallel_info(parallel_mode="", pipeline_stage_num=1, pipeline_stage_id=0):
+    def set_parallel_info(parallel_mode="", stage_num=1):
         """Set parallel info include parallel_mode, pipeline_stage_num and pipeline_stage_id."""
         info = dict()
         info["parallel_mode"] = parallel_mode
-        info["pipeline_stage_num"] = pipeline_stage_num
-        info["pipeline_stage_id"] = pipeline_stage_id
+        info["stage_num"] = stage_num
         ProfilerInfo._profiler_info_dict.update(info)
 
     @staticmethod
@@ -102,7 +101,7 @@ class ProfilerInfo:
         """Save the profiler info to file."""
         ProfilerInfo._file_path = os.path.join(output_path, ProfilerInfo._file_name)
         ProfilerInfo._file_path = validate_and_normalize_path(ProfilerInfo._file_path)
-        with os.fdopen(os.open(ProfilerInfo._file_path,
-                               os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o660), 'w') as json_file:
+        with os.fdopen(os.open(ProfilerInfo._file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
+                       'w') as json_file:
             json.dump(ProfilerInfo._profiler_info_dict, json_file)
         os.chmod(ProfilerInfo._file_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/step_trace_parser.py

@@ -217,12 +217,8 @@ class BaseStepTraceParser:
         average_data = [0] * len(self._header)
         if result_size >= 1:
             for row_info in self._result:
-                average_data = [
-                    Decimal(i) + Decimal(j) for i, j in zip(row_info, average_data)
-                ]
-            average_data = [
-                round(item / result_size) for item in average_data
-            ]
+                average_data = [Decimal(i) + Decimal(j) for i, j in zip(row_info, average_data)]
+            average_data = [round(item / result_size) for item in average_data]
             # change step num info in average_data to None
             step_num_index = self._header.index('step_num')
             average_data[step_num_index] = '-'
@@ -236,7 +232,8 @@ class BaseStepTraceParser:
         if not self._header:
             return
         try:
-            with open(self._output_path, 'w') as file_handle:
+            with os.fdopen(os.open(self._output_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
+                           'w') as file_handle:
                 csv_writer = csv.writer(file_handle)
                 if not self._is_training_mode:
                     self._header[fp_duration] = 'fp'
@@ -250,7 +247,7 @@ class BaseStepTraceParser:
             os.chmod(self._output_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             log.warning('Failed to save step trace raw info. %s', err)
-            raise ProfilerIOException
+            raise ProfilerIOException from err
 
 
 class GpuStepTraceParser(BaseStepTraceParser):
@@ -292,7 +289,7 @@ class GpuStepTraceParser(BaseStepTraceParser):
                 self.get_fp_bp(f_obj, all_step_fp, all_step_bp)
         except (IOError, OSError) as err:
             log.warning(f'Failed to read {self._source_file_path}', err)
-            raise ProfilerIOException
+            raise ProfilerIOException from err
 
         for fp_name, bp_name in zip(all_step_fp, all_step_bp):
             if self._is_training_mode:
@@ -307,7 +304,7 @@ class GpuStepTraceParser(BaseStepTraceParser):
             all_step_points.append(points)
 
         try:
-            with open(output_path, 'w') as json_file:
+            with os.fdopen(os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
                 if self._is_gpu_kernel_async_launch:
                     json.dump(all_step_points, json_file)
                 else:
@@ -315,7 +312,7 @@ class GpuStepTraceParser(BaseStepTraceParser):
             os.chmod(output_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err:
             log.warning('Failed to save point info. %s', err)
-            raise ProfilerIOException
+            raise ProfilerIOException from err
 
         return all_step_points[0]
 
@@ -355,7 +352,7 @@ class GpuStepTraceParser(BaseStepTraceParser):
                 step_trace_info_all.insert(iter_start, iter_start_info)
         except (IOError, OSError) as err:
             log.warning(f'Failed to read {source_file}', err)
-            raise ProfilerIOException
+            raise ProfilerIOException from err
         finally:
             pass
 
@@ -427,7 +424,7 @@ class GpuStepTraceParser(BaseStepTraceParser):
 
         except (IOError, OSError) as err:
             log.warning(f'Failed to read {self._source_file_path}', err)
-            raise ProfilerIOException
+            raise ProfilerIOException from err
 
         self._record_average_info()
         log.info("Finish to parse step trace file.")
@@ -533,7 +530,7 @@ class AscendStepTraceParser(BaseStepTraceParser):
         if os.path.exists(output_path):
             return points
         try:
-            with open(output_path, 'w') as json_file:
+            with os.fdopen(os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
                 json.dump(points, json_file)
             os.chmod(output_path, stat.S_IREAD | stat.S_IWRITE)
         except (IOError, OSError) as err: