mindspore 2.2.14__cp38-cp38-manylinux1_x86_64.whl → 2.3.0rc2__cp38-cp38-manylinux1_x86_64.whl

mindspore/profiler/parser/ascend_msprof_generator.py

@@ -14,232 +14,31 @@
 # ============================================================================
 """msprof data generate model"""
 import csv
-import fnmatch
-import os
-
+import glob
 import numpy as np
-from mindspore import log as logger
-
-
-class AscendMsprofDataGeneratorOld:
-    """Generate ascend data from files."""
-
-    def __init__(self, source_path):
-        self.source_path = source_path
-        self.op_summary = None
-        self.op_statistic = None
-        self.steptrace = []
-
-        self.invalid_index = 1000
-        self.op_summary_basis_name = {
-            'Model ID': {'index': self.invalid_index, 'dtype': ('Model ID', int)},
-            'Task ID': {'index': self.invalid_index, 'dtype': ('Task ID', int)},
-            'Stream ID': {'index': self.invalid_index, 'dtype': ('Stream ID', int)},
-            'Op Name': {'index': self.invalid_index, 'dtype': ('Op Name', object)},
-            'OP Type': {'index': self.invalid_index, 'dtype': ('Op Type', object)},
-            'Task Type': {'index': self.invalid_index, 'dtype': ('Task Type', object)},
-            'Task Start Time(us)': {'index': self.invalid_index, 'dtype': ('Task Start Time', float)},
-            'Task Duration(us)': {'index': self.invalid_index, 'dtype': ('Task Duration', float)},
-            'Task Wait Time(us)': {'index': self.invalid_index, 'dtype': ('Task Wait Time', float)},
-            'Input Shapes': {'index': self.invalid_index, 'dtype': ('Input Shapes', object)},
-            'Input Data Types': {'index': self.invalid_index, 'dtype': ('Input Data Types', object)},
-            'Input Formats': {'index': self.invalid_index, 'dtype': ('Input Formats', object)},
-            'Output Shapes': {'index': self.invalid_index, 'dtype': ('Output Shapes', object)},
-            'Output Data Types': {'index': self.invalid_index, 'dtype': ('Output Data Types', object)},
-            'Output Formats': {'index': self.invalid_index, 'dtype': ('Output Formats', object)},
-        }
-        self.op_summaryA_extend_name = {
-            'vector_fops': {'index': self.invalid_index, 'dtype': ('vector_fops', float)},
-            'cube_fops': {'index': self.invalid_index, 'dtype': ('cube_fops', float)},
-        }
-
-        self.op_summaryB_extend_name = {
-            'aiv_vector_fops': {'index': self.invalid_index, 'dtype': ('vector_fops', float)},
-            'aic_cube_fops': {'index': self.invalid_index, 'dtype': ('cube_fops', float)},
-        }
-        self.op_summary_name = None
-
-        self.op_statistic_name = {
-            'OP Type': {'index': self.invalid_index, 'dtype': ('Op Type', object)},
-            'Count': {'index': self.invalid_index, 'dtype': ('Count', int)},
-            'Total Time(us)': {'index': self.invalid_index, 'dtype': ('Total Time', float)},
-        }
-
-        self.steptrace_name = {
-            'Iteration ID': {'index': self.invalid_index, 'dtype': ('Iteration ID', int)},
-            'FP Start(us)': {'index': self.invalid_index, 'dtype': ('FP Start', float)},
-            'BP End(us)': {'index': self.invalid_index, 'dtype': ('BP End', float)},
-            'Iteration End(us)': {'index': self.invalid_index, 'dtype': ('Iteration End', float)},
-            'Iteration Time(us)': {'index': self.invalid_index, 'dtype': ('Iteration Time', float)},
-            'FP to BP Time(us)': {'index': self.invalid_index, 'dtype': ('FP to BP Time', float)},
-            'Iteration Refresh(us)': {'index': self.invalid_index, 'dtype': ('Iteration Refresh', float)},
-            'Data Aug Bound(us)': {'index': self.invalid_index, 'dtype': ('Data Aug Bound', float)},
-            'Model ID': {'index': self.invalid_index, 'dtype': ('Model ID', int)},
-        }
-
-    @staticmethod
-    def find_files(directory, pattern):
-        """Find files with feature 'pattern' from the directory"""
-
-        for root, _, files in os.walk(directory):
-            files.sort(key=lambda x: os.path.getctime(os.path.join(directory, x)))
-            for basename in files:
-                if fnmatch.fnmatch(basename, pattern):
-                    filename = os.path.join(root, basename)
-                    yield filename
-
-    def link_index_with_name(self, header, name_dict):
-        """link index with row name"""
-        for index, value in enumerate(header):
-            if value in name_dict:
-                name_dict[value]['index'] = index
-        for value in name_dict.values():
-            if value['index'] == self.invalid_index:
-                return False
-        return True
-
-    def parse(self):
-        """read msprof data generate DataFrame data"""
-        self._read_op_summary()
-
-        self._read_op_statistic()
-
-        self._read_steptrace()
-
-        return self.op_summary, self.op_statistic, self.steptrace
-
-    def _read_op_summary(self):
-        """read op summary to memory"""
-        op_summary = []
-        for file in self.find_files(self.source_path, "op_summary*.csv"):
-            with open(file, newline='') as csvfile:
-                iteration = int(file.split('_')[-1].split('.')[0])
-                reader = csv.reader(csvfile, delimiter=',', quotechar='"')
-                header = next(reader)
-                flag = self.link_index_with_name(header, self.op_summary_basis_name)
-                if not flag:
-                    raise RuntimeError("Read op summary failed. The file is missing basic fields.")
-                extend_flag_A = self.link_index_with_name(header, self.op_summaryA_extend_name)
-                extend_flag_B = self.link_index_with_name(header, self.op_summaryB_extend_name)
-                if extend_flag_A:
-                    self.op_summary_name = {**self.op_summary_basis_name, **self.op_summaryA_extend_name}
-                elif extend_flag_B:
-                    self.op_summary_name = {**self.op_summary_basis_name, **self.op_summaryB_extend_name}
-                else:
-                    self.op_summary_name = self.op_summary_basis_name
-                self.op_summary_name['Iteration ID'] = {'index': -1, 'dtype': ('Iteration ID', object)}
-                for row in reader:
-                    try:
-                        row = [row[index.get('index')] for index in self.op_summary_name.values()]
-                        row[self.op_summary_name['Iteration ID']['index']] = iteration
-                        row = ['0' if i == 'N/A' else i for i in row]
-                        row += ['0.000']  # Add one column for Task Start Time(us)
-                        op_summary.append(tuple(row))
-                    except IndexError:
-                        logger.warning(f"Fail to read{file}. Will ignore this file and continue reading")
-
-        op_summary_dt = np.dtype([value['dtype'] for value in self.op_summary_name.values()])
-
-        for i in range(0, len(op_summary)):
-            if len(op_summary[i]) < len(op_summary_dt):
-                new_raw = [j for j in op_summary[i]]
-                new_raw.extend([0 for _ in range(len(op_summary_dt) - len(op_summary[i]))])
-                op_summary[i] = tuple(new_raw)
-        self.op_summary = np.array(op_summary, dtype=op_summary_dt)
-        self.op_summary['Task Start Time'] = self.op_summary['Task Start Time'] * 1e-3
-        self.op_summary['Task Duration'] = self.op_summary['Task Duration'] * 1e-3
-        self.op_summary['Task Wait Time'] = self.op_summary['Task Wait Time'] * 1e-3
-
-    def _read_op_statistic(self):
-        """read op statistic to memory"""
-        op_statistic = []
-        for file in self.find_files(self.source_path, "op_statistic*.csv"):
-            with open(file, newline='') as csvfile:
-                reader = csv.reader(csvfile, delimiter=',', quotechar='"')
-                header = next(reader)
-                flag = self.link_index_with_name(header, self.op_statistic_name)
-                if not flag:
-                    raise RuntimeError("Read op summary failed. The file is missing basic fields.")
-                for row in reader:
-                    row = [row[index.get('index')] for index in self.op_statistic_name.values()]
-                    row = ['0' if i == 'N/A' else i for i in row]
-                    op_statistic.append(tuple(row))
-
-        op_statistic_dt = np.dtype([value['dtype'] for value in self.op_statistic_name.values()])
-        self.op_statistic = np.array(op_statistic, dtype=op_statistic_dt)
-        self.op_statistic['Total Time'] = self.op_statistic['Total Time'] * 1e-3
-
-    def _read_steptrace(self):
-        """read steptrace to memory"""
-        steptrace = []
-        header = []
-        for file in self.find_files(self.source_path, "step_trace*.csv"):
-            with open(file, newline='') as csvfile:
-                reader = csv.reader(csvfile, delimiter=',', quotechar='"')
-                header = next(reader)
-                flag = self.link_index_with_name(header, self.steptrace_name)
-                if not flag:
-                    raise RuntimeError("Read op summary failed. The file is missing basic fields.")
-                for row in reader:
-                    rows = [row[index.get('index')] for index in self.steptrace_name.values()]
-                    if row[9:]:
-                        rows.extend(row[9:len(header)])
-                    if len(rows) < len(header):
-                        rows.extend([0 for _ in range(len(header) - len(rows))])
-                    rows = ['0' if i == 'N/A' else i for i in rows]
-                    steptrace.append(tuple(rows))
-            break
-
-        hccl_data = self.op_summary[(self.op_summary['Task Type'] == 'HCCL') & (self.op_summary['Iteration ID'] == 1)][
-            ['Task ID', 'Stream ID', 'Op Name']]
-
-        index = len(self.steptrace_name)
-        for name in hccl_data:
-            if index >= len(header):
-                break
-            name = f"stream_{name['Stream ID']}_{name['Task ID']}_{name['Op Name']}"
-            self.steptrace_name[name] = {'index': index, 'dtype': (name, float)}
-            index += 1
-            self.steptrace_name[f'{name} duration'] = {'index': index, 'dtype': (f'{name} duration', float)}
-            index += 1
-
-        for i in range(len(self.steptrace_name), len(header), 2):
-            name = f'hccl_{i}'
-            self.steptrace_name[name] = {'index': i, 'dtype': (name, float)}
-            self.steptrace_name[f'{name} duration'] = {'index': i + 1, 'dtype': (f'{name} duration', float)}
-
-        steptrace_dt = np.dtype([value['dtype'] for value in self.steptrace_name.values()])
-
-        self.steptrace = np.array(steptrace, dtype=steptrace_dt)
-        self.steptrace['FP Start'] = self.steptrace['FP Start'] * 1e-3
-        self.steptrace['BP End'] = self.steptrace['BP End'] * 1e-3
-        self.steptrace['Iteration End'] = self.steptrace['Iteration End'] * 1e-3
-        self.steptrace['Iteration Time'] = self.steptrace['Iteration Time'] * 1e-3
-        self.steptrace['FP to BP Time'] = self.steptrace['FP to BP Time'] * 1e-3
-        self.steptrace['Iteration Refresh'] = self.steptrace['Iteration Refresh'] * 1e-3
-        self.steptrace['Data Aug Bound'] = self.steptrace['Data Aug Bound'] * 1e-3
-
-        for name in self.steptrace.dtype.names[9:]:
-            self.steptrace[name] = self.steptrace[name] * 1e-3
+from mindspore.profiler.common.util import get_newest_file
 
 
 class AscendMsprofDataGenerator:
     """Generate ascend data from files."""
 
-    def __init__(self, source_path):
-        self.source_path = source_path
+    _mindspore_model_id = 4294967295
+
+    def __init__(self, mindstudio_profiler_output):
+        self.mindstudio_profiler_output = mindstudio_profiler_output
         self.op_summary = None
         self.op_statistic = None
         self.steptrace = []
+        self.steptrace_model = []
 
         self.op_summary_type = [
-            ('Model ID', int),
+            ('Model ID', float),
             ('Task ID', int),
             ('Stream ID', int),
             ('Op Name', object),
             ('Op Type', object),
             ('Task Type', object),
-            ('Task Start Time', float),
+            ('Task Start Time', object),
             ('Task Duration', float),
             ('Task Wait Time', float),
             ('Input Shapes', object),
@@ -247,7 +46,8 @@ class AscendMsprofDataGenerator:
             ('Input Formats', object),
             ('Output Shapes', object),
             ('Output Data Types', object),
-            ('Output Formats', object)
+            ('Output Formats', object),
+            ('Task Start Time(us)', object)
         ]
 
         self.op_statistic_type = [
@@ -265,20 +65,9 @@ class AscendMsprofDataGenerator:
             ('FP to BP Time', float),
             ('Iteration Refresh', float),
             ('Data Aug Bound', float),
-            ('Model ID', int),
+            ('Model ID', float),
         ]
 
-    @staticmethod
-    def find_files(directory, pattern):
-        """Find files with feature 'pattern' from the directory"""
-
-        for root, _, files in os.walk(directory):
-            files.sort(key=lambda x: os.path.getctime(os.path.join(directory, x)))
-            for basename in files:
-                if fnmatch.fnmatch(basename, pattern):
-                    filename = os.path.join(root, basename)
-                    yield filename
-
     def parse(self):
         """read msprof data generate DataFrame data"""
         self._read_op_summary()
@@ -287,77 +76,76 @@ class AscendMsprofDataGenerator:
 
         self._read_steptrace()
 
-        return self.op_summary, self.op_statistic, self.steptrace
-
-    def _read_op_summary(self):
-        """read op summary to memory"""
-        op_summary = []
-        for file in self.find_files(self.source_path, "op_summary*.csv"):
-            with open(file, newline='') as csvfile:
-                reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
-                for row in reader:
-                    vector_fops = row.get('vector_fops', None)
-                    cube_fops = row.get('cube_fops', None)
-                    aiv_vector_fops = row.get('aiv_vector_fops', None)
-                    aic_cube_fops = row.get('aic_cube_fops', None)
+        self.steptrace_model = self.steptrace[self.steptrace['Model ID'] == self._mindspore_model_id]
 
-                    new_row = [
-                        row.get('Model ID'),
-                        row.get('Task ID'),
-                        row.get('Stream ID'),
-                        row.get('Op Name'),
-                        row.get('OP Type'),
-                        row.get('Task Type'),
-                        row.get('Task Start Time(us)'),
-                        row.get('Task Duration(us)'),
-                        row.get('Task Wait Time(us)'),
-                        row.get('Input Shapes'),
-                        row.get('Input Data Types'),
-                        row.get('Input Formats'),
-                        row.get('Output Shapes'),
-                        row.get('Output Data Types'),
-                        row.get('Output Formats')
-                    ]
+        self.steptrace = self.steptrace[self.steptrace['Model ID'] != self._mindspore_model_id]
 
-                    if vector_fops is not None and cube_fops is not None:
-                        new_row.append(vector_fops)
-                        new_row.append(cube_fops)
+        result = (self.op_summary, self.op_statistic, self.steptrace, self.steptrace_model)
 
-                    elif aic_cube_fops is not None and aiv_vector_fops is not None:
-                        new_row.append(aiv_vector_fops)
-                        new_row.append(aic_cube_fops)
+        return result
 
-                    new_row = tuple(['0' if d == 'N/A' else d for d in new_row])
-                    op_summary.append(new_row)
-            break
+    def _read_op_summary(self):
+        """read op summary to memory"""
+        op_summary = []
+        op_summary_name = fr'{self.mindstudio_profiler_output}/op_summary_*.csv'
+        op_summary_files = glob.glob(op_summary_name)
+        if not op_summary_files:
+            return
+        op_summary_file = get_newest_file(op_summary_files)[0]
+        with open(op_summary_file, newline='') as csvfile:
+            reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
+            for row in reader:
+                vector_fops = row.get('vector_fops', None)
+                cube_fops = row.get('cube_fops', None)
+                aiv_vector_fops = row.get('aiv_vector_fops', None)
+                aic_cube_fops = row.get('aic_cube_fops', None)
+
+                new_row = [row.get('Model ID'), row.get('Task ID'), row.get('Stream ID'), row.get('Op Name'),
+                           row.get('OP Type'), row.get('Task Type'), row.get('Task Start Time(us)'),
+                           row.get('Task Duration(us)'), row.get('Task Wait Time(us)'), row.get('Input Shapes'),
+                           row.get('Input Data Types'), row.get('Input Formats'), row.get('Output Shapes'),
+                           row.get('Output Data Types'), row.get('Output Formats'), '0.000']
+                if vector_fops is not None and cube_fops is not None:
+                    new_row.append(vector_fops)
+                    new_row.append(cube_fops)
+                elif aiv_vector_fops is not None and aic_cube_fops is not None:
+                    new_row.append(aiv_vector_fops)
+                    new_row.append(aic_cube_fops)
+                op_summary.append(tuple(['0' if d == 'N/A' else d for d in new_row]))
 
         if op_summary and len(op_summary[0]) > len(self.op_summary_type):
             self.op_summary_type.extend([
                 ('vector_fops', float),
                 ('cube_fops', float)
             ])
+
         op_summary_dt = np.dtype(self.op_summary_type)
 
         self.op_summary = np.array(op_summary, dtype=op_summary_dt)
-        self.op_summary['Task Start Time'] *= 1e-3
-        self.op_summary['Task Duration'] *= 1e-3
-        self.op_summary['Task Wait Time'] *= 1e-3
+        high_acc_time = self.op_summary['Task Start Time'].copy()
+        self.op_summary['Task Start Time(us)'] = high_acc_time
+        self.op_summary['Task Start Time'] = self.op_summary['Task Start Time'].astype(float) * 1e-3
+        self.op_summary['Task Duration'] = self.op_summary['Task Duration'] * 1e-3
+        self.op_summary['Task Wait Time'] = self.op_summary['Task Wait Time'] * 1e-3
 
     def _read_op_statistic(self):
         """read op statistic to memory"""
         op_statistic = []
-        for file in self.find_files(self.source_path, "op_statistic*.csv"):
-            with open(file, newline='') as csvfile:
-                reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
-                for row in reader:
-                    new_row = (
-                        row.get('OP Type'),
-                        row.get('Count'),
-                        row.get('Total Time(us)'),
-                    )
-                    new_row = tuple(['0' if d == 'N/A' else d for d in new_row])
-                    op_statistic.append(new_row)
-            break
+        op_statistic_name = fr'{self.mindstudio_profiler_output}/op_statistic_*.csv'
+        op_statistic_files = glob.glob(op_statistic_name)
+        if not op_statistic_files:
+            return
+        op_statistic_file = get_newest_file(op_statistic_files)[0]
+        with open(op_statistic_file, newline='') as csvfile:
+            reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
+            for row in reader:
+                new_row = (
+                    row.get('OP Type'),
+                    row.get('Count'),
+                    row.get('Total Time(us)'),
+                )
+                new_row = tuple(['0' if d == 'N/A' else d for d in new_row])
+                op_statistic.append(new_row)
         if not op_statistic:
             return
         op_statistic_dt = np.dtype(self.op_statistic_type)
@@ -366,9 +154,11 @@ class AscendMsprofDataGenerator:
 
     def _read_steptrace(self):
         """read steptrace to memory"""
-        steptrace = []
-        for file in self.find_files(self.source_path, "step_trace*.csv"):
-            with open(file, newline='') as csvfile:
+        step_trace = []
+        step_trace_name = fr'{self.mindstudio_profiler_output}/step_trace_*.csv'
+        step_trace_file_list = get_newest_file(glob.glob(step_trace_name))
+        for step_trace_file in step_trace_file_list:
+            with open(step_trace_file, newline='') as csvfile:
                 reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
                 for row in reader:
                     new_row = [
@@ -382,12 +172,12 @@ class AscendMsprofDataGenerator:
                         row.get('Data Aug Bound(us)'),
                         row.get('Model ID'),
                     ]
-                    new_row = ['0' if i == 'N/A' else i for i in new_row]
-                    steptrace.append(tuple(new_row))
+                    step_trace.append(tuple(['0' if i == 'N/A' else i for i in new_row]))
+            break
 
         steptrace_dt = np.dtype(self.steptrace_type)
 
-        self.steptrace = np.array(steptrace, dtype=steptrace_dt)
+        self.steptrace = np.array(step_trace, dtype=steptrace_dt)
         self.steptrace['FP Start'] = self.steptrace['FP Start'] * 1e-3
         self.steptrace['BP End'] = self.steptrace['BP End'] * 1e-3
         self.steptrace['Iteration End'] = self.steptrace['Iteration End'] * 1e-3