mindspore 2.1.0__cp38-none-any.whl → 2.2.0__cp38-none-any.whl

mindspore/profiler/common/util.py

@@ -23,6 +23,7 @@ import os
 import re
 import stat
 
+from mindspore import log as logger
 
 def to_int(param, param_name):
     """
@@ -80,9 +81,9 @@ def fwrite_format(output_data_path, data_source=None, is_print=False, is_start=F
                 if isinstance(raw_data, (list, tuple)):
                     raw_data = map(str, raw_data)
                     raw_data = " ".join(raw_data)
-                print(raw_data)
+                logger.info(raw_data)
         else:
-            print(data_source)
+            logger.info(data_source)
 
 
 def get_log_slice_id(file_name):

mindspore/profiler/envprofiling.py

@@ -68,7 +68,7 @@ def construct_profiling_options():
         logger.error(
             "The format of MS_PROFILER_OPTIONS is incorrect. "
             "The MS_PROFILER_OPTIONS parameter configuration may refer to "
-            "'https://www.mindspore.cn/mindinsight/docs/zh-CN/master/performance_profiling_ascend.html'."
+            "'https://www.mindspore.cn/mindinsight/docs/zh-CN/r2.2/performance_profiling_ascend.html'."
         )
         return error_config
     conbine_options = combine_profile_options(profiling_options)

mindspore/profiler/parser/aicpu_data_parser.py

@@ -72,9 +72,11 @@ class DataPreProcessParser:
     _source_file_target_old = 'DATA_PREPROCESS.dev.AICPU.'
     _source_file_target = 'DATA_PREPROCESS.AICPU.'
     _dst_file_title = 'title:DATA_PREPROCESS AICPU'
-    _dst_file_column_title = ['serial_number', 'node_type_name', 'total_time(ms)',
-                              'dispatch_time(ms)', 'execution_time(ms)', 'run_start',
-                              'run_end']
+    _dst_file_column_title = [
+        'serial_number', 'node_type_name', 'total_time(ms)',
+        'dispatch_time(ms)', 'execution_time(ms)', 'run_start',
+        'run_end'
+    ]
     _ms_unit = 1000
     _us_unit = 100  # Convert 10ns to 1us.
     _task_id_threshold = 65536

mindspore/profiler/parser/ascend_flops_generator.py

@@ -78,7 +78,7 @@ class AscendFlopsGenerator:
                 writer.writerows(self.flops.tolist())
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write flops file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(flops_path):
             os.chmod(flops_path, stat.S_IREAD | stat.S_IWRITE)
 
@@ -89,6 +89,6 @@ class AscendFlopsGenerator:
                 json.dump(self.flops_summary, json_file)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write step trace point info file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(flops_summary_path):
             os.chmod(flops_summary_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/ascend_fpbp_generator.py

@@ -71,6 +71,6 @@ class AscendFPBPGenerator:
                 json.dump(self.points, json_file)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write step trace point info file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(step_trace_point_info_path):
             os.chmod(step_trace_point_info_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/ascend_hccl_generator.py

@@ -99,7 +99,7 @@ class AscendHCCLGenerator:
     @staticmethod
     def _rdma_analyse(groupby_transport):
         """rdma analyse"""
-        thread_groups, _, _, _ = np.unique(groupby_transport['tid'])
+        thread_groups = np.unique(groupby_transport['tid'])
         thread_information = []
         for thread_index in thread_groups:
             groupby_thread = groupby_transport[groupby_transport['tid'] == thread_index]
@@ -107,7 +107,7 @@ class AscendHCCLGenerator:
             rdma_communication_size = 0
             rdma_communication_wait_time = 0
             start_index = 0
-            end_index = groupby_thread.size - 1
+            end_index = groupby_thread.size - 2
             while start_index < end_index:
                 first_task_type = groupby_thread[start_index]['task_type']
                 if first_task_type == 'RDMASend':
@@ -142,7 +142,8 @@ class AscendHCCLGenerator:
         file_list = find_files(self.root_path, "hccl_*.json")
 
         for hccl_file in file_list:
-            iteration_id = int(hccl_file.split('_')[-1].split(('.'))[0])
+            _, relative_path = os.path.split(hccl_file)
+            iteration_id = int(relative_path.split('_')[3])
             with open(hccl_file) as f:
                 _, hccl_detail_data = self._original_data_analyse(json.load(f))
                 raw = self._iteration_analyse(hccl_detail_data, iteration_id)
@@ -173,7 +174,7 @@ class AscendHCCLGenerator:
                 writer.writerows(self.hccl_raw)
         except (IOError, OSError) as err:
             logging.critical('Errot occurred when write aicore detail file: %s', err)
-            raise ProfilerIOException()
+            raise ProfilerIOException() from err
         if os.path.exists(hccl_raw_path):
             os.chmod(hccl_raw_path, stat.S_IREAD | stat.S_IWRITE)
 
@@ -206,8 +207,8 @@ class AscendHCCLGenerator:
                     tuple([name, pid, tid, ts, te, dur, ph, task_type, link_info, transport_type, size, -1]))
         hccl_data = np.array(target_data, dtype=self.hccl_data_df)
 
-        hccl_abstract_data = hccl_data[hccl_data['tid'] >= 100]
-        hccl_detail_data = hccl_data[hccl_data['tid'] < 100]
+        hccl_abstract_data = hccl_data[hccl_data['task_type'] == '']
+        hccl_detail_data = hccl_data[hccl_data['task_type'] != '']
 
         hccl_abstract_data = hccl_abstract_data[np.argsort(hccl_abstract_data['ts'])]
         hccl_detail_data = hccl_detail_data[np.argsort(hccl_detail_data['ts'])]
@@ -234,7 +235,7 @@ class AscendHCCLGenerator:
             transport_information = dict()
             for transport_index in transport_groups:
                 groupby_transport = groupby_link_info[groupby_link_info['transport_type'] == transport_index]
-                if transport_index == 'SDMA':
+                if transport_index == 'SDMA' and groupby_transport.size > 0:
                     groupby_sdma = \
                         groupby_transport[np.isin(groupby_transport['task_type'], ['Memcpy', 'Reduce Inline'])][
                             ['dur', 'size']]
@@ -243,7 +244,7 @@ class AscendHCCLGenerator:
                     sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
                         if sdma_communication_time != 0 else 0
                     transport_information['SDMA'] = [sdma_communication_time, sdma_communication_size, sdma_bandwidth]
-                elif transport_index == 'RDMA':
+                elif transport_index == 'RDMA' and groupby_transport.size > 0:
                     transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
             link_info_information[link_info_index] = transport_information
         return link_info_information
@@ -271,13 +272,17 @@ class AscendHCCLGenerator:
                         sdma_communication_size = np.sum(groupby_sdma['size']) * 1e-3
                         sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
                             if sdma_communication_time != 0 else 0
-                        transport_information['SDMA'] = [sdma_communication_time, sdma_communication_size,
-                                                         sdma_bandwidth]
+                        transport_information['SDMA'] = [
+                            sdma_communication_time, sdma_communication_size,
+                            sdma_bandwidth
+                        ]
                     elif transport_index == 'RDMA':
                         transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
                     link_info_information[link_info_index] = transport_information
                 communication_cost = np.sum(groupby_tag[groupby_tag['name'] != 'Notify_Wait']['dur'])
                 wait_cost = np.sum(groupby_tag[groupby_tag['name'] == 'Notify_Wait']['dur'])
-                tag_information[tag_index] = [str(iteration_index), communication_cost, wait_cost,
-                                              link_info_information]
+                tag_information[tag_index] = [
+                    str(iteration_index), communication_cost, wait_cost,
+                    link_info_information
+                ]
         return tag_information

mindspore/profiler/parser/ascend_msprof_exporter.py

@@ -14,14 +14,12 @@
 # ============================================================================
 """msprof PROF data export api file"""
 import os
+import shutil
+from collections import defaultdict
 from subprocess import CalledProcessError, TimeoutExpired
 from subprocess import Popen, PIPE
-from typing import List, Optional
-import json
-from json.decoder import JSONDecodeError
 import csv
-from mindspore import log as logger, context
-from mindspore.communication import get_rank
+from mindspore import log as logger
 from mindspore.profiler.common.util import get_file_path
 
 
@@ -30,7 +28,7 @@ class AscendMsprofExporter:
     msprof exporter. export cann edge profiling data.
 
     args:
-        prof_root_dir: the root path of PROF_* files
+       source_path: the root path of PROF_* files
 
     files under prof_root_dir is like:
         profiler/PROF_*/device_{id}/data/xxx
@@ -42,74 +40,37 @@ class AscendMsprofExporter:
         >> ms_exporter.export(start_time)
     """
 
-    _null_info = ""
-    _csv_header_model_id = "Model ID"
-    _csv_header_iter_id = "Iteration ID"
-    _profiling_prefix = "PROF"
+    _hiai_msprof_tail = "Ascend/latest/tools/profiler/bin"
+    _msprof_cmd = "msprof"
+    _ascend_mark = "Ascend"
     _summary_dir = "summary"
+    _timeline_dir = "timeline"
     _step_trace_mark = "step_trace"
     _op_summary_mark = "op_summary"
     _op_statistic_mark = "op_statistic"
-    _device_mark = "device"
-    _msprof_cmd = "msprof"
-    _info_prefix = "info.json"
-    _start_log = "start_info"
-    _rank_id_mark = "rank_id"
-    _dev_info = "DeviceInfo"
-    _dev_index = 0
-    _dev_id = "id"
-    _ascend_mark = "Ascend"
-    _hiai_msprof_tail = "Ascend/latest/tools/profiler/bin"
 
-    def __init__(self, prof_root_dir: str, time_out=3000) -> None:
-        self._prof_root_dir = prof_root_dir
-        self._start_time = 0
-        self._support_step_trace = True
-        self._prof_paths = []
-        self._output_path = None
-        self._device_path = None
-        self._model_ids = []
-        self._iter_ids = []
+    def __init__(self, source_path, time_out=3000):
         self._time_out = time_out
-        self._check_msprof_env()
+        self.source_path = source_path
+        self.prof_root_dir = os.path.abspath(os.path.join(self.source_path, os.path.pardir))
 
-    @staticmethod
-    def _check_readable(file_path: str):
-        """Check whether the file is readable"""
-        if not os.access(file_path, os.R_OK):
-            msg = "The file {} is not readable.".format(file_path)
-            raise PermissionError(msg)
-
-    @staticmethod
-    def _parse_start_info(input_file: str):
-        """Get profiler start time from start info file."""
-        start_time = -1
-        try:
-            with open(input_file, "r") as f:
-                start_time = json.load(f).get("collectionTimeBegin")
-        except (JSONDecodeError, FileNotFoundError, TypeError, PermissionError) as err:
-            logger.warning(err)
-        return int(start_time)
+        self._check_msprof_env()
 
-    def export(self, start_time=0, time_out=3000, support_step_trace=True):
+    def export(self, model_iteration_dict=None):
         """start_time is the time to collect PROF data"""
-        self._start_time = start_time
-        self._time_out = time_out
-        self._support_step_trace = support_step_trace
-        self._init_output_path()
-        if not self._output_path:
-            raise FileNotFoundError("Do not found valid profiling directory")
-        trace_file = self._get_device_trace_file(self._output_path, self._device_path)
-        if trace_file:
-            self._export_whole_prof(self._output_path, trace_file)
-        self._check_export_files(self._device_path, trace_file)
-
-    def get_job_dir(self):
-        """Return matched PROF directory path. Call this function after exporting profiling data."""
-        return self._output_path
-
-    def _run_cmd(self, cmd: List[str], raise_error=True):
-        """run msprof tool shell command"""
+
+        if not model_iteration_dict:
+            model_iteration_dict = self._generate_step_trace(self.prof_root_dir, self.source_path)
+
+        if model_iteration_dict:
+            for model_id, value in model_iteration_dict.items():
+                for iteration_id in value:
+                    msprof_export_cmd = self._msprof_command_generator(self.prof_root_dir, model_id, iteration_id)
+                    self._run_cmd(msprof_export_cmd)
+            self._check_export_files(self.source_path, model_iteration_dict)
+
+    def _run_cmd(self, cmd, raise_error=True):
+        """run shell command"""
         try:
             proc = Popen(cmd, stdout=PIPE, stderr=PIPE, text=True)
         except (FileNotFoundError, PermissionError, CalledProcessError) as exc:
@@ -119,119 +80,26 @@ class AscendMsprofExporter:
         except TimeoutExpired:
             proc.kill()
             msg = "The possible cause is that too much data is collected " \
-                "and the export time is too long."
+                  "and the export time is too long."
             logger.error(msg)
             raise TimeoutError(msg)
         logger.info(outs)
-        if raise_error and errs != self._null_info:
+        if raise_error and errs != "":
             raise RuntimeError(errs)
         return outs
 
-    def _export_helper(self, output, model_id=None, iter_id=None):
+    def _msprof_command_generator(self, output, model_id=None, iter_id=None):
         """msprof export helper"""
         export_cmd = [self._msprof_cmd, "--export=on", "--output={}".format(output)]
-        if model_id:
+        if isinstance(model_id, int) and model_id >= 0:
             export_cmd.append("--model-id={}".format(model_id))
-        if iter_id:
+        if isinstance(iter_id, int) and iter_id >= 0:
             export_cmd.append("--iteration-id={}".format(iter_id))
-        _ = self._run_cmd(export_cmd)
-
-    def _get_model_iter_ids(self, trace_file: str):
-        """Read the step_trace file and get all the model_ids and iteration_ids"""
-        if self._model_ids and self._iter_ids:
-            return
-        self._check_readable(trace_file)
-        with open(trace_file, "r") as f:
-            reader = csv.reader(f)
-            for idx, row in enumerate(reader):
-                if idx == 0:
-                    model_idx = row.index(self._csv_header_model_id)
-                    iter_idx = row.index(self._csv_header_iter_id)
-                else:
-                    self._model_ids.append(int(row[model_idx]))
-                    self._iter_ids.append(int(row[iter_idx]))
-
-    def _check_export_files(self, device_path: str, trace_file: Optional[str]):
-        """Check the existence of op_summary & op_statistic files."""
-        summary_path = os.path.join(device_path, self._summary_dir)
-        if not os.path.isdir(summary_path):
-            raise RuntimeError("Path {} is not a existing directory.".format(summary_path))
-
-        sumary_filess = os.listdir(summary_path)
-        op_summary = set()
-        op_statistic = set()
-
-        for summary_file in sumary_filess:
-            if summary_file.startswith(self._op_summary_mark):
-                op_summary.add(summary_file)
-            elif summary_file.startswith(self._op_statistic_mark):
-                op_statistic.add(summary_file)
-
-        if not op_summary:
-            raise RuntimeError("The op_summary file was not found, " \
-                               "perhaps the original data was not collected.")
-        if not op_statistic:
-            raise RuntimeError("The op_statistics file was not found, " \
-                               "perhaps the original data was not collected.")
-
-        if not trace_file:
-            return
-
-        dev_id = context.get_context("device_id")
-        self._get_model_iter_ids(trace_file)
-
-        for model_id, iter_id in zip(self._model_ids, self._iter_ids):
-            tag = "_{}_{}_{}.csv".format(dev_id, model_id, iter_id)
-            op_sum_file = self._op_summary_mark + tag
-            op_sta_file = self._op_statistic_mark + tag
-            if op_sum_file not in op_summary:
-                logger.warning("[Profiler]The file {} was not found, " \
-                               "perhaps the original data was not collected.".format(op_sum_file))
-            if op_sta_file not in op_statistic:
-                logger.warning("[Profiler]The file {} was not found, " \
-                               "perhaps the original data was not collected.".format(op_sta_file))
-
-        logger.info("Finish checking files.")
-
-    def _export_whole_prof(self, prof: str, trace_file: str):
-        """export all the data under PROF directory"""
-        self._get_model_iter_ids(trace_file)
-
-        for model_id, iter_id in zip(self._model_ids, self._iter_ids):
-            self._export_helper(prof, model_id, iter_id)
-
-    def _get_device_trace_file(self, prof_path: str, device_path: str):
-        """search the step trace csv file under device directory"""
-
-        summary_path = os.path.join(device_path, self._summary_dir)
-
-        if not os.path.exists(summary_path):
-            self._export_helper(output=prof_path)
-
-        if not os.path.isdir(summary_path):
-            msg = "Path {} is not a existing directory. Make sure there is " \
-                "valid profiling data directory!".format(summary_path)
-            raise FileNotFoundError(msg)
-
-        step_trace_file = get_file_path(summary_path, self._step_trace_mark)
-
-        if not step_trace_file and self._support_step_trace:
-            msg = "Do not found step trace csv file in {}.".format(self._output_path)
-            raise FileNotFoundError(msg)
-
-        return step_trace_file
+        return export_cmd
 
     def _check_msprof_env(self):
         """Check the existence of msprof binary tool"""
-        msprof_cmd = ["which", self._msprof_cmd]
-        outs = self._run_cmd(msprof_cmd, raise_error=False)
-        if outs != self._null_info:
-            return
-        logger.warning("[Profiler]The msprof command was not found. Searching from environment variables...")
-        self._search_and_add()
 
-    def _search_and_add(self):
-        """Search msprof and add it to PATH"""
         def _check_msprof(temp_path: str):
             if not os.path.isdir(temp_path):
                 return False
@@ -240,6 +108,12 @@ class AscendMsprofExporter:
                 return True
             return False
 
+        msprof_cmd = ["which", self._msprof_cmd]
+        outs = self._run_cmd(msprof_cmd, raise_error=False)
+        if outs != "":
+            return
+        logger.warning("[Profiler]The msprof command was not found. Searching from environment variables...")
+
         msprof_path = None
         envs = os.environ
         if envs.get("ASCEND_TOOLKIT_HOME"):
@@ -252,8 +126,7 @@ class AscendMsprofExporter:
             for path in path_list:
                 if self._ascend_mark in path:
                     prefix = path.split(self._ascend_mark)[0]
-                    tail = self._hiai_msprof_tail
-                    temp_path = os.path.join(prefix, tail)
+                    temp_path = os.path.join(prefix, self._hiai_msprof_tail)
                     if _check_msprof(temp_path):
                         msprof_path = temp_path
                         break
@@ -264,98 +137,77 @@ class AscendMsprofExporter:
 
         logger.info("The msprof command has been added to the path!")
 
-    def _init_output_path(self):
-        """find all the directories start with PROF"""
-        self._prof_paths = []
+    def _generate_step_trace(self, prof_path, device_path):
+        """"generate model_id iteration_id dict"""
 
-        if not os.path.isdir(self._prof_root_dir):
-            msg = "Path {} is not an existing directory.".format(self._prof_root_dir)
-            raise RuntimeError(msg)
+        summary_path = os.path.join(device_path, self._summary_dir)
+        timeline_path = os.path.join(device_path, self._timeline_dir)
 
-        for loc_root, loc_dirs, _ in os.walk(self._prof_root_dir):
-            for loc_dir in loc_dirs:
-                if loc_dir.startswith(self._profiling_prefix):
-                    self._prof_paths.append(os.path.join(loc_root, loc_dir))
+        msprof_export_cmd = self._msprof_command_generator(prof_path)
+        self._run_cmd(msprof_export_cmd)
 
-        if not self._prof_paths:
-            msg = "Do not found profiling data. Make sure there are directories start with PROF."
+        if not os.path.isdir(summary_path):
+            msg = "Path {} is not a existing directory. Make sure there is " \
+                  "valid profiling data directory!".format(summary_path)
             raise FileNotFoundError(msg)
 
-        # consider there may exists old PROF data, new PROF data have higher priority.
-        self._prof_paths.sort()               # sort by created time
-
-        device_path = self._search_by_rank_id(self._prof_paths)
-        if device_path:
-            dev_par = os.path.join(device_path, os.path.pardir)
-            abs_dev_par = os.path.abspath(dev_par)
-            self._output_path = abs_dev_par
-            self._device_path = device_path
-
-    def _search_by_rank_id(self, prof_paths: str):
-        """search valid device path through rank_id"""
-        device_paths = []
-
-        for prof_path in prof_paths:
-            if not os.path.isdir(prof_path):
-                continue
-            devices = os.listdir(prof_path)
-            for device in devices:
-                if not device.startswith(self._device_mark):
-                    continue
-                dev_path = os.path.join(prof_path, device)
-                device_paths.append(dev_path)
-
-        # search by rank id
-        find_device_path = None
-        rank_id = None
-        device_id = context.get_context("device_id")
+        step_trace_file = get_file_path(summary_path, self._step_trace_mark)
 
-        try:
-            rank_id = get_rank()
-        except RuntimeError:
-            logger.warning("[Profiler]Do not get rank_id in the environment variable, use device_id instead.")
-
-        for dev_path in device_paths:
-            if not os.path.isdir(dev_path):
-                continue
-            start_log = get_file_path(dev_path, self._start_log)
-            if not start_log:
-                continue
-            start_time = self._parse_start_info(start_log)
-            if start_time < self._start_time:
-                continue
-            info_json = get_file_path(dev_path, self._info_prefix)
-            if not info_json:
-                continue
-            temp_rank_id, temp_dev_id = self._parse_info_json(info_json)
-            if rank_id is not None and rank_id == temp_rank_id:
-                find_device_path = dev_path
-                break
-            if (rank_id is None or temp_rank_id == -1) and device_id == temp_dev_id:
-                find_device_path = dev_path
-
-        return find_device_path
-
-    def _parse_info_json(self, info_file: str):
-        """get rank_id from info.json.{device_id} file"""
-        rank_id = -1
-        dev_id = -1
-        info_dict = {}
-        try:
-            with open(info_file, "r") as f:
-                info_dict = json.load(f)
-        except (JSONDecodeError, FileNotFoundError, TypeError, PermissionError) as err:
-            logger.warning(err)
-        if info_dict.get(self._rank_id_mark) is None:
-            msg = "[Profiler]There is no rank_id key in file {}".format(info_file)
-            logger.warning(msg)
-        else:
-            rank_id = info_dict.get(self._rank_id_mark)
+        if not step_trace_file:
+            logger.info("Do not found step trace csv file in {} .".format(summary_path))
+            return None
+
+        step_trace = defaultdict(list)
+        with open(step_trace_file, newline='', mode='r') as csvfile:
+            reader = csv.reader(csvfile, delimiter=',', quotechar='"')
+            header = next(reader)
+            for index, value in enumerate(header):
+                if value == 'Model ID':
+                    Model_ID = index
+                if value == 'Iteration ID':
+                    Iteration_ID = index
+            for row in reader:
+                step_trace[int(row[Model_ID])].append(int(row[Iteration_ID]))
+
+        if os.path.isdir(summary_path):
+            shutil.rmtree(summary_path)
+        if os.path.isdir(timeline_path):
+            shutil.rmtree(timeline_path)
+
+        return step_trace
+
+    def _check_export_files(self, source_path, step_trace):
+        """Check the existence of op_summary & op_statistic files."""
+        summary_path = os.path.join(source_path, self._summary_dir)
+        if not os.path.isdir(summary_path):
+            raise RuntimeError("Path {} is not a existing directory.".format(summary_path))
+        summary_file_list = os.listdir(summary_path)
+        op_summary = set()
+        op_statistic = set()
 
-        if not info_dict.get(self._dev_info):
-            return rank_id, dev_id
+        for summary_file in summary_file_list:
+            if summary_file.startswith(self._op_summary_mark):
+                op_summary.add(summary_file)
+            elif summary_file.startswith(self._op_statistic_mark):
+                op_statistic.add(summary_file)
 
-        dev_info = info_dict.get(self._dev_info)
-        dev_id = dev_info[self._dev_index].get(self._dev_id, -1)
+        if not op_summary:
+            raise RuntimeError("The op_summary file was not found, perhaps the original data was not collected.")
+        if not op_statistic:
+            raise RuntimeError("The op_statistics file was not found, perhaps the original data was not collected.")
+
+        device_id = source_path.split('_')[-1].replace("/", "")
+
+        for model_id, value in step_trace.items():
+            for iteration_id in value:
+                tag = f"_{device_id}_{model_id}_{iteration_id}.csv"
+                op_summary_file_name = self._op_summary_mark + tag
+                op_statistic_file = self._op_statistic_mark + tag
+                if op_summary_file_name not in op_summary:
+                    logger.warning("[Profiler]The file {} was not found, " \
+                                   "perhaps the original data was not collected.".format(op_summary_file_name))
+                if op_statistic_file not in op_statistic:
+                    logger.warning("[Profiler]The file {} was not found, " \
+                                   "perhaps the original data was not collected.".format(op_statistic_file))
 
-        return rank_id, dev_id
+        logger.info("Finish checking files.")

mindspore/profiler/parser/ascend_msprof_generator.py

@@ -37,7 +37,7 @@ class AscendMsprofDataGenerator:
             'Op Name': {'index': self.invalid_index, 'dtype': ('Op Name', object)},
             'OP Type': {'index': self.invalid_index, 'dtype': ('Op Type', object)},
             'Task Type': {'index': self.invalid_index, 'dtype': ('Task Type', object)},
-            'Task Start Time': {'index': self.invalid_index, 'dtype': ('Task Start Time', float)},
+            'Task Start Time(us)': {'index': self.invalid_index, 'dtype': ('Task Start Time', float)},
             'Task Duration(us)': {'index': self.invalid_index, 'dtype': ('Task Duration', float)},
             'Task Wait Time(us)': {'index': self.invalid_index, 'dtype': ('Task Wait Time', float)},
             'Input Shapes': {'index': self.invalid_index, 'dtype': ('Input Shapes', object)},
@@ -61,9 +61,9 @@ class AscendMsprofDataGenerator:
 
         self.steptrace_name = {
             'Iteration ID': {'index': self.invalid_index, 'dtype': ('Iteration ID', int)},
-            'FP Start': {'index': self.invalid_index, 'dtype': ('FP Start', float)},
-            'BP End': {'index': self.invalid_index, 'dtype': ('BP End', float)},
-            'Iteration End': {'index': self.invalid_index, 'dtype': ('Iteration End', float)},
+            'FP Start(us)': {'index': self.invalid_index, 'dtype': ('FP Start', float)},
+            'BP End(us)': {'index': self.invalid_index, 'dtype': ('BP End', float)},
+            'Iteration End(us)': {'index': self.invalid_index, 'dtype': ('Iteration End', float)},
             'Iteration Time(us)': {'index': self.invalid_index, 'dtype': ('Iteration Time', float)},
             'FP to BP Time(us)': {'index': self.invalid_index, 'dtype': ('FP to BP Time', float)},
             'Iteration Refresh(us)': {'index': self.invalid_index, 'dtype': ('Iteration Refresh', float)},
@@ -126,7 +126,7 @@ class AscendMsprofDataGenerator:
         op_summary_dt = np.dtype([value['dtype'] for value in self.op_summary_name.values()])
 
         self.op_summary = np.array(op_summary, dtype=op_summary_dt)
-        self.op_summary['Task Start Time'] = self.op_summary['Task Start Time'] * 1e-6
+        self.op_summary['Task Start Time'] = self.op_summary['Task Start Time'] * 1e-3
         self.op_summary['Task Duration'] = self.op_summary['Task Duration'] * 1e-3
         self.op_summary['Task Wait Time'] = self.op_summary['Task Wait Time'] * 1e-3
 
@@ -171,15 +171,15 @@ class AscendMsprofDataGenerator:
 
         index = len(self.steptrace_name)
         for name in hccl_data:
+            if index >= len(header):
+                break
             name = f"stream_{name['Stream ID']}_{name['Task ID']}_{name['Op Name']}"
             self.steptrace_name[name] = {'index': index, 'dtype': (name, float)}
             index += 1
             self.steptrace_name[f'{name} duration'] = {'index': index, 'dtype': (f'{name} duration', float)}
             index += 1
-            if index >= len(header)-1:
-                break
 
-        for i in range(index, len(header), 2):
+        for i in range(len(self.steptrace_name), len(header), 2):
             name = f'hccl_{i}'
             self.steptrace_name[name] = {'index': i, 'dtype': (name, float)}
             self.steptrace_name[f'{name} duration'] = {'index': i+1, 'dtype': (f'{name} duration', float)}