mindspore 2.1.0__cp38-cp38-manylinux1_x86_64.whl → 2.2.11__cp38-cp38-manylinux1_x86_64.whl

mindspore/profiler/profiling.py

@@ -48,12 +48,12 @@ from mindspore.profiler.parser.msadvisor_analyzer import Msadvisor
 from mindspore.profiler.parser.profiler_info import ProfilerInfo
 from mindspore.common.api import _pynative_executor
 from mindspore.profiler.parser.ascend_msprof_exporter import AscendMsprofExporter
-from mindspore.profiler.parser.ascend_msprof_generator import AscendMsprofDataGenerator
+from mindspore.profiler.parser.ascend_msprof_generator import AscendMsprofDataGenerator, AscendMsprofDataGeneratorOld
 from mindspore.profiler.parser.ascend_fpbp_generator import AscendFPBPGenerator
 from mindspore.profiler.parser.ascend_op_generator import AscendOPGenerator
 from mindspore.profiler.parser.ascend_steptrace_generator import AscendStepTraceGenerator
 from mindspore.profiler.parser.ascend_flops_generator import AscendFlopsGenerator
-from mindspore.profiler.parser.ascend_hccl_generator import AscendHCCLGenerator
+from mindspore.profiler.parser.ascend_hccl_generator import AscendHCCLGenerator, AscendHCCLGeneratorOld
 
 INIT_OP_NAME = 'Default/InitDataSetQueue'
 
@@ -71,15 +71,21 @@ AICORE_METRICS_DICT = {
 class DeviceSupportParam(Enum):
     """The device target enum."""
     CPU = ['start', 'start_profile', 'output_path', 'timeline_limit', 'profile_framework', 'op_time']
-    GPU = ['start', 'start_profile', 'output_path', 'data_process', 'timeline_limit', 'sync_enable', 'op_time',
-           'profile_framework']
-    ASCEND = ['start', 'start_profile', 'output_path', 'data_process', 'timeline_limit', 'profile_memory',
-              'parallel_strategy', 'profile_communication', 'aicore_metrics', 'l2_cache', 'op_time', 'ascend_job_id',
-              'profile_framework']
+    GPU = [
+        'start', 'start_profile', 'output_path', 'data_process', 'timeline_limit', 'sync_enable', 'op_time',
+        'profile_framework'
+    ]
+    ASCEND = [
+        'start', 'start_profile', 'output_path', 'data_process', 'timeline_limit', 'profile_memory',
+        'parallel_strategy', 'profile_communication', 'aicore_metrics', 'l2_cache', 'op_time', 'ascend_job_id',
+        'profile_framework'
+    ]
 
 
-ALWAYS_VALID_PARAM = ['start', 'start_profile', 'output_path', 'data_process', 'parallel_strategy', 'l2_cache',
-                      'ascend_job_id', 'op_time', 'profile_framework']
+ALWAYS_VALID_PARAM = [
+    'start', 'start_profile', 'output_path', 'data_process', 'parallel_strategy', 'l2_cache',
+    'ascend_job_id', 'op_time', 'profile_framework'
+]
 
 
 def _environment_check():
@@ -161,6 +167,7 @@ def _calculate_dataset_execution_time(input_file, output_file):
         csv_writer.writerow(['Operation', 'Stage', 'Occurrences', 'Avg. time (us)', 'Custom Info'])
         for _, v in execution_time_map.items():
             csv_writer.writerow([v.event, v.stage, v.count, v.average_execution, v.custom_info])
+    os.chmod(output_file, modes)
     logger.info('Successfully calculate the execution time and write it to file: %s.', output_file)
 
 
@@ -188,8 +195,10 @@ def _extract_timeline_item(row, time_line, ts_map):
     # Put the instance event into timeline.
     elif start_end == '2':
         title = row['event'] + '::' + row['stage']
-        event = {'name': title, 'cat': row['module_name'], 'ts': int(row['time_stamp(us)']), 'ph': 'i',
-                 'pid': row['pid'], 'tid': row['tid'], 'args': {'parent_pid': row['parent_pid']}}
+        event = {
+            'name': title, 'cat': row['module_name'], 'ts': int(row['time_stamp(us)']), 'ph': 'i',
+            'pid': row['pid'], 'tid': row['tid'], 'args': {'parent_pid': row['parent_pid']}
+        }
         time_line.append(event)
     else:
         logger.warning("Can not map the start time for item: %s.", row)
@@ -209,8 +218,10 @@ def _parse_host_info(input_file, output_timeline_file, output_memory_file, is_de
     time_line = []
     # ts_map is used to store the start time of each event_stage_tid_pid
     ts_map = {}
-    memory_header = ['tid', 'pid', 'parent_pid', 'module_name', 'event', 'stage', 'level', 'start_end', 'custom_info',
-                     'memory_usage(kB)', 'time_stamp(us)']
+    memory_header = [
+        'tid', 'pid', 'parent_pid', 'module_name', 'event', 'stage', 'level', 'start_end', 'custom_info',
+        'memory_usage(kB)', 'time_stamp(us)'
+    ]
     memory_info = []
     with open(input_file, 'r') as f:
         for row in csv.DictReader(f):
@@ -226,12 +237,12 @@ def _parse_host_info(input_file, output_timeline_file, output_memory_file, is_de
                 logger.error("Error occur when analyse line: %s, Details is: %s", row, e)
                 continue
     if memory_info:
-        with os.fdopen(os.open(output_memory_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
-                       'w') as csv_file:
+        with os.fdopen(os.open(output_memory_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as csv_file:
             csv_writer = csv.DictWriter(csv_file, fieldnames=memory_header)
             csv_writer.writeheader()
             for item in memory_info:
                 csv_writer.writerow(item)
+        os.chmod(output_memory_file, stat.S_IREAD | stat.S_IWRITE)
     else:
         logger.warning("No memory_usage is record in file: %s", input_file)
 
@@ -255,14 +266,28 @@ def _parse_host_info(input_file, output_timeline_file, output_memory_file, is_de
 
     if time_line:
         timeline_file = validate_and_normalize_path(output_timeline_file)
-        with os.fdopen(os.open(timeline_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
-                       'w') as json_file:
+        with os.fdopen(os.open(timeline_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as json_file:
             json.dump(time_line, json_file)
+        os.chmod(timeline_file, stat.S_IREAD | stat.S_IWRITE)
     else:
         logger.warning("No valid time_stamp is record in file: %s", input_file)
 
 
-def _ascend_graph_msprof_analyse(source_path):
+def _ascend_graph_msprof_generator(source_path, model_iteration_dict):
+    """Executing the msprof export mode."""
+    try:
+        ProfilerInfo.set_export_start_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+        msprof_exporter = AscendMsprofExporter(source_path)
+        flag = msprof_exporter.export(model_iteration_dict)
+        ProfilerInfo.set_export_end_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+        return flag
+
+    except ProfilerException as err:
+        logger.warning(err.message)
+        return False
+
+
+def _ascend_graph_msprof_analyse(source_path, flag):
     """
     Ascend graph model msprof data analyse.
 
@@ -273,7 +298,10 @@ def _ascend_graph_msprof_analyse(source_path):
     df_op_statistic = []
     df_step_trace = []
     try:
-        msprof_analyser = AscendMsprofDataGenerator(os.path.join(source_path, 'summary'))
+        if flag:
+            msprof_analyser = AscendMsprofDataGenerator(os.path.join(source_path, 'summary'))
+        else:
+            msprof_analyser = AscendMsprofDataGeneratorOld(os.path.join(source_path, 'summary'))
         df_op_summary, df_op_statistic, df_step_trace = msprof_analyser.parse()
     except ProfilerException as err:
         logger.warning(err.message)
@@ -287,7 +315,8 @@ class Profiler:
     This class to enable the profiling of MindSpore neural networks.
     MindSpore users can import the mindspore.Profiler, initialize the Profiler object to start profiling,
     and use Profiler.analyse() to stop profiling and analyse the results.
-    Users can visualize the results using the MindInsight tool.
+    Users can visualize the results using the `MindSpore Insight
+    <https://www.mindspore.cn/mindinsight/docs/en/r2.2/index.html>`_ tool.
     Now, Profiler supports AICORE operator, AICPU operator, HostCPU operator, memory,
     correspondence, cluster, etc data analysis.
 
@@ -330,11 +359,16 @@ class Profiler:
             Default value: ``True`` .
         timeline_limit (int, optional): (Ascend/GPU) Set the maximum storage size of the timeline file (unit M).
             When using this parameter, `op_time` must be set to True. Default value: ``500`` .
-        profile_framework (str, optional): (Ascend/GPU) Whether to collect host memory and time, it must be one of
-            ["all", "time", "memory", None]. When is enabled, a subdirectory host_info will be generated in the
+        profile_framework (str, optional): (Ascend/GPU) The host information to collect, it must be one of
+            ["all", "time", "memory", None], When is not set to None, a subdirectory host_info will be generated in the
             specified profiler directory, which stores the collected memory and time files on the Host side.
             Default: "all".
 
+            - "all": Record both host timestamp and host memory usage.
+            - "time": Only record host timestamp.
+            - "memory": Only record host memory usage.
+            - None: Not record host information.
+
     Raises:
         RuntimeError: When the version of CANN does not match the version of MindSpore,
             MindSpore cannot parse the generated ascend_job_id directory structure.
@@ -407,9 +441,9 @@ class Profiler:
         self._rank_size = 1
         self._rank_id = 0
         self._ascend_profiler = None
-        self._ascend_msprof_exporter = None
         self._timeline_size_limit_byte = 500 * 1024 * 1024  # 500MB
         self._parallel_strategy = True
+        self._model_iteration_dict = None
         _environment_check()
         # default aicore_metrics type is ArithmeticUtilization
         self._aicore_metrics_id = 0
@@ -476,6 +510,25 @@ class Profiler:
 
         return job_start_time
 
+    @staticmethod
+    def _parse_info_json(info_file):
+        """
+        Parse info log file, get the rank id and device id of the job.
+        Args:
+             input_file (str): The file path of the parse info log file.
+
+        Returns:
+            rank id, device id
+        """
+        with open(info_file, "r") as f:
+            info_dict = json.load(f)
+
+            rank_id = info_dict.get("rank_id", 0)
+            dev_info = info_dict.get("DeviceInfo", [])
+            dev_id = dev_info[0].get("id", -1)
+
+            return str(rank_id), str(dev_id)
+
     def op_analyse(self, op_name, device_id=None):
         """
         Profiler users can use this interface to obtain operator performance data.
@@ -487,8 +540,8 @@ class Profiler:
                 parse. If this interface is used for offline data parsing, Default: ``0`` .
 
         Raises:
-            TypeError: If the op_name parameter type is incorrect.
-            TypeError: If the device_id parameter type is incorrect.
+            TypeError: If the `op_name` parameter type is incorrect.
+            TypeError: If the `device_id` parameter type is incorrect.
             RuntimeError: If MindSpore runs on Ascend, this interface cannot be used.
 
         Supported Platforms:
@@ -501,12 +554,12 @@ class Profiler:
             >>> # Profiler init.
             >>> profiler = Profiler()
             >>> # Train Model or eval Model, taking LeNet5 as an example.
-            >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+            >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
             >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
             >>> # Create the dataset taking MNIST as an example.
-            >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/mnist.py
+            >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
             >>> dataloader = create_dataset()
             >>> model = Model(net, loss, optimizer)
             >>> model.train(5, dataloader, dataset_sink_mode=False)
@@ -550,7 +603,34 @@ class Profiler:
                 Offline mode isused in abnormal exit scenario. This parameter should be set to ``None``
                 for online mode. Default: ``None``.
         """
+        self._analyse(offline_path=offline_path)
+
+    def _analyse(self, offline_path=None, model_iteration_dict=None):
+        """
+        Collect and analyze training performance data, support calls during and after training. The example shows above.
+
+        Args:
+            offline_path (Union[str, None], optional): The data path which need to be analysed with offline mode.
+                Offline mode isused in abnormal exit scenario. This parameter should be set to ``None``
+                for online mode. Default: ``None``.
+            model_iteration_dict: Dictionary with model id as the key and iteration id as the value, Default: ``None``.
+        """
+        self._model_iteration_dict = model_iteration_dict
+
+        self._init_profiler_info()
+        self._is_support_step_info_collect()
+        parallel_mode = get_auto_parallel_context("parallel_mode")
+        stage_num = get_auto_parallel_context("pipeline_stages")
+
+        ProfilerInfo.set_parallel_info(parallel_mode, stage_num)
+        ProfilerInfo.set_rank_size(self._rank_size)
+        ProfilerInfo.set_heterogeneous(self._is_heterogeneous)
         if offline_path:
+            if self._is_offline_parser():
+                ProfilerInfo.set_analyse_start_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+                self._ascend_graph_analyse()
+                ProfilerInfo.set_analyse_end_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+                ProfilerInfo.save(self._output_path)
             _offline_parse(offline_path)
             return
         if self._msprof_enable:
@@ -584,15 +664,7 @@ class Profiler:
                 logger.warning("The parameter 'profile_framework' is not support for CPU, so there no host_info"
                                " directory in the output path.")
         logger.info("Profiling: all the data have been analyzed.")
-        self._init_profiler_info()
-        self._is_support_step_info_collect()
-        parallel_mode = get_auto_parallel_context("parallel_mode")
-        stage_num = get_auto_parallel_context("pipeline_stages")
-
-        ProfilerInfo.set_parallel_info(parallel_mode, stage_num)
         ProfilerInfo.set_analyse_end_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
-        ProfilerInfo.set_rank_size(self._rank_size)
-        ProfilerInfo.set_heterogeneous(self._is_heterogeneous)
         ProfilerInfo.save(self._output_path)
 
     def start(self):
@@ -602,7 +674,7 @@ class Profiler:
         Raises:
             RuntimeError: If the profiler has already started.
             RuntimeError: If MD profiling has stopped, repeated start action is not supported.
-            RuntimeError: If the start_profile parameter is not set or is set to True.
+            RuntimeError: If the `start_profile` parameter is not set or is set to ``True``.
 
         Examples:
             >>> from mindspore.train import Callback
@@ -724,6 +796,8 @@ class Profiler:
 
             self._stop_time = int(time.time() * 10000000)
         ProfilerInfo.set_profiling_stop_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+        self._init_profiler_info()
+        ProfilerInfo.save(self._output_path)
         logger.info("Profiling: stop time: %d", self._stop_time)
 
     def _profiler_init(self, kwargs):
@@ -749,7 +823,6 @@ class Profiler:
 
         if self._device_target == DeviceTarget.ASCEND.value:
             self._ascend_profiler = c_expression.Profiler.get_instance("Ascend")
-            self._ascend_msprof_exporter = AscendMsprofExporter(self._output_path)
         self._get_devid_rankid_and_devtarget()
 
     def _init_profiler_info(self):
@@ -827,7 +900,6 @@ class Profiler:
         # use context interface to open profiling, for the new mindspore version(after 2020.5.21)
         self._ascend_profiler = c_expression.Profiler.get_instance("Ascend")
         self._ascend_profiler.init(self._output_path, int(self._dev_id), self._ascend_profiling_options)
-        self._ascend_msprof_exporter = AscendMsprofExporter(self._output_path)
         base_profiling_container_path = os.path.join(self._output_path, "container")
         container_path = os.path.join(base_profiling_container_path, self._dev_id)
         data_path = os.path.join(container_path, "data")
@@ -965,8 +1037,6 @@ class Profiler:
         else:
             logger.info("No need to stop profiler because profiler has been stopped.")
         # export op data before analyse
-        if self._op_time:
-            self._ascend_msprof_exporter.export(self._start_time, support_step_trace=False)
         self._ascend_graph_analyse()
 
     def _minddata_analyse(self, source_path):
@@ -1040,8 +1110,11 @@ class Profiler:
             aicpu_intermediate_detail_path = validate_and_normalize_path(aicpu_intermediate_detail_path)
             framework_raw_path = validate_and_normalize_path(framework_raw_path)
 
-            output_timeline_data_path = os.path.join(self._output_path, f'output_timeline_data_{dev_id}.txt')
-            output_timeline_data_path = validate_and_normalize_path(output_timeline_data_path)
+            if context.get_context("mode") == context.GRAPH_MODE:
+                output_timeline_data_path = os.path.join(self._output_path, f'output_timeline_data_{dev_id}.txt')
+                output_timeline_data_path = validate_and_normalize_path(output_timeline_data_path)
+            else:
+                output_timeline_data_path = None
 
             op_analyser = AscendOPGenerator(op_summary, op_statistic, dynamic_status)
             op_analyser.parse()
@@ -1070,7 +1143,7 @@ class Profiler:
         finally:
             pass
 
-    def _ascend_timeline_analyse(self, source_path, op_summary, steptrace):
+    def _ascend_timeline_analyse(self, op_summary, steptrace):
         """Analyse timeline info."""
         try:
             logger.info("Profiling: analyzing the timeline data")
@@ -1098,7 +1171,7 @@ class Profiler:
 
     def _ascend_flops_analyse(self, op_summary):
         """Get op FLOPs from op_summary, write output_op_flops_x.csv."""
-        if len(op_summary.dtype) != 18:
+        if 'vector_fops' not in op_summary.dtype.names and 'cube_fops' not in op_summary.dtype.names:
             logger.warning("[Profiler] Can not found cube fops and vector fops data in the op summary.")
             return
 
@@ -1135,21 +1208,24 @@ class Profiler:
         finally:
             pass
 
-    def _ascend_graph_hccl_analyse(self, source_path):
+    def _ascend_graph_hccl_analyse(self, source_path, steptrace, flag):
         """Analyse hccl profiler info."""
         if not self._profile_communication:
             return
         if self._profile_communication and context.get_context("mode") == context.PYNATIVE_MODE:
             logger.warning("[Profiler]The parameter profile_communication is not supported on Ascend "
                            "PyNative mode currently.")
+            return
         try:
             logger.info("Profiling: analyzing the hccl profiler info.")
             dev_id = self._rank_id if self._device_target == DeviceTarget.ASCEND.value else self._dev_id
 
             hccl_raw_path = os.path.join(self._output_path, f'hccl_raw_{dev_id}.csv')
             hccl_raw_path = validate_and_normalize_path(hccl_raw_path)
-
-            hccl_analyse = AscendHCCLGenerator(os.path.join(source_path, 'timeline'))
+            if flag:
+                hccl_analyse = AscendHCCLGenerator(os.path.join(source_path, 'timeline'), steptrace)
+            else:
+                hccl_analyse = AscendHCCLGeneratorOld(os.path.join(source_path, 'timeline'))
             hccl_analyse.parse()
             hccl_analyse.write(hccl_raw_path)
 
@@ -1191,9 +1267,14 @@ class Profiler:
         source_path = os.path.join(self._output_path, job_id)
         self._minddata_analyse(source_path)
         if self._op_time:
-            op_summary, op_statistic, steptrace = _ascend_graph_msprof_analyse(source_path)
+            flag = _ascend_graph_msprof_generator(source_path, self._model_iteration_dict)
+            if not flag:
+                logger.warning('Current driver package not support all export mode, use single export mode, '
+                               'this may lead to performance degradation. Suggest upgrading the driver package.')
+            ProfilerInfo.set_export_flag(flag)
+            op_summary, op_statistic, steptrace = _ascend_graph_msprof_analyse(source_path, flag)
             self._ascend_op_analyse(op_summary, op_statistic, self._dynamic_status)
-            self._ascend_timeline_analyse(source_path, op_summary, steptrace)
+            self._ascend_timeline_analyse(op_summary, steptrace)
             graph_ids = np.unique(op_summary['Model ID']).tolist()
             points = self._ascend_fpbp_analyse(op_summary, steptrace)
             if len(graph_ids) == 1:
@@ -1202,7 +1283,7 @@ class Profiler:
                 self._ascend_dynamic_net_analyse(op_summary)
             self._ascend_flops_analyse(op_summary)
             self._ascend_graph_memory_analyse(points)
-            self._ascend_graph_hccl_analyse(source_path)
+            self._ascend_graph_hccl_analyse(source_path, steptrace, flag)
             self._ascend_graph_msadvisor_analyse(job_id)
             ProfilerInfo.set_graph_ids(graph_ids)
 
@@ -1326,29 +1407,37 @@ class Profiler:
         point_info_file_path = validate_and_normalize_path(point_info_file_path)
 
         if self._device_target and self._device_target == DeviceTarget.GPU.value:
-            input_file_path = os.path.join(self._output_path, f'step_trace_profiling_{self._dev_id}.txt')
-            input_file_path = validate_and_normalize_path(input_file_path)
-            parser = GpuStepTraceParser(input_dir=input_file_path,
-                                        output_file_path=step_trace_intermediate_file_path,
-                                        is_training_mode=is_training_mode_flag,
-                                        is_gpu_kernel_async_launch=is_gpu_kernel_async_launch_flag)
-            parser.parse_and_save()
-            point_info = parser.record_point_info(point_info_file_path)
-        else:
-            # whether keep the first step
-            skip_first_step_flag = framework_parser.check_op_name(INIT_OP_NAME)
-            point_info = framework_parser.point_info
-            # recognize inference or training mode
-            is_training_mode_flag = framework_parser.check_op_name("Gradients")
-            # parser the step trace files and save the result to disk
-            source_path = validate_and_normalize_path(source_path)
-            parser = AscendStepTraceParser(input_dir=source_path,
-                                           output_file_path=step_trace_intermediate_file_path,
-                                           skip_first_step=skip_first_step_flag,
-                                           is_training_mode=is_training_mode_flag)
-            parser.set_task_id_op_name_dict(framework_parser.to_task_id_full_op_name_dict())
-            parser.parse_and_save()
-            point_info = parser.record_point_info(point_info_file_path)
+            if context.get_context("mode") != context.PYNATIVE_MODE:
+                input_file_path = os.path.join(self._output_path, f'step_trace_profiling_{self._dev_id}.txt')
+                input_file_path = validate_and_normalize_path(input_file_path)
+                parser = GpuStepTraceParser(input_dir=input_file_path,
+                                            output_file_path=step_trace_intermediate_file_path,
+                                            is_training_mode=is_training_mode_flag,
+                                            is_gpu_kernel_async_launch=is_gpu_kernel_async_launch_flag)
+                parser.parse_and_save()
+                point_info = parser.record_point_info(point_info_file_path)
+                # print parser result
+                parser.show()
+                logger.info("Finish saving the intermediate result: %s", step_trace_intermediate_file_path)
+                logger.info("The point info is: %s", point_info)
+
+                return point_info, is_training_mode_flag
+            return {}, is_training_mode_flag
+
+        # whether keep the first step
+        skip_first_step_flag = framework_parser.check_op_name(INIT_OP_NAME)
+        # recognize inference or training mode
+        is_training_mode_flag = framework_parser.check_op_name("Gradients")
+        # parser the step trace files and save the result to disk
+        source_path = validate_and_normalize_path(source_path)
+        parser = AscendStepTraceParser(input_dir=source_path,
+                                       output_file_path=step_trace_intermediate_file_path,
+                                       skip_first_step=skip_first_step_flag,
+                                       is_training_mode=is_training_mode_flag)
+        parser.set_task_id_op_name_dict(framework_parser.to_task_id_full_op_name_dict())
+        parser.parse_and_save()
+        point_info = parser.record_point_info(point_info_file_path)
+
         # print parser result
         parser.show()
         logger.info("Finish saving the intermediate result: %s", step_trace_intermediate_file_path)
@@ -1389,15 +1478,17 @@ class Profiler:
             job_id = self._ascend_job_id.rstrip('/').split('/')[-1]
             if job_id.startswith('PROF'):
                 device_dir = [dir for dir in os.listdir(self._ascend_job_id) if dir.startswith('device')]
+                info_file_path = get_file_path(os.path.join(self._ascend_job_id, device_dir[0]), "info.json")
+                training_rank_id, _ = self._parse_info_json(info_file_path)
+                self._rank_id = int(training_rank_id)
                 return os.path.join(job_id, device_dir[0])
             return job_id
 
         job_id = ""
-        job_dirs = filter(lambda item: item.startswith('JOB') or item.startswith('PROF') and \
-                                       os.path.isdir(os.path.join(self._output_path, item)),
-                          os.listdir(self._output_path))
-        sorted_job_dirs = sorted(job_dirs, key=lambda x: os.path.getmtime(os.path.join(self._output_path, x)),
-                                 reverse=True)
+        job_dirs = filter(lambda item: item.startswith('JOB') or item.startswith('PROF') and os.path.isdir(
+            os.path.join(self._output_path, item)), os.listdir(self._output_path))
+        sorted_job_dirs = sorted(
+            job_dirs, key=lambda x: os.path.getmtime(os.path.join(self._output_path, x)), reverse=True)
 
         for dir_name in sorted_job_dirs:
             if dir_name.startswith('PROF'):
@@ -1414,22 +1505,21 @@ class Profiler:
                                "profiler will ignore this job dir.", job_dir)
                 continue
 
-            training_device_id = start_file_path.split('.')[-1]
+            info_file_path = get_file_path(job_dir, "info.json")
+            if info_file_path is None:
+                logger.warning("Find profiling job path %s, but info.json not exist, "
+                               "profiler will ignore this job dir.", job_dir)
+                continue
+
+            job_start_time = self._parse_start_log(start_file_path)
+            _, training_device_id = self._parse_info_json(info_file_path)
+
             if self._dev_id != training_device_id:
                 logger.debug("Find profiling find job path %s, but not current training device id. "
                              "Current training device id %s, but job path device id: %s, "
                              "profiler will ignore this job dir.", job_dir, self._dev_id, training_device_id)
                 continue
 
-            if not os.listdir(os.path.join(job_dir, 'data')):
-                continue
-
-            job_start_time = self._parse_start_log(start_file_path)
-            if not job_start_time:
-                logger.warning("Find profiling job path %s, but fail to get job start info, "
-                               "profiler will ignore this job dir.", job_start_time)
-                continue
-
             if int(job_start_time) < self._start_time:
                 logger.warning("Find profiling job path %s, but start_time(%d) is earlier than this training "
                                "start_time(%d), profiler will ignore this job dir.",
@@ -1586,7 +1676,7 @@ class Profiler:
         self._profile_framework = kwargs.pop("profile_framework", "all")
         if self._profile_framework not in ["memory", "time", "all", None]:
             logger.warning(f"For '{self.__class__.__name__}', the parameter profile_framework must be one of ['memory',"
-                           f" 'time', 'all', None]，but got {self._profile_framework}, it will be set to 'all'.")
+                           f" 'time', 'all', None], but got {self._profile_framework}, it will be set to 'all'.")
             self._profile_framework = "all"
 
     def _host_info_analyse(self):