mindspore 1.10.0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/profiler/profiling.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2021 Huawei Technologies Co., Ltd
+# Copyright 2020-2022 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,9 @@ import os
 import stat
 import time
 import json
+import glob
+import subprocess
+from enum import Enum
 
 from mindspore import log as logger, context
 from mindspore.communication.management import GlobalComm, get_rank, get_group_size
@@ -33,7 +36,8 @@ from mindspore.profiler.parser.aicpu_data_parser import DataPreProcessParser
 from mindspore.profiler.parser.framework_parser import FrameworkParser, GpuFrameWorkParser, DynamicFrameWorkParser
 from mindspore.profiler.parser.hwts_log_parser import HWTSLogParser
 from mindspore.profiler.parser.integrator import Integrator, DeviceTarget
-from mindspore.profiler.parser.integrator import GpuTimelineGenerator, CpuTimelineGenerator, AscendTimelineGenerator
+from mindspore.profiler.parser.cpu_gpu_timeline_generator import GpuTimelineGenerator, CpuTimelineGenerator
+from mindspore.profiler.parser.ascend_timeline_generator import AscendTimelineGenerator
 from mindspore.profiler.parser.memory_usage_parser import MemoryUsageParser
 from mindspore.profiler.parser.minddata_parser import MinddataParser
 from mindspore.profiler.parser.minddata_analyzer import MinddataProfilingAnalyzer
@@ -45,9 +49,33 @@ from mindspore.profiler.parser.step_trace_parser import GpuStepTraceParser, Asce
 from mindspore.profiler.parser.hccl_parser import HcclParser
 from mindspore.profiler.parser.op_intermediate_parser import OPIntermediateParser
 from mindspore.profiler.parser.msadvisor_analyzer import Msadvisor
+from mindspore.profiler.parser.profiler_info import ProfilerInfo
+from mindspore.common.api import _pynative_executor
 
 INIT_OP_NAME = 'Default/InitDataSetQueue'
 
+AICORE_METRICS_DICT = {
+    0: "ArithmeticUtilization",
+    1: "PipeUtilization",
+    2: "Memory",
+    3: "MemoryL0",
+    4: "ResourceConflictRatio",
+    5: "MemoryUB",
+    -1: "None"
+}
+
+
+class DeviceSupportParam(Enum):
+    """The device target enum."""
+    CPU = ['start', 'start_profile', 'output_path', 'timeline_limit']
+    GPU = ['start', 'start_profile', 'output_path', 'data_process', 'timeline_limit', 'sync_enable', 'op_time']
+    ASCEND = ['start', 'start_profile', 'output_path', 'data_process', 'timeline_limit', 'profile_memory',
+              'parallel_strategy', 'profile_communication', 'aicore_metrics', 'l2_cache', 'op_time', 'ascend_job_id']
+
+
+ALWAYS_VALID_PARAM = ['start', 'start_profile', 'output_path', 'data_process', 'parallel_strategy', 'l2_cache',
+                      'ascend_job_id', 'op_time']
+
 
 def _environment_check():
     if c_expression.security.enable_security():
@@ -55,7 +83,7 @@ def _environment_check():
 
 
 class Profiler:
-    """
+    r"""
     This class to enable the profiling of MindSpore neural networks.
     MindSpore users can import the mindspore.Profiler, initialize the Profiler object to start profiling,
     and use Profiler.analyse() to stop profiling and analyse the results.
@@ -65,13 +93,42 @@ class Profiler:
 
     Args:
         output_path (str, optional): Output data path. Default: "./data".
+        op_time (bool, optional): (Ascend/GPU) Whether to collect operators performance data. Default value: True.
         profile_communication (bool, optional): (Ascend only) Whether to collect communication performance data in
             a multi devices training,collect when True. Setting this parameter has no effect during single device
-            training. Default: False.
+            training. When using this parameter, `op_time` must be set to True. Default: False.
         profile_memory (bool, optional): (Ascend only) Whether to collect tensor memory data, collect when True.
-            Default: False.
+            When using this parameter, `op_time` must be set to True. Default: False.
+        parallel_strategy (bool, optional): (Ascend only) Whether to collect parallel policy performance data.
+            Default value: true.
         start_profile (bool, optional): The start_profile parameter controls whether to enable or disable performance
             data collection based on conditions. Default: True.
+        aicore_metrics (int, optional): (Ascend only) Types of AICORE performance data collected, when using this
+            parameter, `op_time` must be set to True, and the value must be in [-1, 0, 1, 2, 3, 4, 5], Default: 0, the
+            data items contained in each metric are as follows:
+
+            - -1: Does not collect AICORE data.
+            - 0: ArithmeticUtilization contains mac_fp16/int8_ratio, vec_fp32/fp16/int32_ratio, vec_misc_ratio etc.
+            - 1: PipeUtilization contains vec_ratio, mac_ratio, scalar_ratio, mte1/mte2/mte3_ratio, icache_miss_rate
+              etc.
+            - 2: Memory contains ub_read/write_bw, l1_read/write_bw, l2_read/write_bw, main_mem_read/write_bw etc.
+            - 3: MemoryL0 contains l0a_read/write_bw, l0b_read/write_bw, l0c_read/write_bw etc.
+            - 4: ResourceConflictRatio contains vec_bankgroup/bank/resc_cflt_ratio etc.
+            - 5: MemoryUB contains ub_read/write_bw_mte, ub_read/write_bw_vector, ub\_/write_bw_scalar etc.
+
+        l2_cache (bool, optional): (Ascend only) Whether to collect l2 cache data, collect when True. Default: False.
+        sync_enable (bool, optional): (GPU only) Whether the profiler collects operators in a synchronous way.
+            Default: True.
+
+            - True: The synchronous way. Before sending the operator to the GPU, the CPU records the start timestamp.
+              Then the operator is returned to the CPU after execution, and the end timestamp is recorded,
+              The duration of the operator is the difference between the two timestamps.
+            - False: The asynchronous way. The duration of the operator is that of sending from the CPU to the GPU.
+              This method can reduce the impact of adding profiler on overall training time.
+        data_process (bool, optional): (Ascend/GPU) Whether to collect data to prepare performance data.
+            Default value: True.
+        timeline_limit (int, optional): Set the maximum storage size of the timeline file (unit M). When using this
+            parameter, `op_time` must be set to True. Default value: 500.
 
     Raises:
         RuntimeError: When the version of CANN does not match the version of MindSpore,
@@ -131,15 +188,11 @@ class Profiler:
     _ascend_job_id = ""
 
     def __init__(self, **kwargs):
-        if Profiler._has_initialized:
-            msg = "Do not init twice in the profiler."
-            raise RuntimeError(msg)
-        Profiler._has_initialized = True
-        self._timeline_size_limit_byte = 500 * 1024 * 1024  # 500MB
         self._dev_id = None
         self._cpu_profiler = None
         self._gpu_profiler = None
         self._md_profiler = None
+        self._is_heterogeneous = False
         self._profiler_manager = None
         self._timeline_meta = []
         self._init_time = None
@@ -147,21 +200,40 @@ class Profiler:
         self._job_id_env = None
         self._filt_optype_names = ''
         self._output_path = ''
-        self._rank_size = 0
+        self._rank_size = 1
+        self._rank_id = 0
         self._ascend_profiler = None
+        self._timeline_size_limit_byte = 500 * 1024 * 1024  # 500MB
+        self._parallel_strategy = True
         _environment_check()
-        # get device_id and device_target
-        self._get_devid_rankid_and_devtarget()
-        self._get_output_path(kwargs)
+        # default aicore_metrics type is ArithmeticUtilization
+        self._aicore_metrics_id = 0
+        self._l2_cache = "off"
+        self._data_process = True
+        self._op_time = True
         self._profile_communication = False
         self._has_started = False
         self._has_started_twice = False
         self.start_profile = True
         self._profile_memory = False
+        self._sync_enable = True
         self._stop_time = 0
         self._dynamic_status = False
+        self._msprof_enable = os.getenv("PROFILER_SAMPLECONFIG")
+        if self._msprof_enable:
+            return
+        if kwargs.get("env_enable"):
+            self._profiler_init(kwargs)
+            return
+        if Profiler._has_initialized:
+            msg = "Do not init twice in the profiler."
+            raise RuntimeError(msg)
+        Profiler._has_initialized = True
+        # get device_id and device_target
+        self._get_devid_rankid_and_devtarget()
+        self._parser_kwargs(kwargs)
+        self._get_output_path(kwargs)
         self._decide_device_target(kwargs)
-        self._pynative_profiler = None
         if self.start_profile:
             self.start()
 
@@ -179,7 +251,7 @@ class Profiler:
         return output_path
 
     @staticmethod
-    def _parse_host_start_log(input_file):
+    def _parse_start_log(input_file):
         """
         Parse host start log file, get the start time of the job.
 
@@ -190,12 +262,9 @@ class Profiler:
             str, job start time.
         """
 
-        job_start_time = ""
+        job_start_time = 0
         with open(input_file) as f:
-            for line in f.readlines():
-                if "clock_realtime" in line:
-                    # 16 means the first digit of the timestamp, len(line)-3 means the last.
-                    job_start_time = line[16:len(line) - 3]
+            job_start_time = json.load(f).get("collectionTimeBegin")
 
         return job_start_time
 
@@ -212,7 +281,7 @@ class Profiler:
         Raises:
             TypeError: If the op_name parameter type is incorrect.
             TypeError: If the device_id parameter type is incorrect.
-            RunTimeError: If MindSpore runs on Ascend, this interface cannot be used.
+            RuntimeError: If MindSpore runs on Ascend, this interface cannot be used.
 
         Supported Platforms:
             ``GPU`` ``CPU``
@@ -267,12 +336,22 @@ class Profiler:
         """
         Collect and analyze training performance data, support calls during and after training. The example shows above.
         """
+        if self._msprof_enable:
+            return
+
+        # Stop data collection after all operators are executed.
+        _pynative_executor.sync()
+
         Profiler._has_initialized = False
         self._dynamic_status = self._profiler_manager.dynamic_status()
         _environment_check()
 
         self._cpu_profiler.stop()
 
+        cpu_op_file = glob.glob(os.path.join(self._output_path, 'cpu_op_type_info_*'))
+        if self._device_target and self._device_target != DeviceTarget.CPU.value and cpu_op_file:
+            self._is_heterogeneous = True
+        ProfilerInfo.set_analyse_start_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
         if self._device_target and self._device_target == DeviceTarget.CPU.value:
             self._cpu_analyse()
 
@@ -282,6 +361,11 @@ class Profiler:
         elif self._device_target and self._device_target == DeviceTarget.ASCEND.value:
             self._ascend_analyse()
         logger.info("Profiling: all the data have been analyzed.")
+        self._init_profiler_info()
+        ProfilerInfo.set_analyse_end_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+        ProfilerInfo.set_rank_size(self._rank_size)
+        ProfilerInfo.set_heterogeneous(self._is_heterogeneous)
+        ProfilerInfo.save(self._output_path)
 
     def start(self):
         """
@@ -315,10 +399,10 @@ class Profiler:
              >>>     def end(self, run_context):
              ...         self.profiler.analyse()
         """
-        if not self.start_profile and context.get_context("mode") == context.PYNATIVE_MODE:
-            raise RuntimeError("Pynative mode does not support conditional collection of performance data.")
+        if self._msprof_enable:
+            return
 
-        self._start_time = int(time.time() * 10000000)
+        self._start_time = int(time.time() * 1000000)
         logger.info("Profiling: start time: %d", self._start_time)
 
         if not self._has_started:
@@ -339,14 +423,16 @@ class Profiler:
         self._cpu_profiler.step_profiling_enable(True)
 
         if self._device_target and self._device_target == DeviceTarget.GPU.value:
-            self._md_profiler.start()
-            self._gpu_profiler.step_profiling_enable(True)
+            if self._data_process:
+                self._md_profiler.start()
+                self._gpu_profiler.data_process_enable(True)
+            if self._op_time:
+                self._gpu_profiler.step_profiling_enable(True)
         elif self._device_target and self._device_target == DeviceTarget.ASCEND.value:
-            self._md_profiler.start()
-            if context.get_context("mode") == context.PYNATIVE_MODE:
-                self._ascend_pynative_start()
-            else:
-                self._ascend_graph_start()
+            if self._data_process:
+                self._md_profiler.start()
+            self._ascend_graph_start()
+        ProfilerInfo.set_profiling_start_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
 
     def stop(self):
         """
@@ -378,6 +464,9 @@ class Profiler:
              >>>     def end(self, run_context):
              ...         self.profiler.analyse()
         """
+        if self._msprof_enable:
+            return
+
         if self._has_started:
             self._has_started = False
         else:
@@ -388,18 +477,53 @@ class Profiler:
         if self._is_offline_parser():
             return
 
-        self._md_profiler.stop()
-        self._md_profiler.save(self._output_path)
+        # Stop data collection after all operators are executed.
+        _pynative_executor.sync()
+
+        if self._data_process:
+            self._md_profiler.stop()
+            self._md_profiler.save(self._output_path)
 
         if self._device_target and self._device_target == DeviceTarget.GPU.value:
             self._gpu_profiler.stop()
         elif self._device_target and self._device_target == DeviceTarget.ASCEND.value:
-            if context.get_context("mode") == context.PYNATIVE_MODE:
-                self._pynative_profiler.stop()
             self._ascend_profiler.stop()
 
             self._stop_time = int(time.time() * 10000000)
-            logger.info("Profiling: stop time: %d", self._stop_time)
+        ProfilerInfo.set_profiling_stop_time(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+        logger.info("Profiling: stop time: %d", self._stop_time)
+
+    def _profiler_init(self, kwargs):
+        """Initialize variables when profiler is enabled by environment variables."""
+        options = kwargs.get("env_enable")
+        self._has_started = True
+        self._start_time = options.get("start_time")
+        self._output_path = options.get('file_output_path')
+        self._profile_memory = options.get('profile_memory')
+        self._parallel_strategy = options.get('parallel_strategy')
+        self._timeline_size_limit_byte = options.get('timeline_limit') * 1024 * 1024
+        self._data_process = options.get('data_process')
+        self._profile_communication = options.get('profile_communication')
+        self._op_time = options.get('op_time')
+        self._device_target = context.get_context("device_target").lower()
+        self._profiler_manager = c_expression.ProfilerManager.get_instance()
+        self._cpu_profiler = c_expression.Profiler.get_instance("CPU")
+        if self._data_process:
+            self._md_profiler = cde.GlobalContext.profiling_manager()
+        if self._device_target == DeviceTarget.GPU.value:
+            self._gpu_profiler = c_expression.Profiler.get_instance("GPU")
+
+        if self._device_target == DeviceTarget.ASCEND.value:
+            self._ascend_profiler = c_expression.Profiler.get_instance("Ascend")
+        self._get_devid_rankid_and_devtarget()
+
+    def _init_profiler_info(self):
+        """Init profiler info filer."""
+        mode = "graph"
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            mode = "pynative"
+        store_id = self._dev_id if self._device_target == DeviceTarget.GPU.value else self._rank_id
+        ProfilerInfo.init_info(mode, store_id)
 
     def _decide_device_target(self, kwargs):
         """Complete Profiler initialization according to device_target"""
@@ -421,9 +545,6 @@ class Profiler:
 
     def _cpu_profiler_init(self, kwargs):
         """Cpu profiler init."""
-        if context.get_context("mode") == context.PYNATIVE_MODE:
-            raise RuntimeError("Pynative mode is not supported on CPU currently.")
-
         self.start_profile = kwargs.pop("start_profile", True)
         if not isinstance(self.start_profile, bool):
             raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
@@ -432,24 +553,26 @@ class Profiler:
     def _gpu_profiler_init(self, kwargs):
         """Gpu profiler init."""
         # Setup and start MindData Profiling
-        self._md_profiler = cde.GlobalContext.profiling_manager()
-        self._md_profiler.init()
-        if context.get_context("mode") == context.PYNATIVE_MODE:
-            raise RuntimeError("Pynative mode is not supported on GPU currently.")
+        if self._data_process:
+            self._md_profiler = cde.GlobalContext.profiling_manager()
+            self._md_profiler.init()
         self._parse_parameter_for_gpu(kwargs)
 
         gpu_profiler = c_expression.Profiler
         self._gpu_profiler = gpu_profiler.get_instance("GPU")
         self._gpu_profiler.init(self._output_path)
+        self._gpu_profiler.sync_enable(self._sync_enable)
         if GlobalComm.WORLD_COMM_GROUP == "nccl_world_group":
             self._dev_id = str(get_rank())
         os.environ['DEVICE_ID'] = self._dev_id
+        self._rank_id = self._dev_id
 
     def _ascend_profiler_init(self, kwargs):
         """Ascend profiler init."""
         # Setup and start MindData Profiling
-        self._md_profiler = cde.GlobalContext.profiling_manager()
-        self._md_profiler.init()
+        if self._data_process:
+            self._md_profiler = cde.GlobalContext.profiling_manager()
+            self._md_profiler.init()
         self._init_time = int(time.time() * 10000000)
         logger.info("Profiling: profiling init time: %d", self._init_time)
         self._parse_parameter_for_ascend(kwargs)
@@ -476,13 +599,6 @@ class Profiler:
         """
         Construct profiling options to determine which profiling data should be collected.
         """
-        profile_memory = "off"
-        if self._profile_memory:
-            profile_memory = "on"
-        profiler_communication = "off"
-        if self._profile_communication:
-            profiler_communication = "on"
-
         fp_point = os.environ.get("PROFILING_FP_START", "")
         bp_point = os.environ.get("PROFILING_BP_END", "")
 
@@ -490,41 +606,35 @@ class Profiler:
             "output": self._output_path,
             "fp_point": fp_point,
             "bp_point": bp_point,
-            "training_trace": "on",
-            "task_trace": "on",
-            "aic_metrics": "ArithmeticUtilization",
-            "aicpu": "on",
-            "profile_memory": profile_memory,
-            "hccl": profiler_communication,
-            "parallel_strategy": "on"
+            "training_trace": "on" if self._op_time else "off",
+            "task_trace": "on" if self._op_time else "off",
+            "aic_metrics": AICORE_METRICS_DICT.get(self._aicore_metrics_id, "ArithmeticUtilization"),
+            "aicpu": "on" if self._data_process or self._op_time else "off",
+            "profile_memory": "on" if self._op_time and self._profile_memory else "off",
+            "hccl": "on" if self._op_time and self._profile_communication else "off",
+            "l2_cache": self._l2_cache,
+            "parallel_strategy": "on" if self._parallel_strategy else "off",
+            "op_time": "on" if self._op_time else "off"
         }
 
         return profiling_options
 
     def _parse_parameter_for_gpu(self, kwargs):
         """Parse parameter in Proflier when the device target is GPU."""
-
         self.start_profile = kwargs.pop("start_profile", True)
         if not isinstance(self.start_profile, bool):
             raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
                             f"but got type {type(self.start_profile)}")
 
-        self._profile_communication = kwargs.pop("profile_communication", False)
-        if not isinstance(self._profile_communication, bool):
-            raise TypeError(f"For '{self.__class__.__name__}', the parameter profile_communication must be bool, "
-                            f"but got type {type(self._profile_communication)}")
-        if self._profile_communication:
-            raise RuntimeError(f"The parameter profile_communication is not supported on GPU currently.")
-
-        self._profile_memory = kwargs.pop("profile_memory", False)
-        if not isinstance(self._profile_memory, bool):
-            raise TypeError(f"For '{self.__class__.__name__}', the parameter _profile_memory must be bool, "
-                            f"but got type {type(self._profile_memory)}")
-        if self._profile_memory:
-            raise RuntimeError(f"The parameter profile_memory is not supported on GPU currently.")
+        self._sync_enable = kwargs.pop("sync_enable", True)
+        if not isinstance(self._sync_enable, bool):
+            logger.warning("The parameter sync_enable is an invalid value, it will be set to True.")
+            self._sync_enable = True
 
     def _parse_parameter_for_ascend(self, kwargs):
         """Parse parameter in Proflier when the device target is Ascend."""
+        ascend_job_id = kwargs.pop("ascend_job_id", "")
+        self._set_ascend_job_id(ascend_job_id)
         self.start_profile = kwargs.pop("start_profile", True)
         if not isinstance(self.start_profile, bool):
             raise TypeError(f"For '{self.__class__.__name__}', the parameter start_profile must be bool, "
@@ -532,8 +642,10 @@ class Profiler:
 
         self._profile_communication = kwargs.pop("profile_communication", False)
         if not isinstance(self._profile_communication, bool):
-            raise TypeError(f"For '{self.__class__.__name__}', the parameter profile_communication must be bool, "
-                            f"but got type {type(self._profile_communication)}")
+            logger.warning(f"For '{self.__class__.__name__}', the parameter profile_communication must be bool, "
+                           f"but got type {type(self._profile_communication)}, it will be set to False.")
+            self._profile_communication = False
+
         if self._profile_communication:
             hccl_option = {"output": self._output_path, "task_trace": "on"}
             os.environ['PROFILING_OPTIONS'] = json.dumps(hccl_option)
@@ -543,10 +655,36 @@ class Profiler:
 
         self._profile_memory = kwargs.pop("profile_memory", False)
         if not isinstance(self._profile_memory, bool):
-            raise TypeError(f"For '{self.__class__.__name__}', the parameter profile_memory must be bool, "
-                            f"but got type '{type(self._profile_memory)}'")
-        if kwargs:
-            logger.warning("There are invalid params which don't work.")
+            logger.warning(f"For '{self.__class__.__name__}', the parameter profile_memory must be bool, "
+                           f"but got type {type(self._profile_memory)}, it will be set to False.")
+            self._profile_memory = False
+
+        self._aicore_metrics_id = kwargs.pop("aicore_metrics", 0)
+        if not isinstance(self._aicore_metrics_id, int):
+            logger.warning(f"For '{self.__class__.__name__}', the parameter aicore_metrics must be int, "
+                           f"but got type {type(self._aicore_metrics_id)}, it will be set to 0.")
+            self._aicore_metrics_id = 0
+
+        if self._aicore_metrics_id not in AICORE_METRICS_DICT:
+            logger.warning(f"For '{self.__class__.__name__}', the parameter aicore_metrics must be in "
+                           f"[-1, 0, 1, 2, 3, 4, 5], but got {self._aicore_metrics_id}, it will be set to 0.")
+            self._aicore_metrics_id = 0
+
+        l2_cache_enable = kwargs.pop("l2_cache", False)
+        if not isinstance(l2_cache_enable, bool):
+            logger.warning(f"For '{self.__class__.__name__}', the parameter l2_cache must be bool, "
+                           f"but got type {type(l2_cache_enable)}, it will be set to False.")
+            l2_cache_enable = False
+        if l2_cache_enable:
+            self._l2_cache = "on"
+        else:
+            self._l2_cache = "off"
+
+        self._parallel_strategy = kwargs.pop("parallel_strategy", True)
+        if not isinstance(self._parallel_strategy, bool):
+            logger.warning(f"For '{self.__class__.__name__}', the parameter parallel_strategy must be bool, "
+                           f"but got type {type(self._parallel_strategy)}, it will be set to True.")
+            self._parallel_strategy = True
 
         task_sink = os.getenv("GRAPH_OP_RUN")
         if task_sink and task_sink == "1":
@@ -555,6 +693,8 @@ class Profiler:
 
     def _set_ascend_job_id(self, ascend_job_id):
         """Set output_path for offline parsing performance data."""
+        if not ascend_job_id:
+            return
         self._ascend_job_id = validate_and_normalize_path(ascend_job_id)
         if not os.path.exists(self._ascend_job_id):
             msg = f"Invalid ascend_job_id: {self._ascend_job_id}, Please pass the absolute path of the JOB dir"
@@ -599,34 +739,62 @@ class Profiler:
 
         if GlobalComm.INITED:
             self._rank_size = get_group_size()
+        else:
+            self._rank_size = int(os.getenv('RANK_SIZE', '1'))
 
         if self._has_started:
             self.stop()
         else:
             logger.info("No need to stop profiler because profiler has been stopped.")
+        self._ascend_graph_analyse()
 
-        if context.get_context("mode") == context.PYNATIVE_MODE:
-            self._ascend_pynative_analyse()
-        else:
-            self._ascend_graph_analyse()
+    def _ascend_timeline_analyse(self, aicpu_data_parser, optime_parser, source_path):
+        """Analyse timeline info."""
+        try:
+            self._analyse_timeline(aicpu_data_parser, optime_parser, source_path)
+        except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
+            logger.warning('Fail to write timeline data: %s', err)
+        finally:
+            pass
 
-        # Call MSAdvisor function
+    def _ascend_step_trace_analyse(self, source_path, framework_parser):
+        """Analyse step trace info."""
+        points, is_training_mode_flag = None, False
         try:
-            msadvisor = Msadvisor(self._get_profiling_job_id(), self._rank_id, self._output_path)
-            logger.info("MSAdvisor starts running.")
-            msadvisor.analyse()
-        except (ProfilerFileNotFoundException, ValueError, FileNotFoundError, OSError) as err:
-            if context.get_context("mode") == context.PYNATIVE_MODE:
-                logger.warning("Pynative mode does not support MSAdvisor analyzer currently.")
-            else:
-                logger.warning("MSAdvisor running failed. %s", err)
+            if self._is_support_step_info_collect() and not self._dynamic_status:
+                points, is_training_mode_flag = self._analyse_step_trace(source_path, framework_parser)
+        except ProfilerException as err:
+            logger.warning(err.message)
         finally:
             pass
+        return points, is_training_mode_flag
+
+    def _ascend_dynamic_net_analyse(self):
+        """Analyse dynamic shape network info."""
+        if self._profile_communication:
+            logger.warning(
+                "The profile_communication parameter cannot be set on the dynamic shape network.")
+        if self._profile_memory:
+            logger.warning("The profile_memory parameter cannot be set on the dynamic shape network.")
+        logger.warning(
+            "[Profiler]Dynamic Shape network does not support collecting step trace performance data currently.")
+        dynamic_parser = DynamicFrameWorkParser(self._output_path, self._rank_id)
+        dynamic_parser.write_dynamic_shape_data()
+
+    def _ascend_flops_analyse(self, source_path, op_task_dict, is_training_mode_flag):
+        """Get op FLOPs from aicore.data.x.slice.0 file, and compute FLOPS, write output_op_flops_x.txt."""
+        flops_parser = FlopsParser(source_path, self._output_path, op_task_dict, self._dev_id, self._rank_id,
+                                   is_training_mode_flag)
+        logger.info("Profiling: analyzing the operation FLOPs.")
+        flops_parser.execute()
 
     def _ascend_graph_memory_analyse(self, points):
         """Analyse memory usage info."""
         if not self._profile_memory:
             return
+        if self._profile_memory and context.get_context("mode") == context.PYNATIVE_MODE:
+            logger.warning("[Profiler]The parameter profile_memory is not supported on Ascend "
+                           "PyNative mode currently.")
         try:
             logger.info("Profiling: analyzing the memory usage info.")
             self._analyse_memory_usage(points)
@@ -639,6 +807,9 @@ class Profiler:
         """Analyse hccl profiler info."""
         if not self._profile_communication:
             return
+        if self._profile_communication and context.get_context("mode") == context.PYNATIVE_MODE:
+            logger.warning("[Profiler]The parameter profile_communication is not supported on Ascend "
+                           "PyNative mode currently.")
         try:
             logger.info("Profiling: analyzing the hccl profiler info.")
             self._analyse_hccl_info()
@@ -647,6 +818,26 @@ class Profiler:
         finally:
             pass
 
+    def _ascend_graph_msadvisor_analyse(self, job_id):
+        """Call MSAdvisor function."""
+        logger.info("MSAdvisor starts running.")
+        msadvisor = Msadvisor(job_id, self._rank_id, self._output_path)
+        try:
+            msadvisor.analyse()
+        except FileNotFoundError as err:
+            logger.warning("MSAdvisor: command not found,"
+                           "please check if installed ascend-toolkit and set environment path correctly. %s", err)
+        except OSError as err:
+            logger.warning("Cannot execute binary file: Exec format error. %s", err)
+        except subprocess.CalledProcessError:
+            logger.warning("MSAdvisor running failed, please check MSAdvisor running log.")
+        except (ValueError, ProfilerFileNotFoundException) as err:
+            logger.warning("MSAdvisor running failed. %s", err)
+        finally:
+            pass
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            logger.warning("Pynative mode does not support MSAdvisor analyzer currently.")
+
     def _ascend_graph_op_analyse(self, source_path):
         """
         Ascend graph model hwts analyse.
@@ -690,29 +881,37 @@ class Profiler:
         logger.info("Profiling: analyzing the data preprocess data.")
         aicpu_data_parser.execute()
 
+        # analyse op compute time info
+        try:
+            self._analyser_op_info()
+        except ProfilerException as err:
+            logger.warning(err.message)
+        finally:
+            pass
         return [framework_parser, aicpu_data_parser, optime_parser, op_task_dict]
 
-    def _ascend_graph_minddata_analyse(self, source_path):
+    def _minddata_analyse(self, source_path):
         """Analyse mindadata for ascend graph model."""
+        if not self._data_process:
+            return
+        store_id = self._rank_id if self._device_target == DeviceTarget.ASCEND.value else self._dev_id
         # Parsing minddata AICPU profiling
-        logger.info("Profiling: analyzing the minddata AICPU data.")
-        MinddataParser.execute(source_path, self._output_path, self._rank_id)
+        if self._device_target == DeviceTarget.ASCEND.value:
+            logger.info("Profiling: analyzing the minddata AICPU data.")
+            MinddataParser.execute(source_path, self._output_path, store_id)
 
         # parse minddata pipeline operator and queue
         try:
-            pipeline_parser = MinddataPipelineParser(self._output_path, self._rank_id, self._output_path)
-            logger.info("Profiling: analyzing the minddata pipeline operator and queue.")
-            pipeline_parser.parse()
+            MinddataPipelineParser(self._output_path, store_id, self._output_path).parse()
         except ProfilerException as err:
             logger.warning(err.message)
         finally:
             pass
 
         # Analyze minddata information
+        logger.info("Profiling: analyzing the minddata information.")
         try:
-            md_analyzer = MinddataProfilingAnalyzer(self._output_path, self._rank_id, self._output_path)
-            logger.info("Profiling: analyzing the minddata information.")
-            md_analyzer.analyze()
+            MinddataProfilingAnalyzer(self._output_path, store_id, self._output_path).analyze()
         except ProfilerException as err:
             logger.warning(err.message)
         finally:
@@ -727,63 +926,18 @@ class Profiler:
 
         self._check_output_path(output_path=self._output_path)
         source_path = os.path.join(self._output_path, job_id)
-        framework_parser, aicpu_data_parser, optime_parser, op_task_dict = self._ascend_graph_op_analyse(source_path)
-        self._ascend_graph_minddata_analyse(source_path)
-
-        # analyse op compute time info
-        try:
-            logger.info("Profiling: analyzing the operation compute time.")
-            self._analyser_op_info()
-        except ProfilerException as err:
-            logger.warning(err.message)
-        finally:
-            pass
-
-        if self._dynamic_status and self._profile_communication:
-            raise RuntimeError("The profile_communication parameter cannot be set on the dynamic shape network.")
-        if self._dynamic_status and self._profile_memory:
-            raise RuntimeError("The profile_memory parameter cannot be set on the dynamic shape network.")
-
-        # analyse step trace info
-        points = None
-        is_training_mode_flag = False
-
-        # analyse timeline info
-        try:
-            logger.info("Profiling: analyzing the timeline data.")
-            self._analyse_timeline(aicpu_data_parser, optime_parser, source_path)
-        except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
-            logger.warning('Fail to write timeline data: %s', err)
-        finally:
-            pass
-
-        # get op FLOPs from aicore.data.x.slice.0 file, and compute FLOPS, write output_op_flops_x.txt
-        if not self._dynamic_status:
-            try:
-                logger.info("Profiling: analyzing the step trace data.")
-                points, is_training_mode_flag = self._analyse_step_trace(source_path, framework_parser)
-            except ProfilerException as err:
-                logger.warning(err.message)
-            finally:
-                pass
-
-            flops_parser = FlopsParser(source_path, self._output_path, op_task_dict,
-                                       self._dev_id, self._rank_id, is_training_mode_flag)
-            logger.info("Profiling: analyzing the operation FLOPs.")
-            flops_parser.execute()
-        else:
-            dynamic_parser = DynamicFrameWorkParser(self._output_path, self._rank_id)
-            dynamic_parser.write_dynamic_shape_data()
-
-        self._ascend_graph_memory_analyse(points)
-        self._ascend_graph_hccl_analyse()
-
-    def _ascend_pynative_start(self):
-        """Ascend pynative mode start profiling."""
-        pynative_profiler = c_expression.Profiler
-        self._pynative_profiler = pynative_profiler.get_instance("PyNative")
-        self._pynative_profiler.init(self._output_path)
-        self._ascend_profiler.start()
+        self._minddata_analyse(source_path)
+        if self._op_time:
+            framework_parser, aicpu_data_parser, optime_parser, op_task_dict = self._ascend_graph_op_analyse(
+                source_path)
+            points, is_training_mode_flag = self._ascend_step_trace_analyse(source_path, framework_parser)
+            self._ascend_timeline_analyse(aicpu_data_parser, optime_parser, source_path)
+            if self._dynamic_status:
+                self._ascend_dynamic_net_analyse()
+            self._ascend_flops_analyse(source_path, op_task_dict, is_training_mode_flag)
+            self._ascend_graph_memory_analyse(points)
+            self._ascend_graph_hccl_analyse()
+            self._ascend_graph_msadvisor_analyse(job_id)
 
     def _ascend_graph_start(self):
         """Ascend graph mode start profiling."""
@@ -798,49 +952,58 @@ class Profiler:
 
         if GlobalComm.INITED:
             self._rank_size = get_group_size()
+        else:
+            self._rank_size = int(os.getenv('RANK_SIZE', '1'))
 
         if self._has_started:
             self.stop()
         else:
             logger.info("No need to stop profiler because profiler has been stopped.")
 
-        reduce_op_type = self._get_step_reduce_op_type()
-        timeline_generator = self._generate_timeline(reduce_op_type)
-
-        # parse minddata pipeline operator and queue for GPU
-        try:
-            pipeline_parser = MinddataPipelineParser(self._output_path, self._dev_id, self._output_path)
-            logger.info("Profiling: analyzing the minddata pipeline operator and queue for GPU.")
-            pipeline_parser.parse()
-        except ProfilerException as err:
-            logger.warning(err.message)
+        self._minddata_analyse(self._output_path)
 
-        # Analyze minddata information
         try:
-            md_analyzer = MinddataProfilingAnalyzer(self._output_path, self._dev_id, self._output_path)
-            logger.info("Profiling: analyzing the minddata information.")
-            md_analyzer.analyze()
+            self._analyse_step_relation_info()
         except ProfilerException as err:
             logger.warning(err.message)
+        finally:
+            pass
 
-        # analyse step trace info
-        try:
-            logger.info("Profiling: analyzing the step trace info.")
+    def _is_support_step_info_collect(self, analyse_step_trace=True):
+        """Whether iteration related information needs to be parsed."""
+        profiler_info = ProfilerInfo.get_profiler_info()
+        graph_ids = profiler_info.get("graph_ids")
+        if len(graph_ids) > 1:
+            analyse_step_trace = False
+            logger.warning(
+                "[Profiler]Current model has multiple sub graphs, the segmentation of steps may be inaccurate.")
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            analyse_step_trace = False
+            logger.warning(
+                "[Profiler]Pynative mode does not support collecting step trace performance data currently.")
+        if self._is_heterogeneous:
+            analyse_step_trace = False
+            logger.warning(
+                "[Profiler]Profiler does not support collecting step trace performance data for heterogeneous "
+                "scenarios currently.")
+        return analyse_step_trace
+
+    def _analyse_step_relation_info(self):
+        """Parse iteration related information."""
+        if not self._op_time:
+            return
+        reduce_op_type = self._get_step_reduce_op_type()
+        timeline_generator = self._generate_timeline(reduce_op_type)
+        parser = GpuFrameWorkParser(self._output_path, self._dev_id)
+        graph_ids = parser.get_graph_ids()
+        ProfilerInfo.set_graph_ids(graph_ids)
+        if self._is_support_step_info_collect():
             self._analyse_step_trace(
                 is_training_mode_flag=timeline_generator.check_op_name('Gradients'),
                 is_gpu_kernel_async_launch_flag=timeline_generator.is_gpu_kernel_async_launch()
             )
-        except ProfilerException as err:
-            logger.warning(err.message)
-        finally:
-            pass
-        if self._dynamic_status:
-            parser = GpuFrameWorkParser(self._output_path, self._dev_id)
-            parser.analyse_dynamic_shape_data(self._timeline_meta)
-        logger.warning(
-            '\nThe GPU supports only the training mode or inference mode, '
-            'it does not support train and infer at the same time.'
-        )
+            if self._dynamic_status:
+                parser.analyse_dynamic_shape_data(self._timeline_meta)
 
     def _get_step_reduce_op_type(self):
         """Gets all communication operator names."""
@@ -860,15 +1023,15 @@ class Profiler:
         """Collect and analyse cpu performance data."""
 
         try:
-            timeline_generator = CpuTimelineGenerator(self._output_path, context.get_context("mode"))
+            timeline_generator = CpuTimelineGenerator(self._output_path, self._rank_id, context.get_context("mode"))
             timeline_generator.init_timeline()
             timeline_generator.write_timeline(self._timeline_size_limit_byte)
             timeline_generator.write_timeline_summary()
         except (ProfilerIOException, ProfilerFileNotFoundException, RuntimeError) as err:
             logger.warning('Fail to write timeline data: %s', err)
             raise RuntimeError('Fail to write timeline data.') from err
-        if self._dynamic_status:
-            raise RuntimeError('Profiler does not support dynamic shape network on CPU platform currently.')
+        if context.get_context("mode") == context.PYNATIVE_MODE:
+            raise RuntimeError("Pynative mode is not supported on CPU currently.")
 
     def _analyse_step_trace(self, source_path=None, framework_parser=None, is_training_mode_flag=True,
                             is_gpu_kernel_async_launch_flag=False):
@@ -935,6 +1098,7 @@ class Profiler:
             optime_parser (OPComputeTimeParserParser): The parser instance for AI Core
                 operator execution time calculation.
         """
+        logger.info("Profiling: analyzing the timeline data.")
         timeline_analyser = AscendTimelineGenerator(self._output_path, self._dev_id, self._rank_id,
                                                     self._rank_size, context.get_context("mode"))
         # Get framework info
@@ -1015,13 +1179,13 @@ class Profiler:
             else:
                 job_dir = os.path.join(self._output_path, dir_name)
 
-            host_start_file_path = get_file_path(job_dir, "host_start.log")
-            if host_start_file_path is None:
+            start_file_path = get_file_path(job_dir, "start_info")
+            if start_file_path is None:
                 logger.warning("Find profiling job path %s, but host_start.log not exist, "
                                "profiler will ignore this job dir.", job_dir)
                 continue
 
-            training_device_id = host_start_file_path.split('.')[-1]
+            training_device_id = start_file_path.split('.')[-1]
             if self._dev_id != training_device_id:
                 logger.debug("Find profiling find job path %s, but not current training device id. "
                              "Current training device id %s, but job path device id: %s, "
@@ -1031,7 +1195,7 @@ class Profiler:
             if not os.listdir(os.path.join(job_dir, 'data')):
                 continue
 
-            job_start_time = self._parse_host_start_log(host_start_file_path)
+            job_start_time = self._parse_start_log(start_file_path)
             if not job_start_time:
                 logger.warning("Find profiling job path %s, but fail to get job start info, "
                                "profiler will ignore this job dir.", job_start_time)
@@ -1060,6 +1224,7 @@ class Profiler:
 
     def _analyser_op_info(self):
         """Analyse the operator information."""
+        logger.info("Profiling: analyzing the operation compute time.")
         integrator = Integrator(self._output_path, self._rank_id)
         integrator.integrate()
 
@@ -1177,6 +1342,42 @@ class Profiler:
             logger.warning("The target dir already exists. "
                            "There may be some old profiling data, and they will be rewritten in the end.")
 
+    def _parser_kwargs(self, kwargs):
+        """Parse kwargs vale."""
+        self._op_time = kwargs.get("op_time", True)
+
+        env_run_config = json.loads(os.getenv("MS_PROFILER_RUN_CONFIG", "{}"))
+        params = list(kwargs.keys())
+        if not env_run_config.get("start"):
+            for param in params:
+                if param not in DeviceSupportParam.__getattr__(f'{self._device_target}'.upper()).value:
+                    logger.warning("%s is an invalid param which doesn't work.", param)
+                    kwargs.pop(param)
+                elif not self._op_time and param not in ALWAYS_VALID_PARAM:
+                    logger.warning(f"When op_time is set to False, the parameter '{param}' setting is invalid.")
+
+        if not isinstance(self._op_time, bool):
+            logger.warning(f"For '{self.__class__.__name__}', the parameter op_time must be bool, "
+                           f"but got type {type(self._op_time)}, it will be set to True.")
+            self._op_time = True
+
+        self._data_process = kwargs.pop("data_process", True)
+        if not isinstance(self._data_process, bool):
+            logger.warning(f"For '{self.__class__.__name__}', the parameter data_process must be bool, "
+                           f"but got type {type(self._data_process)}, it will be set to True.")
+            self._data_process = True
+
+        timeline_limit = kwargs.pop("timeline_limit", 500)
+        if isinstance(timeline_limit, bool) or not isinstance(timeline_limit, int):
+            logger.warning(f"For '{self.__class__.__name__}', the parameter timeline_limit must be int, "
+                           f"but got type {type(timeline_limit)}, it will be set to 500.")
+            timeline_limit = 500
+        if timeline_limit <= 0:
+            logger.warning(
+                "[Profiler]The 'timeline_limit' parameter must be greater than 0, it will be set to 500.")
+            timeline_limit = 500
+        self._timeline_size_limit_byte = timeline_limit * 1024 * 1024
+
     def _analyse_hccl_info(self):
         """Analyse hccl info."""
         hccl_path = os.path.join(self._output_path, "hccl_info_{}".format(self._rank_id))
@@ -1195,6 +1396,7 @@ class Profiler:
                             "The hccl_parser-{version}-py3-none-any.whl package is usually located "
                             "in the /usr/local/Ascend/tools Directory", err)
             raise ImportError(err) from err
+
         logger.info("Parse hccl info successfully.")
         logger.info("Start analyse hccl info.")
         hccl_parse = HcclParser(hccl_path, self._dev_id, self._rank_id, self._output_path)