mindspore 2.6.0__cp310-cp310-win_amd64.whl → 2.7.0__cp310-cp310-win_amd64.whl

mindspore/profiler/dynamic_profiler.py

@@ -15,36 +15,25 @@
 """Dynamic Profile Monitor"""
 import os
 import sys
+import json
 import time
 import stat
-import json
 import atexit
-import struct
 import random
 import multiprocessing
 
 from mindspore import log as logger
 from mindspore.train import Callback
-from mindspore.profiler import Profiler, tensorboard_trace_handler, schedule
-from mindspore.communication import get_rank
-from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
-from mindspore.profiler.parser.ascend_analysis.path_manager import PathManager
-from mindspore.profiler.profiler_interface import ProfilerInterface
-from mindspore.profiler.common.constant import (
-    ProfilerActivity,
-    ProfilerLevel,
-    AicoreMetrics,
-    ExportType,
-)
+from mindspore.profiler import tensorboard_trace_handler, schedule
+from mindspore.profiler.profiler import Profile
+from mindspore.profiler.experimental_config import _ExperimentalConfig
+from mindspore.profiler.common.file_manager import FileManager
+from mindspore.profiler.common.path_manager import PathManager
+from mindspore.profiler.dynamic_profile.dynamic_profiler_config_context import DynamicProfilerConfigContext
+from mindspore.profiler.dynamic_profile.dynamic_monitor_proxy import MsDynamicMonitorProxySingleton
+from mindspore.profiler.dynamic_profile.dynamic_profiler_utils import DynamicProfilerUtils
 from mindspore.profiler.common.util import no_exception_func
-
-
-def get_real_rank():
-    """get rank id"""
-    try:
-        return get_rank()
-    except RuntimeError:
-        return int(os.getenv("RANK_ID", "0"))
+from mindspore.profiler.profiler_interface import ProfilerInterface
 
 
 def print_msg(msg):
@@ -52,210 +41,21 @@ def print_msg(msg):
     print("[Dynamic Profiler] " + msg, flush=True)
 
 
-class DynamicProfilerArgs:
-    """
-    Data class for dynamic profile config.
-    """
-    FMT = "i" * 7 + "?" * 6
-    SIZE = struct.calcsize(FMT)
-
-    def __init__(self,
-                 start_step: int = -1,
-                 stop_step: int = -1,
-                 aic_metrics: int = -1,
-                 profiler_level: int = 0,
-                 analyse_mode: int = -1,
-                 activities: int = 0,
-                 export_type: int = 0,
-                 profile_memory: bool = False,
-                 mstx: bool = False,
-                 parallel_strategy: bool = False,
-                 with_stack: bool = False,
-                 data_simplification: bool = True,
-                 is_valid: bool = False,
-                 **kwargs):
-        self._start_step = start_step
-        self._stop_step = stop_step
-        self._aic_metrics = aic_metrics
-        self._profiler_level = profiler_level
-        self._analyse_mode = analyse_mode
-        self._activities = activities
-        self._export_type = export_type
-        self._profile_memory = profile_memory
-        self._mstx = mstx
-        self._parallel_strategy = parallel_strategy
-        self._with_stack = with_stack
-        self._data_simplification = data_simplification
-        self._is_valid = is_valid
-        self._check_params_type()
-
-    def _check_params_type(self):
-        """Check and enforce parameter types with lower complexity."""
-        # Define a parameter check rule. {Parameter name: (expected type, default value)}
-        param_rules = {
-            '_start_step': (int, -1),
-            '_stop_step': (int, -1),
-            '_aic_metrics': (int, -1),
-            '_profiler_level': (int, 0),
-            '_analyse_mode': (int, -1),
-            '_activities': (int, 0),
-            '_export_type': (int, 0),
-            '_profile_memory': (bool, False),
-            '_mstx': (bool, False),
-            '_parallel_strategy': (bool, False),
-            '_with_stack': (bool, False),
-            '_data_simplification': (bool, True),
-            '_is_valid': (bool, False)
-        }
-
-        def _is_valid_type(value, expected_type):
-            """Helper method for type checking."""
-            if expected_type is int and isinstance(value, bool):
-                return False
-            return isinstance(value, expected_type)
-
-        for param, (expected_type, default) in param_rules.items():
-            value = getattr(self, param)
-            if not _is_valid_type(value, expected_type):
-                logger.warning(
-                    f"{param[1:]} should be {expected_type.__name__} type, "
-                    f"will be reset to {default}."
-                )
-                setattr(self, param, default)
-
-    @property
-    def start_step(self):
-        """ get start step value."""
-        return self._start_step
-
-    @property
-    def stop_step(self):
-        """ get stop step value."""
-        return self._stop_step
-
-    @property
-    def is_valid(self):
-        """ get json valid value."""
-        return self._is_valid
-
-    @is_valid.setter
-    def is_valid(self, value):
-        """ set json valid value."""
-        self._is_valid = value
-
-    @property
-    def analyse_mode(self):
-        """ get analyse mode value."""
-        return self._convert_analyse_mode(self._analyse_mode)
-
-    @property
-    def vars(self):
-        """ get all values in DynamicProfilerArgs."""
-        not_supported_args = ['_is_valid']
-        res = {}
-        for key, value in self.__dict__.items():
-            if key not in not_supported_args:
-                res[key.replace('_', '', 1)] = value
-        return res
-
-    @property
-    def args(self):
-        """ get all args in DynamicProfilerArgs."""
-        self._profiler_level = self._convert_profiler_level(self._profiler_level)
-        self._activities = self._convert_activities(self._activities)
-        self._aic_metrics = self._convert_aic_metrics(self._aic_metrics)
-        self._export_type = self._convert_export_type(self._export_type)
-        not_supported_args = ['_start_step', '_stop_step', '_analyse_mode', '_is_valid']
-        res = {}
-        for key, value in self.__dict__.items():
-            if key not in not_supported_args:
-                res[key.replace('_', '', 1)] = value
-        return res
-
-    @classmethod
-    def from_bytes(cls, byte_data):
-        """ unpack bytes to DynamicProfilerArgs."""
-        unpacked = struct.unpack(cls.FMT, byte_data)
-        return cls(*unpacked)
-
-    def to_bytes(self):
-        """ pack DynamicProfilerArgs to bytes."""
-        instance_vars = tuple(self.__dict__.values())
-        if len(instance_vars) != len(self.FMT):
-            raise ValueError("Number of variables does not match format string.")
-        return struct.pack(DynamicProfilerArgs.FMT, *instance_vars)
-
-    def _convert_analyse_mode(self, analyse_mode: int) -> str:
-        """ convert analyse_mode to real args in Profiler."""
-        if analyse_mode == 0:
-            return 'sync'
-        if analyse_mode == 1:
-            return 'async'
-        return None
-
-    def _convert_profiler_level(self, profiler_level: int) -> ProfilerLevel:
-        """ convert profiler_level to real args in Profiler."""
-        if profiler_level == -1:
-            return ProfilerLevel.LevelNone
-        if profiler_level == 0:
-            return ProfilerLevel.Level0
-        if profiler_level == 1:
-            return ProfilerLevel.Level1
-        if profiler_level == 2:
-            return ProfilerLevel.Level2
-        return ProfilerLevel.Level0
-
-    def _convert_activities(self, activities: int) -> ProfilerLevel:
-        """ convert activities to real args in Profiler."""
-        if activities == 0:
-            return [ProfilerActivity.CPU, ProfilerActivity.NPU]
-        if activities == 1:
-            return [ProfilerActivity.CPU]
-        if activities == 2:
-            return [ProfilerActivity.NPU]
-        return [ProfilerActivity.CPU, ProfilerActivity.NPU]
-
-    def _convert_aic_metrics(self, aic_metrics: int) -> AicoreMetrics:
-        """ convert aic_metrics to real args in Profiler."""
-        if aic_metrics == -1:
-            return AicoreMetrics.AiCoreNone
-        if aic_metrics == 0:
-            return AicoreMetrics.PipeUtilization
-        if aic_metrics == 1:
-            return AicoreMetrics.ArithmeticUtilization
-        if aic_metrics == 2:
-            return AicoreMetrics.Memory
-        if aic_metrics == 3:
-            return AicoreMetrics.MemoryL0
-        if aic_metrics == 4:
-            return AicoreMetrics.MemoryUB
-        if aic_metrics == 5:
-            return AicoreMetrics.ResourceConflictRatio
-        if aic_metrics == 6:
-            return AicoreMetrics.L2Cache
-        if aic_metrics == 7:
-            return AicoreMetrics.MemoryAccess
-        return AicoreMetrics.AiCoreNone
-
-    def _convert_export_type(self, export_type: int) -> ExportType:
-        """ convert export_type to real args in Profiler."""
-        if export_type == 0:
-            return [ExportType.Text]
-        if export_type == 1:
-            return [ExportType.Db]
-        if export_type == 2:
-            return [ExportType.Text, ExportType.Db]
-        return [ExportType.Text]
-
 class DynamicProfilerMonitorBase(Callback):
     """
-    Dynamic profile callback base class implementing the dynamic profile functionality.
+    Dynamic profiler callback base class implementing the dynamic profiler functionality.
     """
 
-    def __init__(self, cfg_path, output_path=None, poll_interval=2, **kwargs):
-        self._cfg_path = cfg_path
-        self._cfg_json_path = os.path.join(self._cfg_path, "profiler_config.json")
-        self._cfg_json_path = os.path.realpath(self._cfg_json_path)
+    NPU_MONITOR_START = "NPU_MONITOR_START"
+
+    def __init__(self, cfg_path=None, output_path=None, poll_interval=2, **kwargs):
+        self._is_dyno = DynamicProfilerUtils.is_dyno_mode()
+        self._rank_id = DynamicProfilerUtils.get_real_rank()
+        if not self._is_dyno:
+            self._cfg_path = cfg_path
+            self._cfg_json_path = os.path.join(self._cfg_path, "profiler_config.json")
+            self._cfg_json_path = os.path.realpath(self._cfg_json_path)
+            self._init_cfg_json()
         self._output_path = "dyn_profile_data" if output_path is None else output_path
         self._poll_interval = poll_interval
         if not isinstance(self._poll_interval, int):
@@ -268,7 +68,6 @@ class DynamicProfilerMonitorBase(Callback):
 
         self._kwargs = kwargs
         self._shm_name = time.strftime("DynamicProfileShm%Y%m%d%H", time.localtime())
-        self._rank_id = get_real_rank()
         self._shared_loop_flag = multiprocessing.Value('b', True)
         self._shm = None
         self._process = None
@@ -282,26 +81,35 @@ class DynamicProfilerMonitorBase(Callback):
         self._step_num = 0
 
         self._check_shm_for_killed()
-        self._init_cfg_json()
         self._create_shm()
         self._create_process()
         atexit.register(self._clean_resource)
+        if self._is_dyno:
+            atexit.register(self._finalize_dynolog)
 
     @no_exception_func()
     def step_begin(self, run_context):
         """
-        Start profile at the begin of step.
+        Start profiler at the begin of step.
 
         Args:
             run_context (RunContext): Context of the train running.
         """
-        prof_args = self._get_prof_args()
+        prof_json = self._get_prof_args()
+        if not prof_json:
+            return
+        if self._is_dyno:
+            # Dyno monitor process
+            if self.NPU_MONITOR_START in prof_json:
+                self._call_dyno_monitor(prof_json)
+                return
 
+        prof_args = DynamicProfilerConfigContext(prof_json)
         if not prof_args.is_valid:
-            logger.error("Dynamic profile json is not valid, please check the json file.")
+            logger.error("Dynamic profiler json is not valid, please check the json file.")
             return
 
-        if prof_args.start_step == -1 or prof_args.start_step == self._last_start_step:
+        if prof_args.start_step in (-1, self._last_start_step):
             return
 
         cb_params = run_context.original_args()
@@ -311,16 +119,22 @@ class DynamicProfilerMonitorBase(Callback):
         # Prevent repeated calls of the start function within a complete interval
         if step_num == start_step:
             if self._is_started:
-                logger.error("Dynamic profile is already started at step %d, "
-                             "please wait the first profile finished at step %d.",
+                logger.error("Dynamic profiler is already started at step %d, "
+                             "please wait the first profiler finished at step %d.",
                              self._last_start_step, self._last_stop_step)
                 return
 
             if self._profiler is None:
-                prof_path = os.path.join(self._output_path, f"rank{self._rank_id}_start{start_step}_stop{stop_step}")
+                output_path = prof_args.prof_path if prof_args.prof_path != "./" else self._output_path
+                prof_path = os.path.join(
+                    output_path,
+                    f"rank{self._rank_id}_start{start_step}_stop{stop_step}"
+                )
                 PathManager.check_input_directory_path(prof_path)
-                self._profiler = Profiler(on_trace_ready=tensorboard_trace_handler(dir_name=prof_path),
-                                          start_profile=False, **prof_args.args)
+                profiler_config = self._get_prof_config(prof_args, prof_path, start_step, stop_step,
+                                                        start_profile=False,
+                                                        skip_first=0)
+                self._profiler = Profile(**profiler_config)
                 print_msg(f"Rank {self._rank_id} create output path {prof_path}")
 
             self._profiler.start()
@@ -330,37 +144,82 @@ class DynamicProfilerMonitorBase(Callback):
             print_msg(f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
                       f"will stop at step {stop_step}")
 
+    @staticmethod
+    def _get_prof_config(prof_args, prof_path, start_step, stop_step, start_profile, skip_first):
+        """
+        Get profiler config.
+
+        Args:
+            prof_args: Profiler config.
+            prof_path: Profiler output path.
+            start_step: Start step.
+            stop_step: Stop step.
+            start_profile: enable start_profile.
+            skip_first: skip first step.
+        """
+        profiler_config = {
+            "activities": prof_args.args.get("activities"),
+            "with_stack": prof_args.args.get("with_stack"),
+            "profile_memory": prof_args.args.get("profile_memory"),
+            "parallel_strategy": prof_args.args.get("parallel_strategy"),
+            "start_profile": start_profile,
+            "record_shapes": prof_args.args.get("record_shapes"),
+            "schedule": schedule(
+                wait=0,
+                warmup=0,
+                active=stop_step - start_step + 1,
+                repeat=1,
+                skip_first=skip_first
+            ),
+            "on_trace_ready": tensorboard_trace_handler(
+                dir_name=prof_path,
+                analyse_flag=prof_args.analyse,
+                async_mode=prof_args.analyse_mode == "async",
+            ),
+            "experimental_config": _ExperimentalConfig(
+                profiler_level=prof_args.args.get("profiler_level"),
+                aic_metrics=prof_args.args.get("aic_metrics"),
+                l2_cache=prof_args.args.get("l2_cache"),
+                mstx=prof_args.args.get("mstx"),
+                data_simplification=prof_args.args.get("data_simplification"),
+                export_type=prof_args.args.get("export_type"),
+                mstx_domain_include=prof_args.args.get("mstx_domain_include"),
+                mstx_domain_exclude=prof_args.args.get("mstx_domain_exclude"),
+                sys_io=prof_args.args.get("sys_io"),
+                sys_interconnection=prof_args.args.get("sys_interconnection"),
+                host_sys=prof_args.args.get("host_sys")
+            )
+        }
+        return profiler_config
+
     @no_exception_func()
     def step_end(self, run_context):
         """
-        Stop profile at the end of step.
+        Stop profiler at the end of step.
 
         Args:
             run_context (RunContext): Context of the train running.
         """
-        prof_args = self._get_prof_args()
+        prof_json = self._get_prof_args()
+        prof_args = DynamicProfilerConfigContext(prof_json)
 
         if not prof_args.is_valid:
-            logger.error("Dynamic profile json is not valid, please check the json file.")
+            logger.error("Dynamic profiler json is not valid, please check the json file.")
             return
 
         if prof_args.stop_step == -1:
             return
 
+        if self._profiler:
+            self._profiler.step()
+
         cb_params = run_context.original_args()
         step_num = cb_params.cur_step_num
 
         if step_num == self._last_stop_step and self._is_started:
-            if self._profiler:
-                self._profiler.stop()
-                if prof_args.analyse_mode:
-                    self._profiler.analyse(mode=prof_args.analyse_mode)
-                else:
-                    ProfilerInterface.finalize()
-                    ProfilerInterface.clear()
-                self._profiler = None
-                self._is_started = False
-                print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
+            self._profiler = None
+            self._is_started = False
+            print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
 
     @no_exception_func()
     def step(self):
@@ -415,19 +274,28 @@ class DynamicProfilerMonitorBase(Callback):
             ...      context.set_context(mode=mindspore.PYNATIVE_MODE)
             ...      mindspore.set_device("Ascend")
             ...      data_cfg = {
-            ...          "start_step": 2,
-            ...          "stop_step": 5,
-            ...          "aic_metrics": -1,
-            ...          "profiler_level": 0,
-            ...          "activities": 0,
-            ...          "export_type": 0,
-            ...          "profile_memory": False,
-            ...          "mstx": False,
-            ...          "analyse_mode": 0,
-            ...          "parallel_strategy": False,
-            ...          "with_stack": False,
-            ...          "data_simplification": True,
-            ...          }
+            ...           "start_step": 2,
+            ...           "stop_step": 5,
+            ...           "aic_metrics": "AiCoreNone",
+            ...           "profiler_level": "Level0",
+            ...           "analyse_mode": 0,
+            ...           "activities": ["CPU", "NPU"],
+            ...           "export_type": ["text"],
+            ...           "profile_memory": False,
+            ...           "mstx": False,
+            ...           "parallel_strategy": False,
+            ...           "with_stack": False,
+            ...           "data_simplification": True,
+            ...           "l2_cache": False,
+            ...           "analyse": True,
+            ...           "record_shape": False,
+            ...           "prof_path": "./data",
+            ...           "mstx_domain_include": [],
+            ...           "mstx_domain_exclude": [],
+            ...           "host_sys": [],
+            ...           "sys_io": False,
+            ...           "sys_interconnection": False
+            ...      }
             ...      output_path = "./cfg_path"
             ...      cfg_path = os.path.join(output_path, "profiler_config.json")
             ...      os.makedirs(output_path, exist_ok=True)
@@ -442,7 +310,8 @@ class DynamicProfilerMonitorBase(Callback):
             ...      for i in range(STEP_NUM):
             ...          print(f"step {i}")
             ...          train(net)
-            ...          # Modify the configuration file after step 7. For example, change start_step to 8 and stop_step to 10
+            ...          # Modify the configuration file after step 7
+            ...          # For example, change start_step to 8 and stop_step to 10
             ...          if i == 5:
             ...             # Modify parameters in the JSON file
             ...             change_cfg_json(os.path.join(output_path, "profiler_config.json"))
@@ -451,43 +320,81 @@ class DynamicProfilerMonitorBase(Callback):
         """
 
         self._step_num += 1
-        prof_args = self._get_prof_args()
+        prof_json = self._get_prof_args()
+        if not prof_json:
+            return
+        if self._is_dyno:
+            # Dyno monitor process
+            if self.NPU_MONITOR_START in prof_json:
+                self._call_dyno_monitor(prof_json)
+                return
 
+        prof_args = DynamicProfilerConfigContext(prof_json)
         if not prof_args.is_valid:
-            logger.error("Dynamic profile json is not valid, please check the json file.")
+            logger.error("Dynamic profiler config is not valid, please check the json or dyno config.")
             return
+        self._handle_profiler_setup(prof_args)
 
-        if prof_args.start_step == -1 or prof_args.stop_step == -1:
-            return
+        if self._profiler:
+            self._profiler.step()
 
-        # Skips the number of steps less than start_step
-        if self._step_num < prof_args.start_step:
+    def _handle_profiler_setup(self, args):
+        """Common handler for profiler setup logic shared between dyno and non-dyno paths."""
+        start_step = args.start_step
+        stop_step = args.stop_step
+
+        if not self._is_valid_start_stop_step(self._step_num, start_step, stop_step):
             return
 
-        if self._start_step != prof_args.start_step or self._stop_step != prof_args.stop_step:
-            # Update new start_step and stop_step
-            self._start_step = prof_args.start_step
-            self._stop_step = prof_args.stop_step
-            if self._start_step >= 0 and 0 <= self._start_step <= self._stop_step:
-                prof_path = os.path.join(self._output_path,
-                                         f"rank{self._rank_id}_start{self._start_step}_stop{self._stop_step}")
-                print_msg(f"Rank {self._rank_id} create output path {prof_path}")
-                print_msg(f"Rank {self._rank_id} Dynamic profile start at step {self._start_step}, "
-                          f"will stop at step {self._stop_step}")
-                self._profiler = Profiler(schedule=schedule(wait=0, warmup=0,
-                                                            active=self._stop_step - self._start_step + 1,
-                                                            repeat=1,
-                                                            skip_first=1),
-                                          on_trace_ready=tensorboard_trace_handler(dir_name=prof_path),
-                                          **prof_args.args)
-            else:
+        if self._start_step != start_step or self._stop_step != stop_step:
+            self._start_step = start_step
+            self._stop_step = stop_step
+
+            if not (start_step >= 0 and 0 <= start_step <= stop_step):
                 self._profiler = None
-                logger.error("Rank %d Dynamic profile start at step %d and stop at step %d in config_json must be "
-                             "greater than or equal to 0, and stop step should not be less than start step",
-                             self._rank_id, self._start_step, self._stop_step)
+                logger.error(
+                    "Rank %d Dynamic profiler start at step %d and stop at step %d must be "
+                    "greater than or equal to 0, and stop step should not be less than start step",
+                    self._rank_id, start_step, stop_step
+                )
+                return
 
-        if self._profiler:
-            self._profiler.step()
+            # Setup profiler configuration
+            output_path = args.prof_path if args.prof_path != "./" else self._output_path
+            prof_path = os.path.join(
+                output_path,
+                f"rank{self._rank_id}_start{start_step}_stop{stop_step}"
+            )
+            print_msg(f"Rank {self._rank_id} create output path {prof_path}")
+            print_msg(
+                f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
+                f"will stop at step {stop_step}"
+            )
+            profiler_config = self._get_prof_config(args, prof_path, start_step, stop_step, start_profile=True,
+                                                    skip_first=1)
+            self._profiler = Profile(**profiler_config)
+
+    def _is_valid_start_stop_step(self, step_num, start_step, stop_step):
+        """Verify whether start_step and stop_step are valid parameters."""
+        if start_step < 0 or stop_step < 0:
+            return False
+
+        if step_num < start_step:
+            return False
+
+        if step_num > stop_step != self._stop_step:
+            logger.warning("stop_step must be greater than step_num, "
+                           "but get start_step = %d, stop_step = %d, step_num = %d", start_step, stop_step, step_num)
+            return False
+
+        return True
+
+    @no_exception_func()
+    def _call_dyno_monitor(self, dyno_args):
+        if "is_valid" in dyno_args:
+            del dyno_args["is_valid"]
+        dyno_monitor_proxy = MsDynamicMonitorProxySingleton().get_proxy()
+        dyno_monitor_proxy.enable_dyno_npu_monitor(dyno_args)
 
     @no_exception_func()
     def on_train_end(self, run_context):
@@ -502,12 +409,16 @@ class DynamicProfilerMonitorBase(Callback):
     def _get_prof_args(self):
         """ Get prof_args """
         logger.error("Dynamic profiler _get_prof_args is not implemented")
-        return DynamicProfilerArgs()
+        return {}
 
     def _clean_resource(self):
         """Clean resource"""
         logger.error("Dynamic profiler _clean_resource is not implemented")
 
+    def _finalize_dynolog(self):
+        """finalize dynolog"""
+        logger.error("Dynolog monitor _finalize_dynolog is not implemented")
+
     def _check_step(self, start_step, stop_step, step_num):
         """Check step valid"""
         if start_step <= 0 or stop_step <= 0:
@@ -535,9 +446,11 @@ class DynamicProfilerMonitorBase(Callback):
         """Init config json file"""
         if self._rank_id == 0:
             if not os.path.exists(self._cfg_json_path):
-                logger.warning("cfg_path is not exist, create default cfg json")
-                FileManager.create_json_file(self._cfg_path, DynamicProfilerArgs().vars,
-                                             "profiler_config.json", indent=4)
+                logger.info("cfg_path is not exist, create default cfg json")
+                default_dy_config_context = DynamicProfilerConfigContext({})
+                PathManager.make_dir_safety(self._cfg_path)
+                config_file_path = os.path.join(self._cfg_path, "profiler_config.json")
+                FileManager.create_json_file(config_file_path, default_dy_config_context.vars, indent=4)
         else:
             logger.info("rank_id is not 0, skip init cfg json")
         print_msg(f"Init config json file: {self._cfg_json_path}")
@@ -550,10 +463,12 @@ class DynamicProfilerMonitorBase(Callback):
     def _create_process(self):
         """Create json monitor process, one process will be created at one worker"""
         if self._is_create_process:
+            args = [self._shared_loop_flag, self._poll_interval, self._shm, self._rank_id] if self._is_dyno else \
+                [self._shared_loop_flag, self._poll_interval, self._shm, self._cfg_json_path]
             # daemon need to be set to True, otherwise the process will not be killed when the main process exits.
-            self._process = multiprocessing.Process(target=worker_func, daemon=True,
-                                                    args=(self._shared_loop_flag, self._poll_interval,
-                                                          self._shm, self._cfg_json_path))
+            self._process = multiprocessing.Process(target=worker_dyno_func if self._is_dyno else worker_func,
+                                                    daemon=True,
+                                                    args=args)
             self._process.start()
             logger.info("Config monitor process has been created by rank %d.", self._rank_id)
         else:
@@ -573,7 +488,7 @@ class DynamicProfilerMonitorBase(Callback):
         if not os.path.exists(shm_path):
             return
 
-        MAX_TIME_DIFF = 30 # seconds
+        MAX_TIME_DIFF = 60  # seconds
         time_shm = os.stat(shm_path).st_ctime
         cur_proc_time = self._get_pid_st_ctime(os.getpid())
 
@@ -584,7 +499,7 @@ class DynamicProfilerMonitorBase(Callback):
     def _get_pid_st_ctime(self, pid):
         """Get pid st_ctime"""
         try:
-            fd = os.open("/proc/" + str(pid), os.O_RDONLY, stat.S_IRUSR | stat.S_IRGRP)
+            fd = os.open(os.path.join('/proc', str(pid)), os.O_RDONLY, stat.S_IRUSR | stat.S_IRGRP)
             stat_ino = os.fstat(fd)
             os.close(fd)
             create_time = stat_ino.st_ctime
@@ -593,7 +508,7 @@ class DynamicProfilerMonitorBase(Callback):
             logger.error("Process with PID %d does not exist.", pid)
         except PermissionError:
             logger.error("Permission denied when accessing PID %d.", pid)
-        except Exception as ex: # pylint: disable=W0703
+        except Exception as ex:  # pylint: disable=W0703
             logger.error("An error occurred while getting creation time for PID %d: %s", pid, str(ex))
 
 
@@ -601,7 +516,8 @@ if sys.version_info >= (3, 8):
     @no_exception_func()
     def write_bytes(shm, byte_data):
         """Write bytes to shared memory"""
-        shm.buf[:DynamicProfilerArgs.SIZE] = byte_data
+        shm.buf[:] = b'\x00' * len(shm.buf)
+        shm.buf[:len(byte_data)] = byte_data
 else:
     @no_exception_func()
     def write_bytes(shm, byte_data):
@@ -624,15 +540,13 @@ def worker_func(loop_flag, poll_interval, shm, cfg_path):
                     with open(cfg_path, 'r') as f:
                         data = json.load(f)
 
-                    # convert json to DynamicProfilerArgs
-                    prof_args = DynamicProfilerArgs(**data)
-                    prof_args.is_valid = True
+                    data['is_valid'] = True
                     logger.info("Dynamic profiler process load json success")
                 except json.JSONDecodeError as e:
-                    prof_args = DynamicProfilerArgs()
-                    prof_args.is_valid = False
+                    data = {'is_valid': False}
                     logger.error("Dynamic profiler process load json failed: %s", e)
-                byte_data = prof_args.to_bytes()
+                # convert json to bytes
+                byte_data = DynamicProfilerConfigContext.json_to_bytes(data)
                 write_bytes(shm, byte_data)
         else:
             logger.error("Dynamic profiler cfg json not exists")
@@ -640,6 +554,36 @@ def worker_func(loop_flag, poll_interval, shm, cfg_path):
     logger.info("Dynamic profiler process done")
 
 
+@no_exception_func()
+def worker_dyno_func(loop_flag, poll_interval, shm, rank_id):
+    """ dyno monitor process worker function python version >= 3.8"""
+    proxy = MsDynamicMonitorProxySingleton().get_proxy()
+    ret = proxy.init_dyno(rank_id)
+
+    if not ret:
+        logger.warning("Rank %d init dynolog failed !")
+        return
+    print_msg("Init dynolog success !")
+
+    while loop_flag.value:
+        try:
+            res = proxy.poll_dyno()
+            if not res:
+                continue
+            data = DynamicProfilerUtils.dyno_str_to_dict(res)
+        except Exception as e:  # pylint: disable=broad-except
+            data = {'is_valid': False}
+            logger.error("Dynolog process load config failed: %s", e)
+        else:
+            data['is_valid'] = True
+
+        # convert dyno config json to bytes
+        byte_data = DynamicProfilerConfigContext.json_to_bytes(data)
+        write_bytes(shm, byte_data)
+        time.sleep(poll_interval)
+    logger.info("Dynolog process done")
+
+
 if sys.version_info >= (3, 8):
     from multiprocessing import shared_memory
     from unittest.mock import patch
@@ -647,10 +591,10 @@ if sys.version_info >= (3, 8):
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
-        This class to enable the dynamic profile monitoring of MindSpore neural networks.
+        This class to enable the dynamic profiler monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): (Ascend only) Dynamic profile json config file directory. The requirement is a shared path
+            cfg_path (str): (Ascend only) Dynamic profiler json config file directory. The requirement is a shared path
                 that can be accessed by all nodes. The parameters of the json configuration file are as follows:
 
                 - start_step (int, required) - Sets the step number at which the Profiler starts collecting data.
@@ -660,27 +604,46 @@ if sys.version_info >= (3, 8):
                   a relative value, with the first step of training being 1. The stop_step must be greater than or
                   equal to start_step. The default value is -1, indicating that data collection will not start during
                   the entire training process.
-                - aic_metrics (int, optional) - The range of values corresponds to the Profiler. The default value -1
-                  indicates that AI Core utilization is not collected, and 0 indicates PipeUtilization, 1 indicates
-                  ArithmeticUtilization, 2 stands for Memory, 3 stands for MemoryL0, 4 stands for MemoryUB, 5 indicates
-                  ResourceConflictRatio, 6 indicates L2Cache, 7 indicates MemoryAccess.
-                - profiler_level (int, optional) - Sets the level of performance data collection, where -1 represents
-                  ProfilerLevel.LevelNone, 0 represents ProfilerLevel.Level0, 1 represents ProfilerLevel.Level1, and
-                  2 represents ProfilerLevel.Level2. The default value is 0, indicating the ProfilerLevel.Level0
-                  collection level.
-                - activities (int, optional) - Sets the devices for performance data collection, where 0 represents
-                  CPU+NPU, 1 represents CPU, and 2 represents NPU. The default value is 0, indicating the collection
-                  of CPU+NPU performance data.
-                - export_type (int, optional) -  Sets the data type to export, where 0 represents text, 1 represents db,
-                  and 2 represents text and db. The default value is 0, indicating only export text type data.
+                - aic_metrics (int/str, optional) - Set the collection of AI Core metric data. The current version can
+                  pass in either type int or str. Later, it will be updated to only pass in the str type.
+                  Here, ``0`` and ``"PipeUtilization"`` represent PipeUtilization; ``1`` and ``"ArithmeticUtilization"``
+                  represent ArithmeticUtilization; ``2`` and ``"Memory"`` represent Memory; ``3`` and ``"MemoryL0"``
+                  represent MemoryL0; ``4`` and ``"MemoryUB"`` stand for MemoryUB; ``5`` and ``"ResourceConflictRatio"``
+                  represent ResourceConflictRatio; ``6`` and ``"L2Cache"`` represent L2Cache; ``7`` and
+                  ``"MemoryAccess"`` stand for MemoryAccess. The default value ``"AiCoreNone"`` indicates that the
+                  AI Core metric is not collected.
+                - profiler_level (int/str, optional) - Set the level for collecting performance data. The current
+                  version can pass in either type int or str, and it will be updated to only pass in str type
+                  in the future. Among them, ``-1`` and ``"LevelNone"`` represent ProfilerLevel.LevelNone, ``0``
+                  and ``"Level0"`` represent ProfilerLevel.Level0, and ``1`` and ``"Level1"`` represent
+                  ProfilerLevel.Level1. ``2`` and ``"Level2"`` stand for Profile Level.Level2.
+                  The default value ``"Level0"`` indicates the collection level of ProfilerLevel.Level0.
+                - activities (int/list, optional) - Set the device for collecting performance data.
+                  The current version can pass in either type int or list. Later, it will be updated to only
+                  pass in the list type. Among them, ``0`` and ``["CPU","NPU"]`` represent CPU+NPU, ``1`` and
+                  ``["CPU"]`` represent CPU, and ``2`` and ``["NPU"]`` represent NPU. The default values
+                  ``["CPU","NPU"]`` indicate the collection of  performance data of CPU+NPU.
+                - export_type (int/list, optional) - Set the type of the exported performance data.
+                  The current version can pass in either type int or list, and it will be updated later
+                  to only pass in the list type. Among them, ``0`` and ``["text"]`` represent text, ``1`` and ``["db"]``
+                  represent db, and ``2`` and ``["text","db"]`` represent text and db respectively. The default value
+                  ``["text"]`` indicates that only performance data of the text type is exported.
                 - profile_memory (bool, optional) - Set whether to collect memory performance data, true indicates that
                   memory performance data is collected, false indicates that memory performance data is not collected.
                   The default value is false, indicating that memory performance data is not collected.
                 - mstx (bool, optional) - Set whether to enable mstx, true indicates that mstx is enabled, false
                   indicates that mstx is disabled. The default value is false, indicating that mstx is not enabled.
-                - analyse_mode (int, optional) - Sets the mode for online analysis, corresponding to the analyse_mode
-                  parameter of the mindspore.Profiler.analyse interface, where 0 represents "sync" and 1 represents
-                  "async". The default value is -1, indicating that online analysis is not used.
+                - analyse (bool, optional) - Set whether to enable online analysis. True indicates that online analysis
+                  is enabled, while false indicates that online analysis is disabled. The default value is false,
+                  indicating that online analysis is not enabled. This parameter has a higher priority than the
+                  `analyse_mode` parameter. When this parameter is set to false, the setting of the `analyse_mode`
+                  parameter does not take effect. When this parameter is set to true,
+                  setting the `analyse_mode` parameter to -1 does not take effect.
+                - analyse_mode (int, optional) - Sets the mode for online analysis,
+                  where 0 represents "sync" and 1 represents "async". The default value is -1,
+                  indicating that online analysis is not used. This parameter has a lower priority than the `analyse`
+                  parameter. When the `analyse` parameter is set to false, the setting of this parameter does not take
+                  effect. When the `analyse` parameter is set to true, setting it to -1 does not take effect.
                 - parallel_strategy (bool, optional) - Sets whether to collect parallel strategy performance data,
                   where true means to collect and false means not to collect. The default value is false, indicating
                   that parallel strategy performance data is not collected.
@@ -690,6 +653,44 @@ if sys.version_info >= (3, 8):
                 - data_simplification (bool, optional) - Sets whether to enable data simplification, where true means
                   to enable and false means not to enable. The default value is true, indicating that data
                   simplification is enabled.
+                - record_shapes (bool, optional) - Sets whether to collect operator input tensor shapes data, where true
+                  means that the shape data is collected and false means that the shape data is not collected. The
+                  default value is false, indicating that input tensor shapes data is not collected.
+                - mstx_domain_include (list, optional) - Set the set of enabled domain names when the mstx switch
+                  is turned on. The name must be of str type. Default value: ``[]``, indicating that this parameter
+                  is not used to control the domain. This parameter is mutually exclusive with the mstx_domain_exclude
+                  parameter and cannot be set. simultaneously. If both are set, only the mstx_domain_include parameter
+                  takes effect.
+                - mstx_domain_exclude (list, optional) - Set the set of domain names that are not enabled when the
+                  mstx switch is turned on. The name must be of str type. Default value: ``[]``, indicating that this
+                  parameter is not used to control the domain.
+                - prof_path (str, optional) - Output data path of the dynamic profiler. It is the same as the interface
+                  parameter `output_path`. When both are set, `prof_path` takes effect. Default value:
+                  ``"./"`` .
+                - sys_io (bool, optional) - Set whether to collect NIC and RoCE data. Default value: ``False`` ,
+                  indicating that these data are not collected.
+                - sys_interconnection (bool, optional) - Set whether to collect system interconnection data,
+                  including aggregate collective communication statistics (HCCS), PCIe data, and inter-chip transmission
+                  bandwidth information. Default value: ``False`` , indicating that these data are not collected.
+                - host_sys (list, optional) - Collect the data of system class calls, storage classes and cpu usage
+                  rate on the host side, and pass in the list type. It supports passing in one or more of ``"cpu"``,
+                  ``"mem"``, ``"disk"``, ``"network"`` and ``"osrt"``. Among them, ``"cpu"`` represents the cpu
+                  utilization at the process level, ``"mem"`` represents the memory utilization at the process level,
+                  ``"disk"`` represents the disk I/O utilization at the process level, and ``"network"`` represents the
+                  network I/O utilization at the system level. ``"osrt"`` represents system-level syscall and
+                  pthreadcall. Default value: ``[]``, indicating that system class data on the host side is
+                  not collected. When collecting DISK or OSRT data, it is necessary to install the iotop, perf,
+                  and ltrace third-party tools in advance. For detailed steps, please refer to
+                  `Installing Third-party Tools <https://www.hiascend.com/document/detail/zh/mindstudio/80RC1/T&ITools/
+                  Profiling/atlasprofiling_16_0136.html>`_ .
+                  After the third-party tool is successfully installed, user permissions need to be configured. For
+                  detailed steps, please refer to `Configure User Permissions <https://www.hiascend.com/document/
+                  detail/zh/mindstudio/80RC1/T&ITools/Profiling/atlasprofiling_16_0137.
+                  html>`_ .
+                  Note that in step 3 of configuring user permissions, the content in the msprof_data_collection.sh
+                  script needs to be replaced with `msprof_data_collection.sh
+                  <https://gitee.com/mindspore/mindspore/blob/master/docs/api/api_python/mindspore/script/
+                  msprof_data_collection.sh>`_.
 
             output_path (str, optional): (Ascend only) Output data path. Default: ``"./dyn_profile_data"`` .
             poll_interval (int, optional): (Ascend only) The polling period of the monitoring process, in seconds.
@@ -729,9 +730,13 @@ if sys.version_info >= (3, 8):
             ...     model.train(10, data, callbacks=[dynprof_cb])
         """
 
-        def __init__(self, cfg_path, output_path="./dyn_profile_data", poll_interval=2, **kwargs):
-            if not isinstance(cfg_path, str):
-                raise TypeError("The cfg_path must be a string.")
+        def __init__(self, cfg_path=None, output_path="./dyn_profile_data", poll_interval=2, **kwargs):
+            if DynamicProfilerUtils.is_dyno_mode() and cfg_path is not None:
+                logger.warning("If you export 'MSMONITOR_USE_DAEMON=1', your 'cfg_path' parameter will be invalid!")
+                cfg_path = None
+
+            if not DynamicProfilerUtils.is_dyno_mode() and not isinstance(cfg_path, str):
+                raise TypeError("If you set 'MSMONITOR_USE_DAEMON' to not 1, The cfg_path must be a string.")
             if not isinstance(output_path, str):
                 logger.warning(f"The output_path must be a string, "
                                f"but got type {type(output_path)}, it will be set to './dyn_profile_data'.")
@@ -740,7 +745,21 @@ if sys.version_info >= (3, 8):
 
         def _get_prof_args(self):
             """ Get prof_args py38"""
-            return DynamicProfilerArgs.from_bytes(self._shm.buf[:DynamicProfilerArgs.SIZE])
+            byte_length = self._get_shm_byte_length()
+
+            if byte_length == 0:
+                return {}
+
+            valid_bytes = self._shm.buf[:byte_length]
+            return DynamicProfilerConfigContext.bytes_to_json(bytes(valid_bytes))
+
+        def _get_shm_byte_length(self):
+            byte_length = 0
+            for i, byte in enumerate(self._shm.buf):
+                if byte == 0:
+                    byte_length = i
+                    break
+            return byte_length
 
         @no_exception_func()
         def _clean_resource(self):
@@ -770,6 +789,12 @@ if sys.version_info >= (3, 8):
                     logger.warning("Rank %s unlink shm failed, may be removed", self._rank_id)
                 self._shm = None
 
+        @no_exception_func()
+        def _finalize_dynolog(self):
+            dyno_monitor_proxy = MsDynamicMonitorProxySingleton().get_proxy()
+            dyno_monitor_proxy.finalize_dyno()
+            logger.info("Rank %d finalize dynolog success !", self._rank_id)
+
         @no_exception_func()
         def _create_shm(self):
             """Create a json monitor process based on whether the SharedMemory is successfully created py38"""
@@ -789,7 +814,7 @@ if sys.version_info >= (3, 8):
                     try:
                         # Step 2: only one process can create shm successfully.
                         self._shm = shared_memory.SharedMemory(name=self._shm_name,
-                                                               create=True, size=DynamicProfilerArgs.SIZE)
+                                                               create=True, size=DynamicProfilerUtils.CFG_BUFFER_SIZE)
                         self._is_create_process = True
                         logger.info("Rank %d shared memory is created.", self._rank_id)
                         break
@@ -799,7 +824,7 @@ if sys.version_info >= (3, 8):
                         logger.warning("Rank %d shared memory create failed, "
                                        "retry times = %d.", self._rank_id, try_times)
                         time.sleep(random.uniform(0, 0.02))  # sleep 0 ~ 20 ms
-                except Exception as e: # pylint: disable=W0703
+                except Exception as e:  # pylint: disable=W0703
                     # shm open failed because of other process create shm not finished
                     try_times -= 1
                     logger.warning("Rank %d shared memory open failed, error: %s, retry times = %d",
@@ -815,10 +840,10 @@ else:
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
-        This class to enable the dynamic profile monitoring of MindSpore neural networks.
+        This class to enable the dynamic profiler monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): Dynamic profile json config file directory. The requirement is a shared path
+            cfg_path (str): Dynamic profiler json config file directory. The requirement is a shared path
                 that can be accessed by all nodes.
             output_path (str, optional): Output data path. Default: ``"./dyn_profile_data"`` .
             poll_interval (int, optional): The polling period of the monitoring process, in seconds.
@@ -858,16 +883,24 @@ else:
             ...     model.train(10, data, callbacks=[dynprof_cb])
         """
 
-        def __init__(self, cfg_path, output_path="./dyn_profile_data", poll_interval=2, **kwargs):
-            if not isinstance(cfg_path, str):
-                raise TypeError("The cfg_path must be a string.")
+        def __init__(self, cfg_path=None, output_path="./dyn_profile_data", poll_interval=2, **kwargs):
+            if DynamicProfilerUtils.is_dyno_mode() and cfg_path is not None:
+                logger.warning("If you export 'MSMONITOR_USE_DAEMON=1', your 'cfg_path' parameter will be invalid!")
+                cfg_path = None
+
+            if not DynamicProfilerUtils.is_dyno_mode() and not isinstance(cfg_path, str):
+                raise TypeError("If you set 'MSMONITOR_USE_DAEMON' to not 1, The cfg_path must be a string.")
+
             if not isinstance(output_path, str):
                 logger.warning(f"The output_path must be a string, "
                                f"but got type {type(output_path)}, it will be set to './dyn_profile_data'.")
                 output_path = "./dyn_profile_data"
             self._cfg_path = cfg_path
             self._shm_name = time.strftime("DynamicProfileShm%Y%m%d%H", time.localtime())
-            self._shm_dir = os.path.join(self._cfg_path, "shm")
+            self._shm_dir = (
+                "/dev/shm" if DynamicProfilerUtils.is_dyno_mode()
+                else os.path.join(self._cfg_path, "shm")
+            )
             PathManager.make_dir_safety(self._shm_dir)
             self._shm_path = os.path.realpath(os.path.join(self._shm_dir, self._shm_name))
 
@@ -878,7 +911,8 @@ else:
         def _get_prof_args(self):
             """ Get prof_args py37"""
             self._shm.seek(0)
-            return DynamicProfilerArgs.from_bytes(self._shm.read(DynamicProfilerArgs.SIZE))
+            return DynamicProfilerConfigContext.bytes_to_json(
+                bytes(self._shm.read(DynamicProfilerUtils.CFG_BUFFER_SIZE)))
 
         @no_exception_func()
         def _clean_resource(self):
@@ -923,7 +957,8 @@ else:
                     self.fd = os.open(self._shm_path, os.O_EXCL | os.O_RDWR,
                                       stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP)
                     self._memory_mapped_file = os.fdopen(self.fd, 'rb')
-                    self._shm = mmap.mmap(self._memory_mapped_file.fileno(), length=DynamicProfilerArgs.SIZE)
+                    self._shm = mmap.mmap(self._memory_mapped_file.fileno(),
+                                          length=DynamicProfilerUtils.CFG_BUFFER_SIZE)
                     self._is_create_process = False
                     logger.info("Rank %d shared memory is connected.", self._rank_id)
                     break
@@ -937,7 +972,7 @@ else:
 
                         # Init mmap file need to write data
                         with os.fdopen(fd, 'wb') as f:
-                            data_instance = DynamicProfilerArgs()
+                            data_instance = DynamicProfilerConfigContext({})
                             byte_data = data_instance.to_bytes()
                             f.write(byte_data)
 
@@ -945,7 +980,8 @@ else:
                         self.fd = os.open(self._shm_path, os.O_EXCL | os.O_RDWR,
                                           stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP)
                         self._memory_mapped_file = os.fdopen(self.fd, 'rb')
-                        self._shm = mmap.mmap(self._memory_mapped_file.fileno(), length=DynamicProfilerArgs.SIZE)
+                        self._shm = mmap.mmap(self._memory_mapped_file.fileno(), length=DynamicProfilerUtils.
+                                              CFG_BUFFER_SIZE)
                         self._is_create_process = True
                         logger.info("Rank %d shared memory is created.", self._rank_id)
                         break