mindspore 2.4.10__cp39-cp39-win_amd64.whl → 2.6.0rc1__cp39-cp39-win_amd64.whl

mindspore/profiler/dynamic_profiler.py

@@ -16,6 +16,7 @@
 import os
 import sys
 import time
+import stat
 import json
 import atexit
 import struct
@@ -24,11 +25,18 @@ import multiprocessing
 
 from mindspore import log as logger
 from mindspore.train import Callback
-from mindspore.profiler import Profiler
-from mindspore.profiler import ProfilerLevel
+from mindspore.profiler import Profiler, tensorboard_trace_handler, schedule
 from mindspore.communication import get_rank
 from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
 from mindspore.profiler.parser.ascend_analysis.path_manager import PathManager
+from mindspore.profiler.profiler_interface import ProfilerInterface
+from mindspore.profiler.common.constant import (
+    ProfilerActivity,
+    ProfilerLevel,
+    AicoreMetrics,
+    ExportType,
+)
+from mindspore.profiler.common.util import no_exception_func
 
 
 def get_real_rank():
@@ -48,17 +56,19 @@ class DynamicProfilerArgs:
     """
     Data class for dynamic profile config.
     """
-    FMT = "iiiiii?????"
+    FMT = "i" * 7 + "?" * 6
     SIZE = struct.calcsize(FMT)
 
     def __init__(self,
                  start_step: int = -1,
                  stop_step: int = -1,
-                 aicore_metrics: int = -1,
-                 profiler_level: int = -1,
-                 profile_framework: int = -1,
+                 aic_metrics: int = -1,
+                 profiler_level: int = 0,
                  analyse_mode: int = -1,
-                 profile_communication: bool = False,
+                 activities: int = 0,
+                 export_type: int = 0,
+                 profile_memory: bool = False,
+                 mstx: bool = False,
                  parallel_strategy: bool = False,
                  with_stack: bool = False,
                  data_simplification: bool = True,
@@ -66,11 +76,13 @@ class DynamicProfilerArgs:
                  **kwargs):
         self._start_step = start_step
         self._stop_step = stop_step
-        self._aicore_metrics = aicore_metrics
+        self._aic_metrics = aic_metrics
         self._profiler_level = profiler_level
-        self._profile_framework = profile_framework
         self._analyse_mode = analyse_mode
-        self._profile_communication = profile_communication
+        self._activities = activities
+        self._export_type = export_type
+        self._profile_memory = profile_memory
+        self._mstx = mstx
         self._parallel_strategy = parallel_strategy
         self._with_stack = with_stack
         self._data_simplification = data_simplification
@@ -78,50 +90,38 @@ class DynamicProfilerArgs:
         self._check_params_type()
 
     def _check_params_type(self):
-        """ check params type."""
-        if not isinstance(self._start_step, int):
-            logger.warning("start_step should be int type, start_step will be reset to -1.")
-            self._start_step = -1
-
-        if not isinstance(self._stop_step, int):
-            logger.warning("stop_step should be int type, stop_step will be reset to -1.")
-            self._stop_step = -1
-
-        if not isinstance(self._aicore_metrics, int):
-            logger.warning("aicore_metrics should be int type, aicore_metrics will be reset to -1.")
-            self._aicore_metrics = -1
-
-        if not isinstance(self._profiler_level, int):
-            logger.warning("profiler_level should be int type, profiler_level will be reset to -1.")
-            self._profiler_level = -1
-
-        if not isinstance(self._profile_framework, int):
-            logger.warning("profile_framework should be int type, profile_framework will be reset to -1.")
-            self._profile_framework = -1
-
-        if not isinstance(self._analyse_mode, int):
-            logger.warning("analyse_mode should be int type, analyse_mode will be reset to -1.")
-            self._analyse_mode = -1
-
-        if not isinstance(self._profile_communication, bool):
-            logger.warning("profile_communication should be bool type, profile_communication will be reset to False.")
-            self._profile_communication = False
-
-        if not isinstance(self._parallel_strategy, bool):
-            logger.warning("parallel_strategy should be bool type, parallel_strategy will be reset to False.")
-            self._parallel_strategy = False
-
-        if not isinstance(self._with_stack, bool):
-            logger.warning("with_stack should be bool type, with_stack will be reset to False.")
-            self._with_stack = False
-
-        if not isinstance(self._data_simplification, bool):
-            logger.warning("data_simplification should be bool type, data_simplification will be reset to True.")
-            self._data_simplification = True
-
-        if not isinstance(self._is_valid, bool):
-            logger.warning("is_valid should be bool type, is_valid will be reset to False.")
-            self._is_valid = False
+        """Check and enforce parameter types with lower complexity."""
+        # Define a parameter check rule. {Parameter name: (expected type, default value)}
+        param_rules = {
+            '_start_step': (int, -1),
+            '_stop_step': (int, -1),
+            '_aic_metrics': (int, -1),
+            '_profiler_level': (int, 0),
+            '_analyse_mode': (int, -1),
+            '_activities': (int, 0),
+            '_export_type': (int, 0),
+            '_profile_memory': (bool, False),
+            '_mstx': (bool, False),
+            '_parallel_strategy': (bool, False),
+            '_with_stack': (bool, False),
+            '_data_simplification': (bool, True),
+            '_is_valid': (bool, False)
+        }
+
+        def _is_valid_type(value, expected_type):
+            """Helper method for type checking."""
+            if expected_type is int and isinstance(value, bool):
+                return False
+            return isinstance(value, expected_type)
+
+        for param, (expected_type, default) in param_rules.items():
+            value = getattr(self, param)
+            if not _is_valid_type(value, expected_type):
+                logger.warning(
+                    f"{param[1:]} should be {expected_type.__name__} type, "
+                    f"will be reset to {default}."
+                )
+                setattr(self, param, default)
 
     @property
     def start_step(self):
@@ -162,7 +162,9 @@ class DynamicProfilerArgs:
     def args(self):
         """ get all args in DynamicProfilerArgs."""
         self._profiler_level = self._convert_profiler_level(self._profiler_level)
-        self._profile_framework = self._convert_profile_framework(self._profile_framework)
+        self._activities = self._convert_activities(self._activities)
+        self._aic_metrics = self._convert_aic_metrics(self._aic_metrics)
+        self._export_type = self._convert_export_type(self._export_type)
         not_supported_args = ['_start_step', '_stop_step', '_analyse_mode', '_is_valid']
         res = {}
         for key, value in self.__dict__.items():
@@ -193,22 +195,57 @@ class DynamicProfilerArgs:
 
     def _convert_profiler_level(self, profiler_level: int) -> ProfilerLevel:
         """ convert profiler_level to real args in Profiler."""
+        if profiler_level == -1:
+            return ProfilerLevel.LevelNone
         if profiler_level == 0:
             return ProfilerLevel.Level0
         if profiler_level == 1:
             return ProfilerLevel.Level1
         if profiler_level == 2:
             return ProfilerLevel.Level2
-        return None
-
-    def _convert_profile_framework(self, profile_framework: int) -> str:
-        """ convert profile_framework to real args in Profiler."""
-        if profile_framework == 0:
-            return "time"
-        if profile_framework == 1:
-            return "all"
-        return None
-
+        return ProfilerLevel.Level0
+
+    def _convert_activities(self, activities: int) -> ProfilerLevel:
+        """ convert activities to real args in Profiler."""
+        if activities == 0:
+            return [ProfilerActivity.CPU, ProfilerActivity.NPU]
+        if activities == 1:
+            return [ProfilerActivity.CPU]
+        if activities == 2:
+            return [ProfilerActivity.NPU]
+        return [ProfilerActivity.CPU, ProfilerActivity.NPU]
+
+    def _convert_aic_metrics(self, aic_metrics: int) -> AicoreMetrics:
+        """ convert aic_metrics to real args in Profiler."""
+        if aic_metrics == -1:
+            return AicoreMetrics.AiCoreNone
+        if aic_metrics == 0:
+            return AicoreMetrics.PipeUtilization
+        if aic_metrics == 1:
+            return AicoreMetrics.ArithmeticUtilization
+        if aic_metrics == 2:
+            return AicoreMetrics.Memory
+        if aic_metrics == 3:
+            return AicoreMetrics.MemoryL0
+        if aic_metrics == 4:
+            return AicoreMetrics.MemoryUB
+        if aic_metrics == 5:
+            return AicoreMetrics.ResourceConflictRatio
+        if aic_metrics == 6:
+            return AicoreMetrics.L2Cache
+        if aic_metrics == 7:
+            return AicoreMetrics.MemoryAccess
+        return AicoreMetrics.AiCoreNone
+
+    def _convert_export_type(self, export_type: int) -> ExportType:
+        """ convert export_type to real args in Profiler."""
+        if export_type == 0:
+            return [ExportType.Text]
+        if export_type == 1:
+            return [ExportType.Db]
+        if export_type == 2:
+            return [ExportType.Text, ExportType.Db]
+        return [ExportType.Text]
 
 class DynamicProfilerMonitorBase(Callback):
     """
@@ -240,12 +277,17 @@ class DynamicProfilerMonitorBase(Callback):
         self._last_stop_step = None
         self._is_create_process = None
         self._is_started = False
+        self._start_step = -1
+        self._stop_step = -1
+        self._step_num = 0
 
+        self._check_shm_for_killed()
         self._init_cfg_json()
         self._create_shm()
         self._create_process()
         atexit.register(self._clean_resource)
 
+    @no_exception_func()
     def step_begin(self, run_context):
         """
         Start profile at the begin of step.
@@ -277,7 +319,8 @@ class DynamicProfilerMonitorBase(Callback):
             if self._profiler is None:
                 prof_path = os.path.join(self._output_path, f"rank{self._rank_id}_start{start_step}_stop{stop_step}")
                 PathManager.check_input_directory_path(prof_path)
-                self._profiler = Profiler(output_path=prof_path, start_profile=False, **prof_args.args)
+                self._profiler = Profiler(on_trace_ready=tensorboard_trace_handler(dir_name=prof_path),
+                                          start_profile=False, **prof_args.args)
                 print_msg(f"Rank {self._rank_id} create output path {prof_path}")
 
             self._profiler.start()
@@ -287,6 +330,7 @@ class DynamicProfilerMonitorBase(Callback):
             print_msg(f"Rank {self._rank_id} Dynamic profiler start at step {start_step}, "
                       f"will stop at step {stop_step}")
 
+    @no_exception_func()
     def step_end(self, run_context):
         """
         Stop profile at the end of step.
@@ -312,11 +356,140 @@ class DynamicProfilerMonitorBase(Callback):
                 if prof_args.analyse_mode:
                     self._profiler.analyse(mode=prof_args.analyse_mode)
                 else:
-                    self._profiler._ascend_profiler.finalize()
+                    ProfilerInterface.finalize()
+                    ProfilerInterface.clear()
                 self._profiler = None
                 self._is_started = False
                 print_msg(f"Rank {self._rank_id} Dynamic profiler stop at step {step_num}")
 
+    @no_exception_func()
+    def step(self):
+        """
+        Used for Ascend, distinguish step collection and parsing performance data by dynamic profiler.
+
+        Raises:
+            RuntimeError: If the 'start_step' parameter setting is greater than the 'stop_step' parameter setting.
+
+        Examples:
+            >>> import json
+            >>> import os
+            >>> import numpy as np
+            >>>
+            >>> import mindspore
+            >>> import mindspore.dataset as ds
+            >>> from mindspore import context, nn
+            >>> from mindspore.profiler import DynamicProfilerMonitor
+            >>>
+            >>>
+            >>> class Net(nn.Cell):
+            ...     def __init__(self):
+            ...         super(Net, self).__init__()
+            ...         self.fc = nn.Dense(2, 2)
+            ...
+            ...     def construct(self, x):
+            ...         return self.fc(x)
+            >>>
+            >>> def generator_net():
+            ...     for _ in range(2):
+            ...         yield np.ones([2, 2]).astype(np.float32), np.ones([2]).astype(np.int32)
+            >>>
+            >>> def train(test_net):
+            ...     optimizer = nn.Momentum(test_net.trainable_params(), 1, 0.9)
+            ...     loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+            ...     data = ds.GeneratorDataset(generator_net(), ["data", "label"])
+            ...     model = mindspore.train.Model(test_net, loss, optimizer)
+            ...     model.train(1, data)
+            >>>
+            >>> def change_cfg_json(json_path):
+            ...     with open(json_path, 'r', encoding='utf-8') as file:
+            ...          data = json.load(file)
+            ...
+            ...     data['start_step'] = 6
+            ...     data['stop_step'] = 7
+            ...
+            ...     with open(json_path, 'w', encoding='utf-8') as file:
+            ...          json.dump(data, file, ensure_ascii=False, indent=4)
+            >>>
+            >>> if __name__ == '__main__':
+            ...      # set json configuration file
+            ...      context.set_context(mode=mindspore.PYNATIVE_MODE)
+            ...      mindspore.set_device("Ascend")
+            ...      data_cfg = {
+            ...          "start_step": 2,
+            ...          "stop_step": 5,
+            ...          "aic_metrics": -1,
+            ...          "profiler_level": 0,
+            ...          "activities": 0,
+            ...          "export_type": 0,
+            ...          "profile_memory": False,
+            ...          "mstx": False,
+            ...          "analyse_mode": 0,
+            ...          "parallel_strategy": False,
+            ...          "with_stack": False,
+            ...          "data_simplification": True,
+            ...          }
+            ...      output_path = "./cfg_path"
+            ...      cfg_path = os.path.join(output_path, "profiler_config.json")
+            ...      os.makedirs(output_path, exist_ok=True)
+            ...      # set cfg file
+            ...      with open(cfg_path, 'w') as f:
+            ...           json.dump(data_cfg, f, indent=4)
+            ...      # cfg_path contains the json configuration file path, and output_path is the output path
+            ...      dp = DynamicProfilerMonitor(cfg_path=output_path, output_path=output_path)
+            ...      STEP_NUM = 15
+            ...      # Define a network of training models
+            ...      net = Net()
+            ...      for i in range(STEP_NUM):
+            ...          print(f"step {i}")
+            ...          train(net)
+            ...          # Modify the configuration file after step 7. For example, change start_step to 8 and stop_step to 10
+            ...          if i == 5:
+            ...             # Modify parameters in the JSON file
+            ...             change_cfg_json(os.path.join(output_path, "profiler_config.json"))
+            ...          # Call step collection
+            ...          dp.step()
+        """
+
+        self._step_num += 1
+        prof_args = self._get_prof_args()
+
+        if not prof_args.is_valid:
+            logger.error("Dynamic profile json is not valid, please check the json file.")
+            return
+
+        if prof_args.start_step == -1 or prof_args.stop_step == -1:
+            return
+
+        # Skips the number of steps less than start_step
+        if self._step_num < prof_args.start_step:
+            return
+
+        if self._start_step != prof_args.start_step or self._stop_step != prof_args.stop_step:
+            # Update new start_step and stop_step
+            self._start_step = prof_args.start_step
+            self._stop_step = prof_args.stop_step
+            if self._start_step >= 0 and 0 <= self._start_step <= self._stop_step:
+                prof_path = os.path.join(self._output_path,
+                                         f"rank{self._rank_id}_start{self._start_step}_stop{self._stop_step}")
+                print_msg(f"Rank {self._rank_id} create output path {prof_path}")
+                print_msg(f"Rank {self._rank_id} Dynamic profile start at step {self._start_step}, "
+                          f"will stop at step {self._stop_step}")
+                self._profiler = Profiler(schedule=schedule(wait=0, warmup=0,
+                                                            active=self._stop_step - self._start_step + 1,
+                                                            repeat=1,
+                                                            skip_first=1),
+                                          on_trace_ready=tensorboard_trace_handler(dir_name=prof_path),
+                                          **prof_args.args)
+            else:
+                self._profiler = None
+                logger.error("Rank %d Dynamic profile start at step %d and stop at step %d in config_json must be "
+                             "greater than or equal to 0, and stop step should not be less than start step",
+                             self._rank_id, self._start_step, self._stop_step)
+
+        if self._profiler:
+            self._profiler.step()
+
+    @no_exception_func()
     def on_train_end(self, run_context):
         """
         Callback on trian end
@@ -357,6 +530,7 @@ class DynamicProfilerMonitorBase(Callback):
 
         return start_step, stop_step
 
+    @no_exception_func()
     def _init_cfg_json(self):
         """Init config json file"""
         if self._rank_id == 0:
@@ -372,6 +546,7 @@ class DynamicProfilerMonitorBase(Callback):
         """Create a json monitor process based on whether the SharedMemory is successfully created"""
         logger.error("Dynamic profiler _create_shm is not implemented")
 
+    @no_exception_func()
     def _create_process(self):
         """Create json monitor process, one process will be created at one worker"""
         if self._is_create_process:
@@ -385,18 +560,57 @@ class DynamicProfilerMonitorBase(Callback):
             self._process = None
             logger.info("Rank %d no need to create process.", self._rank_id)
 
+    @no_exception_func()
+    def _check_shm_for_killed(self):
+        """
+        User killed process shm can not clean normally, so check this when create shm.
+        """
+        if sys.version_info >= (3, 8):
+            shm_path = os.path.join("/dev/shm", self._shm_name)
+        else:
+            shm_path = self._shm_path
+
+        if not os.path.exists(shm_path):
+            return
+
+        MAX_TIME_DIFF = 30 # seconds
+        time_shm = os.stat(shm_path).st_ctime
+        cur_proc_time = self._get_pid_st_ctime(os.getpid())
+
+        if cur_proc_time and abs(cur_proc_time - time_shm) > MAX_TIME_DIFF:
+            raise RuntimeError("There maybe exist share memory before this task, if you kill last task, "
+                               "dynamic profiler will not valid, please remove %s, and retry." % shm_path)
+
+    def _get_pid_st_ctime(self, pid):
+        """Get pid st_ctime"""
+        try:
+            fd = os.open("/proc/" + str(pid), os.O_RDONLY, stat.S_IRUSR | stat.S_IRGRP)
+            stat_ino = os.fstat(fd)
+            os.close(fd)
+            create_time = stat_ino.st_ctime
+            return create_time
+        except FileNotFoundError:
+            logger.error("Process with PID %d does not exist.", pid)
+        except PermissionError:
+            logger.error("Permission denied when accessing PID %d.", pid)
+        except Exception as ex: # pylint: disable=W0703
+            logger.error("An error occurred while getting creation time for PID %d: %s", pid, str(ex))
+
 
 if sys.version_info >= (3, 8):
+    @no_exception_func()
     def write_bytes(shm, byte_data):
         """Write bytes to shared memory"""
         shm.buf[:DynamicProfilerArgs.SIZE] = byte_data
 else:
+    @no_exception_func()
     def write_bytes(shm, byte_data):
         """Write bytes to shared memory"""
         shm.seek(0)
         shm.write(byte_data)
 
 
+@no_exception_func()
 def worker_func(loop_flag, poll_interval, shm, cfg_path):
     """ Json monitor process worker function python version >= 3.8"""
     last_file_t = None
@@ -430,15 +644,55 @@ if sys.version_info >= (3, 8):
     from multiprocessing import shared_memory
     from unittest.mock import patch
 
+
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
         This class to enable the dynamic profile monitoring of MindSpore neural networks.
 
         Args:
-            cfg_path (str): Dynamic profile json config file directory. The requirement is a shared path
-                that can be accessed by all nodes.
-            output_path (str, optional): Output data path. Default: ``"./dyn_profile_data"`` .
-            poll_interval (int, optional): The polling period of the monitoring process, in seconds.
+            cfg_path (str): (Ascend only) Dynamic profile json config file directory. The requirement is a shared path
+                that can be accessed by all nodes. The parameters of the json configuration file are as follows:
+
+                - start_step (int, required) - Sets the step number at which the Profiler starts collecting data.
+                  It is a relative value, with the first step of training being 1. The default value is -1, indicating
+                  that data collection will not start during the entire training process.
+                - stop_step (int, required) - Sets the step number at which the Profiler stops collecting data. It is
+                  a relative value, with the first step of training being 1. The stop_step must be greater than or
+                  equal to start_step. The default value is -1, indicating that data collection will not start during
+                  the entire training process.
+                - aic_metrics (int, optional) - The range of values corresponds to the Profiler. The default value -1
+                  indicates that AI Core utilization is not collected, and 0 indicates PipeUtilization, 1 indicates
+                  ArithmeticUtilization, 2 stands for Memory, 3 stands for MemoryL0, 4 stands for MemoryUB, 5 indicates
+                  ResourceConflictRatio, 6 indicates L2Cache, 7 indicates MemoryAccess.
+                - profiler_level (int, optional) - Sets the level of performance data collection, where -1 represents
+                  ProfilerLevel.LevelNone, 0 represents ProfilerLevel.Level0, 1 represents ProfilerLevel.Level1, and
+                  2 represents ProfilerLevel.Level2. The default value is 0, indicating the ProfilerLevel.Level0
+                  collection level.
+                - activities (int, optional) - Sets the devices for performance data collection, where 0 represents
+                  CPU+NPU, 1 represents CPU, and 2 represents NPU. The default value is 0, indicating the collection
+                  of CPU+NPU performance data.
+                - export_type (int, optional) -  Sets the data type to export, where 0 represents text, 1 represents db,
+                  and 2 represents text and db. The default value is 0, indicating only export text type data.
+                - profile_memory (bool, optional) - Set whether to collect memory performance data, true indicates that
+                  memory performance data is collected, false indicates that memory performance data is not collected.
+                  The default value is false, indicating that memory performance data is not collected.
+                - mstx (bool, optional) - Set whether to enable mstx, true indicates that mstx is enabled, false
+                  indicates that mstx is disabled. The default value is false, indicating that mstx is not enabled.
+                - analyse_mode (int, optional) - Sets the mode for online analysis, corresponding to the analyse_mode
+                  parameter of the mindspore.Profiler.analyse interface, where 0 represents "sync" and 1 represents
+                  "async". The default value is -1, indicating that online analysis is not used.
+                - parallel_strategy (bool, optional) - Sets whether to collect parallel strategy performance data,
+                  where true means to collect and false means not to collect. The default value is false, indicating
+                  that parallel strategy performance data is not collected.
+                - with_stack (bool, optional) - Sets whether to collect call stack information, where true means to
+                  collect and false means not to collect. The default value is false, indicating that call stack
+                  information is not collected.
+                - data_simplification (bool, optional) - Sets whether to enable data simplification, where true means
+                  to enable and false means not to enable. The default value is true, indicating that data
+                  simplification is enabled.
+
+            output_path (str, optional): (Ascend only) Output data path. Default: ``"./dyn_profile_data"`` .
+            poll_interval (int, optional): (Ascend only) The polling period of the monitoring process, in seconds.
                 Default value: ``2``.
 
         Raises:
@@ -488,12 +742,14 @@ if sys.version_info >= (3, 8):
             """ Get prof_args py38"""
             return DynamicProfilerArgs.from_bytes(self._shm.buf[:DynamicProfilerArgs.SIZE])
 
+        @no_exception_func()
         def _clean_resource(self):
             """Clean resource py38"""
             # stop profiler when stop_step over all train step
             if self._profiler:
                 self._profiler.stop()
-                self._profiler._ascend_profiler.finalize()
+                ProfilerInterface.finalize()
+                ProfilerInterface.clear()
                 self._profiler = None
                 logger.warning("Rank %d Dynamic profiler stop at end of training", self._rank_id)
 
@@ -505,7 +761,7 @@ if sys.version_info >= (3, 8):
                 logger.info("Rank %s process stop", self._rank_id)
 
             # clear shared memory
-            if self._shm:
+            if self._shm and self._is_create_process:
                 try:
                     self._shm.close()
                     self._shm.unlink()
@@ -514,6 +770,7 @@ if sys.version_info >= (3, 8):
                     logger.warning("Rank %s unlink shm failed, may be removed", self._rank_id)
                 self._shm = None
 
+        @no_exception_func()
         def _create_shm(self):
             """Create a json monitor process based on whether the SharedMemory is successfully created py38"""
             try_times = 10
@@ -542,13 +799,19 @@ if sys.version_info >= (3, 8):
                         logger.warning("Rank %d shared memory create failed, "
                                        "retry times = %d.", self._rank_id, try_times)
                         time.sleep(random.uniform(0, 0.02))  # sleep 0 ~ 20 ms
+                except Exception as e: # pylint: disable=W0703
+                    # shm open failed because of other process create shm not finished
+                    try_times -= 1
+                    logger.warning("Rank %d shared memory open failed, error: %s, retry times = %d",
+                                   self._rank_id, str(e), try_times)
+                    time.sleep(random.uniform(0, 0.02))  # sleep 0 ~ 20 ms
 
             if try_times <= 0:
                 raise RuntimeError(f"Rank {self._rank_id} failed to create shared memory.")
 
 else:
     import mmap
-    import stat
+
 
     class DynamicProfilerMonitor(DynamicProfilerMonitorBase):
         r"""
@@ -617,12 +880,14 @@ else:
             self._shm.seek(0)
             return DynamicProfilerArgs.from_bytes(self._shm.read(DynamicProfilerArgs.SIZE))
 
+        @no_exception_func()
         def _clean_resource(self):
             """Clean resource py37"""
             # stop profiler when stop_step over all train step
             if self._profiler:
                 self._profiler.stop()
-                self._profiler._ascend_profiler.finalize()
+                ProfilerInterface.finalize()
+                ProfilerInterface.clear()
                 self._profiler = None
                 logger.warning("Rank %d Dynamic profiler stop at end of training", self._rank_id)
 
@@ -647,6 +912,7 @@ else:
                     logger.warning("Rank %s unlink shm failed, may be removed", self._rank_id)
                 self._shm = None
 
+        @no_exception_func()
         def _create_shm(self):
             """Create a json monitor process based on whether the SharedMemory is successfully created py37"""