mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/train/callback/_backup_and_restore.py

@@ -34,15 +34,15 @@ class BackupAndRestore(Callback):
 
     Args:
         backup_dir (str): Path to store and load the checkpoint file.
-        save_freq(Union['epoch', int]): When set to 'epoch' the callback saves the checkpoint at the end of
+        save_freq (Union["epoch", int]): When set to ``"epoch"`` the callback saves the checkpoint at the end of
                                         each epoch. When set to an integer, the callback saves the checkpoint
                                         every `save_freq` epoch. Default: ``"epoch"`` .
-        delete_checkpoint(bool): If `delete_checkpoint=True`, the checkpoint will be deleted after
+        delete_checkpoint (bool): If `delete_checkpoint=True`, the checkpoint will be deleted after
                                         training is finished. Default: ``True`` .
 
     Raises:
         ValueError: If backup_dir is not str.
-        ValueError: If save_freq is not 'epoch' or int.
+        ValueError: If save_freq is not ``"epoch"`` or int.
         ValueError: If delete_checkpoint is not bool.
 
     Examples:
@@ -50,13 +50,13 @@ class BackupAndRestore(Callback):
         >>> from mindspore.train import Model, BackupAndRestore, RunContext
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> backup_ckpt = BackupAndRestore("backup")
         >>> model.train(10, dataset, callbacks=backup_ckpt)

mindspore/train/callback/_callback.py

@@ -123,7 +123,7 @@ class Callback:
     recording current attributes. Users can add custimized attributes to the information.
     Training process can also be stopped by calling `request_stop` method. For details
     of custom Callback, please check
-    `Callback tutorial <https://www.mindspore.cn/tutorials/en/r2.2/advanced/model/
+    `Callback tutorial <https://www.mindspore.cn/tutorials/en/master/advanced/model/
     callback.html#customized-callback-mechanism>`_.
 
     Examples:
@@ -493,7 +493,7 @@ class RunContext:
     `RunContext.original_args()` and add extra attributes to the information, but also can stop the
     training process by calling `request_stop` method. For details of custom Callback,
     please check
-    `Callback Mechanism <https://www.mindspore.cn/tutorials/en/r2.2/advanced/model/callback.html>`_.
+    `Callback Mechanism <https://www.mindspore.cn/tutorials/en/master/advanced/model/callback.html>`_.
 
     `RunContext.original_args()` holds the model context information as a dictionary variable, and
     different attributes of the dictionary are stored in training or eval process. Details are as follows:
@@ -575,7 +575,7 @@ class RunContext:
 
         Tutorial Examples:
             - `Callback Mechanism - Customized Callback Mechanism
-              <https://mindspore.cn/tutorials/en/r2.2/advanced/model/callback.html#customized-callback-mechanism>`_
+              <https://mindspore.cn/tutorials/en/master/advanced/model/callback.html#customized-callback-mechanism>`_
         """
         return self._original_args
 
@@ -588,7 +588,7 @@ class RunContext:
 
         Tutorial Examples:
             - `Callback Mechanism - Customized Training Termination Time
-              <https://mindspore.cn/tutorials/en/r2.2/advanced/model/callback.html#
+              <https://mindspore.cn/tutorials/en/master/advanced/model/callback.html#
               customized-training-termination-time>`_
         """
         self._stop_requested = True

mindspore/train/callback/_checkpoint.py

@@ -28,9 +28,15 @@ from mindspore.train._utils import _make_directory
 from mindspore.train.serialization import save_checkpoint, _save_graph
 from mindspore.parallel._cell_wrapper import destroy_allgather_cell
 from mindspore.parallel._recovery_context import _set_recovery_context, _get_recovery_context
+from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
+from mindspore.parallel._utils import _get_device_num
+from mindspore.communication.management import get_rank
+from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy
 from mindspore.train.callback._callback import Callback, set_cur_net
 from mindspore.common.tensor import Tensor
 from mindspore.common.parameter import Parameter
+from mindspore.common.generator import Generator
+from mindspore.common.api import _cell_graph_executor
 from mindspore._c_expression import _collect_host_info
 
 
@@ -39,8 +45,33 @@ SAVE_DIR = _cur_dir
 _info_list = ["epoch_num", "step_num"]
 
 
+def _get_dp_tp_from_redundancy(redundancy_tuple):
+    """From redundancy get dp and tp"""
+    dp = []
+    tp = []
+    for dp_value in redundancy_tuple:
+        dp.append(list(dp_value))
+    for i in range(len(redundancy_tuple[0])):
+        tp.append([v[i] for v in redundancy_tuple])
+    return dp, tp
+
+
+def _get_dp_tp_from_layout(parameter_redundancy_dict):
+    """From layout dict get dp and tp"""
+    tp = []
+    dp = []
+    value_len = 0
+    for _, value in parameter_redundancy_dict.items():
+        if len(value) > value_len:
+            value_len = len(value)
+            dp, tp = _get_dp_tp_from_redundancy(value)
+    return dp, tp
+
+
 def _chg_ckpt_file_name_if_same_exist(directory, prefix, exception=False):
     """Check if there is a file with the same name."""
+    if callable(prefix) or callable(directory):
+        return prefix
     files = os.listdir(directory)
     suffix_num = 0
     pre_len = len(prefix)
@@ -72,14 +103,15 @@ class CheckpointConfig:
     The configuration of model checkpoint.
 
     Note:
-        During the training process, if dataset is transmitted through the data channel,
-        it is suggested to set 'save_checkpoint_steps' to an integer multiple of loop_size.
-        Otherwise, the time to save the checkpoint may be biased.
-        It is recommended to set only one save strategy and one keep strategy at the same time.
-        If both `save_checkpoint_steps` and `save_checkpoint_seconds` are set,
-        `save_checkpoint_seconds` will be invalid.
-        If both `keep_checkpoint_max` and `keep_checkpoint_per_n_minutes` are set,
-        `keep_checkpoint_per_n_minutes` will be invalid.
+        - During the training process, if dataset is transmitted through the data channel,
+          it is suggested to set 'save_checkpoint_steps' to an integer multiple of loop_size.
+          Otherwise, the time to save the checkpoint may be biased.
+          It is recommended to set only one save strategy and one keep strategy at the same time.
+          If both `save_checkpoint_steps` and `save_checkpoint_seconds` are set,
+          `save_checkpoint_seconds` will be invalid.
+          If both `keep_checkpoint_max` and `keep_checkpoint_per_n_minutes` are set,
+          `keep_checkpoint_per_n_minutes` will be invalid.
+        - The `enc_mode` and `crc_check` parameters are mutually exclusive and cannot be configured simultaneously.
 
     Args:
         save_checkpoint_steps (int): Steps to save checkpoint. Default: ``1`` .
@@ -102,6 +134,8 @@ class CheckpointConfig:
         enc_mode (str): This parameter is valid only when enc_key is not set to None. Specifies the encryption
                         mode, currently supports 'AES-GCM', 'AES-CBC' and 'SM4-CBC'. Default: ``'AES-GCM'`` .
         exception_save (bool): Whether to save the current checkpoint when an exception occurs. Default: ``False`` .
+        crc_check (bool): Whether to perform crc32 calculation when saving checkpoint and save the calculation
+                          result to the end of ckpt. Default: ``False`` .
         kwargs (dict): Configuration options dictionary.
 
     Raises:
@@ -112,23 +146,21 @@ class CheckpointConfig:
         >>> from mindspore.train import Model, CheckpointConfig, ModelCheckpoint
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> config = CheckpointConfig(save_checkpoint_seconds=100, keep_checkpoint_per_n_minutes=5, saved_network=net)
         >>> config.save_checkpoint_steps
         1
         >>> config.save_checkpoint_seconds
-        100
         >>> config.keep_checkpoint_max
         5
         >>> config.keep_checkpoint_per_n_minutes
-        5
         >>> config.integrated_save
         True
         >>> config.async_save
@@ -155,6 +187,7 @@ class CheckpointConfig:
                  enc_key=None,
                  enc_mode='AES-GCM',
                  exception_save=False,
+                 crc_check=False,
                  **kwargs):
 
         if save_checkpoint_steps is not None:
@@ -197,7 +230,9 @@ class CheckpointConfig:
         self._append_dict = self._handle_append_info(append_info)
         self._enc_key = Validator.check_isinstance('enc_key', enc_key, (type(None), bytes))
         self._enc_mode = Validator.check_isinstance('enc_mode', enc_mode, str)
+        self._crc_check = Validator.check_isinstance('crc_check', crc_check, bool)
         self._map_param_inc = kwargs.get('incremental', False)
+        self.enable_redundance = kwargs.get('enable_redundance', False)
 
     @property
     def save_checkpoint_steps(self):
@@ -205,7 +240,7 @@ class CheckpointConfig:
         Get the value of steps to save checkpoint.
 
         Returns:
-            Int, steps to save checkpoint.
+            int, steps to save checkpoint.
         """
         return self._save_checkpoint_steps
 
@@ -214,7 +249,7 @@ class CheckpointConfig:
         """Get the value of _save_checkpoint_seconds.
 
         Returns:
-            Int, seconds to save the checkpoint file.
+            int, seconds to save the checkpoint file.
         """
         return self._save_checkpoint_seconds
 
@@ -224,7 +259,7 @@ class CheckpointConfig:
         Get the value of maximum number of checkpoint files can be saved.
 
         Returns:
-            Int, Maximum number of checkpoint files can be saved.
+            int, Maximum number of checkpoint files can be saved.
         """
         return self._keep_checkpoint_max
 
@@ -244,7 +279,7 @@ class CheckpointConfig:
         Get the value of whether to merge and save the split Tensor in the automatic parallel scenario.
 
         Returns:
-            Bool, whether to merge and save the split Tensor in the automatic parallel scenario.
+            bool, whether to merge and save the split Tensor in the automatic parallel scenario.
         """
         return self._integrated_save
 
@@ -254,7 +289,7 @@ class CheckpointConfig:
         Get the value of whether asynchronous execution saves the checkpoint to a file.
 
         Returns:
-            Bool, whether asynchronous execution saves the checkpoint to a file.
+            bool, whether asynchronous execution saves the checkpoint to a file.
         """
         return self._async_save
 
@@ -288,13 +323,23 @@ class CheckpointConfig:
         """
         return self._enc_mode
 
+    @property
+    def crc_check(self):
+        """
+        Get the value of the whether to enable crc check.
+
+        Returns:
+            bool, whether to enable crc check.
+        """
+        return self._crc_check
+
     @property
     def append_dict(self):
         """
         Get the value of information dict saved to checkpoint file.
 
         Returns:
-            Dict, the information saved to checkpoint file.
+            dict, the information saved to checkpoint file.
         """
         return self._append_dict
 
@@ -304,7 +349,7 @@ class CheckpointConfig:
         Get the value of whether to save map Parameter incrementally.
 
         Returns:
-            Bool, whether to save map Parameter incrementally.
+            bool, whether to save map Parameter incrementally.
         """
         return self._map_param_inc
 
@@ -313,7 +358,7 @@ class CheckpointConfig:
         Get the policy of checkpoint.
 
         Returns:
-            Dict, the information of checkpoint policy.
+            dict, the information of checkpoint policy.
         """
         checkpoint_policy = {'save_checkpoint_steps': self.save_checkpoint_steps,
                              'save_checkpoint_seconds': self.save_checkpoint_seconds,
@@ -353,7 +398,8 @@ class CheckpointConfig:
                     raise TypeError(f"For 'CheckpointConfig', the element of 'append_info' must has only one dict, "
                                     "but got {dict_num}")
                 for key, value in element.items():
-                    if isinstance(key, str) and isinstance(value, (int, float, bool, str, Parameter, Tensor)):
+                    if isinstance(key, str) and isinstance(value,
+                                                           (int, float, bool, str, Parameter, Tensor, Generator)):
                         handle_append_info[key] = value
                     else:
                         raise TypeError(f"For 'CheckpointConfig', the key type of the dict 'append_info' "
@@ -376,14 +422,16 @@ class ModelCheckpoint(Callback):
         parameters of the optimizer by default.
 
     Args:
-        prefix (str): The prefix name of checkpoint files. Default: ``'CKP'`` .
-        directory (str): The path of the folder which will be saved in the checkpoint file.
-            By default, the file is saved in the current directory. Default: ``None`` .
+        prefix (Union[str, callable object]): The prefix name or callable object to generate name of checkpoint files.
+            Default: ``'CKP'`` .
+        directory (Union[str, callable object]): The folder path where the checkpoint is stored, or the callable object
+            used to generate the path. By default, the file is saved in the current directory.
+            Default: ``None`` .
         config (CheckpointConfig): Checkpoint strategy configuration. Default: ``None`` .
 
     Raises:
-        ValueError: If `prefix` is not str or contains the '/' character.
-        ValueError: If `directory` is not str.
+        ValueError: If `prefix` is not str or contains the '/' character and is not a callable object.
+        ValueError: If `directory` is not str and is not a callable object.
         TypeError: If the config is not CheckpointConfig type.
 
     Examples:
@@ -409,19 +457,29 @@ class ModelCheckpoint(Callback):
         self._last_time = time.time()
         self._last_time_for_keep = time.time()
         self._last_triggered_step = 0
+        """a callable for users to set self-defined prefix."""
+        self._prefix_func = None
+        """a callable for users to set self-defined directory."""
+        self._directory_func = None
 
-        if not isinstance(prefix, str) or prefix.find('/') >= 0:
+        if not callable(prefix) and (not isinstance(prefix, str) or prefix.find('/') >= 0):
             raise ValueError("For 'ModelCheckpoint', the argument 'prefix' "
                              "for checkpoint file name is invalid, it must be "
-                             "string and does not contain '/', but got {}.".format(prefix))
+                             "callable or string that does not contain '/', but got {}.".format(prefix))
         self._prefix = prefix
         self._exception_prefix = prefix
 
         if directory is not None:
-            self._directory = _make_directory(directory)
+            if callable(directory):
+                self._directory_func = directory
+            else:
+                self._directory = _make_directory(directory)
         else:
             self._directory = _cur_dir
 
+        if callable(prefix):
+            self._prefix_func = prefix
+
         if _get_recovery_context("enable_recovery"):
             _set_recovery_context(ckpt_path=self._directory)
 
@@ -434,9 +492,15 @@ class ModelCheckpoint(Callback):
                                 "but got {}.".format(type(config)))
             self._config = config
 
+        self._aiturbo_init_flag = os.getenv("AITURBO") == "1"
         # get existing checkpoint files
-        self._manager = CheckpointManager()
-        self._prefix = _chg_ckpt_file_name_if_same_exist(self._directory, self._prefix)
+        if self._aiturbo_init_flag:
+            import aiturbo
+            self._manager = aiturbo.CheckpointShmManager()
+        else:
+            self._manager = CheckpointManager()
+        if not callable(directory) and not callable(prefix):
+            self._prefix = _chg_ckpt_file_name_if_same_exist(self._directory, self._prefix)
         self._append_dict = self._config.append_dict or {}
         self._append_epoch_num = self._append_dict.get("epoch_num") if "epoch_num" in self._append_dict else 0
         self._append_step_num = self._append_dict.get("step_num") if "step_num" in self._append_dict else 0
@@ -452,6 +516,36 @@ class ModelCheckpoint(Callback):
             run_context (RunContext): Context of the train running.
         """
         cb_params = run_context.original_args()
+        if self._aiturbo_init_flag:
+            import aiturbo
+            ckpt_storage_path = self._directory
+            rank_id = get_rank()
+            stage_num = _get_auto_parallel_context("pipeline_stages")
+            stage_rank_num = _get_device_num() // stage_num
+            param_layout = cb_params.train_network.parameter_layout_dict
+            if not param_layout:
+                layout = {"stage_num": stage_num, "stage_rank_num": stage_rank_num, "stage_layout": None}
+                aiturbo.init(ckpt_storage_path, rank_id, layout, None, False, None)
+            else:
+                device_num = _get_device_num()
+                chunk_size = device_num // stage_num
+                initial_rank = (rank_id // chunk_size) * chunk_size
+                param_redundancy_dict = get_parameter_redundancy(param_layout, initial_rank)
+                dp, _ = _get_dp_tp_from_layout(param_redundancy_dict)
+                layout = {"stage_num": stage_num, "stage_rank_num": stage_rank_num,
+                          "stage_layout": param_redundancy_dict}
+                single_params = remove_param_redundancy(param_redundancy_dict)
+                single_params = {device_id: list(params) for device_id, params in single_params.items()}
+                aiturbo.init(ckpt_storage_path, rank_id, layout, single_params, self._config.enable_redundance, dp)
+            self._aiturbo_init_flag = False
+        if self._prefix_func:
+            self._prefix = self._prefix_func(cb_params)
+            if not isinstance(self._prefix, str) or self._prefix.find('/') >= 0:
+                raise ValueError("For 'ModelCheckpoint', the argument 'prefix' "
+                                 "for checkpoint file name is callable, it must return a "
+                                 "string that does not contain '/', but got {}.".format(self._prefix))
+        if self._directory_func:
+            self._directory = self._directory_func(cb_params)
         _collect_host_info("Callback", "ModelCheckpoint", "step_end", level=1)
         # In disaster recovery scenario, the training process may be rolled back to the last step where
         # the ckpt was successfully saved, so the _last_triggered_step should be updated.
@@ -520,8 +614,11 @@ class ModelCheckpoint(Callback):
         step_num_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num + 1)
 
         if save_ckpt:
-            cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \
-                + str(step_num_in_epoch) + ".ckpt"
+            if self._prefix_func:
+                cur_ckpoint_file = self._prefix + ".ckpt"
+            else:
+                cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \
+                    + str(step_num_in_epoch) + ".ckpt"
             # update checkpoint file list.
             self._manager.update_ckpoint_filelist(self._directory, self._prefix)
             # keep checkpoint files number equal max number.
@@ -545,15 +642,22 @@ class ModelCheckpoint(Callback):
             if context.get_context("enable_ge") and os.getenv('MS_DISABLE_REF_MODE') \
                     and context.get_context("mode") == context.GRAPH_MODE:
                 set_cur_net(cb_params.train_network)
-                cb_params.train_network.exec_checkpoint_graph()
+                cb_params.train_network.add_flags(ge_sync_data=True)
+                _cell_graph_executor(cb_params.train_network, phase='save')
             if "epoch_num" in self._append_dict:
                 self._append_dict["epoch_num"] = self._append_epoch_num + cb_params.cur_epoch_num
             if "step_num" in self._append_dict:
                 self._append_dict["step_num"] = self._append_step_num + cb_params.cur_step_num
             network = self._config.saved_network if self._config.saved_network is not None else cb_params.train_network
-            save_checkpoint(network, cur_file, self._config.integrated_save, self._config.async_save,
-                            self._append_dict, self._config.enc_key, self._config.enc_mode,
-                            incremental=self._map_param_inc)
+            if os.getenv("AITURBO") == "1":
+                save_checkpoint(network, cur_file, self._config.integrated_save, self._config.async_save,
+                                self._append_dict, self._config.enc_key, self._config.enc_mode,
+                                crc_check=self._config.crc_check, incremental=self._map_param_inc,
+                                global_step_num=cb_params.cur_step_num)
+            else:
+                save_checkpoint(network, cur_file, self._config.integrated_save, self._config.async_save,
+                                self._append_dict, self._config.enc_key, self._config.enc_mode,
+                                crc_check=self._config.crc_check, incremental=self._map_param_inc)
 
             self._latest_ckpt_file_name = cur_file
 
@@ -573,6 +677,16 @@ class ModelCheckpoint(Callback):
         """Return the latest checkpoint path and file name."""
         return self._latest_ckpt_file_name
 
+    @property
+    def _get_save_checkpoint_steps(self):
+        """Return save ckpt steps"""
+        return self._config.save_checkpoint_steps
+
+    @property
+    def _get_last_trigger_step(self):
+        """Return last triggered steps"""
+        return self._last_triggered_step
+
 
 class CheckpointManager:
     """Manage checkpoint files according to train_config of checkpoint."""

mindspore/train/callback/_cluster_monitor.py

@@ -0,0 +1,201 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""ClusterMonitor Callback class."""
+from __future__ import absolute_import
+
+import os
+import stat
+import glob
+import time
+from threading import RLock
+
+from mindspore.train.callback._callback import Callback
+from mindspore.communication.management import get_rank, get_local_rank
+from mindspore import log as logger
+from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
+from mindspore.parallel._utils import _get_device_num
+from mindspore.train._utils import get_parameter_redundancy
+
+_perf_mutex = RLock()
+
+
+def _get_dp_tp_from_redundancy(redundancy_tuple):
+    """From redundancy get dp and tp"""
+    dp = []
+    tp = []
+    for dp_value in redundancy_tuple:
+        dp.append(list(dp_value))
+    for i in range(len(redundancy_tuple[0])):
+        tp.append([v[i] for v in redundancy_tuple])
+    return dp, tp
+
+
+def _get_dp_tp_from_layout(parameter_layout_dict, initial_rank=0):
+    """From layout dict get dp and tp"""
+    tp = []
+    dp = []
+    parameter_redundancy_dict = get_parameter_redundancy(parameter_layout_dict, initial_rank)
+    value_len = 0
+    for _, value in parameter_redundancy_dict.items():
+        if len(value) > value_len:
+            value_len = len(value)
+            dp, tp = _get_dp_tp_from_redundancy(value)
+    return dp, tp
+
+
+def _check_perf_config(perf_config):
+    """Check if the format of perf_config is correct."""
+    enabled = perf_config.get("enable", None)
+    if enabled is None or not isinstance(enabled, bool):
+        raise TypeError(f"For cluster monitor, enabled should be bool, but got {type(enabled)}.")
+    enable_step_time = perf_config.get("steptime", None)
+    if enable_step_time is None or not isinstance(enable_step_time, bool):
+        raise TypeError(f"For cluster monitor, enable_step_time should be bool, but got {type(enable_step_time)}.")
+    enabled_dtp_group = perf_config.get("dtpGroup", None)
+    if enabled_dtp_group is None or not isinstance(enabled_dtp_group, bool):
+        raise TypeError(f"For cluster monitor, enabled_dtp_group should be bool, but got {type(enabled_dtp_group)}.")
+
+
+def _parse_perf_config():
+    """parse perf config"""
+    perf_config = os.getenv("PERF_DUMP_CONFIG")
+    perf_config_dict = {}
+    if perf_config is None:
+        return perf_config_dict
+    pairs = perf_config.split(',')
+    for pair in pairs:
+        key, value = pair.split(':')
+        if value.lower() == 'true':
+            perf_config_dict[key] = True
+        elif value.lower() == 'false':
+            perf_config_dict[key] = False
+        elif value.isdigit():
+            perf_config_dict[key] = int(value)
+        else:
+            perf_config_dict[key] = value
+    _check_perf_config(perf_config_dict)
+    return perf_config_dict
+
+
+def _remove_pre_log():
+    """Remove the previously saved log files."""
+    directory = os.getenv("PERF_DUMP_PATH")
+    device_id = get_local_rank()
+    pattern = os.path.join(directory, f"perf_ms_*_{device_id}.log")
+    files_to_delete = glob.glob(pattern)
+    for file_path in files_to_delete:
+        if os.path.islink(file_path):
+            continue
+        try:
+            os.remove(file_path)
+        except OSError as e:
+            logger.warning(f"When CCAE is opening, {file_path} need to be removed, but failed to remove.")
+            raise e
+
+
+class ClusterMonitor(Callback):
+    """
+    Monitor the cluster in train process.
+    """
+
+    def __init__(self):
+        super(ClusterMonitor, self).__init__()
+        self.perf_config = _parse_perf_config()
+        self.enabled = self.perf_config.get("enable")
+        self.enable_step_time = self.perf_config.get("steptime")
+        self.enabled_dtp_group = self.perf_config.get("dtpGroup")
+        self.data_time_start = None
+        self.data_time_end = None
+        self.frame_work = "MindSpore"
+        self.ms_sched_host = os.getenv("MS_SCHED_HOST", "127.0.0.1")
+        self.ms_sched_port = os.getenv("MS_SCHED_PORT", "8888")
+        self.uuid_value = self.ms_sched_host + "_" + self.ms_sched_port
+        self.global_rank = get_rank()
+        self.process_id = os.getpid()
+        self.device_id = get_local_rank()
+        self.log_name = "perf_ms" + "_" + str(self.process_id) + "_" + str(self.device_id) + ".log"
+        self.log_path = os.getenv("PERF_DUMP_PATH")
+        if not self.log_path.endswith(os.path.sep):
+            self.log_path += os.path.sep
+        self.full_path = self.log_path + self.log_name
+
+        self.write_dp_tp_flag = True
+        self.initial_rank = 0
+
+    def begin(self, run_context):
+        _remove_pre_log()
+        pp_num = _get_auto_parallel_context("pipeline_stages")
+        device_num = _get_device_num()
+
+        original_list = list(range(device_num))
+        chunk_size = device_num // pp_num
+        split_pp_lists = []
+        for i in range(0, device_num, chunk_size):
+            end_index = i + chunk_size if i + chunk_size <= device_num else device_num
+            split_pp_lists.append(original_list[i:end_index])
+
+        self.initial_rank = (self.global_rank // chunk_size) * chunk_size
+        with _perf_mutex:
+            dir_path = os.path.dirname(self.full_path)
+            if not os.path.exists(dir_path):
+                os.makedirs(dir_path)
+            if os.path.exists(self.full_path):
+                os.chmod(self.full_path, stat.S_IWUSR)
+                os.remove(self.full_path)
+            with open(self.full_path, 'w') as file:
+                log_message = f'UUID:{self.uuid_value}\nFRAMEWORK:{self.frame_work}\nGLOBAL RANKID:{self.global_rank}\n'
+                file.write(log_message)
+                for _, split_pp_list in enumerate(split_pp_lists):
+                    file.write(f'PP:{split_pp_list}\n')
+            os.chmod(self.full_path, stat.S_IRUSR)
+
+    def step_begin(self, run_context):
+        """
+        Record time at the beginning of step.
+
+        Args:
+            run_context (RunContext): Context of the process running. For more details,
+                    please refer to :class:`mindspore.train.RunContext`.
+        """
+        self.data_time_start = time.time()
+
+    def step_end(self, run_context):
+        """
+        Record time at the end of step.
+
+        Args:
+            run_context (RunContext): Context of the process running. For more details,
+                    please refer to :class:`mindspore.train.RunContext`.
+        """
+        self.data_time_end = time.time()
+        if self.enabled and self.enabled_dtp_group and self.write_dp_tp_flag:
+            cb_params = run_context.original_args()
+            param_layout_dict = cb_params.train_network.parameter_layout_dict
+            dp, tp = _get_dp_tp_from_layout(param_layout_dict, self.initial_rank)
+            with _perf_mutex:
+                os.chmod(self.full_path, stat.S_IWUSR)
+                with open(self.full_path, 'a') as file:
+                    for dp_value in dp:
+                        file.write(f'dp:{dp_value}\n')
+                    for tp_value in tp:
+                        file.write(f'tp:{tp_value}\n')
+                os.chmod(self.full_path, stat.S_IRUSR)
+            self.write_dp_tp_flag = False
+        if self.enabled and self.enable_step_time:
+            with _perf_mutex:
+                os.chmod(self.full_path, stat.S_IWUSR)
+                with open(self.full_path, 'a') as file:
+                    file.write(f"STEPTIME:{int(self.data_time_start * 1000)},{int(self.data_time_end * 1000)}\n")
+                os.chmod(self.full_path, stat.S_IRUSR)