mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.6.0__cp311-cp311-win_amd64.whl

mindspore/train/callback/_early_stop.py

@@ -198,7 +198,7 @@ class EarlyStopping(Callback):
         """
         Get the monitor value at the end of epoch during training.
 
-        If `mindspore.train.callback.ReduceLROnPlateau` used with `model.train`, no evaluation process
+        If :class:`mindspore.train.callback.ReduceLROnPlateau` used with `model.train`, no evaluation process
         during training, only monitor="loss" is valid; if it used with `model.fit`, evaluation process will be
         performed at the end of epoch, valid monitor is "loss", "eval_loss" and metrics passed to `Model`.
 

mindspore/train/callback/_flops_collector.py

@@ -61,10 +61,13 @@ class FlopsUtilizationCollector(Callback):
         computility (int): The peak flops of each compute card. Default: ``1`` .
         full_flops(bool): Whether to count the full model flops. If set full_flops to False,
             FlopsUtilizationCollector would count the shard model flops in each device. Default: ``True`` .
+        enable_ma_collector(bool): Whether to write flops into the log and provide them to tasks
+            on the cloud for retrieval. Default: ``False`` .
 
     Raises:
         TypeError: If data_size is not positive int.
         TypeError: If full_flops is not bool.
+        TypeError: If enable_ma_collector is not bool.
         AssertionError: If the training mode is not a static graph or not a static shape.
 
     Examples:
@@ -86,7 +89,7 @@ class FlopsUtilizationCollector(Callback):
         Train per step time: 135.572 ms, mfu:0.47% hfu:0.47%
         Train per step time: 1.317 ms, mfu:48.59% hfu:48.59%
     """
-    def __init__(self, data_size, computility=1, full_flops=True):
+    def __init__(self, data_size, computility=1, full_flops=True, enable_ma_collector=False):
         super(FlopsUtilizationCollector, self).__init__()
         self.step_time = time.time()
         self.computility = computility
@@ -101,9 +104,12 @@ class FlopsUtilizationCollector(Callback):
         self.mfu_calculated = False
         self.data_size = data_size
         self.time_step_path = ''
-        self.verbose = full_flops
-        self.ma = os.environ.get("ENABLE_FLOPS_UTILIZATION_COLLECTOR") == "1"
-        Validator.check_bool(full_flops, "verbose")
+        self.full_flops = full_flops
+        self.verbose = not(computility == 1 and enable_ma_collector)
+        self.ma = enable_ma_collector
+        self.batch_step_size = None
+        Validator.check_bool(full_flops, "full_flops")
+        Validator.check_bool(enable_ma_collector, "enable_ma_collector")
         Validator.check_positive_int(data_size, "data_size")
 
     def step_begin(self, run_context):
@@ -115,6 +121,14 @@ class FlopsUtilizationCollector(Callback):
             run_context (RunContext): Context of the process running. For more details,
                     please refer to :class:`mindspore.train.RunContext`.
         """
+        if self.batch_step_size is None:
+            self.batch_step_size = self.data_size
+            cb_params = run_context.original_args()
+            if hasattr(cb_params, "batch_num"):
+                batch_num = cb_params.batch_num
+                if isinstance(batch_num, int) and batch_num > 0:
+                    self.batch_step_size = cb_params.batch_num
+            Validator.check_positive_int(self.batch_step_size)
         self.step_time = time.time()
 
     def _get_pipeline_group(self):
@@ -134,6 +148,40 @@ class FlopsUtilizationCollector(Callback):
         rank_list_str = "-".join(rank_str_list)
         return rank_list, rank_list_str
 
+    def _check_run_mode_valid(self, run_context):
+        """
+        Check whether FlopsUtilizationCollector is working in the current environment
+        """
+        if context.get_context("mode") != context.GRAPH_MODE:
+            if self.verbose:
+                raise ValueError("FlopsUtilizationCollector now only support graph mode.")
+            logger.info("FlopsUtilizationCollector now only support graph mode.")
+            return False
+        cb_params = run_context.original_args()
+        if cb_params.mode == 'train':
+            network = cb_params.train_network
+        elif cb_params.mode == 'eval':
+            network = cb_params.eval_network
+        else:
+            if self.verbose:
+                raise ValueError('FlopsUtilizationCollector only support train and eval mode!')
+            logger.info('FlopsUtilizationCollector only support train and eval mode!')
+            return False
+        try:
+            self.full_model_flops, self.full_hardware_flops, self.shard_model_flops, \
+            self.shard_hardware_flops, is_dynamic_shape = flops_collection(network.current_phase)
+        except Exception as e:
+            if self.verbose:
+                raise ValueError("FlopsUtilizationCollector is not supported because {}.".format(e))
+            logger.info("FlopsUtilizationCollector is not supported because {}.".format(e))
+            return False
+        if is_dynamic_shape:
+            if self.verbose:
+                raise ValueError("FlopsUtilizationCollector now do not support dynamic shape.")
+            logger.info("FlopsUtilizationCollector now do not support dynamic shape.")
+            return False
+        return True
+
     def step_end(self, run_context):
         """
         Print mfu and hfu time at the end of step.
@@ -142,84 +190,67 @@ class FlopsUtilizationCollector(Callback):
            run_context (RunContext): Context of the process running. For more details,
                    please refer to :class:`mindspore.train.RunContext`.
         """
-        if context.get_context("mode") != context.GRAPH_MODE:
-            logger.warning("FlopsUtilizationCollector now only support graph mode.")
-            return
-
         step_seconds = (time.time() - self.step_time) * 1000
         if not self.mfu_calculated:
-            cb_params = run_context.original_args()
-            if cb_params.mode == 'train':
-                network = cb_params.train_network
-            elif cb_params.mode == 'eval':
-                network = cb_params.eval_network
-            else:
-                logger.warning('FlopsUtilizationCollector only support train and eval mode!')
+            if not self._check_run_mode_valid(run_context):
                 return
-            full_model_flops, full_hardware_flops, shard_model_flops, \
-                shard_hardware_flops, is_dynamic_shape = flops_collection(network.current_phase)
-            if is_dynamic_shape:
-                logger.warning("FlopsUtilizationCollector now do not support dynamic shape.")
-                return
-            self.full_mfu = full_model_flops / self.computility
-            self.full_hfu = full_hardware_flops / self.computility
-
-            self.shard_mfu = shard_model_flops / self.computility
-            self.shard_hfu = shard_hardware_flops / self.computility
-            self.full_model_flops = full_model_flops
-            self.full_hardware_flops = full_hardware_flops
-            self.shard_model_flops = shard_model_flops
-            self.shard_hardware_flops = shard_hardware_flops
+            self.full_mfu = self.full_model_flops / self.computility
+            self.full_hfu = self.full_hardware_flops / self.computility
+            self.shard_mfu = self.shard_model_flops / self.computility
+            self.shard_hfu = self.shard_hardware_flops / self.computility
             self.mfu_calculated = True
             shard_mf_dir = os.path.realpath(os.getenv('MA_LOG_DIR', './'))
             if self.ma:
-                flops_path = os.path.join(shard_mf_dir, "flops.txt")
-                self.time_step_path = os.path.join(shard_mf_dir, "time_step.txt")
-                if auto_parallel_context().get_parallel_mode() != "stand_alone":
-                    flops_path = os.path.join(
-                        shard_mf_dir, "flops_rank_" + str(get_rank())) + ".txt"
-                    self.time_step_path = os.path.join(
-                        shard_mf_dir, "time_step_rank_" + str(get_rank())) + ".txt"
+                rank_id = get_rank() if auto_parallel_context().get_parallel_mode() != "stand_alone" else 0
+                flops_path = os.path.join(
+                    shard_mf_dir, "flops_rank_" + str(rank_id)) + ".txt"
+                self.time_step_path = os.path.join(
+                    shard_mf_dir, "time_step_rank_" + str(rank_id)) + ".txt"
                 time_stamp = time.time()
-                model_flops_log = "flops{{type=\"model_flops\"}} {} {}\n".\
-                    format(shard_model_flops, int(round(time_stamp * 1000)))
-                hardware_flops_log = "flops{{type=\"hardware_flops\"}} {} {}\n".\
-                    format(shard_hardware_flops, int(round(time_stamp * 1000)))
+                model_flops_log = "flops{{type=\"model_flops\", rank_id=\"{}\"}} {} {}\n".\
+                    format(str(rank_id), self.shard_model_flops, int(round(time_stamp * 1000)))
+                hardware_flops_log = "flops{{type=\"hardware_flops\", rank_id=\"{}\"}} {} {}\n".\
+                    format(str(rank_id), self.shard_hardware_flops, int(round(time_stamp * 1000)))
                 flags = os.O_WRONLY | os.O_CREAT
                 modes = stat.S_IWUSR | stat.S_IRUSR
                 with os.fdopen(os.open(flops_path, flags, modes), 'w') as f:
                     f.write(model_flops_log)
                     f.write(hardware_flops_log)
             if self.verbose:
-                flops_log = f"Full model flops is {full_model_flops}, Full hardware flops is {full_hardware_flops}, " \
-                            f"Shard model flops is {shard_model_flops}, Shard hardware flops is {shard_hardware_flops}."
+                if self.full_flops:
+                    pipeline_num = auto_parallel_context().get_pipeline_stages()
+                    if pipeline_num > 1:
+                        pipeline_group_list, pipeline_group_name = self._get_pipeline_group()
+                        auto_parallel_context().set_pipeline_stages(1)
+                        hashed = hashlib.md5(
+                            pipeline_group_name.encode()).hexdigest()[:48]
+                        pipeline_group_name = str(hashed)
+                        create_group(pipeline_group_name, pipeline_group_list)
+                        self.full_mfu = AllReduceNet(pipeline_group_name)(
+                            Tensor([self.full_mfu])).asnumpy()[0]
+                        self.full_hfu = AllReduceNet(pipeline_group_name)(
+                            Tensor([self.full_hfu])).asnumpy()[0]
+                        auto_parallel_context().set_pipeline_stages(pipeline_num)
+                    full_model_flops = self.full_mfu * self.computility
+                    full_hardware_flops = self.full_hfu * self.computility
+                    if auto_parallel_context().get_parallel_mode() != "stand_alone":
+                        self.full_mfu = self.full_mfu / get_group_size()
+                        self.full_hfu = self.full_hfu / get_group_size()
+                    flops_log = f"Full model flops is {full_model_flops}, " \
+                                f"Full hardware flops is {full_hardware_flops}, " \
+                                f"Shard model flops is {self.shard_model_flops}, " \
+                                f"Shard hardware flops is {self.shard_hardware_flops}."
+                else:
+                    flops_log = f"Shard model flops is {self.shard_model_flops}, " \
+                                f"Shard hardware flops is {self.shard_hardware_flops}."
                 print(flops_log, flush=True)
-                if auto_parallel_context().get_pipeline_stages() > 1:
-                    pipeline_group_list, pipeline_group_name = self._get_pipeline_group()
-                    auto_parallel_context().set_pipeline_stages(1)
-                    hashed = hashlib.md5(
-                        pipeline_group_name.encode()).hexdigest()[:48]
-                    pipeline_group_name = str(hashed)
-                    create_group(pipeline_group_name, pipeline_group_list)
-                    self.full_mfu = AllReduceNet(pipeline_group_name)(
-                        Tensor([self.full_mfu])).asnumpy()[0]
-                    self.full_hfu = AllReduceNet(pipeline_group_name)(
-                        Tensor([self.full_hfu])).asnumpy()[0]
-                if auto_parallel_context().get_parallel_mode() != "stand_alone":
-                    self.full_mfu = self.full_mfu / get_group_size()
-                    self.full_hfu = self.full_hfu / get_group_size()
-
-        step_size = self.data_size
         cb_params = run_context.original_args()
-        if hasattr(cb_params, "batch_num"):
-            batch_num = cb_params.batch_num
-            if isinstance(batch_num, int) and batch_num > 0:
-                step_size = cb_params.batch_num
-        Validator.check_positive_int(step_size)
         if cb_params.dataset_sink_mode:
-            step_seconds = step_seconds / step_size
+            step_seconds = step_seconds / self.batch_step_size
         time_stamp = time.time()
-        train_log = "time_monitor{{type=\"per_step_time\"}} {} {}".format(step_seconds, int(round(time_stamp * 1000)))
+        rank_id = get_rank() if auto_parallel_context().get_parallel_mode() != "stand_alone" else 0
+        train_log = "time_monitor{{type=\"per_step_time\", rank_id=\"{}\"}} {} {}".format(
+            str(rank_id), step_seconds, int(round(time_stamp * 1000)))
         if self.ma:
             flags = os.O_WRONLY | os.O_CREAT
             modes = stat.S_IWUSR | stat.S_IRUSR
@@ -227,9 +258,13 @@ class FlopsUtilizationCollector(Callback):
                 f.write(train_log + '\n')
         train_log = "{} per step time: {:5.3f} ms".format(
             cb_params.mode.title(), step_seconds)
-        if self.verbose:
-            mfu = 1000 * self.full_mfu / step_seconds
-            hfu = 1000 * self.full_hfu / step_seconds
+        if self.verbose and cb_params.cur_step_num % self.data_size:
+            if self.full_flops:
+                mfu = 1000 * self.full_mfu / step_seconds
+                hfu = 1000 * self.full_hfu / step_seconds
+            else:
+                mfu = 1000 * self.shard_mfu / step_seconds
+                hfu = 1000 * self.shard_hfu / step_seconds
 
             def floored_percentage(index, val, digits):
                 val *= 10 ** (digits + 2)

mindspore/train/callback/_history.py

@@ -25,13 +25,16 @@ class History(Callback):
     """
     Records the network outputs and metrics information into a `History` object.
 
-    The network outputs information will be the loss value if not custimizing the train network or eval network;
-    if the custimized network returns a `Tensor` or `numpy.ndarray`, the mean value of network output
-    will be recorded, if the custimized network returns a `tuple` or `list`, the first element of network
-    outputs will be recorded.
+    - The network outputs information will be the loss value if not custimizing the train network or eval network;
+    - If the train network or eval network is custimized:
+
+      - if the custimized network returns a `Tensor` or `numpy.ndarray`, the mean value of network output
+        will be recorded.
+      - if the custimized network returns a `tuple` or `list`, the first element of network
+        outputs will be recorded.
 
     Note:
-        Normally used in `mindspore.train.Model.train` or `mindspore.train.Model.fit`.
+        Normally used in :func:`mindspore.train.Model.train` or :func:`mindspore.train.Model.fit`.
 
     Examples:
         >>> import numpy as np

mindspore/train/callback/_lambda_callback.py

@@ -36,8 +36,8 @@ class LambdaCallback(Callback):
         on_train_step_end (Function): called at each train step end. Default: ``None`` .
         on_train_begin (Function): called at the beginning of model train. Default: ``None`` .
         on_train_end (Function): called at the end of model train. Default: ``None`` .
-        on_eval_epoch_begin (Function): called at eval epoch begin. Default: ``None`` .
-        on_eval_epoch_end (Function): called at eval epoch end. Default: ``None`` .
+        on_eval_epoch_begin (Function): called at each eval epoch begin. Default: ``None`` .
+        on_eval_epoch_end (Function): called at each eval epoch end. Default: ``None`` .
         on_eval_step_begin (Function): called at each eval step begin. Default: ``None`` .
         on_eval_step_end (Function): called at each eval step end. Default: ``None`` .
         on_eval_begin (Function): called at the beginning of model eval. Default: ``None`` .

mindspore/train/callback/_landscape.py

@@ -256,9 +256,6 @@ class SummaryLandscape:
         """
         Clean the checkpoint.
 
-        Tutorial Examples:
-            - `Training Optimization Process Visualization
-              <https://www.mindspore.cn/mindinsight/docs/en/master/landscape.html>`_
         """
         shutil.rmtree(self._ckpt_dir, ignore_errors=True)
 

mindspore/train/callback/_loss_monitor.py

@@ -93,7 +93,8 @@ class LossMonitor(Callback):
 
     def on_train_epoch_end(self, run_context):
         """
-        When LossMonitor used in `model.fit`, print eval metrics at the end of epoch if current epoch
+        When LossMonitor used in :func:`mindspore.train.Model.fit`, print eval metrics
+        at the end of epoch if current epoch
         should do evaluation.
 
         Args:

mindspore/train/callback/_on_request_exit.py

@@ -26,6 +26,7 @@ from mindspore.common.tensor import Tensor
 from mindspore.train._utils import _make_directory
 from mindspore import _checkparam as Validator
 from mindspore.train.serialization import load_checkpoint, save_checkpoint, export
+from mindspore.communication.management import get_group_size
 from mindspore.train.callback._callback import Callback
 from mindspore.parallel._utils import _get_parallel_mode
 from mindspore.context import ParallelMode
@@ -37,7 +38,7 @@ class OnRequestExit(Callback):
 
     Register OnRequestExit Callback before training, when the user want to exit the training process
     and save the training data, could send the registered exit signal 'sig' to the training process or modify the
-    'GracefulExit' that a key in the json file specified by the 'config_file' to '1'.
+    'GracefulExit' that a key in the JSON file specified by the 'config_file' to '1'.
     After the training process executes the current step, saves the current training status,
     including checkpoint and mindir, and then exit the training process.
 
@@ -58,7 +59,7 @@ class OnRequestExit(Callback):
         ValueError: If the 'save_mindir' is not a bool.
         ValueError: If the 'file_name' is not a str.
         ValueError: If the 'directory' is not a str.
-        ValueError: If the 'sig' is not an int or the 'sig' is signal.SIGKILL.
+        ValueError: If the 'sig' is not an int or the 'sig' is ``signal.SIGTERM``.
 
     Examples:
         >>> from mindspore import nn
@@ -92,10 +93,8 @@ class OnRequestExit(Callback):
         self.key = "GracefulExit"
         self.remote_config_file = config_file  # used config file to save checkpoint and exit training process
         self.use_graceful = os.environ.get("MS_ENABLE_GRACEFUL_EXIT") == "1"
-        self.is_distributed = _get_parallel_mode() != ParallelMode.STAND_ALONE
+        self.is_distributed = get_group_size() > 1
         self.integrated_save = True
-        if self.is_distributed:
-            self.integrated_save = _get_parallel_mode() == ParallelMode.AUTO_PARALLEL
         self.stop_train = False
         self.need_do_step_end = False
         if self.save_ckpt or self.save_mindir:
@@ -250,6 +249,8 @@ class OnRequestExit(Callback):
                         else:
                             global_step = int(call_params.network.optimizer.global_step.data)
                         append_dict["global_step"] = global_step
+                        if self.is_distributed:
+                            self.integrated_save = _get_parallel_mode() == ParallelMode.AUTO_PARALLEL
                         save_checkpoint(net, self.train_name, integrated_save=self.integrated_save,
                                         append_dict=append_dict)
                     if self.save_mindir:

mindspore/train/callback/_reduce_lr_on_plateau.py

@@ -63,12 +63,17 @@ class ReduceLROnPlateau(Callback):
             will be reduced. Default: ``10`` .
         verbose (bool): If False: quiet, if True: print related information.
             Default: ``False`` .
-        mode (str): one of `{'auto', 'min', 'max'}`. In "min" mode,
-            the learning rate will be reduced when the
-            quantity monitored has stopped decreasing; in "max" mode it will be
-            reduced when the quantity monitored has stopped increasing; in "auto"
-            mode, the direction is automatically inferred from the name of the
-            monitored quantity. Default: ``'auto'`` .
+        mode (str): one of `{'auto', 'min', 'max'}`. Default: ``'auto'`` .
+
+            - In ``'min'`` mode,
+              the learning rate will be reduced when the
+              quantity monitored has stopped decreasing.
+            - In ``'max'`` mode it will be
+              reduced when the quantity monitored has stopped increasing.
+            - In ``'auto'``
+              mode, the direction is automatically inferred from the name of the
+              monitored quantity.
+
         min_delta (float): threshold for measuring the new optimum, to only focus on
             significant changes. Default: ``1e-4`` .
         cooldown (int): number of epochs to wait before resuming normal operation after

mindspore/train/callback/_summary_collector.py

@@ -16,6 +16,7 @@
 from __future__ import absolute_import
 
 import os
+import platform
 import stat
 import re
 import json
@@ -43,6 +44,9 @@ from mindspore.train._utils import check_value_type, _make_directory
 from mindspore._c_expression import security
 from mindspore._c_expression import collect_host_info, get_clock_syscnt
 
+if platform.system() == "Linux":
+    import fcntl
+
 HYPER_CONFIG_ENV_NAME = "MINDINSIGHT_HYPER_CONFIG"
 HYPER_CONFIG_LEN_LIMIT = 100000
 
@@ -103,22 +107,18 @@ class SummaryCollector(Callback):
               The first output will be treated as the loss and it will be averaged. Default: ``True`` .
             - collect_graph (bool): Whether to collect the computational graph. Currently, only
               training computational graph is collected. Default: ``True`` .
-            - collect_train_lineage (bool): Whether to collect lineage data for the training phase,
-              this field will be displayed on the `lineage page \
-              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
-              of MindInsight. Default: ``True`` .
-            - collect_eval_lineage (bool): Whether to collect lineage data for the evaluation phase,
-              this field will be displayed on the `lineage page
-              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
-              of MindInsight. Default: ``True`` .
+            - collect_train_lineage (bool): Whether to collect lineage data for the training phase.
+              Default: ``True`` .
+            - collect_eval_lineage (bool): Whether to collect lineage data for the evaluation phase.
+              Default: ``True`` .
             - collect_input_data (bool): Whether to collect dataset for each training.
               Currently only image data is supported.
               If there are multiple columns of data in the dataset, the first column should be image data.
               Default: ``True`` .
             - collect_dataset_graph (bool): Whether to collect dataset graph for the training phase.
               Default: ``True`` .
-            - histogram_regular (Union[str, None]): Collect weight and bias for parameter distribution page
-              and displayed in MindInsight. This field allows regular strings to control which parameters to collect.
+            - histogram_regular (Union[str, None]): Collect weight and bias for parameter distribution page.
+              This field allows regular strings to control which parameters to collect.
               It is not recommended to collect too many parameters at once, as it can affect performance.
               Note that if you collect too many parameters and run out of memory, the training will fail.
               Default: ``None`` , it means only the first five parameters are collected.
@@ -149,8 +149,7 @@ class SummaryCollector(Callback):
             True: it means that after specified data is set, non-specified data is collected as the default behavior.
             False: it means that after specified data is set, only the specified data is collected,
             and the others are not collected. Default: ``True`` .
-        custom_lineage_data (Union[dict, None]): Allows you to customize the data and present it on the MingInsight
-            `lineage page <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_ .
+        custom_lineage_data (Union[dict, None]): Allows you to customize the data.
             In the custom data, the type of the key supports str, and the type of value supports str, int
             and float. Default: ``None`` , it means there is no custom data.
         collect_tensor_freq (Optional[int]): The same semantics as the `collect_freq`, but controls TensorSummary only.
@@ -164,7 +163,7 @@ class SummaryCollector(Callback):
             affect the number of steps TensorSummary will be collected.
             Default: ``None`` , which means to follow the behavior as described above.
         max_file_size (Optional[int]): The maximum size in bytes of each file that can be written to the disk.
-            For example, to write not larger than 4GB, specify `max_file_size=4*1024**3`.
+            For example, to write not larger than 4GB, specify `max_file_size=4*1024*3`.
             Default: ``None`` , which means no limit.
         export_options (Union[None, dict]): Perform custom operations on the export data.
             Note that the size of export files is not limited by the max_file_size.
@@ -606,13 +605,32 @@ class SummaryCollector(Callback):
             "landscape_size": landscape_size,
             "create_landscape": create_landscape
         }
+
         meta_path = os.path.join(self._ckpt_dir, 'train_metadata.json')
+        if platform.system() != "Linux":
+            try:
+                with open(meta_path, 'w') as file:
+                    json.dump(data, file)
+                os.chmod(meta_path, stat.S_IRUSR)
+            except OSError as e:
+                logger.error("Write meta data %s failed, detail: %s" % (meta_path, str(e)))
+            return
+
+        lock_file = f"{meta_path}.lock"
         try:
-            with open(meta_path, 'w') as file:
-                json.dump(data, file)
-            os.chmod(meta_path, stat.S_IRUSR)
+            with os.fdopen(os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR), 'w') as f:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                if not os.path.isfile(meta_path):
+                    with open(meta_path, 'w') as file:
+                        json.dump(data, file)
+                    os.chmod(meta_path, stat.S_IRUSR)
         except OSError as e:
             logger.error("Write meta data %s failed, detail: %s" % (meta_path, str(e)))
+        try:
+            if os.path.isfile(lock_file):
+                os.remove(lock_file)
+        except OSError:
+            logger.warning("The lock file %s has been removed.", lock_file)
 
     def _save_model_params(self, cur_num, unit, backbone):
         """Save model params."""
@@ -629,12 +647,27 @@ class SummaryCollector(Callback):
 
         ckpt_file_name = f"{type(backbone).__name__}_{cur_num}_{unit}.ckpt"
         file_path = os.path.join(self._ckpt_dir, ckpt_file_name)
+        self._model_params_file_map[str(cur_num)] = file_path
+        if platform.system() != "Linux":
+            try:
+                save_checkpoint(param_list, file_path)
+            except OSError as e:
+                logger.error(str(e))
+            return
+
+        lock_file = f"{file_path}.lock"
         try:
-            save_checkpoint(param_list, file_path)
+            with os.fdopen(os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR), 'w') as f:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                if not os.path.isfile(file_path):
+                    save_checkpoint(param_list, file_path)
         except OSError as e:
             logger.error(str(e))
-
-        self._model_params_file_map[str(cur_num)] = file_path
+        try:
+            if os.path.isfile(lock_file):
+                os.remove(lock_file)
+        except OSError:
+            logger.warning("The lock file %s has been removed.", lock_file)
 
     def _save_model_params_for_landscape(self, cb_params):
         """Save model params for landscape."""

mindspore/train/callback/_time_monitor.py

@@ -28,7 +28,8 @@ class TimeMonitor(Callback):
     Args:
         data_size (int): How many steps are the intervals between print information each time.
             if the program get `batch_num` during training, `data_size` will be set to `batch_num`,
-            otherwise `data_size` will be used. Default: ``None`` .
+            otherwise `data_size` will be used. If the program does not get `batch_num` during training,
+            meanwhile `data_size` does not set, the program will report an error. Default: ``None`` .
 
         data_time (bool): Whether to show the average time of fetching data in Host.
             Note that data fetch and network compute are processed sequentially in non dataset sink mode, while