mindspore 2.5.0__cp39-cp39-win_amd64.whl → 2.6.0rc1__cp39-cp39-win_amd64.whl

mindspore/train/_utils.py

@@ -16,6 +16,7 @@
 from __future__ import absolute_import
 
 import os
+import sys
 import json
 from collections.abc import Iterable
 
@@ -23,7 +24,7 @@ import time
 import numpy as np
 
 from mindspore.common.tensor import Tensor
-from mindspore._c_expression import Tensor as Tensor_
+from mindspore._c_expression import TensorPy as Tensor_
 from mindspore._c_expression import MSContext, ms_ctx_param
 from mindspore.common.dtype import dtype_to_nptype, pytype_to_dtype
 from mindspore.common import dtype as mstype
@@ -31,7 +32,7 @@ from mindspore import context
 from mindspore import log as logger
 from mindspore import _checkparam as Validator
 from mindspore.common.api import _cell_graph_executor
-from mindspore.communication import get_group_size
+from mindspore.communication.management import get_rank, get_group_size
 from mindspore.train.mind_ir_pb2 import ModelProto as mindir_model
 from mindspore.train.checkpoint_pb2 import Checkpoint
 from mindspore.train.node_strategy_pb2 import ParallelStrategyMap as ckpt_strategy
@@ -64,6 +65,7 @@ def _get_types_and_shapes(dataset):
     dataset_shapes = dataset.output_shapes()
     return dataset_types, dataset_shapes
 
+
 def enable_data_broadcast():
     """Get status to indicate if enable dataset broadcast."""
     return MSContext.get_instance().get_param(ms_ctx_param.dataset_broadcast_opt_level) > 0
@@ -375,20 +377,40 @@ def _get_parameter_redundancy_without_opt_shard(parameter_layout, param_redundan
         param_redundancy_dict[key] = tuple(redundancy_list)
 
 
-def get_parameter_redundancy(layout_obj, initial_rank=0):
+def _get_initial_rank(parameter_layout):
+    """Get the initial rank of pp."""
+    for k, _ in parameter_layout.items():
+        dev_matrix = parameter_layout[k][0]
+        break
+    dev_num = 1
+    if dev_matrix:
+        for i in dev_matrix:
+            dev_num *= i
+    rank_id = get_rank()
+    initial_rank = (rank_id // dev_num) * dev_num
+    return initial_rank
+
+
+def _get_pp_size_from_redundancy_map(param_redundancy):
+    """Get pp size from redundancy map."""
+    for _, v in param_redundancy.items():
+        return len(v) * len(v[0])
+
+
+def get_parameter_redundancy(layout_obj, initial_rank=None):
     """
     Get parameter redundancy map.
 
     Args:
         layout_obj (Union[str, layout): File name of `strategy.ckpt` or net.parameter_layout_dict.
-        initial_rank (int): Start rank id for each pipeline. Default: 0.
+        initial_rank (int): Start rank id for each pipeline. Default: ``None``.
 
     Returns:
         Dict, dict of parameter redundancy info.
 
     Examples:
         >>> from mindspore.train.utils import get_parameter_redundancy
-        >>> param_redundancy_dict = get_parameter_redundancy("/path/to/strategy.ckpt")
+        >>> param_redundancy_dict = get_parameter_redundancy("/path/to/strategy.ckpt", initial_rank=0)
         {'param1': ((0, 1, 2, 3, 4, 5, 6, 7),),
          'param2': ((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)),
          'param3': ((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)),
@@ -405,7 +427,8 @@ def get_parameter_redundancy(layout_obj, initial_rank=0):
         from mindspore.communication.management import get_process_group_ranks
         groups_ranks = (tuple(get_process_group_ranks()),)
         param_redundancy_dict = {param.name: groups_ranks for _, param in layout_obj.parameters_and_names()}
-        return param_redundancy_dict
+        sorted_param_redundancy_dict = {key: param_redundancy_dict[key] for key in sorted(param_redundancy_dict.keys())}
+        return sorted_param_redundancy_dict
     else:
         parameter_layout = {}
         for k, v in layout_obj.items():
@@ -413,6 +436,9 @@ def get_parameter_redundancy(layout_obj, initial_rank=0):
 
     param_redundancy_dict = {}
 
+    if initial_rank is None:
+        initial_rank = _get_initial_rank(parameter_layout)
+
     _get_parameter_redundancy_without_opt_shard(parameter_layout, param_redundancy_dict, initial_rank)
 
     if isinstance(layout_obj, str):
@@ -420,7 +446,8 @@ def get_parameter_redundancy(layout_obj, initial_rank=0):
     else:
         _get_layout_opt_shard(layout_obj, param_redundancy_dict)
 
-    return param_redundancy_dict
+    sorted_param_redundancy_dict = {key: param_redundancy_dict[key] for key in sorted(param_redundancy_dict.keys())}
+    return sorted_param_redundancy_dict
 
 
 def _collect_settings_by_rank(redundancy_map):
@@ -539,6 +566,7 @@ def _progress_bar(iterable, total=None):
         elapsed_time_str = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
         remaining_time_str = time.strftime("%H:%M:%S", time.gmtime(remaining_time))
 
+        sys.stdout.reconfigure(encoding="utf-8")
         print(f'\r{percent}%|{bar}|[{elapsed_time_str}<{remaining_time_str}]', end='')
         if iteration == total:
             print()

mindspore/train/amp.py

@@ -638,7 +638,7 @@ def _add_loss_network(network, loss_fn, cast_model_type):
 
 
 def _is_grad_accumulation(mcell):
-    if mcell.cls_name == "GradAccumulationCell":
+    if mcell.cls_name == "GradAccumulationCell" or mcell.cls_name == "GradAccumulation":
         return True
     for cell in mcell.cells():
         if _is_grad_accumulation(cell):

mindspore/train/callback/__init__.py

@@ -36,9 +36,9 @@ from mindspore.train.callback._reduce_lr_on_plateau import ReduceLROnPlateau
 from mindspore.train.callback._on_request_exit import OnRequestExit
 from mindspore.train.callback._backup_and_restore import BackupAndRestore
 from mindspore.train.callback._flops_collector import FlopsUtilizationCollector
-from mindspore.train.callback._tft_register import TFTRegister
+from mindspore.train.callback._train_fault_tolerance import TrainFaultTolerance
 
 __all__ = ["Callback", "LossMonitor", "TimeMonitor", "ModelCheckpoint", "FlopsUtilizationCollector",
            "SummaryCollector", "CheckpointConfig", "RunContext", "LearningRateScheduler", "SummaryLandscape",
            "History", "LambdaCallback", "ReduceLROnPlateau", "EarlyStopping", "OnRequestExit", "BackupAndRestore",
-           "TFTRegister"]
+           "TrainFaultTolerance"]

mindspore/train/callback/_callback.py

@@ -121,10 +121,7 @@ class Callback:
     When creating a custom Callback, model context information can be obtained in Callback
     methods by calling `RunContext.original_args()`, which is a dictionary varivable
     recording current attributes. Users can add custimized attributes to the information.
-    Training process can also be stopped by calling `request_stop` method. For details
-    of custom Callback, please check
-    `Callback tutorial <https://www.mindspore.cn/docs/en/master/model_train/train_process/model/
-    callback.html#customized-callback-mechanism>`_.
+    Training process can also be stopped by calling `request_stop` method.
 
     Examples:
         >>> import numpy as np
@@ -491,9 +488,7 @@ class RunContext:
 
     Callback objects not only can obtain the Model context information by calling by
     `RunContext.original_args()` and add extra attributes to the information, but also can stop the
-    training process by calling `request_stop` method. For details of custom Callback,
-    please check
-    `Callback Mechanism <https://www.mindspore.cn/docs/en/master/model_train/train_process/model/callback.html>`_.
+    training process by calling `request_stop` method.
 
     `RunContext.original_args()` holds the model context information as a dictionary variable, and
     different attributes of the dictionary are stored in training or eval process. Details are as follows:
@@ -572,10 +567,6 @@ class RunContext:
 
         Returns:
            Dict, an object that holds the original arguments of model.
-
-        Tutorial Examples:
-            - `Callback Mechanism - Customized Callback Mechanism
-              <https://mindspore.cn/docs/en/master/model_train/train_process/model/callback.html#customized-callback-mechanism>`_
         """
         return self._original_args
 
@@ -585,11 +576,6 @@ class RunContext:
 
         Callbacks can use this function to request stop of iterations.
         model.train() checks whether this is called or not.
-
-        Tutorial Examples:
-            - `Callback Mechanism - Customized Training Termination Time
-              <https://mindspore.cn/docs/en/master/model_train/train_process/model/callback.html#
-              customized-training-termination-time>`_
         """
         self._stop_requested = True
 

mindspore/train/callback/_checkpoint.py

@@ -28,15 +28,12 @@ from mindspore.train.serialization import save_checkpoint, _save_graph, _wait_as
     _wait_async_thread_save_ckpt, _check_async_save
 from mindspore.parallel._cell_wrapper import destroy_allgather_cell
 from mindspore.parallel._recovery_context import _set_recovery_context, _get_recovery_context
-from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
-from mindspore.parallel._utils import _get_device_num
-from mindspore.communication.management import get_rank
-from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy
-from mindspore.train.callback._callback import Callback, set_cur_net
+from mindspore.communication.management import get_rank, get_group_size
+from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy, _get_pp_size_from_redundancy_map
+from mindspore.train.callback._callback import Callback
 from mindspore.common.tensor import Tensor
 from mindspore.common.parameter import Parameter
 from mindspore.common.generator import Generator
-from mindspore.common.api import _cell_graph_executor
 from mindspore._c_expression import collect_host_info, get_clock_syscnt
 
 _cur_dir = os.getcwd()
@@ -87,7 +84,7 @@ def _chg_ckpt_file_name_if_same_exist(directory, prefix, exception=False):
         name_ext = os.path.splitext(filename)
         if exception and filename[-16:] != "_breakpoint.ckpt":
             continue
-        if not exception and (name_ext[-1] != ".ckpt" or filename[-16:] == "_breakpoint.ckpt"):
+        if not exception and (name_ext[-1] not in (".ckpt", ".safetensors") or filename[-16:] == "_breakpoint.ckpt"):
             continue
         # find same prefix file
         if filename.find(prefix) == 0 and not filename[pre_len].isalpha():
@@ -106,10 +103,10 @@ def _chg_ckpt_file_name_if_same_exist(directory, prefix, exception=False):
     return prefix
 
 
-def _check_format_and_other_params(format, enc_key, enc_mode, crc_check=False, async_save=False, exception_save=False,
+def _check_format_and_other_params(format, enc_key, enc_mode, crc_check=False, exception_save=False,
                                    map_param_inc=False, global_step_num=None):
-    param_not_default = (enc_key is not None or enc_mode != "AES-GCM" or crc_check or async_save
-                         or exception_save or map_param_inc or global_step_num is not None)
+    param_not_default = (enc_key is not None or enc_mode != "AES-GCM" or crc_check or exception_save or map_param_inc
+                         or global_step_num is not None)
     if format == "safetensors" and param_not_default:
         raise ValueError("For 'save_checkpoint', when format is 'safetensors', other param must be default.")
 
@@ -139,9 +136,9 @@ class CheckpointConfig:
         integrated_save (bool): Whether to merge and save the split Tensor in the automatic parallel scenario.
             Integrated save function is only supported in automatic parallel scene, not supported
             in manual parallel. Default: ``True`` .
-        async_save (Union[bool, str]):Whether to use asynchronous saving of the checkpoint file, if True,
-                                    the asynchronous thread is used by default. If the type is string,
-                                    the method of asynchronous saving, it can be "process" or "thread".
+        async_save (Union[bool, str], optional):Whether to use asynchronous saving of the checkpoint file or
+                                    safetensors file, if True, the asynchronous thread is used by default. If the type
+                                    is string, the method of asynchronous saving, it can be "process" or "thread".
                                     Default: ``False`` .
         saved_network (Cell): Network to be saved in checkpoint file. If the saved_network has no relation
             with the network in training, the initial value of saved_network will be saved. Default: ``None`` .
@@ -261,8 +258,7 @@ class CheckpointConfig:
         self.enable_redundance = kwargs.get('enable_redundance', False)
         self.remove_redundancy = Validator.check_isinstance('remove_redundancy', remove_redundancy, bool)
 
-        _check_format_and_other_params(format, enc_key, enc_mode, crc_check, async_save, exception_save,
-                                       self._map_param_inc)
+        _check_format_and_other_params(format, enc_key, enc_mode, crc_check, exception_save, self._map_param_inc)
 
     @property
     def save_checkpoint_steps(self):
@@ -452,8 +448,9 @@ class ModelCheckpoint(Callback):
     Note:
         In the distributed training scenario, please specify different directories for each training process
         to save the checkpoint file. Otherwise, the training may fail.
-        If this callback is used in the `model` function, the checkpoint file will saved
-        parameters of the optimizer by default.
+        If this callback is used in the
+        `Model <https://www.mindspore.cn/docs/en/master/api_python/train/mindspore.train.Model.html>`_ function,
+        the checkpoint file will saved parameters of the optimizer by default.
 
     Args:
         prefix (Union[str, callable object]): The prefix name or callable object to generate name of checkpoint files.
@@ -514,7 +511,7 @@ class ModelCheckpoint(Callback):
         if callable(prefix):
             self._prefix_func = prefix
 
-        if _get_recovery_context("enable_recovery"):
+        if context.get_context("device_target") == "GPU" and _get_recovery_context("enable_recovery"):
             _set_recovery_context(ckpt_path=self._directory)
 
         if config is None:
@@ -556,19 +553,17 @@ class ModelCheckpoint(Callback):
             from aiturbo.checkpoint import aiturbo_mindspore as aiturbo
             ckpt_storage_path = self._directory
             rank_id = get_rank()
-            stage_num = _get_auto_parallel_context("pipeline_stages")
-            stage_rank_num = _get_device_num() // stage_num
+            device_num = get_group_size()
             param_layout = cb_params.train_network.parameter_layout_dict
             if not param_layout:
-                layout = {"stage_num": stage_num, "stage_rank_num": stage_rank_num, "stage_layout": None}
+                layout = {"stage_num": 1, "stage_rank_num": device_num, "stage_layout": None}
                 aiturbo.init(ckpt_storage_path, rank_id, layout, None, False, None)
             else:
-                device_num = _get_device_num()
-                chunk_size = device_num // stage_num
-                initial_rank = (rank_id // chunk_size) * chunk_size
-                param_redundancy_dict = get_parameter_redundancy(param_layout, initial_rank)
+                param_redundancy_dict = get_parameter_redundancy(param_layout)
+                pp_size = _get_pp_size_from_redundancy_map(param_redundancy_dict)
+                stage_num = device_num // pp_size
                 dp, _ = _get_dp_tp_from_layout(param_redundancy_dict)
-                layout = {"stage_num": stage_num, "stage_rank_num": stage_rank_num,
+                layout = {"stage_num": stage_num, "stage_rank_num": pp_size,
                           "stage_layout": param_redundancy_dict}
                 single_params = remove_param_redundancy(param_redundancy_dict)
                 single_params = {device_id: list(params) for device_id, params in single_params.items()}
@@ -684,12 +679,6 @@ class ModelCheckpoint(Callback):
             self._last_time_for_keep = time.time()
             self._last_triggered_step = cb_params.cur_step_num
 
-            # TODO(MS_DISABLE_REF_MODE): Delete when remove MS_DISABLE_REF_MODE env.
-            if context.get_context("enable_ge") and os.getenv('MS_DISABLE_REF_MODE') \
-                    and context.get_context("mode") == context.GRAPH_MODE:
-                set_cur_net(cb_params.train_network)
-                cb_params.train_network.add_flags(ge_sync_data=True)
-                _cell_graph_executor(cb_params.train_network, phase='save')
             self._append_dict_content(cb_params.cur_epoch_num, cb_params.cur_step_num)
             network = self._config.saved_network if self._config.saved_network is not None else cb_params.train_network
             if os.getenv("AITURBO") == "1":
@@ -698,18 +687,13 @@ class ModelCheckpoint(Callback):
                                 crc_check=self._config.crc_check, incremental=self._map_param_inc,
                                 global_step_num=cb_params.cur_step_num)
             elif self._config.remove_redundancy:
-                parallel_mode = context.get_auto_parallel_context("parallel_mode")
-                if parallel_mode == "stand_alone":
+                if get_group_size() == 1:
                     raise TypeError(f"The deduplication feature for saving checkpoint can only be used "
-                                    f"in parallel scenarios, but got {parallel_mode}.")
+                                    f"in parallel scenarios, but got 'stand_alone'.")
                 param_layout = network.parameter_layout_dict
                 rank_id = get_rank()
                 if param_layout:
-                    device_num = _get_device_num()
-                    stage_num = _get_auto_parallel_context("pipeline_stages")
-                    chunk_size = device_num // stage_num
-                    initial_rank = (rank_id // chunk_size) * chunk_size
-                    param_redundancy_dict = get_parameter_redundancy(param_layout, initial_rank)
+                    param_redundancy_dict = get_parameter_redundancy(param_layout)
                     single_params = remove_param_redundancy(param_redundancy_dict)
                     save_param_names = single_params.get(rank_id)
                     param_layout_set = set(param_layout.keys())

mindspore/train/callback/_cluster_monitor.py

@@ -24,9 +24,8 @@ from threading import RLock
 from mindspore.train.callback._callback import Callback
 from mindspore.communication.management import get_rank, get_local_rank
 from mindspore import log as logger
-from mindspore.parallel._auto_parallel_context import _get_auto_parallel_context
 from mindspore.parallel._utils import _get_device_num
-from mindspore.train._utils import get_parameter_redundancy
+from mindspore.train._utils import get_parameter_redundancy, _get_pp_size_from_redundancy_map
 
 _perf_mutex = RLock()
 
@@ -42,7 +41,7 @@ def _get_dp_tp_from_redundancy(redundancy_tuple):
     return dp, tp
 
 
-def _get_dp_tp_from_layout(parameter_layout_dict, initial_rank=0):
+def _get_dp_tp_from_layout(parameter_layout_dict, initial_rank=None):
     """From layout dict get dp and tp"""
     tp = []
     dp = []
@@ -132,21 +131,9 @@ class ClusterMonitor(Callback):
         self.full_path = self.log_path + self.log_name
 
         self.write_dp_tp_flag = True
-        self.initial_rank = 0
 
     def begin(self, run_context):
         _remove_pre_log()
-        pp_num = _get_auto_parallel_context("pipeline_stages")
-        device_num = _get_device_num()
-
-        original_list = list(range(device_num))
-        chunk_size = device_num // pp_num
-        split_pp_lists = []
-        for i in range(0, device_num, chunk_size):
-            end_index = i + chunk_size if i + chunk_size <= device_num else device_num
-            split_pp_lists.append(original_list[i:end_index])
-
-        self.initial_rank = (self.global_rank // chunk_size) * chunk_size
         with _perf_mutex:
             dir_path = os.path.dirname(self.full_path)
             if not os.path.exists(dir_path):
@@ -157,8 +144,6 @@ class ClusterMonitor(Callback):
             with open(self.full_path, 'w') as file:
                 log_message = f'UUID:{self.uuid_value}\nFRAMEWORK:{self.frame_work}\nGLOBAL RANKID:{self.global_rank}\n'
                 file.write(log_message)
-                for _, split_pp_list in enumerate(split_pp_lists):
-                    file.write(f'PP:{split_pp_list}\n')
             os.chmod(self.full_path, stat.S_IRUSR)
 
     def step_begin(self, run_context):
@@ -183,10 +168,21 @@ class ClusterMonitor(Callback):
         if self.enabled and self.enabled_dtp_group and self.write_dp_tp_flag:
             cb_params = run_context.original_args()
             param_layout_dict = cb_params.train_network.parameter_layout_dict
-            dp, tp = _get_dp_tp_from_layout(param_layout_dict, self.initial_rank)
+            device_num = _get_device_num()
+            original_list = list(range(device_num))
+            param_redundancy_dict = get_parameter_redundancy(param_layout_dict)
+            pp_size = _get_pp_size_from_redundancy_map(param_redundancy_dict)
+            split_pp_lists = []
+            for i in range(0, device_num, pp_size):
+                end_index = i + pp_size if i + pp_size <= device_num else device_num
+                split_pp_lists.append(original_list[i:end_index])
+            dp, tp = _get_dp_tp_from_layout(param_layout_dict)
+
             with _perf_mutex:
                 os.chmod(self.full_path, stat.S_IWUSR)
                 with open(self.full_path, 'a') as file:
+                    for _, split_pp_list in enumerate(split_pp_lists):
+                        file.write(f'PP:{split_pp_list}\n')
                     for dp_value in dp:
                         file.write(f'dp:{dp_value}\n')
                     for tp_value in tp:

mindspore/train/callback/_flops_collector.py

@@ -89,7 +89,7 @@ class FlopsUtilizationCollector(Callback):
         Train per step time: 135.572 ms, mfu:0.47% hfu:0.47%
         Train per step time: 1.317 ms, mfu:48.59% hfu:48.59%
     """
-    def __init__(self, data_size=None, computility=1, full_flops=True, enable_ma_collector=False):
+    def __init__(self, data_size, computility=1, full_flops=True, enable_ma_collector=False):
         super(FlopsUtilizationCollector, self).__init__()
         self.step_time = time.time()
         self.computility = computility
@@ -110,8 +110,7 @@ class FlopsUtilizationCollector(Callback):
         self.batch_step_size = None
         Validator.check_bool(full_flops, "full_flops")
         Validator.check_bool(enable_ma_collector, "enable_ma_collector")
-        if data_size:
-            Validator.check_positive_int(data_size, "data_size")
+        Validator.check_positive_int(data_size, "data_size")
 
     def step_begin(self, run_context):
         """

mindspore/train/callback/_history.py

@@ -25,10 +25,13 @@ class History(Callback):
     """
     Records the network outputs and metrics information into a `History` object.
 
-    The network outputs information will be the loss value if not custimizing the train network or eval network;
-    if the custimized network returns a `Tensor` or `numpy.ndarray`, the mean value of network output
-    will be recorded, if the custimized network returns a `tuple` or `list`, the first element of network
-    outputs will be recorded.
+    - The network outputs information will be the loss value if not custimizing the train network or eval network;
+    - If the train network or eval network is custimized:
+
+      - if the custimized network returns a `Tensor` or `numpy.ndarray`, the mean value of network output
+        will be recorded.
+      - if the custimized network returns a `tuple` or `list`, the first element of network
+        outputs will be recorded.
 
     Note:
         Normally used in :func:`mindspore.train.Model.train` or :func:`mindspore.train.Model.fit`.

mindspore/train/callback/_lambda_callback.py

@@ -36,8 +36,8 @@ class LambdaCallback(Callback):
         on_train_step_end (Function): called at each train step end. Default: ``None`` .
         on_train_begin (Function): called at the beginning of model train. Default: ``None`` .
         on_train_end (Function): called at the end of model train. Default: ``None`` .
-        on_eval_epoch_begin (Function): called at eval epoch begin. Default: ``None`` .
-        on_eval_epoch_end (Function): called at eval epoch end. Default: ``None`` .
+        on_eval_epoch_begin (Function): called at each eval epoch begin. Default: ``None`` .
+        on_eval_epoch_end (Function): called at each eval epoch end. Default: ``None`` .
         on_eval_step_begin (Function): called at each eval step begin. Default: ``None`` .
         on_eval_step_end (Function): called at each eval step end. Default: ``None`` .
         on_eval_begin (Function): called at the beginning of model eval. Default: ``None`` .

mindspore/train/callback/_landscape.py

@@ -256,9 +256,6 @@ class SummaryLandscape:
         """
         Clean the checkpoint.
 
-        Tutorial Examples:
-            - `Training Optimization Process Visualization
-              <https://www.mindspore.cn/mindinsight/docs/en/master/landscape.html>`_
         """
         shutil.rmtree(self._ckpt_dir, ignore_errors=True)
 

mindspore/train/callback/_loss_monitor.py

@@ -93,7 +93,8 @@ class LossMonitor(Callback):
 
     def on_train_epoch_end(self, run_context):
         """
-        When LossMonitor used in `model.fit`, print eval metrics at the end of epoch if current epoch
+        When LossMonitor used in :func:`mindspore.train.Model.fit`, print eval metrics
+        at the end of epoch if current epoch
         should do evaluation.
 
         Args:

mindspore/train/callback/_on_request_exit.py

@@ -26,6 +26,7 @@ from mindspore.common.tensor import Tensor
 from mindspore.train._utils import _make_directory
 from mindspore import _checkparam as Validator
 from mindspore.train.serialization import load_checkpoint, save_checkpoint, export
+from mindspore.communication.management import get_group_size
 from mindspore.train.callback._callback import Callback
 from mindspore.parallel._utils import _get_parallel_mode
 from mindspore.context import ParallelMode
@@ -37,7 +38,7 @@ class OnRequestExit(Callback):
 
     Register OnRequestExit Callback before training, when the user want to exit the training process
     and save the training data, could send the registered exit signal 'sig' to the training process or modify the
-    'GracefulExit' that a key in the json file specified by the 'config_file' to '1'.
+    'GracefulExit' that a key in the JSON file specified by the 'config_file' to '1'.
     After the training process executes the current step, saves the current training status,
     including checkpoint and mindir, and then exit the training process.
 
@@ -58,7 +59,7 @@ class OnRequestExit(Callback):
         ValueError: If the 'save_mindir' is not a bool.
         ValueError: If the 'file_name' is not a str.
         ValueError: If the 'directory' is not a str.
-        ValueError: If the 'sig' is not an int or the 'sig' is signal.SIGKILL.
+        ValueError: If the 'sig' is not an int or the 'sig' is ``signal.SIGTERM``.
 
     Examples:
         >>> from mindspore import nn
@@ -92,10 +93,8 @@ class OnRequestExit(Callback):
         self.key = "GracefulExit"
         self.remote_config_file = config_file  # used config file to save checkpoint and exit training process
         self.use_graceful = os.environ.get("MS_ENABLE_GRACEFUL_EXIT") == "1"
-        self.is_distributed = _get_parallel_mode() != ParallelMode.STAND_ALONE
+        self.is_distributed = get_group_size() > 1
         self.integrated_save = True
-        if self.is_distributed:
-            self.integrated_save = _get_parallel_mode() == ParallelMode.AUTO_PARALLEL
         self.stop_train = False
         self.need_do_step_end = False
         if self.save_ckpt or self.save_mindir:
@@ -250,6 +249,8 @@ class OnRequestExit(Callback):
                         else:
                             global_step = int(call_params.network.optimizer.global_step.data)
                         append_dict["global_step"] = global_step
+                        if self.is_distributed:
+                            self.integrated_save = _get_parallel_mode() == ParallelMode.AUTO_PARALLEL
                         save_checkpoint(net, self.train_name, integrated_save=self.integrated_save,
                                         append_dict=append_dict)
                     if self.save_mindir:

mindspore/train/callback/_reduce_lr_on_plateau.py

@@ -63,12 +63,17 @@ class ReduceLROnPlateau(Callback):
             will be reduced. Default: ``10`` .
         verbose (bool): If False: quiet, if True: print related information.
             Default: ``False`` .
-        mode (str): one of `{'auto', 'min', 'max'}`. In "min" mode,
-            the learning rate will be reduced when the
-            quantity monitored has stopped decreasing; in "max" mode it will be
-            reduced when the quantity monitored has stopped increasing; in "auto"
-            mode, the direction is automatically inferred from the name of the
-            monitored quantity. Default: ``'auto'`` .
+        mode (str): one of `{'auto', 'min', 'max'}`. Default: ``'auto'`` .
+
+            - In ``'min'`` mode,
+              the learning rate will be reduced when the
+              quantity monitored has stopped decreasing.
+            - In ``'max'`` mode it will be
+              reduced when the quantity monitored has stopped increasing.
+            - In ``'auto'``
+              mode, the direction is automatically inferred from the name of the
+              monitored quantity.
+
         min_delta (float): threshold for measuring the new optimum, to only focus on
             significant changes. Default: ``1e-4`` .
         cooldown (int): number of epochs to wait before resuming normal operation after

mindspore/train/callback/_summary_collector.py

@@ -107,22 +107,18 @@ class SummaryCollector(Callback):
               The first output will be treated as the loss and it will be averaged. Default: ``True`` .
             - collect_graph (bool): Whether to collect the computational graph. Currently, only
               training computational graph is collected. Default: ``True`` .
-            - collect_train_lineage (bool): Whether to collect lineage data for the training phase,
-              this field will be displayed on the `lineage page \
-              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
-              of MindInsight. Default: ``True`` .
-            - collect_eval_lineage (bool): Whether to collect lineage data for the evaluation phase,
-              this field will be displayed on the `lineage page
-              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
-              of MindInsight. Default: ``True`` .
+            - collect_train_lineage (bool): Whether to collect lineage data for the training phase.
+              Default: ``True`` .
+            - collect_eval_lineage (bool): Whether to collect lineage data for the evaluation phase.
+              Default: ``True`` .
             - collect_input_data (bool): Whether to collect dataset for each training.
               Currently only image data is supported.
               If there are multiple columns of data in the dataset, the first column should be image data.
               Default: ``True`` .
             - collect_dataset_graph (bool): Whether to collect dataset graph for the training phase.
               Default: ``True`` .
-            - histogram_regular (Union[str, None]): Collect weight and bias for parameter distribution page
-              and displayed in MindInsight. This field allows regular strings to control which parameters to collect.
+            - histogram_regular (Union[str, None]): Collect weight and bias for parameter distribution page.
+              This field allows regular strings to control which parameters to collect.
               It is not recommended to collect too many parameters at once, as it can affect performance.
               Note that if you collect too many parameters and run out of memory, the training will fail.
               Default: ``None`` , it means only the first five parameters are collected.
@@ -153,8 +149,7 @@ class SummaryCollector(Callback):
             True: it means that after specified data is set, non-specified data is collected as the default behavior.
             False: it means that after specified data is set, only the specified data is collected,
             and the others are not collected. Default: ``True`` .
-        custom_lineage_data (Union[dict, None]): Allows you to customize the data and present it on the MingInsight
-            `lineage page <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_ .
+        custom_lineage_data (Union[dict, None]): Allows you to customize the data.
             In the custom data, the type of the key supports str, and the type of value supports str, int
             and float. Default: ``None`` , it means there is no custom data.
         collect_tensor_freq (Optional[int]): The same semantics as the `collect_freq`, but controls TensorSummary only.
@@ -168,7 +163,7 @@ class SummaryCollector(Callback):
             affect the number of steps TensorSummary will be collected.
             Default: ``None`` , which means to follow the behavior as described above.
         max_file_size (Optional[int]): The maximum size in bytes of each file that can be written to the disk.
-            For example, to write not larger than 4GB, specify `max_file_size=4*1024**3`.
+            For example, to write not larger than 4GB, specify `max_file_size=4*1024*3`.
             Default: ``None`` , which means no limit.
         export_options (Union[None, dict]): Perform custom operations on the export data.
             Note that the size of export files is not limited by the max_file_size.

mindspore/train/callback/_time_monitor.py

@@ -28,7 +28,8 @@ class TimeMonitor(Callback):
     Args:
         data_size (int): How many steps are the intervals between print information each time.
             if the program get `batch_num` during training, `data_size` will be set to `batch_num`,
-            otherwise `data_size` will be used. Default: ``None`` .
+            otherwise `data_size` will be used. If the program does not get `batch_num` during training,
+            meanwhile `data_size` does not set, the program will report an error. Default: ``None`` .
 
         data_time (bool): Whether to show the average time of fetching data in Host.
             Note that data fetch and network compute are processed sequentially in non dataset sink mode, while