mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/train/model.py

@@ -29,7 +29,8 @@ from mindspore.train.serialization import save_checkpoint, load_checkpoint
 from mindspore.train.callback._checkpoint import ModelCheckpoint, _chg_ckpt_file_name_if_same_exist
 from mindspore.common.tensor import Tensor
 from mindspore.train.metrics import get_metrics, get_metric_fn
-from mindspore._checkparam import check_input_data, check_output_data, Validator
+from mindspore._checkparam import check_input_data, check_output_data
+from mindspore import _checkparam as Validator
 from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager, Callback, TimeMonitor
 from mindspore.train.callback import __all__ as internal_cb_names
 from mindspore import context
@@ -45,6 +46,7 @@ from mindspore.context import ParallelMode
 from mindspore.parallel._recovery_context import _set_recovery_context, _get_recovery_context
 from mindspore.train.dataset_helper import DatasetHelper, connect_network_with_dataset
 from mindspore.common.api import _pynative_executor
+from mindspore.dataset.core.config import get_debug_mode
 from mindspore.dataset.engine.datasets import _set_training_dataset, _reset_training_dataset
 from mindspore.train import amp
 
@@ -125,13 +127,15 @@ class Model:
                              three elements: the positions of loss value, predicted value and label in outputs of the
                              `eval_network`. In this case, the loss value will be passed to the `Loss` metric, the
                              predicted value and label will be passed to other metrics.
-                             :func:`mindspore.train.metric.set_indexes` is recommended instead of `eval_indexes`.
+                             :func:`mindspore.train.Metric.set_indexes` is recommended instead of `eval_indexes`.
                              Default: None.
-        amp_level (str): Option for argument `level` in :func:`mindspore.build_train_network`, level for mixed
+        amp_level (str): Option for argument `level` in :func:`mindspore.amp.build_train_network`, level for mixed
             precision training. Supports ["O0", "O1", "O2", "O3", "auto"]. Default: "O0".
 
             - "O0": Do not change.
             - "O1": Cast the operators in white_list to float16, the remaining operators are kept in float32.
+              The operators in the whitelist: [Conv1d, Conv2d, Conv3d, Conv1dTranspose, Conv2dTranspose,
+              Conv3dTranspose, Dense, LSTMCell, RNNCell, GRUCell, MatMul, BatchMatMul, PReLU, ReLU, Ger].
             - "O2": Cast network to float16, keep BatchNorm run in float32, using dynamic loss scale.
             - "O3": Cast network to float16, the BatchNorm is also cast to float16, loss scale will not be used.
             - auto: Set level to recommended level in different devices. Set level to "O2" on GPU, set
@@ -142,7 +146,7 @@ class Model:
             The BatchNorm strategy can be changed by `keep_batchnorm_fp32` settings in `kwargs`. `keep_batchnorm_fp32`
             must be a bool. The loss scale strategy can be changed by `loss_scale_manager` setting in `kwargs`.
             `loss_scale_manager` should be a subclass of :class:`mindspore.amp.LossScaleManager`.
-            The more detailed explanation of `amp_level` setting can be found at `mindspore.build_train_network`.
+            The more detailed explanation of `amp_level` setting can be found at `mindspore.amp.build_train_network`.
 
         boost_level (str): Option for argument `level` in `mindspore.boost`, level for boost mode
             training. Supports ["O0", "O1", "O2"]. Default: "O0".
@@ -192,7 +196,7 @@ class Model:
         >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None)
         >>> # For details about how to build the dataset, please refer to the variable `dataset_train` in tutorial
         >>> # document on the official website:
-        >>> # https://www.mindspore.cn/tutorials/zh-CN/r2.0.0-alpha/beginner/quick_start.html
+        >>> # https://www.mindspore.cn/tutorials/zh-CN/r2.0/beginner/quick_start.html
         >>> dataset = create_custom_dataset()
         >>> model.train(2, dataset)
     """
@@ -393,13 +397,12 @@ class Model:
     def _get_metrics(self):
         """Get metrics local values."""
         metrics = dict()
-        # Embedding cache server as a storage service, no need to execute eval, just give fake metrics.
-        is_embedding_cache_server = _is_role_pserver() and _cache_enable()
+        # There's no need for server to execute eval, just give fake metrics.
         for key, value in self._metric_fns.items():
-            if not is_embedding_cache_server:
+            if not _is_role_pserver():
                 metrics[key] = value.eval()
             else:
-                metrics[key] = 0
+                metrics[key] = 1
         return metrics
 
     def _get_scaling_sens(self):
@@ -429,7 +432,7 @@ class Model:
         if dataset_sink_mode:
             network = connect_network_with_dataset(network, dataset_helper)
 
-        if is_train:
+        if _get_recovery_context("enable_recovery") and is_train:
             _set_training_dataset(dataset_helper)
 
 
@@ -675,7 +678,7 @@ class Model:
             cb_params.train_network = train_network
 
             # Perform recovery for process which is restarted.
-            self._reset_training_step_for_abnormal_process(cb_params)
+            self._reset_training_step_for_abnormal_process(cb_params, dataset_helper)
             # Perform recovery for process which is not restarted.
             self._reset_training_step_for_normal_process(cb_params, dataset_helper)
 
@@ -691,13 +694,15 @@ class Model:
                 train_network = self._check_network_mode(train_network, True)
                 outputs = train_network(*inputs)
                 cb_params.net_outputs = outputs
+
+                if _is_role_sched():
+                    os._exit(0)
+
                 # In disaster recovery scenarios, need not to execute callbacks if this step executes failed.
                 need_exec_callback_step_end = not (self.enable_recovery and _get_recovery_context("need_reset"))
                 if need_exec_callback_step_end:
                     list_callback.on_train_step_end(run_context)
 
-                if _is_role_sched():
-                    os._exit(0)
                 # Embedding cache server only run one step.
                 if is_embedding_cache_server:
                     break
@@ -804,7 +809,7 @@ class Model:
         else:
             self.need_load_ckpt = False
 
-    def _reset_training_step_for_abnormal_process(self, cb_params):
+    def _reset_training_step_for_abnormal_process(self, cb_params, dataset_helper):
         """
         Execute recovery for abnormal exit process when restart.
 
@@ -819,7 +824,7 @@ class Model:
                 os.remove(cb_params.latest_ckpt_file)
                 raise RuntimeError(e.__str__() + ", load ckpt failed and remove the ckpt: "\
                                    + cb_params.latest_ckpt_file) from e
-            _reset_training_dataset(cb_params.cur_step_num, cb_params.cur_epoch_num)
+            _reset_training_dataset(cb_params.cur_step_num, dataset_helper.sink_size())
             self.need_load_ckpt = False
 
     def _reset_training_step_for_normal_process(self, cb_params, dataset_helper):
@@ -848,9 +853,9 @@ class Model:
                 self.epoch_iter = recovery_epoch_num
                 cb_params.cur_epoch_num = self.epoch_iter + 1
                 cb_params.last_save_ckpt_step = cb_params.cur_step_num
-                _reset_training_dataset(cb_params.cur_step_num, cb_params.cur_epoch_num)
+                _reset_training_dataset(cb_params.cur_step_num, dataset_helper.sink_size())
             else:
-                _reset_training_dataset(0, 0)
+                _reset_training_dataset(0, dataset_helper.sink_size())
 
             _set_recovery_context(need_reset=False)
 
@@ -999,12 +1004,17 @@ class Model:
             >>> dataset = create_custom_dataset()
             >>> net = Net()
             >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-            >>> loss_scale_manager = ms.FixedLossScaleManager()
+            >>> loss_scale_manager = ms.FixedLossScaleManager(1024., False)
             >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
             >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None,
             ...                  loss_scale_manager=loss_scale_manager)
             >>> model.train(2, dataset)
         """
+        device_target = context.get_context("device_target")
+        if _is_ps_mode() and not _cache_enable() and (device_target in ["Ascend", "CPU"]) and dataset_sink_mode:
+            logger.info("For PS mode, reset datasink mode to False when using Ascend or CPU backend.")
+            dataset_sink_mode = False
+
         Validator.check_bool(dataset_sink_mode)
         if isinstance(self._train_network, nn.GraphCell) and dataset_sink_mode:
             raise ValueError("Dataset sink mode is currently not supported when training with a GraphCell.")
@@ -1017,11 +1027,10 @@ class Model:
 
         # Parameter server and embedding cache mode check.
         if _is_ps_mode():
-            if dataset_sink_mode and not _cache_enable():
-                raise ValueError("Parameter server mode does not support 'data_sink_mode=True'.")
             if not dataset_sink_mode and _cache_enable():
-                raise ValueError("Embedding cache mode should run with 'data_sink_mode=True'.")
+                raise ValueError("Embedding cache mode should run with 'dataset_sink_mode=True'.")
 
+        self._check_sink_mode_for_ds_debug_mode(dataset_sink_mode)
 
         Validator.check_is_int(sink_size)
         Validator.check_non_negative_int(epoch)
@@ -1057,6 +1066,12 @@ class Model:
         if _enable_distributed_mindrt():
             _reset_op_id_with_offset()
 
+    @staticmethod
+    def _check_sink_mode_for_ds_debug_mode(dataset_sink_mode):
+        if get_debug_mode() and dataset_sink_mode:
+            raise ValueError("Dataset sink mode is not supported when dataset pipeline debug mode is on. "
+                             "Please manually turn off sink mode.")
+
     @staticmethod
     def _check_methods_for_custom_callbacks(callbacks, current_mode):
         """
@@ -1091,7 +1106,8 @@ class Model:
 
         Evaluation process will be performed during training process if `valid_dataset` is provided.
 
-        More details please refer to `mindspore.train.Model.train` and `mindspore.train.Model.eval`.
+        More details please refer to :func:`mindspore.train.Model.train` and
+        :func:`mindspore.train.Model.eval`.
 
         Args:
             epoch (int): Total training epochs. Generally, train network will be trained on complete dataset per epoch.
@@ -1142,6 +1158,10 @@ class Model:
             >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={"accuracy"})
             >>> model.fit(2, train_dataset, valid_dataset)
         """
+        device_target = context.get_context("device_target")
+        if _is_ps_mode() and not _cache_enable() and (device_target in ["Ascend", "CPU"]) and dataset_sink_mode:
+            logger.info("For PS mode, reset datasink mode to False when using Ascend or CPU backend.")
+            dataset_sink_mode = False
 
         dataset_sink_mode = Validator.check_bool(dataset_sink_mode)
         valid_dataset_sink_mode = Validator.check_bool(valid_dataset_sink_mode)
@@ -1154,9 +1174,6 @@ class Model:
                              "should be equal to value in Model.fit, but got {} and {} separately."
                              .format(train_dataset._warmup_epoch, epoch))
 
-        if dataset_sink_mode and _is_ps_mode() and not _cache_enable():
-            raise ValueError("Parameter server mode does not support 'data_sink_mode=True'.")
-
         Validator.check_is_int(sink_size)
         Validator.check_non_negative_int(epoch)
         Validator.check_non_negative_int(initial_epoch)
@@ -1175,7 +1192,7 @@ class Model:
         _device_number_check(self._parallel_mode, self._device_number)
 
         if not isinstance(valid_frequency, (int, list)):
-            raise TypeError(f"For 'Model.fit', the type of 'valid_frequency' must be a list or a integer, but got "
+            raise TypeError(f"For 'Model.fit', the type of 'valid_frequency' must be a list or an integer, but got "
                             f"type {type(valid_frequency)}.")
 
         if valid_dataset and not self._metric_fns:
@@ -1198,7 +1215,7 @@ class Model:
         Build computational graphs and data graphs with the sink mode.
 
         .. warning::
-            This is an experimental prototype that is subject to change or deletion.
+            This is an experimental API that is subject to change or deletion.
 
         Note:
             The interface builds the computational graphs, when the interface is executed first, 'Model.train' only
@@ -1520,7 +1537,7 @@ class Model:
         Only dataset sink mode is supported for now.
 
         .. warning::
-            This is an experimental prototype that is subject to change and/or deletion.
+            This is an experimental API that is subject to change or deletion.
 
         Note:
             This is a pre-compile function. The arguments should be the same as model.train() function.