mindspore 2.2.11__cp37-cp37m-manylinux1_x86_64.whl → 2.3.0rc1__cp37-cp37m-manylinux1_x86_64.whl

mindspore/train/callback/_checkpoint.py

@@ -41,6 +41,8 @@ _info_list = ["epoch_num", "step_num"]
 
 def _chg_ckpt_file_name_if_same_exist(directory, prefix, exception=False):
     """Check if there is a file with the same name."""
+    if callable(prefix) or callable(directory):
+        return prefix
     files = os.listdir(directory)
     suffix_num = 0
     pre_len = len(prefix)
@@ -112,13 +114,13 @@ class CheckpointConfig:
         >>> from mindspore.train import Model, CheckpointConfig, ModelCheckpoint
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> config = CheckpointConfig(save_checkpoint_seconds=100, keep_checkpoint_per_n_minutes=5, saved_network=net)
         >>> config.save_checkpoint_steps
@@ -205,7 +207,7 @@ class CheckpointConfig:
         Get the value of steps to save checkpoint.
 
         Returns:
-            Int, steps to save checkpoint.
+            int, steps to save checkpoint.
         """
         return self._save_checkpoint_steps
 
@@ -214,7 +216,7 @@ class CheckpointConfig:
         """Get the value of _save_checkpoint_seconds.
 
         Returns:
-            Int, seconds to save the checkpoint file.
+            int, seconds to save the checkpoint file.
         """
         return self._save_checkpoint_seconds
 
@@ -224,7 +226,7 @@ class CheckpointConfig:
         Get the value of maximum number of checkpoint files can be saved.
 
         Returns:
-            Int, Maximum number of checkpoint files can be saved.
+            int, Maximum number of checkpoint files can be saved.
         """
         return self._keep_checkpoint_max
 
@@ -244,7 +246,7 @@ class CheckpointConfig:
         Get the value of whether to merge and save the split Tensor in the automatic parallel scenario.
 
         Returns:
-            Bool, whether to merge and save the split Tensor in the automatic parallel scenario.
+            bool, whether to merge and save the split Tensor in the automatic parallel scenario.
         """
         return self._integrated_save
 
@@ -254,7 +256,7 @@ class CheckpointConfig:
         Get the value of whether asynchronous execution saves the checkpoint to a file.
 
         Returns:
-            Bool, whether asynchronous execution saves the checkpoint to a file.
+            bool, whether asynchronous execution saves the checkpoint to a file.
         """
         return self._async_save
 
@@ -294,7 +296,7 @@ class CheckpointConfig:
         Get the value of information dict saved to checkpoint file.
 
         Returns:
-            Dict, the information saved to checkpoint file.
+            dict, the information saved to checkpoint file.
         """
         return self._append_dict
 
@@ -304,7 +306,7 @@ class CheckpointConfig:
         Get the value of whether to save map Parameter incrementally.
 
         Returns:
-            Bool, whether to save map Parameter incrementally.
+            bool, whether to save map Parameter incrementally.
         """
         return self._map_param_inc
 
@@ -313,7 +315,7 @@ class CheckpointConfig:
         Get the policy of checkpoint.
 
         Returns:
-            Dict, the information of checkpoint policy.
+            dict, the information of checkpoint policy.
         """
         checkpoint_policy = {'save_checkpoint_steps': self.save_checkpoint_steps,
                              'save_checkpoint_seconds': self.save_checkpoint_seconds,
@@ -376,14 +378,16 @@ class ModelCheckpoint(Callback):
         parameters of the optimizer by default.
 
     Args:
-        prefix (str): The prefix name of checkpoint files. Default: ``'CKP'`` .
-        directory (str): The path of the folder which will be saved in the checkpoint file.
-            By default, the file is saved in the current directory. Default: ``None`` .
+        prefix (Union[str, callable object]): The prefix name or callable object to generate name of checkpoint files.
+            Default: ``'CKP'`` .
+        directory (Union[str, callable object]): The folder path where the checkpoint is stored, or the callable object
+            used to generate the path. By default, the file is saved in the current directory.
+            Default: ``None`` .
         config (CheckpointConfig): Checkpoint strategy configuration. Default: ``None`` .
 
     Raises:
-        ValueError: If `prefix` is not str or contains the '/' character.
-        ValueError: If `directory` is not str.
+        ValueError: If `prefix` is not str or contains the '/' character and is not a callable object.
+        ValueError: If `directory` is not str and is not a callable object.
         TypeError: If the config is not CheckpointConfig type.
 
     Examples:
@@ -409,19 +413,29 @@ class ModelCheckpoint(Callback):
         self._last_time = time.time()
         self._last_time_for_keep = time.time()
         self._last_triggered_step = 0
+        """a callable for users to set self-defined prefix."""
+        self._prefix_func = None
+        """a callable for users to set self-defined directory."""
+        self._directory_func = None
 
-        if not isinstance(prefix, str) or prefix.find('/') >= 0:
+        if not callable(prefix) and (not isinstance(prefix, str) or prefix.find('/') >= 0):
             raise ValueError("For 'ModelCheckpoint', the argument 'prefix' "
                              "for checkpoint file name is invalid, it must be "
-                             "string and does not contain '/', but got {}.".format(prefix))
+                             "callable or string that does not contain '/', but got {}.".format(prefix))
         self._prefix = prefix
         self._exception_prefix = prefix
 
         if directory is not None:
-            self._directory = _make_directory(directory)
+            if callable(directory):
+                self._directory_func = directory
+            else:
+                self._directory = _make_directory(directory)
         else:
             self._directory = _cur_dir
 
+        if callable(prefix):
+            self._prefix_func = prefix
+
         if _get_recovery_context("enable_recovery"):
             _set_recovery_context(ckpt_path=self._directory)
 
@@ -436,7 +450,8 @@ class ModelCheckpoint(Callback):
 
         # get existing checkpoint files
         self._manager = CheckpointManager()
-        self._prefix = _chg_ckpt_file_name_if_same_exist(self._directory, self._prefix)
+        if not callable(directory) and not callable(prefix):
+            self._prefix = _chg_ckpt_file_name_if_same_exist(self._directory, self._prefix)
         self._append_dict = self._config.append_dict or {}
         self._append_epoch_num = self._append_dict.get("epoch_num") if "epoch_num" in self._append_dict else 0
         self._append_step_num = self._append_dict.get("step_num") if "step_num" in self._append_dict else 0
@@ -452,6 +467,14 @@ class ModelCheckpoint(Callback):
             run_context (RunContext): Context of the train running.
         """
         cb_params = run_context.original_args()
+        if self._prefix_func:
+            self._prefix = self._prefix_func(cb_params)
+            if not isinstance(self._prefix, str) or self._prefix.find('/') >= 0:
+                raise ValueError("For 'ModelCheckpoint', the argument 'prefix' "
+                                 "for checkpoint file name is callable, it must return a "
+                                 "string that does not contain '/', but got {}.".format(self._prefix))
+        if self._directory_func:
+            self._directory = self._directory_func(cb_params)
         _collect_host_info("Callback", "ModelCheckpoint", "step_end", level=1)
         # In disaster recovery scenario, the training process may be rolled back to the last step where
         # the ckpt was successfully saved, so the _last_triggered_step should be updated.
@@ -520,8 +543,11 @@ class ModelCheckpoint(Callback):
         step_num_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num + 1)
 
         if save_ckpt:
-            cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \
-                + str(step_num_in_epoch) + ".ckpt"
+            if self._prefix_func:
+                cur_ckpoint_file = self._prefix + ".ckpt"
+            else:
+                cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \
+                    + str(step_num_in_epoch) + ".ckpt"
             # update checkpoint file list.
             self._manager.update_ckpoint_filelist(self._directory, self._prefix)
             # keep checkpoint files number equal max number.

mindspore/train/callback/_early_stop.py

@@ -85,13 +85,13 @@ class EarlyStopping(Callback):
         >>> from mindspore import nn
         >>> from mindspore.train import Model, EarlyStopping
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={"acc"})
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> cb = EarlyStopping(monitor="acc", patience=3, verbose=True)
         >>> model.fit(10, dataset, callbacks=cb)

mindspore/train/callback/_landscape.py

@@ -1,4 +1,4 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
+# Copyright 2021-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -170,8 +170,7 @@ class SummaryLandscape:
     It can create landscape in PCA direction or random direction by calculating loss.
 
     Note:
-        1. When using SummaryLandscape, you need to run the code in `if __name__ == "__main__"` .
-        2. SummaryLandscape only supports Linux systems.
+        SummaryLandscape only supports Linux systems.
 
     Args:
         summary_dir (str): The path of summary is used to save the model weight,
@@ -187,10 +186,10 @@ class SummaryLandscape:
         ...     # If the device_target is Ascend, set the device_target to "Ascend"
         ...     ms.set_context(mode=ms.GRAPH_MODE, device_target="GPU")
         ...     # Create the dataset taking MNIST as an example. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        ...     # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         ...     ds_train = create_dataset()
         ...     # Define the network structure of LeNet5. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        ...     # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         ...     network = LeNet5()
         ...     net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
         ...     net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
@@ -210,13 +209,13 @@ class SummaryLandscape:
         ...     # Simple usage for visualization landscape:
         ...     def callback_fn():
         ...         # Define the network structure of LeNet5. Refer to
-        ...         # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        ...         # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         ...         network = LeNet5()
         ...         net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
         ...         metrics = {"Loss": Loss()}
         ...         model = Model(network, net_loss, metrics=metrics)
         ...         # Create the dataset taking MNIST as an example. Refer to
-        ...         # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        ...         # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         ...         ds_eval = create_dataset()
         ...         return model, network, ds_eval, metrics
         ...
@@ -254,7 +253,13 @@ class SummaryLandscape:
                 self._epoch_group[i].append(j)
 
     def clean_ckpt(self):
-        """Clean the checkpoint."""
+        """
+        Clean the checkpoint.
+
+        Tutorial Examples:
+            - `Training Optimization Process Visualization
+              <https://www.mindspore.cn/mindinsight/docs/en/master/landscape.html>`_
+        """
         shutil.rmtree(self._ckpt_dir, ignore_errors=True)
 
     def gen_landscapes_with_multi_process(self, callback_fn, collect_landscape=None,
@@ -281,7 +286,7 @@ class SummaryLandscape:
                   Default: ``40`` . Optional values: between 3 and 256.
                 - create_landscape (dict): Select how to create loss landscape.
                   Training process loss landscape(train) and training result loss landscape(result).
-                  Default: {"train": True, "result": True}. Optional: ``True`` / ``False`` .
+                  Default: ``{"train": True, "result": True}``. Optional: ``True`` / ``False`` .
                 - num_samples (int): The size of the dataset used to create the loss landscape.
                   For example, in image dataset, You can set num_samples is 2048,
                   which means that 2048 images are used to create loss landscape.
@@ -816,7 +821,7 @@ class SummaryLandscape:
             #device_id should be between 0 and 7.
             if i < 0 or i > 7:
                 raise ValueError(f'For "{self.__class__.__name__}.gen_landscapes_with_multi_process", the parameter '
-                                 f'"device_ids" should be between 0 and 7,but got {i}.')
+                                 f'"device_ids" should be between 0 and 7, but got {i}.')
 
     def _check_collect_landscape_data(self, collect_landscape):
         """Check collect landscape data type and value."""

mindspore/train/callback/_loss_monitor.py

@@ -43,13 +43,13 @@ class LossMonitor(Callback):
         >>> from mindspore.train import Model, LossMonitor
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> loss_monitor = LossMonitor()
         >>> model.train(10, dataset, callbacks=loss_monitor)

mindspore/train/callback/_on_request_exit.py

@@ -55,13 +55,13 @@ class OnRequestExit(Callback):
         >>> import mindspore as ms
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> on_request_exit = ms.train.OnRequestExit(file_name='LeNet5')
         >>> model.train(10, dataset, callbacks=on_request_exit)

mindspore/train/callback/_reduce_lr_on_plateau.py

@@ -84,13 +84,13 @@ class ReduceLROnPlateau(Callback):
         >>> from mindspore import nn
         >>> from mindspore.train import Model, ReduceLROnPlateau
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={"acc"})
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> cb = ReduceLROnPlateau(monitor="acc", patience=3, verbose=True)
         >>> model.fit(10, dataset, callbacks=cb)

mindspore/train/callback/_summary_collector.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -77,12 +77,11 @@ class SummaryCollector(Callback):
     SummaryCollector also enables the summary operator to collect data to summary files.
 
     Note:
-        1. When using SummaryCollector, you need to run the code in `if __name__ == "__main__"` .
-        2. Multiple SummaryCollector instances in callback list are not allowed.
-        3. Not all information is collected at the training phase or at the eval phase.
-        4. SummaryCollector always record the data collected by the summary operator.
-        5. SummaryCollector only supports Linux systems.
-        6. The Summary is not supported when compile source with `-s on` option.
+        1. Multiple SummaryCollector instances in callback list are not allowed.
+        2. Not all information is collected at the training phase or at the eval phase.
+        3. SummaryCollector always record the data collected by the summary operator.
+        4. SummaryCollector only supports Linux systems.
+        5. The Summary is not supported when compile source with `-s on` option.
 
     Args:
         summary_dir (str): The collected data will be persisted to this directory.
@@ -106,11 +105,11 @@ class SummaryCollector(Callback):
               training computational graph is collected. Default: ``True`` .
             - collect_train_lineage (bool): Whether to collect lineage data for the training phase,
               this field will be displayed on the `lineage page \
-              <https://www.mindspore.cn/mindinsight/docs/en/r2.2/lineage_and_scalars_comparison.html>`_
+              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
               of MindInsight. Default: ``True`` .
             - collect_eval_lineage (bool): Whether to collect lineage data for the evaluation phase,
               this field will be displayed on the `lineage page
-              <https://www.mindspore.cn/mindinsight/docs/en/r2.2/lineage_and_scalars_comparison.html>`_
+              <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_
               of MindInsight. Default: ``True`` .
             - collect_input_data (bool): Whether to collect dataset for each training.
               Currently only image data is supported.
@@ -135,7 +134,7 @@ class SummaryCollector(Callback):
                 Optional: epoch/step.
               - create_landscape (dict): Select how to create loss landscape.
                 Training process loss landscape(train) and training result loss landscape(result).
-                Default: {"train": True, "result": True}. Optional: ``True`` / ``False`` .
+                Default: ``{"train": True, "result": True}``. Optional: ``True`` / ``False`` .
               - num_samples (int): The size of the dataset used to create the loss landscape.
                 For example, in image dataset, You can set num_samples is 128,
                 which means that 128 images are used to create loss landscape.
@@ -151,7 +150,7 @@ class SummaryCollector(Callback):
             False: it means that after specified data is set, only the specified data is collected,
             and the others are not collected. Default: ``True`` .
         custom_lineage_data (Union[dict, None]): Allows you to customize the data and present it on the MingInsight
-            `lineage page <https://www.mindspore.cn/mindinsight/docs/en/r2.2/lineage_and_scalars_comparison.html>`_ .
+            `lineage page <https://www.mindspore.cn/mindinsight/docs/en/master/lineage_and_scalars_comparison.html>`_ .
             In the custom data, the type of the key supports str, and the type of value supports str, int
             and float. Default: ``None`` , it means there is no custom data.
         collect_tensor_freq (Optional[int]): The same semantics as the `collect_freq`, but controls TensorSummary only.
@@ -191,10 +190,10 @@ class SummaryCollector(Callback):
         ...     ms.set_context(mode=ms.GRAPH_MODE, device_target="Ascend")
         ...     mnist_dataset_dir = '/path/to/mnist_dataset_directory'
         ...     # Create the dataset taking MNIST as an example. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        ...     # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         ...     ds_train = create_dataset()
         ...     # Define the network structure of LeNet5. Refer to
-        ...     # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        ...     # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         ...     network = LeNet5(10)
         ...     net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
         ...     net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9)
@@ -1157,7 +1156,7 @@ class SummaryCollector(Callback):
         except TypeError as exc:
             logger.warning("Summary cannot collect the type of metrics, currently support type: dict, list, tuple, "
                            "str, int, float, bool and None. %s.", str(exc))
-        _ = self._parse_dataset(cb_params, eval_lineage)
+        self._parse_dataset(cb_params, eval_lineage)
 
         eval_lineage_message = self._package_eval_lineage_message(eval_lineage)
         self._record.add_value(PluginEnum.EVAL_LINEAGE.value, 'eval_lineage', eval_lineage_message)

mindspore/train/callback/_time_monitor.py

@@ -43,13 +43,13 @@ class TimeMonitor(Callback):
         >>> from mindspore.train import Model, TimeMonitor
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
         >>> optim = nn.Momentum(net.trainable_params(), 0.01, 0.9)
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> time_monitor = TimeMonitor()
         >>> model.train(10, dataset, callbacks=time_monitor)

mindspore/train/data_sink.py

@@ -145,7 +145,7 @@ def data_sink(fn, dataset, sink_size=1, jit_config=None, input_signature=None):
         input_signature (Union[Tensor, List or Tuple of Tensors]): The Tensor which describes the input arguments.
             The shape and dtype of the Tensor will be supplied to this function. If input_signature is specified,
             each input to `fn` must be a `Tensor`. And the input parameters of `fn` cannot accept `**kwargs`. The shape
-            and dtype of actual inputs should keep the same as input_signature. Otherwise, TypeError will be raised.
+            and dtype of actual inputs should keep the same as `input_signature`. Otherwise, TypeError will be raised.
             Default: ``None`` .
 
     Returns:

mindspore/train/dataset_helper.py

@@ -27,7 +27,8 @@ from mindspore.dataset.engine import offload
 from mindspore import context, nn
 from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, _construct_tensor_list
 from mindspore.parallel._utils import _get_device_num, _get_global_rank, _need_to_full, \
-    _to_full_shapes, _get_pipeline_stages
+    _to_full_shapes, _get_pipeline_stages, _change_symbols_for_parallel, _is_in_auto_parallel_mode, \
+    _origin_shapes
 from mindspore.parallel._ps_context import _is_role_sched
 from mindspore.ops import operations as P
 from mindspore.common.auto_dynamic_shape import _auto_dynamic_shape
@@ -95,6 +96,11 @@ class _DataWrapper(nn.Cell):
         self.add_flags(**flags)
         self.get_next = P.GetNext(
             dataset_types, dataset_shapes, len(dataset_types), queue_name)
+        if network.get_inputs() is not None:
+            symbol_inputs = [getattr(inp, "symbolic_shape", None) for inp in network.get_inputs()]
+            symbol_inputs = _change_symbols_for_parallel(dataset_shapes, symbol_inputs)
+            if any((s is not None for s in symbol_inputs)):
+                self.get_next.add_prim_attr("symbols", symbol_inputs)
         self.network = network
         self._get_attr_from_cell(network)
 
@@ -129,7 +135,10 @@ def _generate_network_with_dataset(network, dataset_helper, queue_name):
         return _generate_dataset_sink_mode_net(network, new_shapes, dataset_types, queue_name)
 
     if network.get_inputs() and None not in network.get_inputs():
-        _check_inputs(network.get_inputs(), dataset_shapes, dataset_types)
+        if _is_in_auto_parallel_mode():
+            _check_inputs(network.get_inputs(), _origin_shapes(dataset_shapes), dataset_types)
+        else:
+            _check_inputs(network.get_inputs(), dataset_shapes, dataset_types)
     elif context.get_context("mode") == context.PYNATIVE_MODE:
         dataset_shapes = tuple([(-2,)] * len(dataset_shapes))
     network = _generate_dataset_sink_mode_net(
@@ -182,7 +191,7 @@ def _get_dataset_aux(dataset):
 def connect_network_with_dataset(network, dataset_helper):
     """
     Connect the `network` with dataset in `dataset_helper`. Only supported in `sink mode
-    <https://mindspore.cn/tutorials/experts/en/r2.2/optimize/execution_opt.html>`_, (dataset_sink_mode=True).
+    <https://mindspore.cn/tutorials/experts/en/r2.3.q1/optimize/execution_opt.html>`_, (dataset_sink_mode=True).
 
     Args:
         network (Cell): The training network for dataset.
@@ -282,7 +291,7 @@ class DatasetHelper:
 
     Args:
         dataset (Dataset): The dataset iterator. The dataset can be generated by dataset generator API in
-                           :class:`mindspore.dataset`, such as :class:`mindspore.dataset.ImageFolderDataset`.
+                           `mindspore.dataset` module, such as :class:`mindspore.dataset.ImageFolderDataset`.
         dataset_sink_mode (bool): If the value is True, GetNext is employed to fetch the data at device through the
                                   dataset pipeline, otherwise fetch the data at host by iterating through the dataset.
                                   Default: ``True``.
@@ -444,6 +453,11 @@ class DatasetHelper:
         # Generally, it works in dynamic shape scenarios.
         return self.iter.get_data_info()
 
+    # pylint: disable=missing-docstring
+    def get_mbuf_queue_size(self):
+        # In sink mode, it returns the element numbers inside mbuf channel.
+        return self.iter.get_mbuf_queue_size()
+
     # pylint: disable=missing-docstring
     def get_send_info(self, run_context):
         # In sink mode, it returns the send information of dataset at this moment.
@@ -526,6 +540,7 @@ class _DatasetIter:
         self.release = dataset.__transfer_dataset__.release
         self.continue_send = dataset.__transfer_dataset__.continue_send
         self.get_data_info = dataset.__transfer_dataset__.get_data_info
+        self.get_mbuf_queue_size = dataset.__transfer_dataset__.get_mbuf_queue_size
         self.get_send_info = dataset.__transfer_dataset__.get_send_info
         if hasattr(dataset.__transfer_dataset__, "_reset"):
             self._reset = dataset.__transfer_dataset__._reset  # pylint: disable=protected-access

mindspore/train/loss_scale_manager.py

@@ -62,7 +62,7 @@ class FixedLossScaleManager(LossScaleManager):
         >>> from mindspore import amp, nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_scale = 1024.0
         >>> loss_scale_manager = amp.FixedLossScaleManager(loss_scale, False)
@@ -136,7 +136,7 @@ class DynamicLossScaleManager(LossScaleManager):
         >>> from mindspore import amp, nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_scale_manager = amp.DynamicLossScaleManager()
         >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)

mindspore/train/metrics/accuracy.py

@@ -32,9 +32,9 @@ class Accuracy(EvaluationBase):
         {\text{true_positive} + \text{true_negative} + \text{false_positive} + \text{false_negative}}
 
     Args:
-        eval_type (str): The metric to calculate the accuracy over a dataset. Supports 'classification' and
-          'multilabel'. 'classification' means the dataset label is single. 'multilabel' means the dataset has multiple
-          labels. Default: ``'classification'`` .
+        eval_type (str): The metric to calculate the accuracy over a dataset. Supports ``'classification'`` and
+          ``'multilabel'``. ``'classification'`` means the dataset label is single.
+          ``'multilabel'`` means the dataset has multiple labels. Default: ``'classification'`` .
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -67,19 +67,19 @@ class Accuracy(EvaluationBase):
     @rearrange_inputs
     def update(self, *inputs):
         """
-        Updates the local variables. For 'classification', if the index of the maximum of the predict value
-        matches the label, the predict result is correct. For 'multilabel', the predict value match the label,
+        Updates the local variables. For ``'classification'``, if the index of the maximum of the predict value
+        matches the label, the predict result is correct. For ``'multilabel'``, the predict value match the label,
         the predict result is correct.
 
         Args:
             inputs: Logits and labels. `y_pred` stands for logits, `y` stands for labels. `y_pred` and `y` must be a
                 `Tensor`, a list or an array.
 
-                - For the 'classification' evaluation type, `y_pred` is a list of floating numbers in range
+                - For the ``'classification'`` evaluation type, `y_pred` is a list of floating numbers in range
                   :math:`[0, 1]` and the shape is :math:`(N, C)` in most cases (not strictly), where :math:`N`
                   is the number of cases and :math:`C` is the number of categories. `y` must be in one-hot format
                   that shape is :math:`(N, C)`, or can be transformed to one-hot format that shape is :math:`(N,)`.
-                - For 'multilabel' evaluation type, the value of `y_pred` and `y` can only be 0 or 1, indices with 1
+                - For ``'multilabel'`` evaluation type, the value of `y_pred` and `y` can only be 0 or 1, indices with 1
                   indicate the positive category. The shape of `y_pred` and `y` are both :math:`(N, C)`.
 
         Raises:

mindspore/train/metrics/confusion_matrix.py

@@ -33,10 +33,10 @@ class ConfusionMatrix(Metric):
         num_classes (int): Number of classes in the dataset.
         normalize (str): Normalization mode for confusion matrix. Default: ``"no_norm"`` . Choose from:
 
-            - **"no_norm"** (None) - No Normalization is used. Default: ``None``.
-            - **"target"** (str) - Normalization based on target value.
-            - **"prediction"** (str) - Normalization based on predicted value.
-            - **"all"** (str) - Normalization over the whole matrix.
+            - ``"no_norm"`` : No Normalization is used. Default: ``None``.
+            - ``"target"`` : Normalization based on target value.
+            - ``"prediction"`` : Normalization based on predicted value.
+            - ``"all"`` : Normalization over the whole matrix.
 
         threshold (float): The threshold used to compare with the input tensor. Default: ``0.5`` .
 
@@ -224,8 +224,10 @@ class ConfusionMatrixMetric(Metric):
 
                 - y_pred (ndarray): The batch data shape is :math:`(N, C, ...)`
                   or :math:`(N, ...)`, representing onehot format
-                  or category index format respectively. As for classification tasks, y_pred should have the shape [BN]
-                  where N is larger than 1. As for segmentation tasks, the shape should be [BNHW] or [BNHWD].
+                  or category index format respectively. As for classification tasks,
+                  y_pred should have the shape :math:`(B, N)`
+                  where N is larger than 1. As for segmentation tasks,
+                  the shape should be :math:`(B, N, H, W)` or :math:`(B, N, H, W, D)`.
                 - y (ndarray): It must be one-hot format. The batch data shape is :math:`(N, C, ...)`.
 
         Raises:

mindspore/train/metrics/cosine_similarity.py

@@ -26,9 +26,11 @@ class CosineSimilarity(Metric):
     Computes representation similarity.
 
     Args:
-        similarity (str): 'dot' or 'cosine'. Default: ``'cosine'`` .
-        reduction (str): ``'none'``, 'sum', ``'mean'`` (all along dim -1). Default: ``'none'`` .
-        zero_diagonal (bool): If True,  diagonals of results will be set to zero. Default: ``True`` .
+        similarity (str): the computation logit. ``'cosine'`` means computing similarity.
+            ``'dot'`` means computing dots of arrays, Default: ``'cosine'`` .
+        reduction (str): Specifies the reduction to be applied to the output. Support ``'none'``,
+            ``'sum'``, ``'mean'`` (all along dim -1). Default: ``'none'`` .
+        zero_diagonal (bool): If ``True``,  diagonals of results will be set to zero. Default: ``True`` .
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -71,7 +73,7 @@ class CosineSimilarity(Metric):
         Updates the internal evaluation result with 'inputs'.
 
         Args:
-            inputs (Union[Tensor, list, numpy.ndarray]): The input matrix.
+            inputs (Union[Tensor, list, numpy.ndarray]): The input matrix including `y` and `y_pred`.
         """
         input_data = self._convert_data(inputs)