mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/nn/optim/ftrl.py

@@ -191,8 +191,7 @@ class FTRL(Optimizer):
 
     FTRL is an online convex optimization algorithm that adaptively chooses its regularization function
     based on the loss functions. Refer to paper `Adaptive Bound Optimization for Online Convex Optimization
-    <https://arxiv.org/abs/1002.4908>`_. Refer to paper `Ad Click Prediction: a View from the Trenches
-    <https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf>`_ for engineering document.
+    <https://arxiv.org/abs/1002.4908>`_.
 
     The updating formulas are as follows,
 
@@ -250,20 +249,21 @@ class FTRL(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        initial_accum (float): The starting value for accumulators `m`, must be zero or positive values. Default: 0.1.
+        initial_accum (float): The starting value for accumulators `m`, must be zero or positive values.
+            Default: ``0.1`` .
         learning_rate (float): The learning rate value, must be zero or positive, dynamic learning rate is currently
-            not supported. Default: 0.001.
+            not supported. Default: ``0.001`` .
         lr_power (float): Learning rate power controls how the learning rate decreases during training, must be less
-            than or equal to zero. Use fixed learning rate if lr_power is zero. Default: -0.5.
-        l1 (float): l1 regularization strength, must be greater than or equal to zero. Default: 0.0.
-        l2 (float): l2 regularization strength, must be greater than or equal to zero. Default: 0.0.
-        use_locking (bool): If true, use locks for updating operation. Default: False.
+            than or equal to zero. Use fixed learning rate if lr_power is zero. Default: ``-0.5`` .
+        l1 (float): l1 regularization strength, must be greater than or equal to zero. Default: ``0.0`` .
+        l2 (float): l2 regularization strength, must be greater than or equal to zero. Default: ``0.0`` .
+        use_locking (bool): If true, use locks for updating operation. Default: ``False`` .
         loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+            Default: ``1.0`` .
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -295,7 +295,9 @@ class FTRL(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.FTRL(params=net.trainable_params())
         >>>
@@ -313,7 +315,7 @@ class FTRL(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
 
     @opt_init_args_register
@@ -357,6 +359,7 @@ class FTRL(Optimizer):
         grads = self.scale_grad(grads)
         grads = self._grad_sparse_indices_deduplicate(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         if self.use_dist_optimizer:
             success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull,
@@ -377,12 +380,12 @@ class FTRL(Optimizer):
         optimizer operation.
         """
         if not isinstance(value, str):
-            raise TypeError("For 'FTRL', the property 'target' must be string type, "
-                            "but got type {}.".format(type(value)))
+            raise TypeError(f"For 'FTRL', the property 'target' must be string type, "
+                            f"but got type {type(value)}.")
 
         if value not in ('CPU', 'Ascend', 'GPU'):
-            raise ValueError("For 'FTRL', the property 'target' must be 'CPU', 'Ascend' or 'GPU', "
-                             "but got {}".format(value))
+            raise ValueError(f"For 'FTRL', the property 'target' must be 'CPU', 'Ascend' or 'GPU', "
+                             f"but got {value}.")
 
         if value == 'CPU':
             self.sparse_opt = P.FusedSparseFtrl(self.lr, self.l1, self.l2, self.lr_power, self.use_locking)

mindspore/nn/optim/lamb.py

@@ -132,7 +132,7 @@ class Lamb(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means. Refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.0/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
@@ -140,6 +140,10 @@ class Lamb(Optimizer):
         parameters are grouped, each group can set `weight_decay`. If not, the `weight_decay` in optimizer will be
         applied.
 
+    .. warning::
+        The update process of the Lamb optimizer is not completely elementwise, and the sharding of weights in
+        distributed parallel may affect the update result.
+
     Args:
         params (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
             `params` is a list of `dict`, the string "params", "lr", "weight_decay", "grad_centralization" and
@@ -181,14 +185,14 @@ class Lamb(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
-        beta1 (float): The exponential decay rate for the 1st moment estimations. Default: 0.9.
+        beta1 (float): The exponential decay rate for the 1st moment estimations. Default: ``0.9`` .
             Should be in range (0.0, 1.0).
-        beta2 (float): The exponential decay rate for the 2nd moment estimations. Default: 0.999.
+        beta2 (float): The exponential decay rate for the 2nd moment estimations. Default: ``0.999`` .
             Should be in range (0.0, 1.0).
-        eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6.
+        eps (float): Term added to the denominator to improve numerical stability. Default: ``1e-6`` .
             Should be greater than 0.
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -201,7 +205,7 @@ class Lamb(Optimizer):
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
 
     Outputs:
-        tuple[bool], all elements are True.
+        tuple[bool], all elements are ``True`` .
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -218,14 +222,15 @@ class Lamb(Optimizer):
     Examples:
         >>> import mindspore as ms
         >>> from mindspore import nn
-        >>> from mindspore.nn import learning_rate_schedule
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Lamb(params=net.trainable_params(), learning_rate=0.1)
         >>>
         >>> #2) Use parameter groups and set different values
-        >>> poly_decay_lr = learning_rate_schedule.PolynomialDecayLR(learning_rate=0.1, end_learning_rate=0.01,
+        >>> poly_decay_lr = nn.PolynomialDecayLR(learning_rate=0.1, end_learning_rate=0.01,
         ...                                                    decay_steps=4, power = 0.5)
         >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params()))
         >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params()))
@@ -240,7 +245,7 @@ class Lamb(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
     _support_parallel_optimizer = True
 
@@ -262,8 +267,7 @@ class Lamb(Optimizer):
     def construct(self, gradients):
         weight_decay = self.get_weight_decay()
         lr = self.get_lr()
-        if not self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         lamb_opt = _lamb_opt
         gradients = self.flatten_gradients(gradients)
         gradients = self.gradients_centralization(gradients)

mindspore/nn/optim/lars.py

@@ -87,10 +87,10 @@ class LARS(Optimizer):
     :math:`\gamma` represents `learning_rate` in `optimizer`, :math:`\eta` represents `coefficient`.
 
     Args:
-        optimizer (Optimizer): MindSpore optimizer for which to wrap and modify gradients.
-        epsilon (float): Term added to the denominator to improve numerical stability. Default: 1e-05.
-        coefficient (float): Trust coefficient for calculating the local learning rate. Default: 0.001.
-        use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: False.
+        optimizer (:class:`mindspore.nn.Optimizer`): MindSpore optimizer for which to wrap and modify gradients.
+        epsilon (float): Term added to the denominator to improve numerical stability. Default: ``1e-05`` .
+        coefficient (float): Trust coefficient for calculating the local learning rate. Default: ``0.001`` .
+        use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: ``False`` .
         lars_filter (Function): A function to determine which of the network parameters to use LARS algorithm. Default:
                                 lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name.
 
@@ -108,11 +108,13 @@ class LARS(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
         >>> opt = nn.Momentum(net.trainable_params(), 0.1, 0.9)
         >>> opt_lars = nn.LARS(opt, epsilon=1e-08, coefficient=0.02)
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=opt_lars, metrics=None)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=opt_lars, metrics=None)
     """
 
     @opt_init_args_register

mindspore/nn/optim/lazyadam.py

@@ -270,14 +270,15 @@ class LazyAdam(Optimizer):
         \begin{array}{ll} \\
             m_{t+1} = \beta_1 * m_{t} + (1 - \beta_1) * g \\
             v_{t+1} = \beta_2 * v_{t} + (1 - \beta_2) * g * g \\
-            l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\
-            w_{t+1} = w_{t} - l * \frac{m_{t+1}}{\sqrt{v_{t+1}} + \epsilon}
+            \widehat{m_{t+1}} = \frac{m_{t+1}}{1-\beta_1^t} \\
+            \widehat{v_{t+1}} = \frac{v_{t+1}}{1-\beta_2^t} \\
+            w_{t+1} = w_{t} - \gamma * \frac{\widehat{m_{t+1}}}{\sqrt{\widehat{v_{t+1}}} + \epsilon}
         \end{array}
 
     :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`,
-    :math:`g` represents `gradients`, :math:`l` represents scaling factor, :math:`\beta_1, \beta_2` represent
+    :math:`g` represents `gradients`, :math:`\gamma` represents `learning_rate`, :math:`\beta_1, \beta_2` represent
     `beta1` and `beta2`, :math:`t` represents the current step while :math:`beta_1^t` and :math:`beta_2^t` represent
-    `beta1_power` and `beta2_power`, :math:`\alpha` represents `learning_rate`, :math:`w` represents `params`,
+    `beta1_power` and `beta2_power`, :math:`w` represents `params`,
     :math:`\epsilon` represents `eps`.
 
     Note:
@@ -320,7 +321,7 @@ class LazyAdam(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
+        learning_rate (Union[float, int, Tensor, Iterable, :class:`~.train.LearningRateScheduler`]): Default: ``1e-3`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -335,19 +336,19 @@ class LazyAdam(Optimizer):
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.9.
+                       Default: ``0.9`` .
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.999.
-        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. Default:
-                     1e-8.
+                       Default: ``0.999`` .
+        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0.
+                     Default: ``1e-8`` .
         use_locking (bool): Whether to enable a lock to protect the updating process of variable tensors.
-            If true, updates of the `w`, `m`, and `v` tensors will be protected by a lock.
-            If false, the result is unpredictable. Default: False.
+            If ``true`` , updates of the `w`, `m`, and `v` tensors will be protected by a lock.
+            If ``false`` , the result is unpredictable. Default: ``False`` .
         use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients.
-            If true, update the gradients using NAG.
-            If false, update the gradients without using NAG. Default: False.
+            If ``true`` , update the gradients using NAG.
+            If ``false`` , update the gradients without using NAG. Default: ``False`` .
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -358,18 +359,19 @@ class LazyAdam(Optimizer):
 
         loss_scale (float): A floating point value for the loss scale. Should be equal to or greater than 1. In general,
             use the default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update`
-            in `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            in `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
+            Default: ``1.0`` .
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
 
     Outputs:
-        Tensor[bool], the value is True.
+        Tensor[bool], the value is ``True`` .
 
     Raises:
-        TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
+        TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable,
+            :class:`~.train.LearningRateScheduler`.
         TypeError: If element of `parameters` is neither Parameter nor dict.
         TypeError: If `beta1`, `beta2`, `eps` or `loss_scale` is not a float.
         TypeError: If `weight_decay` is neither float nor int.
@@ -385,7 +387,9 @@ class LazyAdam(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.LazyAdam(params=net.trainable_params())
         >>>
@@ -403,7 +407,7 @@ class LazyAdam(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
 
     @deprecated("2.0", "Adam", False)
@@ -442,6 +446,7 @@ class LazyAdam(Optimizer):
         gradients = self.scale_grad(gradients)
         gradients = self._grad_sparse_indices_deduplicate(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         beta1_power = self.beta1_power * self.beta1
         self.beta1_power = beta1_power

mindspore/nn/optim/momentum.py

@@ -134,7 +134,7 @@ class Momentum(Optimizer):
         momentum (float): Hyperparameter of type float, means momentum for the moving average.
             It must be at least 0.0.
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -145,16 +145,16 @@ class Momentum(Optimizer):
 
         loss_scale (float): A floating point value for the loss scale. It must be greater than 0.0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
-        use_nesterov (bool): Enable Nesterov momentum. Default: False.
+            Default: ``1.0`` .
+        use_nesterov (bool): Enable Nesterov momentum. Default: ``False`` .
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
 
     Outputs:
-        tuple[bool]. All elements are True.
+        tuple[bool]. All elements are ``True`` .
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -172,7 +172,9 @@ class Momentum(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
         >>>
@@ -190,7 +192,7 @@ class Momentum(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim, metrics=None)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim, metrics=None)
     """
     @opt_init_args_register
     def __init__(self, params, learning_rate, momentum, weight_decay=0.0, loss_scale=1.0, use_nesterov=False):
@@ -218,6 +220,7 @@ class Momentum(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.use_dist_optimizer:
             if self.is_group_lr:
                 success = self.hyper_map_reverse(F.partial(_momentum_opt, self.opt, self.momentum),

mindspore/nn/optim/optimizer.py

@@ -121,13 +121,13 @@ class Optimizer(Cell):
 
         weight_decay (Union[float, int]): An int or a floating point value for the weight decay.
             It must be equal to or greater than 0.
-            If the type of `weight_decay` input is int, it will be converted to float. Default: 0.0.
+            If the type of `weight_decay` input is int, it will be converted to float. Default: ``0.0`` .
         loss_scale (float): A floating point value for the loss scale. It must be greater than 0. If the
             type of `loss_scale` input is int, it will be converted to float. In general, use the default value. Only
             when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
+            Default: ``1.0`` .
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -140,6 +140,57 @@ class Optimizer(Cell):
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import nn
+        >>> import numpy as np
+        >>> import mindspore
+        >>> from mindspore import nn, ops, Tensor
+        >>>
+        >>> class MyMomentum(nn.Optimizer):
+        ...     def __init__(self, params, learning_rate, momentum=0.9):
+        ...         super(MyMomentum, self).__init__(learning_rate, params)
+        ...         self.moments = self.parameters.clone(prefix="moments", init="zeros")
+        ...         self.momentum = momentum
+        ...         self.opt = ops.ApplyMomentum()
+        ...
+        ...     def construct(self, gradients):
+        ...         params = self.parameters
+        ...         lr = self.get_lr()
+        ...         gradients = self.flatten_gradients(gradients)
+        ...         gradients = self.decay_weight(gradients)
+        ...         gradients = self.gradients_centralization(gradients)
+        ...         gradients = self.scale_grad(gradients)
+        ...
+        ...         success = None
+        ...         for param, mom, grad in zip(params, self.moments, gradients):
+        ...             success = self.opt(param, mom, lr, grad, self.momentum)
+        ...         return success
+        >>>
+        >>> net = nn.Dense(2, 3)
+        >>> loss_fn = nn.MAELoss()
+        >>> opt = MyMomentum(net.trainable_params(), 0.01)
+        >>>
+        >>> device_target = opt.target
+        >>> opt_unique = opt.unique
+        >>> weight_decay_value = opt.get_weight_decay()
+        >>>
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>>
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, opt.parameters, has_aux=True)
+        >>>
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     opt(grads)
+        ...     return loss
+        >>>
+        >>> data = Tensor(np.random.rand(4, 10, 2), mindspore.dtype.float32)
+        >>> label = Tensor(np.random.rand(4, 10, 3), mindspore.dtype.float32)
+        >>> train_step(data, label)
     """
     _support_parallel_optimizer = False
 
@@ -148,6 +199,8 @@ class Optimizer(Cell):
         parameters = self._parameters_base_check(parameters, "parameters")
         self.param_rank = None
         self.optim_filter = None
+        if not isinstance(parameters, list):
+            raise TypeError(f"For 'Optimizer' argument 'parameters' must be 'list', but got {type(parameters)}.")
         if not all(isinstance(x, Parameter) for x in parameters) and not all(isinstance(x, dict) for x in parameters):
             raise TypeError("For 'Optimizer', all elements of the argument 'parameters' must be 'Parameter' or 'dict',"
                             " please check the 'parameters'.")
@@ -231,7 +284,7 @@ class Optimizer(Cell):
         self.cache_enable = tuple(cache_filter(x) for x in self._parameters)
         self.reciprocal_scale = Tensor(1.0 / self.loss_scale, mstype.float32)
         self.need_scale = self.loss_scale != 1.0
-        self.global_step_increase_tensor = Tensor(1, mstype.int32)
+        self.global_step_increase_tensor = Tensor([1], mstype.int32)
         self.param_length = len(self._parameters)
         self.map_ = C.Map()
         self.map_reverse = C.Map(None, True)
@@ -700,8 +753,6 @@ class Optimizer(Cell):
                     lr += (current_dynamic_lr,)
             else:
                 lr = self.learning_rate(self.global_step).reshape(())
-        if self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
         return lr
 
     def get_lr_parameter(self, param):
@@ -719,8 +770,9 @@ class Optimizer(Cell):
 
         Examples:
             >>> from mindspore import nn
-            >>> # net = LeNet5()
-            >>> net = Net()
+            >>> # Define the network structure of LeNet5. Refer to
+            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+            >>> net = LeNet5()
             >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params()))
             >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params()))
             >>> group_params = [{'params': conv_params, 'lr': 0.05},
@@ -922,7 +974,7 @@ def rowtensor_deduplicate_indices_slices(grad):
     values = grad.values
 
     unique_indices, index_position = P.Unique()(indices)
-    summed_values = P.UnsortedSegmentSum()(values, index_position, P.TensorShape()(unique_indices)[0])
+    summed_values = P.UnsortedSegmentSum()(values, index_position, P.Shape()(unique_indices)[0])
 
     return RowTensorInner(unique_indices, summed_values, grad.dense_shape)
 

mindspore/nn/optim/proximal_ada_grad.py

@@ -55,9 +55,7 @@ def _check_param_value(accum, l1, l2, use_locking, prim_name=None):
 
 class ProximalAdagrad(Optimizer):
     r"""
-    Implements the ProximalAdagrad algorithm.
-
-    ProximalAdagrad is an online Learning and Stochastic Optimization.
+    Implements the ProximalAdagrad algorithm that is an online Learning and Stochastic Optimization.
     Refer to paper `Efficient Learning using Forward-Backward Splitting
     <http://papers.nips.cc//paper/3793-efficient-learning-using-forward-backward-splitting.pdf>`_.
 
@@ -110,8 +108,8 @@ class ProximalAdagrad(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        accum (float): The starting value for accumulators `accum`, must be zero or positive values. Default: 0.1.
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 0.001.
+        accum (float): The starting value for accumulators `accum`, must be zero or positive values. Default: ``0.1`` .
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``0.001`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -125,15 +123,15 @@ class ProximalAdagrad(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of the current step.
 
-        l1 (float): l1 regularization strength, must be greater than or equal to zero. Default: 0.0.
-        l2 (float): l2 regularization strength, must be greater than or equal to zero. Default: 0.0.
-        use_locking (bool): If true, use locks for updating operation. Default: False.
+        l1 (float): l1 regularization strength, must be greater than or equal to zero. Default: ``0.0`` .
+        l2 (float): l2 regularization strength, must be greater than or equal to zero. Default: ``0.0`` .
+        use_locking (bool): If true, use locks for updating operation. Default: ``False`` .
         loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+            Default: ``1.0`` .
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -164,7 +162,9 @@ class ProximalAdagrad(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.ProximalAdagrad(params=net.trainable_params())
         >>>
@@ -182,7 +182,7 @@ class ProximalAdagrad(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
 
     @opt_init_args_register
@@ -207,6 +207,7 @@ class ProximalAdagrad(Optimizer):
         grads = self.scale_grad(grads)
         grads = self._grad_sparse_indices_deduplicate(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.map_reverse(F.partial(_proximal_ada_grad_opt, self.opt, self.sparse_opt, self.l1, self.l2),
                                        lr, grads, params, accum)

mindspore/nn/optim/rmsprop.py

@@ -47,8 +47,8 @@ class RMSProp(Optimizer):
     Implements Root Mean Squared Propagation (RMSProp) algorithm.
 
     Update `params` according to the RMSProp algorithm.
-    The 29th of the original presentation slide
-    [http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf] proposes RMSProp.
+    The 29th of the original `presentation slide
+    <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_ proposes RMSProp.
     The equation is as follows:
 
     .. math::
@@ -120,7 +120,7 @@ class RMSProp(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 0.1.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``0.1`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -134,20 +134,21 @@ class RMSProp(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of the current step.
 
-        decay (float): Decay rate. Should be equal to or greater than 0. Default: 0.9.
+        decay (float): Decay rate. Should be equal to or greater than 0. Default: ``0.9`` .
         momentum (float): Hyperparameter of type float, means momentum for the moving average. Should be equal to or
-                          greater than 0. Default: 0.0.
+                          greater than 0. Default: ``0.0`` .
         epsilon (float): Term added to the denominator to improve numerical stability. Should be greater than
-                         0. Default: 1e-10.
+                         0. Default: ``1e-10`` .
         use_locking (bool):  Whether to enable a lock to protect the updating process of variable tensors.
-            Default: False.
-        centered (bool): If True, gradients are normalized by the estimated variance of the gradient. Default: False.
+            Default: ``False`` .
+        centered (bool): If True, gradients are normalized by the estimated variance of the gradient.
+            Default: ``False`` .
         loss_scale (float): A floating point value for the loss scale. Should be greater than 0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+            Default: ``1.0`` .
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -178,7 +179,9 @@ class RMSProp(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.RMSProp(params=net.trainable_params(), learning_rate=0.1)
         >>>
@@ -196,7 +199,7 @@ class RMSProp(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
 
     @opt_init_args_register
@@ -233,6 +236,7 @@ class RMSProp(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.centered:
             if self.is_group_lr:
                 success = self.hyper_map_reverse(F.partial(_centered_rmsprop_opt, self.opt, self.decay, self.epsilon,