mindspore 2.1.0__cp39-cp39-win_amd64.whl → 2.2.10__cp39-cp39-win_amd64.whl

mindspore/nn/optim/ada_grad.py

@@ -166,7 +166,7 @@ class Adagrad(Optimizer):
         >>> import mindspore.nn as nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Adagrad(params=net.trainable_params())
@@ -205,6 +205,7 @@ class Adagrad(Optimizer):
         grads = self.gradients_centralization(grads)
         grads = self.scale_grad(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.map_reverse(F.partial(_ada_grad_opt, self.opt), lr, params, accum,
                                        grads)

mindspore/nn/optim/adadelta.py

@@ -194,6 +194,7 @@ class Adadelta(Optimizer):
         grads = self.gradients_centralization(grads)
         grads = self.scale_grad(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.map_reverse(F.partial(_adadelta_opt, self.opt, self.rho, self.epsilon), lr, params,
                                        self.accum, self.accum_update, grads)

mindspore/nn/optim/adafactor.py

@@ -264,7 +264,7 @@ class AdaFactor(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) Parameters use the default learning rate with None and weight decay with 0.
         >>> optim = nn.AdaFactor(params=net.trainable_params())
@@ -410,6 +410,7 @@ class AdaFactor(Optimizer):
     def construct(self, gradients):
         gradients = self.flatten_gradients(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         step = F.assign_add(self.step, 1)
         if self.scale_lr and self.relative_step:
             if self.warmup_init:

mindspore/nn/optim/adam.py

@@ -719,7 +719,7 @@ class Adam(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Adam(params=net.trainable_params())
@@ -918,6 +918,7 @@ class Adam(Optimizer):
         gradients = self.scale_grad(gradients)
         gradients = self._grad_sparse_indices_deduplicate(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         beta1_power = self.beta1_power * self.beta1
         self.beta1_power = beta1_power
@@ -985,7 +986,7 @@ class AdamWeightDecay(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means, refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.1/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
@@ -1069,7 +1070,7 @@ class AdamWeightDecay(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdamWeightDecay(params=net.trainable_params())
@@ -1109,6 +1110,7 @@ class AdamWeightDecay(Optimizer):
         gradients = self.flatten_gradients(gradients)
         weight_decay = self.get_weight_decay()
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         if self.use_fused_opt:
             if self.is_group:
@@ -1282,7 +1284,7 @@ class AdamOffload(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdamOffload(params=net.trainable_params())
@@ -1330,6 +1332,7 @@ class AdamOffload(Optimizer):
         gradients = self.decay_weight(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         beta1_power = self.beta1_power * self.beta1
         self.beta1_power = beta1_power

mindspore/nn/optim/adamax.py

@@ -66,7 +66,7 @@ class AdaMax(Optimizer):
     :math:`m` represents the 1st moment vector, :math:`v` represents the 2nd moment vector,
     :math:`g` represents `gradients`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`,
     :math:`t` represents the current step, :math:`beta_1^t` represent `beta1_power`,
-    :math:`\l` represents `learning_rate`, :math:`w` represents `params`,
+    :math:`l` represents `learning_rate`, :math:`w` represents `params`,
     :math:`\epsilon` represents `eps`.
 
     Note:
@@ -161,7 +161,7 @@ class AdaMax(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdaMax(params=net.trainable_params())
@@ -204,6 +204,7 @@ class AdaMax(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         self.beta1_power *= self.beta1
 

mindspore/nn/optim/adasum.py

@@ -445,7 +445,7 @@ class AdaSumByGradWrapCell(Cell):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> optim = nn.AdaSumByGradWrapCell(nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9))
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
@@ -514,7 +514,7 @@ class AdaSumByDeltaWeightWrapCell(Cell):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> optim = nn.AdaSumByDeltaWeightWrapCell(nn.Momentum(params=net.trainable_params(),
         ...                                                 learning_rate=0.1, momentum=0.9))

mindspore/nn/optim/asgd.py

@@ -128,7 +128,7 @@ class ASGD(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.ASGD(params=net.trainable_params())
@@ -185,8 +185,7 @@ class ASGD(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lrs = self.get_lr()
-        if not self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         success = True
         params = self._parameters
         for index, (grad, param, mu, eta, ax) in enumerate(zip(gradients, params, self.mu, self.eta, self.ax)):

mindspore/nn/optim/ftrl.py

@@ -296,7 +296,7 @@ class FTRL(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.FTRL(params=net.trainable_params())
@@ -359,6 +359,7 @@ class FTRL(Optimizer):
         grads = self.scale_grad(grads)
         grads = self._grad_sparse_indices_deduplicate(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         if self.use_dist_optimizer:
             success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull,
@@ -379,12 +380,12 @@ class FTRL(Optimizer):
         optimizer operation.
         """
         if not isinstance(value, str):
-            raise TypeError("For 'FTRL', the property 'target' must be string type, "
-                            "but got type {}.".format(type(value)))
+            raise TypeError(f"For 'FTRL', the property 'target' must be string type, "
+                            f"but got type {type(value)}.")
 
         if value not in ('CPU', 'Ascend', 'GPU'):
-            raise ValueError("For 'FTRL', the property 'target' must be 'CPU', 'Ascend' or 'GPU', "
-                             "but got {}".format(value))
+            raise ValueError(f"For 'FTRL', the property 'target' must be 'CPU', 'Ascend' or 'GPU', "
+                             f"but got {value}.")
 
         if value == 'CPU':
             self.sparse_opt = P.FusedSparseFtrl(self.lr, self.l1, self.l2, self.lr_power, self.use_locking)

mindspore/nn/optim/lamb.py

@@ -132,7 +132,7 @@ class Lamb(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means. Refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.1/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
@@ -140,6 +140,10 @@ class Lamb(Optimizer):
         parameters are grouped, each group can set `weight_decay`. If not, the `weight_decay` in optimizer will be
         applied.
 
+    .. warning::
+        The update process of the Lamb optimizer is not completely elementwise, and the sharding of weights in
+        distributed parallel may affect the update result.
+
     Args:
         params (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
             `params` is a list of `dict`, the string "params", "lr", "weight_decay", "grad_centralization" and
@@ -220,7 +224,7 @@ class Lamb(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Lamb(params=net.trainable_params(), learning_rate=0.1)
@@ -263,8 +267,7 @@ class Lamb(Optimizer):
     def construct(self, gradients):
         weight_decay = self.get_weight_decay()
         lr = self.get_lr()
-        if not self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         lamb_opt = _lamb_opt
         gradients = self.flatten_gradients(gradients)
         gradients = self.gradients_centralization(gradients)

mindspore/nn/optim/lars.py

@@ -109,7 +109,7 @@ class LARS(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
         >>> opt = nn.Momentum(net.trainable_params(), 0.1, 0.9)

mindspore/nn/optim/lazyadam.py

@@ -321,7 +321,7 @@ class LazyAdam(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1e-3`` .
+        learning_rate (Union[float, int, Tensor, Iterable, :class:`~.train.LearningRateScheduler`]): Default: ``1e-3`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -370,7 +370,8 @@ class LazyAdam(Optimizer):
         Tensor[bool], the value is ``True`` .
 
     Raises:
-        TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
+        TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable,
+            :class:`~.train.LearningRateScheduler`.
         TypeError: If element of `parameters` is neither Parameter nor dict.
         TypeError: If `beta1`, `beta2`, `eps` or `loss_scale` is not a float.
         TypeError: If `weight_decay` is neither float nor int.
@@ -387,7 +388,7 @@ class LazyAdam(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.LazyAdam(params=net.trainable_params())
@@ -445,6 +446,7 @@ class LazyAdam(Optimizer):
         gradients = self.scale_grad(gradients)
         gradients = self._grad_sparse_indices_deduplicate(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         beta1_power = self.beta1_power * self.beta1
         self.beta1_power = beta1_power

mindspore/nn/optim/momentum.py

@@ -173,7 +173,7 @@ class Momentum(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
@@ -220,6 +220,7 @@ class Momentum(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.use_dist_optimizer:
             if self.is_group_lr:
                 success = self.hyper_map_reverse(F.partial(_momentum_opt, self.opt, self.momentum),

mindspore/nn/optim/optimizer.py

@@ -140,6 +140,57 @@ class Optimizer(Cell):
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import nn
+        >>> import numpy as np
+        >>> import mindspore
+        >>> from mindspore import nn, ops, Tensor
+        >>>
+        >>> class MyMomentum(nn.Optimizer):
+        ...     def __init__(self, params, learning_rate, momentum=0.9):
+        ...         super(MyMomentum, self).__init__(learning_rate, params)
+        ...         self.moments = self.parameters.clone(prefix="moments", init="zeros")
+        ...         self.momentum = momentum
+        ...         self.opt = ops.ApplyMomentum()
+        ...
+        ...     def construct(self, gradients):
+        ...         params = self.parameters
+        ...         lr = self.get_lr()
+        ...         gradients = self.flatten_gradients(gradients)
+        ...         gradients = self.decay_weight(gradients)
+        ...         gradients = self.gradients_centralization(gradients)
+        ...         gradients = self.scale_grad(gradients)
+        ...
+        ...         success = None
+        ...         for param, mom, grad in zip(params, self.moments, gradients):
+        ...             success = self.opt(param, mom, lr, grad, self.momentum)
+        ...         return success
+        >>>
+        >>> net = nn.Dense(2, 3)
+        >>> loss_fn = nn.MAELoss()
+        >>> opt = MyMomentum(net.trainable_params(), 0.01)
+        >>>
+        >>> device_target = opt.target
+        >>> opt_unique = opt.unique
+        >>> weight_decay_value = opt.get_weight_decay()
+        >>>
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>>
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, opt.parameters, has_aux=True)
+        >>>
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     opt(grads)
+        ...     return loss
+        >>>
+        >>> data = Tensor(np.random.rand(4, 10, 2), mindspore.dtype.float32)
+        >>> label = Tensor(np.random.rand(4, 10, 3), mindspore.dtype.float32)
+        >>> train_step(data, label)
     """
     _support_parallel_optimizer = False
 
@@ -233,7 +284,7 @@ class Optimizer(Cell):
         self.cache_enable = tuple(cache_filter(x) for x in self._parameters)
         self.reciprocal_scale = Tensor(1.0 / self.loss_scale, mstype.float32)
         self.need_scale = self.loss_scale != 1.0
-        self.global_step_increase_tensor = Tensor(1, mstype.int32)
+        self.global_step_increase_tensor = Tensor([1], mstype.int32)
         self.param_length = len(self._parameters)
         self.map_ = C.Map()
         self.map_reverse = C.Map(None, True)
@@ -702,8 +753,6 @@ class Optimizer(Cell):
                     lr += (current_dynamic_lr,)
             else:
                 lr = self.learning_rate(self.global_step).reshape(())
-        if self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
         return lr
 
     def get_lr_parameter(self, param):
@@ -722,7 +771,7 @@ class Optimizer(Cell):
         Examples:
             >>> from mindspore import nn
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params()))
             >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params()))

mindspore/nn/optim/proximal_ada_grad.py

@@ -55,9 +55,7 @@ def _check_param_value(accum, l1, l2, use_locking, prim_name=None):
 
 class ProximalAdagrad(Optimizer):
     r"""
-    Implements the ProximalAdagrad algorithm.
-
-    ProximalAdagrad is an online Learning and Stochastic Optimization.
+    Implements the ProximalAdagrad algorithm that is an online Learning and Stochastic Optimization.
     Refer to paper `Efficient Learning using Forward-Backward Splitting
     <http://papers.nips.cc//paper/3793-efficient-learning-using-forward-backward-splitting.pdf>`_.
 
@@ -165,7 +163,7 @@ class ProximalAdagrad(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.ProximalAdagrad(params=net.trainable_params())
@@ -209,6 +207,7 @@ class ProximalAdagrad(Optimizer):
         grads = self.scale_grad(grads)
         grads = self._grad_sparse_indices_deduplicate(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.map_reverse(F.partial(_proximal_ada_grad_opt, self.opt, self.sparse_opt, self.l1, self.l2),
                                        lr, grads, params, accum)

mindspore/nn/optim/rmsprop.py

@@ -47,8 +47,8 @@ class RMSProp(Optimizer):
     Implements Root Mean Squared Propagation (RMSProp) algorithm.
 
     Update `params` according to the RMSProp algorithm.
-    The 29th of the original presentation slide
-    [http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf] proposes RMSProp.
+    The 29th of the original `presentation slide
+    <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_ proposes RMSProp.
     The equation is as follows:
 
     .. math::
@@ -180,7 +180,7 @@ class RMSProp(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.RMSProp(params=net.trainable_params(), learning_rate=0.1)
@@ -236,6 +236,7 @@ class RMSProp(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.centered:
             if self.is_group_lr:
                 success = self.hyper_map_reverse(F.partial(_centered_rmsprop_opt, self.opt, self.decay, self.epsilon,

mindspore/nn/optim/rprop.py

@@ -135,7 +135,7 @@ class Rprop(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Rprop(params=net.trainable_params())
@@ -189,8 +189,8 @@ class Rprop(Optimizer):
         self.prev = self._parameters.clone(prefix="prev", init='zeros')
         self.step_size = self._parameters.clone(prefix="step_size", init='zeros')
 
-        self.fill = P.Fill()
         self.sign = P.Sign()
+        self.fill = P.FillV2()
         self.assign = P.Assign()
         self.assignadd = P.AssignAdd()
         self.cast = P.Cast()
@@ -204,8 +204,7 @@ class Rprop(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lrs = self.get_lr()
-        if not self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         success = True
 
         for index, (grad, param, prev, step_size) in enumerate(zip(gradients, self._parameters,
@@ -221,14 +220,26 @@ class Rprop(Optimizer):
             param_fp32 = self.cast(param, mstype.float32)
 
             sign = self.sign(gradient_fp32 * prev)
-            sign = self.select(sign > 0, self.fill(mstype.float32, sign.shape, self.etaplus), sign)
-            sign = self.select(sign < 0, self.fill(mstype.float32, sign.shape, self.etaminus), sign)
-            sign = self.select(sign == 0, self.fill(mstype.float32, sign.shape, 1.), sign)
-
-            step_size_fp32 = ops.clip_by_value(step_size_fp32 * sign, self.step_size_min, self.step_size_max)
-
-            gradient_update = self.select(sign == self.etaminus, self.fill(mstype.float32, sign.shape, 0.),
-                                          gradient_fp32)
+            sign = self.select(
+                sign > 0,
+                self.fill(sign.shape, self.cast(self.etaplus, mstype.float32)),
+                sign)
+            sign = self.select(
+                sign < 0,
+                self.fill(sign.shape, self.cast(self.etaminus,
+                                                mstype.float32)), sign)
+            sign = self.select(
+                sign == 0, self.fill(sign.shape,
+                                     self.cast(1., mstype.float32)), sign)
+
+            step_size_fp32 = ops.clip_by_value(step_size_fp32 * sign,
+                                               self.step_size_min,
+                                               self.step_size_max)
+
+            gradient_update = self.select(
+                sign == self.etaminus,
+                self.fill(sign.shape, self.cast(0., mstype.float32)),
+                gradient_fp32)
             next_param = param_fp32 - self.sign(gradient_update) * step_size_fp32
 
             self.assign(param, self.cast(next_param, param.dtype))

mindspore/nn/optim/sgd.py

@@ -132,7 +132,7 @@ class SGD(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.1/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.SGD(params=net.trainable_params())
@@ -163,29 +163,29 @@ class SGD(Optimizer):
         if isinstance(momentum, int):
             momentum = float(momentum)
         if not isinstance(momentum, float):
-            raise TypeError("For 'SGD', the argument 'momentum' must be float type, "
-                            "but got {}.".format(type(momentum)))
+            raise TypeError(f"For 'SGD', the argument 'momentum' must be float type, "
+                            f"but got {type(momentum)}.")
 
         if isinstance(momentum, float) and momentum < 0.0:
-            raise ValueError("For 'SGD', the argument 'momentum' must be at least 0.0, "
-                             "but got {}.".format(momentum))
+            raise ValueError(f"For 'SGD', the argument 'momentum' must be at least 0.0, "
+                             f"but got {momentum}.")
 
         if isinstance(dampening, int):
             dampening = float(dampening)
         if not isinstance(dampening, float):
-            raise TypeError("For 'SGD', the argument 'dampening' must be float type, "
-                            "but got {}.".format(type(dampening)))
+            raise TypeError(f"For 'SGD', the argument 'dampening' must be float type, "
+                            f"but got {type(dampening)}.")
 
         if dampening < 0.0:
-            raise ValueError("For 'SGD', the argument 'dampening' must be at least 0.0, "
-                             "but got 'dampening' {}".format(dampening))
+            raise ValueError(f"For 'SGD', the argument 'dampening' must be at least 0.0, "
+                             f"but got 'dampening' {dampening}")
         self.dampening = dampening
 
         validator.check_value_type("nesterov", nesterov, [bool], self.cls_name)
 
         if nesterov and (momentum <= 0.0 or dampening != 0.0):
-            raise ValueError("For 'SGD', if 'nesterov' is true, 'momentum' must be > 0.0 and 'dampening' must "
-                             "equal to 0.0, but got 'momentum' {}, 'dampening' {}".format(momentum, dampening))
+            raise ValueError(f"For 'SGD', if 'nesterov' is true, 'momentum' must be > 0.0 and 'dampening' must "
+                             f"equal to 0.0, but got 'momentum' {momentum}, 'dampening' {dampening}.")
         self.nesterov = nesterov
 
         if self.dynamic_weight_decay:
@@ -198,9 +198,23 @@ class SGD(Optimizer):
             self.opt = tuple([P.SGD(dampening, float(weight_decay), nesterov)] * len(self._parameters))
 
         self.momentum = Parameter(Tensor(momentum, mstype.float32), name="momentum")
+
+        if not momentum > 0.0:
+            enable_cache_param_list = []
+            for param in self._parameters:
+                if param.cache_enable:
+                    enable_cache_param_list.append(param)
+                    param.cache_enable = False
+
         self.accum = self._parameters.clone(prefix="accum", init='zeros')
         self.stat = self._parameters.clone(prefix="stat", init='ones')
 
+
+        if not momentum > 0.0:
+            for param in enable_cache_param_list:
+                param.cache_enable = True
+
+
     @jit
     def construct(self, gradients):
         params = self._parameters
@@ -210,6 +224,7 @@ class SGD(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.hyper_map_reverse(F.partial(_sgd_opt, self.momentum),
                                              lr, gradients, params, accum, stat, self.opt)