mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/nn/optim/ada_grad.py

@@ -112,8 +112,8 @@ class Adagrad(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        accum (float): The starting value for :math:`h`, must be zero or positive values. Default: 0.1.
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 0.001.
+        accum (float): The starting value for :math:`h`, must be zero or positive values. Default: ``0.1`` .
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``0.001`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -127,13 +127,13 @@ class Adagrad(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
-        update_slots (bool): Whether the :math:`h` will be updated. Default: True.
+        update_slots (bool): Whether the :math:`h` will be updated. Default: ``True`` .
         loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
             `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+            Default: ``1.0`` .
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -147,7 +147,7 @@ class Adagrad(Optimizer):
           in optimizer.
 
     Outputs:
-        Tensor[bool], the value is True.
+        Tensor[bool], the value is ``True`` .
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -162,10 +162,12 @@ class Adagrad(Optimizer):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore as ms
-        >>> from mindspore import nn
+        >>> import mindspore
+        >>> import mindspore.nn as nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Adagrad(params=net.trainable_params())
         >>>
@@ -203,6 +205,7 @@ class Adagrad(Optimizer):
         grads = self.gradients_centralization(grads)
         grads = self.scale_grad(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.map_reverse(F.partial(_ada_grad_opt, self.opt), lr, params, accum,
                                        grads)

mindspore/nn/optim/adadelta.py

@@ -84,8 +84,8 @@ class Adadelta(Optimizer):
               to get the weight decay value of current step.
 
             - grad_centralization: Optional. Must be Boolean. If "grad_centralization" is in the keys, the set value
-              will be used. If not, the `grad_centralization` is False by default. This configuration only works on the
-              convolution layer.
+              will be used. If not, the `grad_centralization` is ``False``  by default. This configuration only works
+              on the convolution layer.
 
             - order_params: Optional. When parameters is grouped, this usually is used to maintain the order of
               parameters that appeared in the network to improve performance. The value should be parameters whose
@@ -93,7 +93,7 @@ class Adadelta(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1.0.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1.0`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -107,14 +107,14 @@ class Adadelta(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
-        rho (float): Decay rate, must be in range [0.0, 1.0]. Default: 0.9.
-        epsilon (float):  A small value added for numerical stability, must be non-negative. Default: 1e-6.
+        rho (float): Decay rate, must be in range [0.0, 1.0]. Default: ``0.9`` .
+        epsilon (float):  A small value added for numerical stability, must be non-negative. Default: ``1e-6`` .
         loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+            Default: ``1.0`` .
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -128,7 +128,7 @@ class Adadelta(Optimizer):
           the `params` in optimizer. With float16 or float32 data type.
 
     Outputs:
-        Tensor[bool], the value is True.
+        Tensor[bool], the value is ``True`` .
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -143,7 +143,9 @@ class Adadelta(Optimizer):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> from mindspore import nn, Model
+        >>> import mindspore as ms
+        >>> import mindspore.nn as nn
+        >>>
         >>> class Net(nn.Cell):
         ...    def __init__(self):
         ...        super(Net, self).__init__()
@@ -171,7 +173,7 @@ class Adadelta(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
 
     @opt_init_args_register
@@ -192,6 +194,7 @@ class Adadelta(Optimizer):
         grads = self.gradients_centralization(grads)
         grads = self.scale_grad(grads)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         if self.is_group_lr:
             success = self.map_reverse(F.partial(_adadelta_opt, self.opt, self.rho, self.epsilon), lr, params,
                                        self.accum, self.accum_update, grads)

mindspore/nn/optim/adafactor.py

@@ -216,27 +216,29 @@ class AdaFactor(Optimizer):
 
         learning_rate (Union[float, Tensor]): A value or a graph for the learning rate.
             When the learning_rate is a Tensor in a 1D dimension.
-            If the type of `learning_rate` is int, it will be converted to float. Default: None.
+            If the type of `learning_rate` is int, it will be converted to float. Default: ``None`` .
         eps (tuple): The regularization constans for square gradient and parameter scale respectively.
-            default: (1e-30, 1e-3)
-        clip_threshold (Union[float, Tensor]): The threshold of root mean square of final gradient update. default: 1.0
+            default: ``(1e-30, 1e-3)`` .
+        clip_threshold (Union[float, Tensor]): The threshold of root mean square of final gradient update.
+            default: ``1.0``.
         decay_rate (Union[float, Tensor]): The coefficient used to compute running averages of square gradient.
-            default: 0.8
+            default: ``0.8`` .
         beta1 (float): The coefficient to computing running averages of gradient. Should be in range (0.0, 1.0).
-               Default: None.
-        weight_decay (float): Weight decay (L2 penalty). It must be equal to or greater than 0. Default: 0.0.
-        scale_parameter (bool): If True, learning rate is scaled by root mean square of parameter. default: True
+            Default: ``None`` .
+        weight_decay (float): Weight decay (L2 penalty). It must be equal to or greater than 0. Default: ``0.0`` .
+        scale_parameter (bool): If True, learning rate is scaled by root mean square of parameter.
+            default: ``True`` .
         relative_step (bool): If True, time-dependent learning rate is computed instead of external learning rate.
-            default: True
+            default: ``True`` .
         warmup_init (bool): The time-dependent learning rate computation depends on whether warm-up
-            initialization is being used. default: False
+            initialization is being used. default: ``False`` .
         compression (bool): If True, the data type of the running averages exponent will be compression to float16.
-            default: False
+            default: ``False`` .
         loss_scale (float): A floating point value for the loss scale. Should be greater than 0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
+            Default: ``1.0`` .
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
@@ -261,7 +263,9 @@ class AdaFactor(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) Parameters use the default learning rate with None and weight decay with 0.
         >>> optim = nn.AdaFactor(params=net.trainable_params())
         >>>
@@ -329,7 +333,6 @@ class AdaFactor(Optimizer):
         self.weight_decay = trans_to_tensor(weight_decay)
         self.weight_decay_flag = bool(weight_decay)
 
-        self.step = Parameter(Tensor(0, dtype=mstype.float32), name="train_step")
         self.scale_parameter = scale_parameter
         self.relative_step = relative_step
         self.warmup_init = warmup_init
@@ -407,8 +410,8 @@ class AdaFactor(Optimizer):
     def construct(self, gradients):
         gradients = self.flatten_gradients(gradients)
         lr = self.get_lr()
-        F.assign_add(self.step, 1)
-        step = self.step
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
+        step = F.assign_add(self.step, 1)
         if self.scale_lr and self.relative_step:
             if self.warmup_init:
                 min_step = 1e-6 * step

mindspore/nn/optim/adam.py

@@ -499,7 +499,8 @@ def _run_opt_with_one_number_use_amsgrad(opt, sparse_opt, push, pull,
                                               (op_shape(param), op_shape(moment1), op_shape(moment2),
                                                op_shape(vhat))), param))
     else:
-        success = F.depend(success, opt(param, moment1, moment2, vhat, beta1_power, beta2_power, lr, gradient))
+        success = F.depend(success, opt(param, moment1, moment2, vhat, beta1_power, beta2_power,
+                                        lr, beta1, beta2, eps, gradient))
     return success
 
 
@@ -634,7 +635,7 @@ class Adam(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1e-3`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -649,22 +650,22 @@ class Adam(Optimizer):
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.9.
+                       Default: ``0.9`` .
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.999.
-        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. Default:
-                     1e-8.
+                       Default: ``0.999`` .
+        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0.
+                     Default: ``1e-8`` .
         use_locking (bool): Whether to enable a lock to protect the updating process of variable tensors.
-            If true, updates of the `w`, `m`, and `v` tensors will be protected by a lock.
-            If false, the result is unpredictable. Default: False.
+            If ``true`` , updates of the `w`, `m`, and `v` tensors will be protected by a lock.
+            If ``false`` , the result is unpredictable. Default: ``False`` .
         use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients.
-            If true, update the gradients using NAG.
-            If false, update the gradients without using NAG. Default: False.
+            If ``true`` , update the gradients using NAG.
+            If ``false`` , update the gradients without using NAG. Default: ``False`` .
         use_amsgrad (bool): Whether to use Amsgrad algorithm to update the gradients.
-            If true, update the gradients using Amsgrad.
-            If false, update the gradients without using Amsgrad. Default: False.
+            If ``true`` , update the gradients using Amsgrad.
+            If ``false`` , update the gradients without using Amsgrad. Default: ``False`` .
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -681,14 +682,14 @@ class Adam(Optimizer):
 
         kwargs:
 
-            - use_lazy (bool): Whether to use Lazy Adam algorithm. Default: False.
-              If true, apply lazy adam algorithm.
-              If false, apply normal adam algorithm.
+            - use_lazy (bool): Whether to use Lazy Adam algorithm. Default: ``False`` .
+              If ``true`` , apply lazy adam algorithm.
+              If ``false`` , apply normal adam algorithm.
 
             - use_offload (bool): Whether to offload adam optimizer to host CPU and keep parameters being updated on
-              the device in order to minimize the memory cost. Default: False.
-              If true, apply offload adam.
-              If false, apply normal adam.
+              the device in order to minimize the memory cost. Default: ``False`` .
+              If ``true`` , apply offload adam.
+              If ``false`` , apply normal adam.
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
@@ -697,6 +698,7 @@ class Adam(Optimizer):
         Tensor[bool], the value is True.
 
     Raises:
+        KeyError: If kwargs got keys other than 'use_lazy' or 'use_offload'.
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
         TypeError: If element of `parameters` is neither Parameter nor dict.
         TypeError: If `beta1`, `beta2`, `eps` or `loss_scale` is not a float.
@@ -705,8 +707,8 @@ class Adam(Optimizer):
         ValueError: If `loss_scale` or `eps` is less than or equal to 0.
         ValueError: If `beta1`, `beta2` is not in range (0.0, 1.0).
         ValueError: If `weight_decay` is less than 0.
-        ValueError: If `use_lazy` and `use_offload` are both true.
-        ValueError: If `use_amsgrad` is true and (`use_lazy` or `use_offload` is true).
+        ValueError: If `use_lazy` and `use_offload` are both ``true`` .
+        ValueError: If `use_amsgrad` is ``true`` and (`use_lazy` or `use_offload` is ``true`` ).
         ValueError: If `use_amsgrad` while using distributed training.
 
     Supported Platforms:
@@ -716,7 +718,9 @@ class Adam(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Adam(params=net.trainable_params())
         >>>
@@ -741,6 +745,10 @@ class Adam(Optimizer):
     def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False,
                  use_nesterov=False, weight_decay=0.0, loss_scale=1.0, use_amsgrad=False, **kwargs):
         super(Adam, self).__init__(learning_rate, params, weight_decay, loss_scale)
+        valid_keys = {'use_lazy', 'use_offload'}
+        if set(kwargs.keys()) - valid_keys:
+            raise KeyError(f"For 'Adam', invalid keys are passed as kwargs, supported keys are 'use_lazy' and"
+                           f"'use_offload', but got {kwargs.keys()}.")
         use_lazy = kwargs.get('use_lazy', False)
         use_offload = kwargs.get('use_offload', False)
         _check_param_value(beta1, beta2, eps, self.cls_name)
@@ -790,7 +798,7 @@ class Adam(Optimizer):
         else:
             self._is_device = True
             if use_amsgrad:
-                self.opt = P.ApplyAdamWithAmsgrad(beta1, beta2, eps, use_locking)
+                self.opt = P.ApplyAdamWithAmsgradV2(use_locking)
             else:
                 self.opt = P.Adam(use_locking, use_nesterov)
             self.sparse_opt = P.FusedSparseAdam(use_locking, use_nesterov)
@@ -910,6 +918,7 @@ class Adam(Optimizer):
         gradients = self.scale_grad(gradients)
         gradients = self._grad_sparse_indices_deduplicate(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         beta1_power = self.beta1_power * self.beta1
         self.beta1_power = beta1_power
@@ -977,7 +986,7 @@ class AdamWeightDecay(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means, refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.0/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
@@ -1008,7 +1017,7 @@ class AdamWeightDecay(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1e-3`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -1022,14 +1031,14 @@ class AdamWeightDecay(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
-        beta1 (float): The exponential decay rate for the 1st moment estimations. Default: 0.9.
+        beta1 (float): The exponential decay rate for the 1st moment estimations. Default: ``0.9`` .
             Should be in range (0.0, 1.0).
-        beta2 (float): The exponential decay rate for the 2nd moment estimations. Default: 0.999.
+        beta2 (float): The exponential decay rate for the 2nd moment estimations. Default: ``0.999`` .
             Should be in range (0.0, 1.0).
-        eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6.
+        eps (float): Term added to the denominator to improve numerical stability. Default: ``1e-6`` .
             Should be greater than 0.
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -1060,7 +1069,9 @@ class AdamWeightDecay(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdamWeightDecay(params=net.trainable_params())
         >>>
@@ -1089,16 +1100,17 @@ class AdamWeightDecay(Optimizer):
         self.moments1 = self._parameters.clone(prefix="adam_m", init='zeros')
         self.moments2 = self._parameters.clone(prefix="adam_v", init='zeros')
         self.fused_opt = P.AdamWeightDecay()
-        if context.get_context("device_target") == "CPU":
-            self.use_fused_opt = True
-        else:
+        if context.get_context("device_target") == "Ascend":
             self.use_fused_opt = False
+        else:
+            self.use_fused_opt = True
 
     @jit
     def construct(self, gradients):
         gradients = self.flatten_gradients(gradients)
         weight_decay = self.get_weight_decay()
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         if self.use_fused_opt:
             if self.is_group:
@@ -1206,7 +1218,7 @@ class AdamOffload(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1e-3`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -1221,19 +1233,19 @@ class AdamOffload(Optimizer):
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.9.
+                       Default: ``0.9`` .
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.999.
-        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. Default:
-                     1e-8.
+                       Default: ``0.999`` .
+        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0.
+                       Default: ``1e-8`` .
         use_locking (bool): Whether to enable a lock to protect the updating process of variable tensors.
-            If true, updates of the `w`, `m`, and `v` tensors will be protected by a lock.
-            If false, the result is unpredictable. Default: False.
+            If ``true`` , updates of the `w`, `m`, and `v` tensors will be protected by a lock.
+            If ``false`` , the result is unpredictable. Default: ``False`` .
         use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients.
-            If true, update the gradients using NAG.
-            If false, update the gradients without using NAG. Default: False.
+            If ``true`` , update the gradients using NAG.
+            If ``false`` , update the gradients without using NAG. Default: ``False`` .
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -1244,15 +1256,15 @@ class AdamOffload(Optimizer):
 
         loss_scale (float): A floating point value for the loss scale. Should be greater than 0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
+            Default: ``1.0`` .
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
 
     Outputs:
-        Tensor[bool], the value is True.
+        Tensor[bool], the value is ``True`` .
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -1271,7 +1283,9 @@ class AdamOffload(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdamOffload(params=net.trainable_params())
         >>>
@@ -1318,6 +1332,7 @@ class AdamOffload(Optimizer):
         gradients = self.decay_weight(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         beta1_power = self.beta1_power * self.beta1
         self.beta1_power = beta1_power

mindspore/nn/optim/adamax.py

@@ -66,7 +66,7 @@ class AdaMax(Optimizer):
     :math:`m` represents the 1st moment vector, :math:`v` represents the 2nd moment vector,
     :math:`g` represents `gradients`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`,
     :math:`t` represents the current step, :math:`beta_1^t` represent `beta1_power`,
-    :math:`\l` represents `learning_rate`, :math:`w` represents `params`,
+    :math:`l` represents `learning_rate`, :math:`w` represents `params`,
     :math:`\epsilon` represents `eps`.
 
     Note:
@@ -102,7 +102,7 @@ class AdaMax(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 0.001.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``0.001`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -117,13 +117,13 @@ class AdaMax(Optimizer):
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.9.
+                       Default: ``0.9`` .
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
-                       Default: 0.999.
-        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. Default:
-                     1e-8.
+                       Default: ``0.999`` .
+        eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0.
+                     Default: ``1e-08`` .
 
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -134,9 +134,9 @@ class AdaMax(Optimizer):
 
         loss_scale (float): A floating point value for the loss scale. Should be greater than 0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
-            `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
+            `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
-            Default: 1.0.
+            Default: ``1.0`` .
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
@@ -160,7 +160,9 @@ class AdaMax(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdaMax(params=net.trainable_params())
         >>>
@@ -202,6 +204,7 @@ class AdaMax(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         self.beta1_power *= self.beta1
 

mindspore/nn/optim/adasum.py

@@ -442,12 +442,14 @@ class AdaSumByGradWrapCell(Cell):
         ``Ascend`` ``GPU``
 
     Examples:
+        >>> import mindspore as ms
         >>> from mindspore import nn
-        >>> from mindspore.nn import AdaSumByGradWrapCell
-        >>> net = Net()
-        >>> optim = AdaSumByGradWrapCell(nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9))
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> optim = nn.AdaSumByGradWrapCell(nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9))
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim, metrics=None)
     """
     def __init__(self, optimizer):
         super(AdaSumByGradWrapCell, self).__init__(auto_prefix=False)
@@ -509,13 +511,15 @@ class AdaSumByDeltaWeightWrapCell(Cell):
         ``Ascend`` ``GPU``
 
     Examples:
+        >>> import mindspore as ms
         >>> from mindspore import nn
-        >>> from mindspore.nn import AdaSumByDeltaWeightWrapCell
-        >>> net = Net()
-        >>> optim = AdaSumByDeltaWeightWrapCell(nn.Momentum(params=net.trainable_params(),
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> optim = nn.AdaSumByDeltaWeightWrapCell(nn.Momentum(params=net.trainable_params(),
         ...                                                 learning_rate=0.1, momentum=0.9))
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim, metrics=None)
     """
     def __init__(self, optimizer):
         super(AdaSumByDeltaWeightWrapCell, self).__init__(auto_prefix=False)

mindspore/nn/optim/asgd.py

@@ -81,7 +81,7 @@ class ASGD(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): learning_rate. Default: 0.1.
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): learning_rate. Default: ``0.1`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -95,10 +95,10 @@ class ASGD(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
-        lambd (float): The decay term. Default: 1e-4.
-        alpha (float): The power for :math:`\eta` update. Default: 0.75.
-        t0 (float): The point of starting averaging. Default: 1e6.
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
+        lambd (float): The decay term. Default: ``1e-4`` .
+        alpha (float): The power for :math:`\eta` update. Default: ``0.75`` .
+        t0 (float): The point of starting averaging. Default: ``1e6`` .
+        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -127,7 +127,9 @@ class ASGD(Optimizer):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>>
-        >>> net = Net()
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.ASGD(params=net.trainable_params())
         >>>
@@ -145,7 +147,7 @@ class ASGD(Optimizer):
         >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'.
         >>>
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
-        >>> model = ms.Model(net, loss_fn=loss, optimizer=optim)
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
     """
 
     @opt_init_args_register
@@ -183,8 +185,7 @@ class ASGD(Optimizer):
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)
         lrs = self.get_lr()
-        if not self._is_dynamic_lr_or_weight_decay():
-            self.assignadd(self.global_step, self.global_step_increase_tensor)
+        self.assignadd(self.global_step, self.global_step_increase_tensor)
         success = True
         params = self._parameters
         for index, (grad, param, mu, eta, ax) in enumerate(zip(gradients, params, self.mu, self.eta, self.ax)):