mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/nn/optim/ada_grad.py

@@ -16,7 +16,7 @@
 from __future__ import absolute_import
 
 from mindspore.ops import functional as F, composite as C, operations as P
-from mindspore._checkparam import Validator as validator
+from mindspore import _checkparam as validator
 from mindspore.common.api import jit
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
@@ -53,7 +53,7 @@ class Adagrad(Optimizer):
     The updating Pseudo codes are as follows:
 
     .. math::
-       \begin{aligned} \\
+        \begin{aligned} \\
             &\newline
             &\hline \\
             &\textbf{Parameters}: \text{learning rate } \gamma, \:  \text{ params } w_0, \:
@@ -73,7 +73,7 @@ class Adagrad(Optimizer):
             &\bf{return} \:  w_t \\[-1.ex]
             &\newline
             &\hline \\
-       \end{aligned}
+        \end{aligned}
 
     :math:`state\_sum` stands for the accumulated squared sum of the gradients :math:`accum`.
     :math:`g` stands for `grads`, :math:`\lambda` stands for `weight_decay`.
@@ -112,7 +112,7 @@ class Adagrad(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        accum (float): The starting value for `h`, must be zero or positive values. Default: 0.1.
+        accum (float): The starting value for :math:`h`, must be zero or positive values. Default: 0.1.
         learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 0.001.
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
@@ -127,7 +127,7 @@ class Adagrad(Optimizer):
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
-        update_slots (bool): Whether the `h` will be updated. Default: True.
+        update_slots (bool): Whether the :math:`h` will be updated. Default: True.
         loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
             `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
@@ -159,7 +159,7 @@ class Adagrad(Optimizer):
         ValueError: If `accum` or `weight_decay` is less than 0.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore as ms

mindspore/nn/optim/adadelta.py

@@ -16,8 +16,7 @@
 from __future__ import absolute_import
 
 from mindspore.ops import functional as F, composite as C, operations as P
-from mindspore._checkparam import Validator as validator
-from mindspore._checkparam import Rel
+from mindspore import _checkparam as validator
 from mindspore.common.tensor import Tensor
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
@@ -37,7 +36,7 @@ def _check_param_value(rho, epsilon, prim_name=None):
     """Check inputs param."""
     validator.check_value_type("rho", rho, [float], prim_name)
     validator.check_value_type("epsilon", epsilon, [float], prim_name)
-    validator.check_float_range(rho, 0.0, 1.0, Rel.INC_BOTH, "rho", prim_name)
+    validator.check_float_range(rho, 0.0, 1.0, validator.INC_BOTH, "rho", prim_name)
     validator.check_non_negative_float(epsilon, "epsilon", prim_name)
 
 

mindspore/nn/optim/adafactor.py

@@ -25,8 +25,7 @@ from mindspore.ops import composite as C
 from mindspore.ops import functional as F
 from mindspore.common.parameter import Parameter, ParameterTuple
 from mindspore.common.tensor import Tensor
-from mindspore._checkparam import Validator as validator
-from mindspore._checkparam import Rel
+from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import opt_init_args_register
 from mindspore.nn.optim.optimizer import Optimizer
 
@@ -153,7 +152,7 @@ class AdaFactor(Optimizer):
     Cost <https://arxiv.org/abs/1804.04235>`_.
 
     .. warning::
-        This is an experimental prototype that is subject to change and/or deletion.
+        This is an experimental API that is subject to change or deletion.
 
     Adafactor for weight vector are as follows,
 
@@ -316,8 +315,8 @@ class AdaFactor(Optimizer):
         validator.check_value_type("clip_threshold", clip_threshold, [float], self.cls_name)
         validator.check_non_negative_float(clip_threshold, "clip_threshold", self.cls_name)
         validator.check_value_type("decay_rate", decay_rate, [float], self.cls_name)
-        validator.check_float_range(decay_rate, 0, 1, Rel.INC_NEITHER, "decay_rate", self.cls_name)
-        validator.check_float_range(weight_decay, 0, 1, Rel.INC_LEFT, "weight_decay", self.cls_name)
+        validator.check_float_range(decay_rate, 0, 1, validator.INC_NEITHER, "decay_rate", self.cls_name)
+        validator.check_float_range(weight_decay, 0, 1, validator.INC_LEFT, "weight_decay", self.cls_name)
         validator.check_value_type("scale_parameter", scale_parameter, [bool], self.cls_name)
         validator.check_value_type("relative_step", relative_step, [bool], self.cls_name)
         validator.check_value_type("compression", compression, [bool], self.cls_name)

mindspore/nn/optim/adam.py

@@ -26,8 +26,7 @@ from mindspore.ops import composite as C
 from mindspore.ops import functional as F
 from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
-from mindspore._checkparam import Validator as validator
-from mindspore._checkparam import Rel
+from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
 from mindspore.nn.optim._dist_optimizer_registry import _register_dist_optimizer
@@ -93,6 +92,46 @@ def _run_lazy_opt_with_sparse_dist(opt, sparse_opt, push, pull, use_locking, use
     return success
 
 
+@_lazy_adam_opt.register("Function", "Function", "Function", "Function", "Bool", "Bool", "Bool", "Tensor", "Tensor",
+                         "Tensor", "Tensor", "Tensor", "Tensor", "MapTensor", "MapTensor", "MapTensor", "MapTensor",
+                         "Bool", "Bool", "Function", "Bool", "Function", "Bool")
+def _run_map_tensor_lazy_opt_with_sparse_dist(opt, sparse_opt, push, pull, use_locking, use_nesterov, target,
+                                              beta1_power, beta2_power, beta1, beta2, eps, lr, gradient, params, m, v,
+                                              ps_parameter, cache_enable, distributed_opt, use_flag,
+                                              distributed_sparse_opt, use_sparse_flag):
+    """Apply sparse lazy adam optimizer to the weight parameter when the gradient is sparse."""
+    success = True
+    indices, values = gradient.get_data()
+    if use_sparse_flag:
+        # PS Mode.
+        success = F.depend(success, distributed_sparse_opt(params, m, v, beta1_power, beta2_power, lr, beta1, beta2,
+                                                           eps, values, indices))
+    else:
+        # PS Cache mode.
+        op_sqrt = P.Sqrt()
+
+        m_slice = m.get(indices)
+        v_slice = v.get(indices)
+
+        next_m = m_slice * beta1 + values * (1 - beta1)
+        next_v = v_slice * beta2 + values * values * (1 - beta2)
+
+        lr_t = lr * op_sqrt(1 - beta2_power) / (1 - beta1_power)
+
+        if use_nesterov:
+            m_temp = beta1 * next_m + values * (1 - beta1)
+            param_update = m_temp / (op_sqrt(next_v) + eps)
+        else:
+            param_update = next_m / (op_sqrt(next_v) + eps)
+
+        params_need_update = params.get(indices)
+        params.put(indices, params_need_update - lr_t * param_update)
+        m.put(indices, next_m)
+        v.put(indices, next_v)
+
+    return success
+
+
 @_lazy_adam_opt.register("Function", "Function", "Function", "Function", "Bool", "Bool", "Bool", "Tensor", "Tensor",
                          "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool",
                          "Function", "Bool", "Function", "Bool")
@@ -360,18 +399,14 @@ def _run_opt_with_one_number_dist(opt, sparse_opt, push, pull, use_locking, use_
 
 
 @_adam_opt.register("Function", "Function", "Function", "Function",
-                    "Bool", "Bool", "Bool", "Bool",
+                    "Bool", "Bool", "Bool",
                     "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
-                    "RowTensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
+                    "RowTensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
 def _run_opt_with_sparse(opt, sparse_opt, push, pull,
-                         use_locking, use_nesterov, use_amsgrad, target,
+                         use_locking, use_nesterov, target,
                          beta1_power, beta2_power, beta1, beta2, eps, lr,
-                         gradient, param, m, v, vhat, ps_parameter, cache_enable):
+                         gradient, param, m, v, ps_parameter, cache_enable):
     """Apply sparse adam optimizer to the weight parameter when the gradient is sparse."""
-    if use_amsgrad:
-        raise Exception("""Adam with amsgrad is currently not supported when the gradients are sparse!
-                        Please set use_amsgrad=False for sparse gradients.""")
-
     success = True
     indices = gradient.indices
     values = gradient.values
@@ -429,30 +464,42 @@ def _run_opt_with_sparse(opt, sparse_opt, push, pull,
 
 
 @_adam_opt.register("Function", "Function", "Function", "Function",
-                    "Bool", "Bool", "Bool", "Bool",
+                    "Bool", "Bool", "Bool",
                     "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
-                    "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
+                    "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
 def _run_opt_with_one_number(opt, sparse_opt, push, pull,
-                             use_locking, use_nesterov, use_amsgrad, target,
+                             use_locking, use_nesterov, target,
                              beta1_power, beta2_power, beta1, beta2, eps, lr,
-                             gradient, param, moment1, moment2, vhat, ps_parameter, cache_enable):
+                             gradient, param, moment1, moment2, ps_parameter, cache_enable):
     """Apply adam optimizer to the weight parameter using Tensor."""
     success = True
     if ps_parameter and not cache_enable:
         op_shape = P.Shape()
-        if use_amsgrad:
-            success = F.depend(success, pull(push((beta1_power, beta2_power, lr, gradient),
-                                                  (op_shape(param), op_shape(moment1), op_shape(moment2),
-                                                   op_shape(vhat))), param))
-        else:
-            success = F.depend(success, pull(push((beta1_power, beta2_power, lr, beta1, beta2, eps, gradient),
-                                                  (op_shape(param), op_shape(moment1), op_shape(moment2))), param))
+        success = F.depend(success, pull(push((beta1_power, beta2_power, lr, beta1, beta2, eps, gradient),
+                                              (op_shape(param), op_shape(moment1), op_shape(moment2))), param))
     else:
-        if use_amsgrad:
-            success = F.depend(success, opt(param, moment1, moment2, vhat, beta1_power, beta2_power, lr, gradient))
-        else:
-            success = F.depend(success, opt(param, moment1, moment2, beta1_power, beta2_power, lr, beta1, beta2,
-                                            eps, gradient))
+        success = F.depend(success, opt(param, moment1, moment2, beta1_power, beta2_power, lr, beta1, beta2,
+                                        eps, gradient))
+    return success
+
+
+@_adam_opt.register("Function", "Function", "Function", "Function",
+                    "Bool", "Bool", "Bool",
+                    "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
+                    "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
+def _run_opt_with_one_number_use_amsgrad(opt, sparse_opt, push, pull,
+                                         use_locking, use_nesterov, target,
+                                         beta1_power, beta2_power, beta1, beta2, eps, lr,
+                                         gradient, param, moment1, moment2, vhat, ps_parameter, cache_enable):
+    """Apply adam optimizer to the weight parameter using Tensor and use amsgrad."""
+    success = True
+    if ps_parameter and not cache_enable:
+        op_shape = P.Shape()
+        success = F.depend(success, pull(push((beta1_power, beta2_power, lr, gradient),
+                                              (op_shape(param), op_shape(moment1), op_shape(moment2),
+                                               op_shape(vhat))), param))
+    else:
+        success = F.depend(success, opt(param, moment1, moment2, vhat, beta1_power, beta2_power, lr, gradient))
     return success
 
 
@@ -484,8 +531,8 @@ def _check_param_value(beta1, beta2, eps, prim_name):
     validator.check_value_type("beta1", beta1, [float], prim_name)
     validator.check_value_type("beta2", beta2, [float], prim_name)
     validator.check_value_type("eps", eps, [float], prim_name)
-    validator.check_float_range(beta1, 0.0, 1.0, Rel.INC_NEITHER, "beta1", prim_name)
-    validator.check_float_range(beta2, 0.0, 1.0, Rel.INC_NEITHER, "beta2", prim_name)
+    validator.check_float_range(beta1, 0.0, 1.0, validator.INC_NEITHER, "beta1", prim_name)
+    validator.check_float_range(beta2, 0.0, 1.0, validator.INC_NEITHER, "beta2", prim_name)
     validator.check_positive_float(eps, "eps", prim_name)
 
 
@@ -723,7 +770,8 @@ class Adam(Optimizer):
         self.use_offload = use_offload
         self.moment1 = self._parameters.clone(prefix="moment1", init='zeros')
         self.moment2 = self._parameters.clone(prefix="moment2", init='zeros')
-        self.vhat = self._parameters.clone(prefix="vhat", init='zeros')
+        if use_amsgrad:
+            self.vhat = self._parameters.clone(prefix="vhat", init='zeros')
 
         if use_offload:
             self.opt = P.AdamNoUpdateParam(use_locking, use_nesterov)
@@ -756,25 +804,8 @@ class Adam(Optimizer):
 
             self._init_distributed_opts(use_locking, use_nesterov)
 
-    @jit
-    def construct(self, gradients):
-        params = self._parameters
-        moment1 = self.moment1
-        moment2 = self.moment2
-        vhat = self.vhat
-        gradients = self.flatten_gradients(gradients)
-        gradients = self.decay_weight(gradients)
-        if not self.use_offload:
-            gradients = self.gradients_centralization(gradients)
-        gradients = self.scale_grad(gradients)
-        gradients = self._grad_sparse_indices_deduplicate(gradients)
-        lr = self.get_lr()
-
-        beta1_power = self.beta1_power * self.beta1
-        self.beta1_power = beta1_power
-        beta2_power = self.beta2_power * self.beta2
-        self.beta2_power = beta2_power
-
+    def _apply_adam(self, params, beta1_power, beta2_power, moment1, moment2, lr, gradients):
+        """Execute Adam optimizer and its variants."""
         if self.use_offload:
             if self.is_group_lr:
                 success = self.map_reverse(F.partial(_adam_opt, self.opt, beta1_power, beta2_power, self.beta1,
@@ -831,13 +862,19 @@ class Adam(Optimizer):
                                                       self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
                                                       self.eps), lr, gradients, params, moment1, moment2,
                                             self.ps_parameters, self.cache_enable)
-
                     else:
-                        success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push,
-                                                      self._ps_pull, self.use_locking, self.use_nesterov,
-                                                      self.use_amsgrad, self._is_device, beta1_power, beta2_power,
-                                                      self.beta1, self.beta2, self.eps), lr, gradients, params,
-                                            moment1, moment2, vhat, self.ps_parameters, self.cache_enable)
+                        if self.use_amsgrad:
+                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push,
+                                                          self._ps_pull, self.use_locking, self.use_nesterov,
+                                                          self._is_device, beta1_power, beta2_power,
+                                                          self.beta1, self.beta2, self.eps), lr, gradients, params,
+                                                moment1, moment2, self.vhat, self.ps_parameters, self.cache_enable)
+                        else:
+                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push,
+                                                          self._ps_pull, self.use_locking, self.use_nesterov,
+                                                          self._is_device, beta1_power, beta2_power,
+                                                          self.beta1, self.beta2, self.eps), lr, gradients, params,
+                                                moment1, moment2, self.ps_parameters, self.cache_enable)
                 else:
                     if self.use_lazy:
                         success = self.map_(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt, self._ps_push,
@@ -845,15 +882,42 @@ class Adam(Optimizer):
                                                       self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
                                                       self.eps, lr), gradients, params, moment1, moment2,
                                             self.ps_parameters, self.cache_enable)
-
                     else:
-                        success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push,
-                                                      self._ps_pull, self.use_locking, self.use_nesterov,
-                                                      self.use_amsgrad, self._is_device, beta1_power, beta2_power,
-                                                      self.beta1, self.beta2, self.eps, lr), gradients, params,
-                                            moment1, moment2, vhat, self.ps_parameters, self.cache_enable)
+                        if self.use_amsgrad:
+                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push,
+                                                          self._ps_pull, self.use_locking, self.use_nesterov,
+                                                          self._is_device, beta1_power, beta2_power,
+                                                          self.beta1, self.beta2, self.eps, lr), gradients, params,
+                                                moment1, moment2, self.vhat, self.ps_parameters, self.cache_enable)
+                        else:
+                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push,
+                                                          self._ps_pull, self.use_locking, self.use_nesterov,
+                                                          self._is_device, beta1_power, beta2_power,
+                                                          self.beta1, self.beta2, self.eps, lr), gradients, params,
+                                                moment1, moment2, self.ps_parameters, self.cache_enable)
+
         return success
 
+    @jit
+    def construct(self, gradients):
+        params = self._parameters
+        moment1 = self.moment1
+        moment2 = self.moment2
+        gradients = self.flatten_gradients(gradients)
+        gradients = self.decay_weight(gradients)
+        if not self.use_offload:
+            gradients = self.gradients_centralization(gradients)
+        gradients = self.scale_grad(gradients)
+        gradients = self._grad_sparse_indices_deduplicate(gradients)
+        lr = self.get_lr()
+
+        beta1_power = self.beta1_power * self.beta1
+        self.beta1_power = beta1_power
+        beta2_power = self.beta2_power * self.beta2
+        self.beta2_power = beta2_power
+
+        return self._apply_adam(params, beta1_power, beta2_power, moment1, moment2, lr, gradients)
+
     @Optimizer.target.setter
     def target(self, value):
         """
@@ -907,13 +971,13 @@ class AdamWeightDecay(Optimizer):
     :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`,
     :math:`g` represents `gradients`, :math:`\gamma` represents `learning_rate`,
     :math:`\beta_1, \beta_2` represent `beta1` and `beta2`, :math:`t` represents the current step,
-    :math:`w` represents `params`, :math:`\gamma` represents `weight_decay`.
+    :math:`w` represents `params`, :math:`\lambda` represents `weight_decay`.
 
     Note:
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means, refer
-        document `LossScale <https://www.mindspore.cn/tutorials/zh-CN/r2.0.0-alpha/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.0/advanced/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without

mindspore/nn/optim/adamax.py

@@ -23,10 +23,9 @@ from mindspore.ops import composite as C
 from mindspore.ops import functional as F
 from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
-from mindspore._checkparam import Validator as validator
+from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
-from mindspore._checkparam import Rel
 
 _ada_max_opt = C.MultitypeFuncGraph("ada_max_opt")
 
@@ -44,8 +43,8 @@ def _check_param_value(beta1, beta2, eps, prim_name):
     validator.check_value_type("beta1", beta1, [float], prim_name)
     validator.check_value_type("beta2", beta2, [float], prim_name)
     validator.check_value_type("eps", eps, [float], prim_name)
-    validator.check_float_range(beta1, 0.0, 1.0, Rel.INC_NEITHER, "beta1", prim_name)
-    validator.check_float_range(beta2, 0.0, 1.0, Rel.INC_NEITHER, "beta2", prim_name)
+    validator.check_float_range(beta1, 0.0, 1.0, validator.INC_NEITHER, "beta1", prim_name)
+    validator.check_float_range(beta2, 0.0, 1.0, validator.INC_NEITHER, "beta2", prim_name)
     validator.check_positive_float(eps, "eps", prim_name)
 
 

mindspore/nn/optim/adasum.py

@@ -22,7 +22,7 @@ import math
 import mindspore.nn as nn
 import mindspore.log as logger
 from mindspore import context
-from mindspore._checkparam import Validator as validator
+from mindspore import _checkparam as validator
 from mindspore.nn.cell import Cell
 from mindspore.common.parameter import ParameterTuple, Parameter
 from mindspore.parallel._utils import _get_global_rank, _get_stage_device_num
@@ -114,7 +114,7 @@ def _adasum_opt_forward_process(left_send, allreduce, parameter_divisibility, al
     if parameter_divisibility:
         delta_w = P.Squeeze()(delta_w)
         ori_len = F.shape(delta_w)[0]
-        divide_len = ori_len / 2
+        divide_len = ori_len // 2
         left_part = delta_w[:divide_len]
         right_part = delta_w[divide_len:]
     else:
@@ -412,8 +412,8 @@ class AdaSumByGradWrapCell(Cell):
     .. math::
         \begin{array}{ll}
           w_{t+1}=w_{t} - \alpha \cdot Adasum(g_{1}, g_{2})  \\
-          w_{t+1}=w_{t} - \alpha \cdot [(1 - \frac{g_2^{T}\cdot g_1}{2\cdot \left \| g_1 \right \|^2 })\cdot g_1 +
-           (1 - \frac{g_1^{T}\cdot g_2}{2\cdot \left \| g_2 \right \|^2 })\cdot g_2]  \\
+          w_{t+1}=w_{t} - \alpha \cdot [(1 - \frac{g_2^{T}\cdot g_1}{2\cdot \left \| g_1 \right \|^2 })\cdot g_1 + (1 -
+          \frac{g_1^{T}\cdot g_2}{2\cdot \left \| g_2 \right \|^2 })\cdot g_2]  \\
         \end{array}
 
     In this implementation, :math:`g` represents the gradient of the weights,
@@ -479,8 +479,8 @@ class AdaSumByDeltaWeightWrapCell(Cell):
     .. math::
         \begin{array}{ll}
           w_{t+1}=w_{t} - \alpha \cdot Adasum(g_{1}, g_{2})  \\
-          w_{t+1}=w_{t} - \alpha \cdot [(1 - \frac{g_2^{T}\cdot g_1}{2\cdot \left \| g_1 \right \|^2 })\cdot g_1 +
-           (1 - \frac{g_1^{T}\cdot g_2}{2\cdot \left \| g_2 \right \|^2 })\cdot g_2]  \\
+          w_{t+1}=w_{t} - \alpha \cdot [(1 - \frac{g_2^{T}\cdot g_1}{2\cdot \left \| g_1 \right \|^2 })\cdot g_1 + (1 -
+          \frac{g_1^{T}\cdot g_2}{2\cdot \left \| g_2 \right \|^2 })\cdot g_2]  \\
         \end{array}
 
     In this implementation, :math:`g` represents the weight difference before and after the updating of optimizer,

mindspore/nn/optim/asgd.py

@@ -21,7 +21,7 @@ from mindspore.common.api import jit
 from mindspore.common.tensor import Tensor
 import mindspore.common.dtype as mstype
 import mindspore
-from mindspore._checkparam import Validator as validator
+from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
 
@@ -96,7 +96,7 @@ class ASGD(Optimizer):
               LearningRateSchedule with step as the input to get the learning rate of current step.
 
         lambd (float): The decay term. Default: 1e-4.
-        alpha (float): The power for :math:`eta` update. Default: 0.75.
+        alpha (float): The power for :math:`\eta` update. Default: 0.75.
         t0 (float): The point of starting averaging. Default: 1e6.
         weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
 

mindspore/nn/optim/ftrl.py

@@ -18,8 +18,7 @@ from __future__ import absolute_import
 from mindspore.ops import functional as F, composite as C, operations as P
 from mindspore.ops.composite.multitype_ops.zeros_like_impl import zeros_like
 from mindspore.common.api import jit
-from mindspore._checkparam import Validator as validator
-from mindspore._checkparam import Rel
+from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
 from mindspore.nn.optim._dist_optimizer_registry import _register_dist_optimizer
@@ -48,6 +47,62 @@ def _tensor_run_opt_with_sparse_dist(opt, spars_opt, push, pull, l1, l2, lr_powe
     return success
 
 
+def _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, moment, indices, values):
+    """Apllpy ftrl optimizer for map parameter"""
+    success = True
+    linear_slice = linear.get(indices)
+    moment_slice = moment.get(indices)
+    weight_slice = weight.get(indices)
+
+    op_pow = P.Pow()
+    op_sign = P.Sign()
+    op_greater = P.Greater()
+    op_select = P.Select()
+    op_abs = P.Abs()
+
+    lr_power_val = -lr_power
+    accu_pow = op_pow(moment_slice, lr_power_val)
+    moment_slice = F.depend(moment_slice, accu_pow)
+    cur_accu = moment_slice + values * values
+    cur_accu_pow = op_pow(cur_accu, lr_power_val)
+    sigma = (cur_accu_pow - accu_pow) / learning_rate
+
+    linear_slice = linear_slice + values - sigma * weight_slice
+
+    update_weight_cond = op_greater(op_abs(linear_slice), l1)
+    updated_weight = (l1 * op_sign(linear_slice) - linear_slice) / (cur_accu_pow / learning_rate + 2 * l2)
+    zeros = zeros_like(weight_slice)
+
+    weight_slice = op_select(update_weight_cond, updated_weight, zeros)
+    moment_slice = cur_accu
+
+    moment.put(indices, moment_slice)
+    linear.put(indices, linear_slice)
+    weight.put(indices, weight_slice)
+
+    return success
+
+
+@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
+                    "MapTensor", "MapTensor", "MapTensor", "Bool", "Bool",
+                    "Function", "Bool", "Function", "Bool")
+def _run_map_tensor_opt_with_sparse_dist(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
+                                         gradient, weight, moment, ps_parameter, cache_enable,
+                                         distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
+    """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
+    success = True
+    indices, values = gradient.get_data()
+    if use_sparse_flag:
+        # PS Mode.
+        success = F.depend(success, distributed_sparse_opt(weight, moment, linear, values, indices))
+    elif cache_enable:
+        # PS Cache mode.
+        _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, moment, indices, values)
+    else:
+        raise Exception("Unexpected mode for distributed optimizer.")
+    return success
+
+
 @_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
                     "Tensor", "Tensor", "Tensor", "Bool", "Bool",
                     "Function", "Bool", "Function", "Bool")
@@ -91,36 +146,7 @@ def _run_map_tensor_opt_with_sparse(opt, spars_opt, push, pull, l1, l2, lr_power
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
     indices, values = gradient.get_data()
-
-    linear_slice = linear.get(indices)
-    moment_slice = moment.get(indices)
-    weight_slice = weight.get(indices)
-
-    op_pow = P.Pow()
-    op_sign = P.Sign()
-    op_greater = P.Greater()
-    op_select = P.Select()
-
-    lr_power_val = -lr_power
-    accu_pow = op_pow(moment_slice, lr_power_val)
-    moment_slice = F.depend(moment_slice, accu_pow)
-    cur_accu = moment_slice + values * values
-    cur_accu_pow = op_pow(cur_accu, lr_power_val)
-    sigma = (cur_accu_pow - accu_pow) / learning_rate
-
-    linear_slice = linear_slice + values - sigma * weight_slice
-
-    update_weight_cond = op_greater(linear_slice, l1)
-    updated_weight = (l1 * op_sign(linear_slice) - linear_slice) / (cur_accu_pow / learning_rate + 2 * l2)
-    zeros = zeros_like(weight_slice)
-
-    weight_slice = op_select(update_weight_cond, updated_weight, zeros)
-    moment_slice = cur_accu
-
-    moment.put(indices, moment_slice)
-    linear.put(indices, linear_slice)
-    weight.put(indices, weight_slice)
-
+    _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, moment, indices, values)
     return success
 
 
@@ -139,19 +165,22 @@ def _tensor_run_opt(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate,
     return success
 
 
-def _check_param(initial_accum, lr_power, l1, l2, use_locking, prim_name=None):
+def _check_param(initial_accum, learning_rate, lr_power, l1, l2, use_locking, prim_name=None):
     """Check param."""
     validator.check_value_type("initial_accum", initial_accum, [float], prim_name)
-    validator.check_number("initial_accum", initial_accum, 0.0, Rel.GE, prim_name)
+    validator.check_number("initial_accum", initial_accum, 0.0, validator.GE, prim_name)
+
+    validator.check_value_type("learning_rate", learning_rate, [float], prim_name)
+    validator.check_positive_float(learning_rate, "learning_rate", prim_name)
 
     validator.check_value_type("lr_power", lr_power, [float], prim_name)
-    validator.check_number("lr_power", lr_power, 0.0, Rel.LE, prim_name)
+    validator.check_number("lr_power", lr_power, 0.0, validator.LE, prim_name)
 
     validator.check_value_type("l1", l1, [float], prim_name)
-    validator.check_number("l1", l1, 0.0, Rel.GE, prim_name)
+    validator.check_number("l1", l1, 0.0, validator.GE, prim_name)
 
     validator.check_value_type("l2", l2, [float], prim_name)
-    validator.check_number("l2", l2, 0.0, Rel.GE, prim_name)
+    validator.check_number("l2", l2, 0.0, validator.GE, prim_name)
 
     validator.check_value_type("use_locking", use_locking, [bool], prim_name)
 
@@ -295,7 +324,7 @@ class FTRL(Optimizer):
             raise ValueError(f"For 'FTRL', dynamic learning rate and group learning rate are currently not supported "
                              f"in FTRL, they should all be false, but got dynamic learning rate {self.dynamic_lr} and"
                              f" group learning rate {self.is_group_lr}.")
-        _check_param(initial_accum, lr_power, l1, l2, use_locking, self.cls_name)
+        _check_param(initial_accum, learning_rate, lr_power, l1, l2, use_locking, self.cls_name)
         self.moments = self._parameters.clone(prefix="moments", init=initial_accum)
         self.linear = self._parameters.clone(prefix="linear", init='zeros')
         self.l1 = l1

mindspore/nn/optim/lamb.py

@@ -23,8 +23,7 @@ from mindspore.ops import functional as F
 from mindspore.ops.operations import _inner_ops as inner
 from mindspore.common.tensor import Tensor
 from mindspore.common.api import jit
-from mindspore._checkparam import Validator as validator
-from mindspore._checkparam import Rel
+from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
 
@@ -69,8 +68,8 @@ def _check_param_value(beta1, beta2, eps, prim_name):
     validator.check_value_type("beta1", beta1, [float], prim_name)
     validator.check_value_type("beta2", beta2, [float], prim_name)
     validator.check_value_type("eps", eps, [float], prim_name)
-    validator.check_float_range(beta1, 0.0, 1.0, Rel.INC_NEITHER, "beta1", prim_name)
-    validator.check_float_range(beta2, 0.0, 1.0, Rel.INC_NEITHER, "beta2", prim_name)
+    validator.check_float_range(beta1, 0.0, 1.0, validator.INC_NEITHER, "beta1", prim_name)
+    validator.check_float_range(beta2, 0.0, 1.0, validator.INC_NEITHER, "beta2", prim_name)
     validator.check_positive_float(eps, "eps", prim_name)
 
 
@@ -133,7 +132,7 @@ class Lamb(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means. Refer
-        document `LossScale <https://www.mindspore.cn/tutorials/zh-CN/r2.0.0-alpha/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.0/advanced/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without