mindspore 2.4.0__cp311-cp311-macosx_10_15_x86_64.whl

mindspore/experimental/optim/nadam.py

@@ -0,0 +1,157 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""nadam"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common import Parameter, Tensor
+import mindspore.common.dtype as mstype
+from mindspore import _checkparam as validator
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
+from mindspore import jit
+
+_nadam_opt = C.MultitypeFuncGraph("nadam_opt")
+
+op_sqrt = P.Sqrt()
+
+
+@_nadam_opt.register("Number", "Number", "Number", "Number", "Tensor", "Tensor", "Tensor",
+                     "Tensor", "Tensor", "Tensor", "Tensor")
+def _tensor_run_opt(beta1, beta2, momentum_decay, eps, step_t, lr, param, grad, exp_avg, exp_avg_sq, mu_product):
+    """Apply nadam optimizer to the weight parameter."""
+    bias_correction2 = 1 - beta2 ** step_t
+    mu = beta1 * (1. - 0.5 * (0.96 ** (step_t * momentum_decay)))
+    mu_next = beta1 * (1. - 0.5 * (0.96 ** ((step_t + 1) * momentum_decay)))
+    F.assign(mu_product, mu_product * mu)
+    F.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
+    F.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
+
+    denom = op_sqrt(exp_avg_sq / bias_correction2) + eps
+
+    mu_product_next = mu_product * mu_next
+    F.assign(param, param - lr * (1. - mu) / (1. - mu_product) * grad / denom)
+    F.assign(param, param - (lr * mu_next) / (1. - mu_product_next) * exp_avg / denom)
+
+    return True
+
+
+class NAdam(Optimizer):
+    r"""
+    Implements NAdam algorithm.
+
+    .. _Incorporating Nesterov Momentum into Adam:
+        https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``2e-3``.
+        betas (Tuple[float, float], optional): coefficients used for computing
+            running averages of gradient and its square. Default: ``(0.9, 0.999)``.
+        eps (float, optional): term added to the denominator to improve
+            numerical stability. Default: ``1e-8``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
+        momentum_decay (float, optional): momentum momentum_decay. Default: ``4e-3``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `eps` is less than 0.0.
+        ValueError: If the `weight_decay` is less than 0.
+        ValueError: If the `momentum_decay` is less than 0.
+        ValueError: If elements of `betas` not in the range of [0, 1).
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.NAdam(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=2e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.0, momentum_decay=4e-3):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than_without_equal(eps, "eps", self.cls_name)
+        check_not_less_than(weight_decay, "weight_decay", self.cls_name)
+        check_not_less_than(momentum_decay, "momentum_decay", self.cls_name)
+
+        validator.check_float_range(betas[0], 0., 1., validator.INC_LEFT, "betas[0]", self.cls_name)
+        validator.check_float_range(betas[1], 0., 1., validator.INC_LEFT, "betas[1]", self.cls_name)
+
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, momentum_decay=momentum_decay)
+        super(NAdam, self).__init__(params, defaults)
+        self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
+        self.exp_avg = self.parameters.clone(prefix="exp_avg", init='zeros')
+        self.exp_avg_sq = self.parameters.clone(prefix="exp_avg_sq", init='zeros')
+        self.mu_product = [Parameter(Tensor(1.), "mu_product_" + param.name) for param in self.parameters]
+
+        self.increase_tensor = Tensor(1, mstype.int32)
+        self.assignadd = P.AssignAdd()
+        self.op_cast = P.Cast()
+
+    @jit
+    def implementation(self, lr, beta1, beta2, weight_decay, momentum_decay, eps, start_id, end_id, gradients):
+        """Extract the common computing part for acceleration"""
+        params = self.parameters[start_id: end_id]
+        grads = gradients[start_id: end_id]
+        grads = self._decay_weight(weight_decay, params, grads)
+        exp_avg = self.exp_avg[start_id: end_id]
+        exp_avg_sq = self.exp_avg_sq[start_id: end_id]
+        mu_product = self.mu_product[start_id: end_id]
+
+        self.hyper_map(F.partial(_nadam_opt, beta1, beta2, momentum_decay, eps, self.step_t, lr),
+                       params, grads, exp_avg, exp_avg_sq, mu_product)
+        return True
+
+    def construct(self, gradients):
+        self.assignadd(self.step_t, self.increase_tensor)
+        for group_id, group in enumerate(self.param_groups):
+
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+
+            beta1, beta2 = group["betas"]
+            start_id = self.group_start_id[group_id]
+            end_id = self.group_start_id[group_id + 1]
+            weight_decay = group["weight_decay"]
+            momentum_decay = group["momentum_decay"]
+            eps = group["eps"]
+            self.implementation(lr, beta1, beta2, weight_decay, momentum_decay, eps, start_id, end_id, gradients)
+        return True

mindspore/experimental/optim/optimizer.py

@@ -0,0 +1,262 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""optimizer"""
+from __future__ import absolute_import
+from collections import defaultdict
+from typing import Iterable
+from mindspore.ops import functional as F, composite as C, operations as P
+
+from mindspore.nn.cell import Cell
+from mindspore.common.parameter import Parameter, ParameterTuple
+from mindspore.common import Tensor
+import mindspore.common.dtype as mstype
+from mindspore import _checkparam as validator
+from mindspore import log as logger
+
+
+__all__ = ['Optimizer']
+
+
+class Optimizer(Cell):
+    r"""
+    Base class for all optimizers.
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): an iterable of :class:`mindspore.Parameter` or
+            dict. Specifies what Tensors should be optimized.
+        defaults (dict): a dict containing default values of optimization
+            options (used when a parameter group doesn't specify them).
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import numpy as np
+        >>> import mindspore
+        >>> from mindspore import nn, Tensor, Parameter
+        >>> from mindspore import ops
+        >>> from mindspore.experimental import optim
+        >>>
+        >>> class MySGD(optim.Optimizer):
+        ...    def __init__(self, params, lr):
+        ...        defaults = dict(lr=lr)
+        ...        super(MySGD, self).__init__(params, defaults)
+        ...
+        ...    def construct(self, gradients):
+        ...         for group_id, group in enumerate(self.param_groups):
+        ...            id = self.group_start_id[group_id]
+        ...            for i, param in enumerate(group["params"]):
+        ...                next_param = param + gradients[id+i] * group["lr"]
+        ...                ops.assign(param, next_param)
+        >>>
+        >>> net = nn.Dense(8, 2)
+        >>> data = Tensor(np.random.rand(20, 8).astype(np.float32))
+        >>> label = Tensor(np.random.rand(20, 2).astype(np.float32))
+        >>>
+        >>> optimizer = MySGD(net.trainable_params(), 0.01)
+        >>> optimizer.add_param_group({"params": Parameter([0.01, 0.02])})
+        >>>
+        >>> criterion = nn.MAELoss(reduction="mean")
+        >>>
+        >>> def forward_fn(data, label):
+        ...    logits = net(data)
+        ...    loss = criterion(logits, label)
+        ...    return loss, logits
+        >>>
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>>
+        >>> def train_step(data, label):
+        ...    (loss, _), grads = grad_fn(data, label)
+        ...    optimizer(grads)
+        ...    print(loss)
+        >>>
+        >>> train_step(data, label)
+    """
+    def __init__(self, params, defaults):
+        super(Optimizer, self).__init__(auto_prefix=False)
+
+        param_groups = self._parameters_base_check(params, "params")
+        self.defaults = defaults
+        self.state = defaultdict(dict)
+        self.param_groups = []
+        self.parameters = []
+        self.lrs = []
+        self.map_ = C.Map()
+        self.group_start_id = [0]
+        if not isinstance(param_groups[0], dict):
+            param_groups = [{'params': param_groups}]
+
+        for param_group in param_groups:
+            self.add_param_group(param_group)
+        self.parameters = ParameterTuple(self.parameters)
+        self.hyper_map = C.HyperMap()
+        self.enable_tuple_broaden = True
+
+    def __repr__(self):
+        format_string = self.__class__.__name__ + ' ('
+        for i, group in enumerate(self.param_groups):
+            format_string += '\n'
+            format_string += 'Parameter Group {0}\n'.format(i)
+            for key in sorted(group.keys()):
+                if key != 'params':
+                    format_string += '    {0}: {1}\n'.format(key, group[key].value()) \
+                        if key == "lr" and isinstance(group[key], Parameter) \
+                        else '    {0}: {1}\n'.format(key, group[key])
+        format_string += ')'
+        return format_string
+
+    def add_param_group(self, param_group):
+        r"""
+        Add a param group to the `Optimizer.param_groups`.
+
+        Args:
+            param_group (dict): Specifies what Parameters should be optimized along with group
+                specific optimization options.
+        """
+        group_id = len(self.param_groups)
+        param_group = self._preprocess_param_group(param_group)
+        self.parameters += tuple(param_group.get("params"))
+
+        for name, default in self.defaults.items():
+            if name not in param_group:
+                param_group.setdefault(name, default)
+
+        lr = self._build_single_lr(param_group.get("lr"), 'learning_rate_group_' + str(group_id))
+        weight_decay = self._preprocess_weight_decay(param_group.get("weight_decay", 0.0))
+        self.lrs.append(lr)
+        param_group["lr"] = lr
+        param_group["weight_decay"] = weight_decay
+        if "amsgrad" in param_group and param_group.get("amsgrad") and hasattr(self, 'max_v_group'):
+            param_items = ParameterTuple(tuple(param_group.get("params")))
+            param_group["max_exp_avg_sq"] = param_items.clone(prefix="max_exp_avg_sq", init='zeros')
+        self.param_groups.append(param_group)
+        self.group_start_id.append(self.group_start_id[-1] + len(param_group.get("params")))
+
+    @staticmethod
+    def _parameters_base_check(parameters, param_info):
+        """Parameters base check."""
+        if parameters is None:
+            raise ValueError(f"For 'Optimizer', the argument {param_info} can not be None.")
+        if not isinstance(parameters, Iterable):
+            raise TypeError(f"For 'Optimizer', the argument {param_info} must be Iterable type, "
+                            f"but got {type(parameters)}.")
+        parameters = list(parameters)
+
+        if not parameters:
+            raise ValueError(f"For 'Optimizer', the argument {param_info} must not be empty.")
+        return parameters
+
+    def _decay_weight(self, weight_decay, params, gradients):
+        """Apply weight decay."""
+        if weight_decay != 0.:
+            weight_decay = Tensor(weight_decay, mstype.float32)
+            gradients = self.map_(F.partial(_apply_decay, weight_decay), params, gradients)
+        return gradients
+
+    def _preprocess_param_group(self, param_group):
+        """Preprocess param groups."""
+        if not isinstance(param_group, dict):
+            raise TypeError('Param group must be a dict.')
+
+        params = param_group['params']
+        if isinstance(params, Parameter):
+            param_group['params'] = [params]
+        elif isinstance(params, set):
+            raise TypeError('Optimizer parameters need to be organized in ordered collections, but '
+                            'the ordering of tensors in sets will change between runs. '
+                            'Please use a list instead.')
+        else:
+            param_group['params'] = list(params)
+
+        for param in param_group['params']:
+            if not isinstance(param, Parameter):
+                raise TypeError("Optimizer can only optimize Parameters, but one of the params is " + type(param))
+
+        if len(param_group['params']) != len(set(param_group['params'])):
+            logger.warning("Optimizer contains a parameter group with duplicate parameters.")
+
+        param_set = set()
+        for group in self.param_groups:
+            param_set.update(set(group['params']))
+        if not param_set.isdisjoint(set(param_group['params'])):
+            raise ValueError("some parameters appear in more than one parameter group.")
+        return param_group
+
+    def _build_single_lr(self, learning_rate, name):
+        """Check lr value, and convert lr to a float or a Tensor."""
+        if isinstance(learning_rate, (float, int)):
+            learning_rate = float(learning_rate)
+            validator.check_non_negative_float(learning_rate, "learning rate", self.cls_name)
+            return Parameter(Tensor(learning_rate, mstype.float32), name)
+
+        if isinstance(learning_rate, Tensor):
+            if learning_rate.ndim == 0:
+                return Parameter(learning_rate.astype(mstype.float32), name)
+            raise ValueError(f"For 'Optimizer', if 'learning_rate' is a Tensor, "
+                             f"then it should be scalar Tensor")
+
+        raise TypeError("For 'Optimizer', the argument 'learning_rate' must be int, float or Tensor, "
+                        "but got {}.".format(type(learning_rate)))
+
+    def _preprocess_weight_decay(self, weight_decay):
+        """preprocess weight decay"""
+        if isinstance(weight_decay, (float, int)):
+            weight_decay = float(weight_decay)
+            validator.check_non_negative_float(weight_decay, "weight_decay", self.cls_name)
+        else:
+            raise TypeError("For 'Optimizer', the argument 'Weight_decay' must be int or "
+                            "float.but got {}".format(type(weight_decay)))
+        return weight_decay
+
+    def construct(self, *hyper_params):
+        raise NotImplementedError
+
+op_add = P.AddN()
+op_gather = P.Gather()
+op_mul = P.Mul()
+
+_apply_decay = C.MultitypeFuncGraph("apply_decay")
+
+
+@_apply_decay.register("Tensor", "Tensor", "RowTensor")
+def _tensor_apply_decay_with_sparse(weight_decay, weight, gradient):
+    """Get grad with weight_decay."""
+    indices = gradient.indices
+    values = op_add((op_gather(weight, indices, 0) * F.cast(weight_decay, F.dtype(weight)), gradient.values))
+    shape = gradient.dense_shape
+    return RowTensorInner(indices, values, shape)
+
+
+@_apply_decay.register("Tensor", "Tensor", "Tensor")
+def _tensor_apply_decay(weight_decay, weight, gradient):
+    """Get grad with weight_decay."""
+    return op_add((op_mul(weight, F.cast(weight_decay, F.dtype(weight))), gradient))
+
+
+def check_not_less_than(arg_value, arg_name, prim, value=0.0):
+    if arg_value < value:
+        raise ValueError("For {}, the {} must be greater than or equal to {}, "
+                         "but got {}.".format(prim, arg_name, value, arg_value))
+
+
+def check_not_less_than_without_equal(arg_value, arg_name, prim, value=0.0):
+    if arg_value <= value:
+        raise ValueError("For {}, the {} must be greater than {}, "
+                         "but got {}.".format(prim, arg_name, value, arg_value))

mindspore/experimental/optim/radam.py

@@ -0,0 +1,194 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""radam"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common import Tensor, Parameter
+import mindspore.common.dtype as mstype
+from mindspore import _checkparam as validator
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
+from mindspore import jit
+
+_radam_opt = C.MultitypeFuncGraph("radam_opt")
+
+op_pow = P.Pow()
+op_sqrt = P.Sqrt()
+op_cast = P.Cast()
+
+
+@_radam_opt.register("Number", "Number", "Number", "Tensor", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
+                     "Tensor", "Tensor")
+def _tensor_run_opt(beta1, beta2, eps, lr, rho_inf, rho_t, bias_correction1, bias_correction2, param, grad, exp_avg,
+                    exp_avg_sq):
+    """Apply radam optimizer to the weight parameter."""
+
+    F.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
+    F.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
+    bias_corrected_exp_avg = exp_avg / bias_correction1
+
+    if rho_t > 5.0:
+        rect = op_sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t))
+        exp_avg_sq_sqrt = op_sqrt(exp_avg_sq) + eps
+        adaptive_lr = op_sqrt(bias_correction2) / exp_avg_sq_sqrt
+        F.assign(param, param - bias_corrected_exp_avg * lr * adaptive_lr * rect)
+    else:
+        F.assign(param, param - bias_corrected_exp_avg * lr)
+
+    return True
+
+
+class RAdam(Optimizer):
+    r"""
+    Implements RAdam algorithm.
+
+    .. math::
+        \begin{aligned}
+            &\rule{110mm}{0.4pt}                                                                 \\
+            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
+                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
+                \lambda \text{ (weightdecay)},                                                   \\
+            &\hspace{13mm} \epsilon \text{ (epsilon)}                                            \\
+            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
+                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
+            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
+            &\rule{110mm}{0.4pt}  \\
+            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
+            &\hspace{6mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
+            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
+            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
+            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
+            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
+            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
+            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
+                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                     \\[-1.ex]
+            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
+            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
+            &\hspace{12mm} r_t \leftarrow
+                \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
+            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t} r_t l_t        \\
+            &\hspace{6mm}\textbf{else}                                                           \\
+            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}                \\
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+        \end{aligned}
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-3``.
+        betas (Tuple[float, float], optional): coefficients used for computing
+            running averages of gradient and its square. Default: ``(0.9, 0.999)``.
+        eps (float, optional): term added to the denominator to improve
+            numerical stability. Default: ``1e-8``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.0``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `eps` is less than 0.0.
+        ValueError: If the `weight_decay` is less than 0.
+        ValueError: If elements of `betas` not in the range of [0, 1).
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.RAdam(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.0):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than(weight_decay, "weight_decay", self.cls_name)
+        check_not_less_than_without_equal(eps, "eps", self.cls_name)
+        validator.check_float_range(betas[0], 0., 1., validator.INC_LEFT, "betas[0]", self.cls_name)
+        validator.check_float_range(betas[1], 0., 1., validator.INC_LEFT, "betas[1]", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            betas=betas,
+            eps=eps,
+            weight_decay=weight_decay,
+        )
+        super(RAdam, self).__init__(params, defaults)
+        self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
+        self.exp_avg = self.parameters.clone(prefix="exp_avg", init='zeros')
+        self.exp_avg_sq = self.parameters.clone(prefix="exp_avg_sq", init='zeros')
+        self.increase_tensor = Tensor(1, mstype.int32)
+        self.assignadd = P.AssignAdd()
+
+    @jit
+    def implementation(self, lr, beta1, beta2, weight_decay, eps, start_id, end_id, gradients):
+        """Extract the common computing part for acceleration"""
+        params = self.parameters[start_id: end_id]
+        grads = gradients[start_id: end_id]
+        grads = self._decay_weight(weight_decay, params, grads)
+        exp_avg = self.exp_avg[start_id: end_id]
+        exp_avg_sq = self.exp_avg_sq[start_id: end_id]
+
+        bias_correction1 = 1 - op_pow(beta1, self.step_t.value())
+        bias_correction2 = 1 - op_pow(beta2, self.step_t.value())
+
+        rho_inf = 2 / (1 - beta2) - 1
+        beta2_pow = op_pow(beta2, self.step_t.value())
+        right = 2 * self.step_t.value() * beta2_pow / bias_correction2
+
+        rho_t = rho_inf - right
+
+        self.hyper_map(F.partial(_radam_opt, beta1, beta2, eps, lr, rho_inf, rho_t, bias_correction1, bias_correction2),
+                       params, grads, exp_avg, exp_avg_sq)
+        return True
+
+    def construct(self, gradients):
+        self.assignadd(self.step_t, self.increase_tensor)
+        for group_id, group in enumerate(self.param_groups):
+
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = op_cast(group.get("lr"), mstype.float32)
+
+            beta1, beta2 = group["betas"]
+            start_id = self.group_start_id[group_id]
+            end_id = self.group_start_id[group_id + 1]
+            weight_decay = group["weight_decay"]
+            eps = group["eps"]
+            self.implementation(lr, beta1, beta2, weight_decay, eps, start_id, end_id, gradients)
+
+        return True