mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/experimental/optim/optimizer.py

@@ -36,7 +36,7 @@ class Optimizer(Cell):
     .. warning::
         This is an experimental optimizer API that is subject to change.
         This module must be used with lr scheduler module in `LRScheduler Class
-        <https://www.mindspore.cn/docs/en/r2.2/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
 
     Args:
         params (Union[list(Parameter), list(dict)]): an iterable of :class:`mindspore.Parameter` or
@@ -44,13 +44,6 @@ class Optimizer(Cell):
         defaults (dict): a dict containing default values of optimization
             options (used when a parameter group doesn't specify them).
 
-    Raises:
-        TypeError: If `learning_rate` is not one of int, float, Tensor.
-        TypeError: If element of `parameters` is neither Parameter nor dict.
-        TypeError: If `weight_decay` is neither float nor int.
-        ValueError: If `weight_decay` is less than 0.
-        ValueError: If `learning_rate` is a Tensor, but the dimension of tensor is greater than 1.
-
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
@@ -104,6 +97,7 @@ class Optimizer(Cell):
         self.state = defaultdict(dict)
         self.param_groups = []
         self.parameters = []
+        self.lrs = []
         self.map_ = C.Map()
         self.group_start_id = [0]
         if not isinstance(param_groups[0], dict):
@@ -146,8 +140,12 @@ class Optimizer(Cell):
 
         lr = self._build_single_lr(param_group.get("lr"), 'learning_rate_group_' + str(group_id))
         weight_decay = self._preprocess_weight_decay(param_group.get("weight_decay", 0.0))
+        self.lrs.append(lr)
         param_group["lr"] = lr
         param_group["weight_decay"] = weight_decay
+        if "amsgrad" in param_group and param_group.get("amsgrad") and hasattr(self, 'max_v_group'):
+            param_items = ParameterTuple(tuple(param_group.get("params")))
+            param_group["max_exp_avg_sq"] = param_items.clone(prefix="max_exp_avg_sq", init='zeros')
         self.param_groups.append(param_group)
         self.group_start_id.append(self.group_start_id[-1] + len(param_group.get("params")))
 
@@ -250,3 +248,15 @@ def _tensor_apply_decay_with_sparse(weight_decay, weight, gradient):
 def _tensor_apply_decay(weight_decay, weight, gradient):
     """Get grad with weight_decay."""
     return op_add((op_mul(weight, F.cast(weight_decay, F.dtype(weight))), gradient))
+
+
+def check_not_less_than(arg_value, arg_name, prim, value=0.0):
+    if arg_value < value:
+        raise ValueError("For {}, the {} must be greater than or equal to {}, "
+                         "but got {}.".format(prim, arg_name, value, arg_value))
+
+
+def check_not_less_than_without_equal(arg_value, arg_name, prim, value=0.0):
+    if arg_value <= value:
+        raise ValueError("For {}, the {} must be greater than {}, "
+                         "but got {}.".format(prim, arg_name, value, arg_value))

mindspore/experimental/optim/radam.py

@@ -0,0 +1,194 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""radam"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common import Tensor, Parameter
+import mindspore.common.dtype as mstype
+from mindspore import _checkparam as validator
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
+from mindspore import jit
+
+_radam_opt = C.MultitypeFuncGraph("radam_opt")
+
+op_pow = P.Pow()
+op_sqrt = P.Sqrt()
+op_cast = P.Cast()
+
+
+@_radam_opt.register("Number", "Number", "Number", "Tensor", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
+                     "Tensor", "Tensor")
+def _tensor_run_opt(beta1, beta2, eps, lr, rho_inf, rho_t, bias_correction1, bias_correction2, param, grad, exp_avg,
+                    exp_avg_sq):
+    """Apply radam optimizer to the weight parameter."""
+
+    F.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
+    F.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
+    bias_corrected_exp_avg = exp_avg / bias_correction1
+
+    if rho_t > 5.0:
+        rect = op_sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t))
+        exp_avg_sq_sqrt = op_sqrt(exp_avg_sq) + eps
+        adaptive_lr = op_sqrt(bias_correction2) / exp_avg_sq_sqrt
+        F.assign(param, param - bias_corrected_exp_avg * lr * adaptive_lr * rect)
+    else:
+        F.assign(param, param - bias_corrected_exp_avg * lr)
+
+    return True
+
+
+class RAdam(Optimizer):
+    r"""
+    Implements RAdam algorithm.
+
+    .. math::
+        \begin{aligned}
+            &\rule{110mm}{0.4pt}                                                                 \\
+            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
+                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
+                \lambda \text{ (weightdecay)},                                                   \\
+            &\hspace{13mm} \epsilon \text{ (epsilon)}                                            \\
+            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
+                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
+            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
+            &\rule{110mm}{0.4pt}  \\
+            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
+            &\hspace{6mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
+            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
+            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
+            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
+            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
+            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
+            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
+                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                     \\[-1.ex]
+            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
+            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
+            &\hspace{12mm} r_t \leftarrow
+                \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
+            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t} r_t l_t        \\
+            &\hspace{6mm}\textbf{else}                                                           \\
+            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}                \\
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+        \end{aligned}
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-3``.
+        betas (Tuple[float, float], optional): coefficients used for computing
+            running averages of gradient and its square. Default: ``(0.9, 0.999)``.
+        eps (float, optional): term added to the denominator to improve
+            numerical stability. Default: ``1e-8``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.0``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `eps` is less than 0.0.
+        ValueError: If the `weight_decay` is less than 0.
+        ValueError: If elements of `betas` not in the range of [0, 1).
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.RAdam(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.0):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than(weight_decay, "weight_decay", self.cls_name)
+        check_not_less_than_without_equal(eps, "eps", self.cls_name)
+        validator.check_float_range(betas[0], 0., 1., validator.INC_LEFT, "betas[0]", self.cls_name)
+        validator.check_float_range(betas[1], 0., 1., validator.INC_LEFT, "betas[1]", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            betas=betas,
+            eps=eps,
+            weight_decay=weight_decay,
+        )
+        super(RAdam, self).__init__(params, defaults)
+        self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
+        self.exp_avg = self.parameters.clone(prefix="exp_avg", init='zeros')
+        self.exp_avg_sq = self.parameters.clone(prefix="exp_avg_sq", init='zeros')
+        self.increase_tensor = Tensor(1, mstype.int32)
+        self.assignadd = P.AssignAdd()
+
+    @jit
+    def implementation(self, lr, beta1, beta2, weight_decay, eps, start_id, end_id, gradients):
+        """Extract the common computing part for acceleration"""
+        params = self.parameters[start_id: end_id]
+        grads = gradients[start_id: end_id]
+        grads = self._decay_weight(weight_decay, params, grads)
+        exp_avg = self.exp_avg[start_id: end_id]
+        exp_avg_sq = self.exp_avg_sq[start_id: end_id]
+
+        bias_correction1 = 1 - op_pow(beta1, self.step_t.value())
+        bias_correction2 = 1 - op_pow(beta2, self.step_t.value())
+
+        rho_inf = 2 / (1 - beta2) - 1
+        beta2_pow = op_pow(beta2, self.step_t.value())
+        right = 2 * self.step_t.value() * beta2_pow / bias_correction2
+
+        rho_t = rho_inf - right
+
+        self.hyper_map(F.partial(_radam_opt, beta1, beta2, eps, lr, rho_inf, rho_t, bias_correction1, bias_correction2),
+                       params, grads, exp_avg, exp_avg_sq)
+        return True
+
+    def construct(self, gradients):
+        self.assignadd(self.step_t, self.increase_tensor)
+        for group_id, group in enumerate(self.param_groups):
+
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = op_cast(group.get("lr"), mstype.float32)
+
+            beta1, beta2 = group["betas"]
+            start_id = self.group_start_id[group_id]
+            end_id = self.group_start_id[group_id + 1]
+            weight_decay = group["weight_decay"]
+            eps = group["eps"]
+            self.implementation(lr, beta1, beta2, weight_decay, eps, start_id, end_id, gradients)
+
+        return True

mindspore/experimental/optim/rmsprop.py

@@ -0,0 +1,154 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""rmsprop"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+import mindspore.common.dtype as mstype
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
+from mindspore import ops
+from mindspore import jit
+
+_rmsprop_opt = C.MultitypeFuncGraph("rmsprop_opt")
+
+op_mul = P.Mul()
+op_sqrt = P.Sqrt()
+
+
+@_rmsprop_opt.register("Bool", "Number", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
+def _run_rmsprop_opt(centered, alpha, eps, momentum, lr, weight, mean_square, mean_grad, mom, grad):
+    """Apply rmsprop optimizer to the weight parameter using dynamic learning rate."""
+    F.assign(mean_square, ops.addcmul(op_mul(mean_square, alpha), grad, grad, 1 - alpha))
+
+    if centered:
+        F.assign(mean_grad, op_mul(mean_grad, alpha) + op_mul(grad, 1 - alpha))
+        avg = op_sqrt(ops.addcmul(mean_square, mean_grad, mean_grad, -1.)) + eps
+    else:
+        avg = op_sqrt(mean_square) + eps
+
+    if momentum > 0:
+        F.assign(mom, op_mul(mom, momentum) + grad / avg)
+        F.assign(weight, weight - mom * lr)
+    else:
+        F.assign(weight, weight - lr * grad / avg)
+    return True
+
+
+class RMSprop(Optimizer):
+    r"""
+    Implements RMSprop algorithm.
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-2``.
+        alpha (float, optional): smoothing constant. Default: ``0.99``.
+        eps (float, optional): term added to the denominator to improve numerical stability. Default: ``1e-8``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
+        momentum (float, optional): momentum factor. Default: ``0.``.
+        centered (bool, optional) : if ``True``, compute the centered RMSProp,
+            the gradient is normalized by an estimation of its variance. Default: ``False``.
+        maximize (bool, optional): maximize the params based on the objective, instead of minimizing.
+            Default: ``False``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `momentum` is less than 0.0.
+        ValueError: If the `alpha` is less than 0.0.
+        ValueError: If the `eps` is less than 0.0.
+        ValueError: If the `weight_decay` is less than 0.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.RMSprop(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-2, alpha=0.99, eps=1e-8, weight_decay=0.0, momentum=0.0,
+                 centered=False, maximize=False):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than(alpha, "alpha", self.cls_name)
+        check_not_less_than_without_equal(eps, "eps", self.cls_name)
+        check_not_less_than(momentum, "momentum", self.cls_name)
+        check_not_less_than(weight_decay, "weight_decay", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            momentum=momentum,
+            alpha=alpha,
+            eps=eps,
+            centered=centered,
+            weight_decay=weight_decay,
+            maximize=maximize,
+        )
+        super(RMSprop, self).__init__(params, defaults)
+
+        self.mean_grad = self.parameters.clone(prefix="mean_grad", init='zeros')
+        self.mean_square = self.parameters.clone(prefix="mean_square", init='zeros')
+        self.moment = self.parameters.clone(prefix="moment", init='zeros')
+        self.op_cast = P.Cast()
+
+    @jit
+    def implementation(self, group_id, lr, gradients, maximize, weight_decay, centered, alpha, eps, momentum):
+        """Extract the common computing part for acceleration"""
+        start_id = self.group_start_id[group_id]
+        end_id = self.group_start_id[group_id + 1]
+        params = self.parameters[start_id: end_id]
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = self._decay_weight(weight_decay, params, grads)
+        mean_grad = self.mean_grad[start_id: end_id]
+        mean_square = self.mean_square[start_id: end_id]
+        moment = self.moment[start_id: end_id]
+        self.hyper_map(F.partial(_rmsprop_opt, centered, alpha, eps, momentum, lr),
+                       params, mean_square, mean_grad, moment, grads)
+        return True
+
+    def construct(self, gradients):
+        for group_id, group in enumerate(self.param_groups):
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+            maximize = group.get("maximize")
+
+            self.implementation(group_id, lr, gradients, maximize, group["weight_decay"], group["centered"],
+                                group["alpha"], group["eps"],
+                                group["momentum"])
+        return True

mindspore/experimental/optim/rprop.py

@@ -0,0 +1,164 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""rprop"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common import Tensor, Parameter
+import mindspore.common.dtype as mstype
+from mindspore import _checkparam as validator
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than_without_equal
+from mindspore import ops
+from mindspore import jit
+
+_rprop_opt = C.MultitypeFuncGraph("rprop_opt")
+
+op_sign = P.Sign()
+op_fill = P.FillV2()
+op_assign = P.Assign()
+op_assignadd = P.AssignAdd()
+op_cast = P.Cast()
+op_select = P.Select()
+op_oneslike = P.OnesLike()
+
+
+@_rprop_opt.register("Tensor", "Tensor", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
+def _tensor_run_opt(etaminus, etaplus, step_size_min, step_size_max, step, lr, param, prev, step_size, gradient):
+    """Apply rprop optimizer to the weight parameter."""
+    if step == 1:
+        step_size_value = op_oneslike(step_size) * lr
+    else:
+        step_size_value = step_size.value()
+
+    sign = op_sign(gradient * prev)
+
+    sign[sign.gt(0)] = etaplus
+    sign[sign.lt(0)] = etaminus
+    sign[sign.eq(0)] = 1
+
+    step_size_clip = ops.clip_by_value(step_size_value * sign, step_size_min, step_size_max)
+    op_assign(step_size, step_size_clip)
+
+    gradient_update = op_select(sign == etaminus, op_fill(sign.shape, op_cast(0., mstype.float32)), gradient)
+
+    op_assign(prev, gradient_update)
+    next_param = param - op_sign(gradient_update) * step_size_clip
+
+    op_assign(param, next_param)
+
+    return True
+
+
+class Rprop(Optimizer):
+    r"""
+    Implements Rprop algorithm.
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-2``.
+        etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that
+            are multiplicative increase and decrease factors. Default:``(0.5, 1.2)``
+        step_sizes (Tuple[float, float], optional): a pair of minimal and
+            maximal allowed step sizes. Default:``(1e-6, 50)``
+
+    Keyword Args:
+        maximize (bool, optional): maximize the params based on the objective, instead of minimizing.
+            Default: ``False``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `etas[1]` is less than or equal to 1.0.
+        ValueError: If the `etas[0]` not in the range of 0-1.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.Rprop(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-2, etas=(0.5, 1.2), step_sizes=(1e-6, 50), *, maximize=False):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than_without_equal(etas[1], "etas[1]", self.cls_name, 1.)
+        validator.check_float_range(etas[0], 0., 1., validator.INC_NEITHER, "etas[0]", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            etas=etas,
+            step_sizes=step_sizes,
+            maximize=maximize,
+        )
+        super(Rprop, self).__init__(params, defaults)
+        self.prev = self.parameters.clone(prefix="prev", init='zeros')
+        self.step_size = self.parameters.clone(prefix="step_size", init='zeros')
+        self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
+        self.increase_tensor = Tensor(1, mstype.int32)
+        self.op_cast = P.Cast()
+
+    @jit
+    def implementation(self, etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max):
+        """Extract the common computing part for acceleration"""
+        etaminus, etaplus = op_cast(etaminus, mstype.float32), op_cast(etaplus, mstype.float32)
+        start_id = self.group_start_id[group_id]
+        end_id = self.group_start_id[group_id + 1]
+
+        params = self.parameters[start_id: end_id]
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        prev = self.prev[start_id: end_id]
+        step_size = self.step_size[start_id: end_id]
+        self.hyper_map(F.partial(_rprop_opt, etaminus, etaplus, step_size_min, step_size_max, self.step_t, lr),
+                       params, prev, step_size, grads)
+        return True
+
+    def construct(self, gradients):
+        op_assignadd(self.step_t, self.increase_tensor)
+        for group_id, group in enumerate(self.param_groups):
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+            maximize = group.get("maximize")
+
+            etaminus, etaplus = group["etas"]
+            step_size_min, step_size_max = group["step_sizes"]
+
+            self.implementation(etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max)
+
+        return True

mindspore/experimental/optim/sgd.py

@@ -20,16 +20,16 @@ from mindspore.common.tensor import Tensor
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as Validator
 from mindspore.experimental.optim.optimizer import Optimizer
+from mindspore import jit
 
 _sgd_opt = C.MultitypeFuncGraph("sgd_opt")
 
 
-@_sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",)
+@_sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
 def _tensor_run_opt_ext(opt, momentum, learning_rate, gradient, weight, accum, stat):
     """Apply sgd optimizer to the weight parameter using Tensor."""
-    success = True
-    success = F.depend(success, opt(weight, gradient, learning_rate, accum, momentum, stat))
-    return success
+    opt(weight, gradient, learning_rate, accum, momentum, stat)
+    return True
 
 
 class SGD(Optimizer):
@@ -56,14 +56,14 @@ class SGD(Optimizer):
     .. warning::
         This is an experimental optimizer API that is subject to change.
         This module must be used with lr scheduler module in `LRScheduler Class
-        <https://www.mindspore.cn/docs/en/r2.2/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
 
     Args:
         params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
             parameter groups.
         lr (Union[int, float, Tensor]): learning rate.
         momentum (Union[int, float], optional): momentum factor. Default: ``0``.
-        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
         dampening (Union[int, float], optional): dampening for momentum. Default: ``0``.
         nesterov (bool, optional): enables Nesterov momentum. Default: ``False``.
 
@@ -90,7 +90,7 @@ class SGD(Optimizer):
         >>> from mindspore import nn
         >>> from mindspore.experimental import optim
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
         >>> optimizer = optim.SGD(net.trainable_params(), lr=0.1)
@@ -104,7 +104,8 @@ class SGD(Optimizer):
         ...     optimizer(grads)
         ...     return loss
     """
-    def __init__(self, params, lr, momentum=0, dampening=0, weight_decay=0, nesterov=False, *,
+
+    def __init__(self, params, lr, momentum=0, dampening=0, weight_decay=0.0, nesterov=False, *,
                  maximize=False):
         Validator.check_value_type("lr", lr, [float, int, Tensor], self.cls_name)
         if lr < 0.0:
@@ -130,18 +131,26 @@ class SGD(Optimizer):
         self.stat = self.parameters.clone(prefix="stat", init='ones')
         self.op_cast = P.Cast()
 
+    @jit
+    def implementation(self, momentum, lr, group_id, gradients, maximize, dampening, weight_decay, nesterov):
+        """Extract the common computing part for acceleration"""
+        start_id = self.group_start_id[group_id]
+        end_id = self.group_start_id[group_id + 1]
+        momentum = self.op_cast(momentum, mstype.float32)
+        opt = P.SGD(dampening, weight_decay, nesterov)
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        self.hyper_map(F.partial(_sgd_opt, opt, momentum, lr), grads,
+                       self.parameters[start_id: end_id], self.accum[start_id: end_id],
+                       self.stat[start_id: end_id])
+        return True
+
     def construct(self, gradients):
         for group_id, group in enumerate(self.param_groups):
-            opt = P.SGD(group.get("dampening"), group.get("weight_decay"), group.get("nesterov"))
-            lr = group.get("lr")
-            if isinstance(lr, float):
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
                 lr = self.op_cast(group.get("lr"), mstype.float32)
-            maximize = group.get("maximize")
-            momentum = self.op_cast(group.get("momentum"), mstype.float32)
-            start_id = self.group_start_id[group_id]
-            end_id = self.group_start_id[group_id+1]
-            grads = gradients[start_id: end_id] if not maximize else -gradients[start_id: end_id]
-            self.hyper_map(F.partial(_sgd_opt, opt, momentum, lr), grads,
-                           self.parameters[start_id: end_id], self.accum[start_id: end_id],
-                           self.stat[start_id: end_id])
+
+            self.implementation(group.get("momentum"), lr, group_id, gradients, group.get("maximize"),
+                                group.get("dampening"),
+                                group.get("weight_decay"), group.get("nesterov"))
         return True