mindspore 2.4.0__cp311-cp311-win_amd64.whl

mindspore/experimental/map_parameter.py

@@ -0,0 +1,309 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""MapParameter implementation."""
+from __future__ import absolute_import
+
+__all__ = ['MapParameter']
+
+import os
+import sys
+from copy import copy
+import numbers
+import mindspore as ms
+from mindspore.common.parameter import Parameter, _get_unique_parameter_key
+from mindspore._c_expression import Tensor as Tensor_
+from mindspore._c_expression import MapTensor_
+from mindspore.ops.operations import _map_tensor_ops
+
+
+class MapParameter(Parameter):
+    """
+    MapParameter is a parameter that stores a map like data structure.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        key_dtype (:class:`mindspore.dtype`): The data type of the key. The argument should be defined in
+            `mindspore.dtype`, currently only integer types are supported. Default: int32.
+        value_dtype (:class:`mindspore.dtype`): The data type of the value Tensor. The argument should
+            be defined in `mindspore.dtype`. Default: float32.
+        value_shape (Union[tuple, list, int]): Used to indicate the shape of the value Tensor. The argument should be
+            a list of integers, a tuple of integers or an integer. Default: 1.
+        key_tensor (:class:`mindspore.tensor`): The key Tensor.
+        value_tensor (:class:`mindspore.tensor`): The value Tensor.
+        default_value (Union[numbers.Number, str]): The default value number or initializer name. Default: 'normal'.
+        permit_filter_value (numbers.Number): The permit filter value number. Default: 1.
+        evict_filter_value (numbers.Number): The evict filter value number. Default: MAX_SIZE.
+        name (str): Name of the map parameter. Default: ``None``.
+        requires_grad (bool): True if the parameter requires gradient. Default: True.
+
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import Tensor
+        >>> from mindspore.experimental import MapParameter
+        >>>
+        >>> m = MapParameter(key_dtype=ms.int32, value_dtype=ms.float32, value_shape=(3), default_value='zeros')
+        >>> t = m.get(Tensor([1, 2, 3], dtype=ms.int32))
+        [[0. 0. 0.]
+         [0. 0. 0.]
+         [0. 0. 0.]]
+        >>> m.put(Tensor([1, 2], dtype=ms.int32), Tensor([[1, 1, 1], [2, 2, 2]], dtype=ms.float32))
+        >>> t = m.get(Tensor([1, 2, 3], dtype=ms.int32))
+        >>> print(t)
+        [[1. 1. 1.]
+         [2. 2. 2.]
+         [0. 0. 0.]]
+        >>> m.erase(Tensor([2, 3], dtype=ms.int32))
+        >>> print(t)
+        [[1. 1. 1.]]
+
+    """
+
+    def __new__(cls, key_dtype=None, value_dtype=None, value_shape=None, key_tensor=None, value_tensor=None,
+                default_value='normal', permit_filter_value=1, evict_filter_value=sys.maxsize, **kwargs):
+        if value_dtype is not None:
+            if isinstance(value_shape, numbers.Number):
+                value_shape = (value_shape,)
+            data = Tensor_(value_dtype, value_shape)
+        elif value_tensor is not None:
+            data = Tensor_(value_tensor.dtype, value_tensor.shape)
+        else:
+            # default
+            data = Tensor_(ms.float32, (1,))
+        obj = Tensor_.__new__(cls)
+        Tensor_.__init__(obj, data)
+        # Compatible attributes with Parameter.
+        obj.has_init = False
+        obj.init_mode = None
+        obj.is_default_input_init = False
+        # MapParameter added attributes.
+        MapParameter._check_map_parameter_args(key_tensor, key_dtype, value_tensor, value_dtype, value_shape)
+        if key_tensor is not None:
+            obj.key_dtype = key_tensor.dtype
+        else:
+            obj.key_dtype = key_dtype if key_dtype is not None else ms.int32
+
+        if value_tensor is not None:
+            obj.value_dtype = value_tensor.dtype
+        else:
+            obj.value_dtype = value_dtype if value_dtype is not None else ms.float32
+
+        if value_tensor is not None:
+            obj.value_shape = value_tensor.shape
+        else:
+            obj.value_shape = value_shape if value_shape is not None else (1,)
+
+        obj.default_value = default_value
+        obj.permit_filter_value = permit_filter_value
+        obj.evict_filter_value = evict_filter_value
+        obj.key_tensor = key_tensor
+        obj.value_tensor = value_tensor
+        return obj
+
+    def __init__(self, name=None, requires_grad=True, **kwargs):
+        Parameter.__init__(self, self, name=name, requires_grad=requires_grad)
+        if self.key_tensor is not None and self.value_tensor is not None:
+            self._map_tensor = MapTensor_(self.key_tensor, self.value_tensor, self.default_value,
+                                          self.permit_filter_value, self.evict_filter_value)
+        else:
+            self._map_tensor = MapTensor_(self.key_dtype, self.value_dtype, self.value_shape, self.default_value,
+                                          self.permit_filter_value, self.evict_filter_value)
+        self.map_put = _map_tensor_ops.put
+        self.map_erase = _map_tensor_ops.erase
+
+    def __getitem__(self, key_tensor):
+        return self.get(key_tensor, True)
+
+    def __setitem__(self, key_tensor, value_tensor):
+        return self.put(key_tensor, value_tensor)
+
+    def __str__(self):
+        return 'MapParameter(' + str(self._map_tensor) + ')'
+
+    def __copy__(self):
+        x = type(self)()
+        x.__dict__.update(self.__dict__)
+        return x
+
+    @staticmethod
+    def _check_map_parameter_args(key_tensor, key_dtype, value_tensor, value_dtype, value_shape):
+        if key_dtype is not None and key_tensor is not None and key_dtype != key_tensor.dtype:
+            raise ValueError(f"When initializing a MapParameter, 'key_dtype' and 'key_tensor.dtype' should be set the"
+                             f" same.")
+        if value_dtype is not None and value_tensor is not None and value_dtype != value_tensor.dtype:
+            raise ValueError(f"When initializing a MapParameter, 'value_dtype' and 'value_tensor.dtype' should be set "
+                             f"the same.")
+        if value_shape is not None and value_tensor is not None and value_shape != value_tensor.shape:
+            raise ValueError(f"When initializing a map_parameter, 'value_shape' and 'value_tensor.shape' should be set "
+                             f"the same.")
+
+    def clone(self, init='same'):
+        """
+        Clone the MapParameter.
+
+        Args:
+            init (Union[str, numbers.Number]): Initialize the default value of the new map parameter.
+                If `init` is a `numbers.Number`, clone a new map parameter with the same key value shape
+                and dtype, and the default value of the new map parameter will be set according to `init`.
+                If `init` is a `str`, the `init` should be the alias of the class inheriting from `Initializer`.
+                If `init` is 'same', clone a new map parameter with the same default value. Default: 'same'.
+
+        Returns:
+            MapParameter, the new map parameter.
+        """
+        x = copy(self)
+        x.param_info = self.param_info.clone()
+        info = self.param_info
+        if hasattr(info, "cloned_obj"):
+            info.cloned_obj.append(x)
+        else:
+            info.cloned_obj = [x]
+        self.param_info = info
+        if init != 'same':
+            x.default_value = init
+        x._map_tensor = MapTensor_(x.key_dtype, x.value_dtype, x.value_shape, x.default_value, x.permit_filter_value,
+                                   x.evict_filter_value)
+        x.cache_enable = self.cache_enable
+        if x.cache_enable:
+            x.key = _get_unique_parameter_key()
+        return x
+
+    def get(self, key_tensor, insert_default_value=True):
+        """
+        Get value tensor according the key tensor, fill and return the default value in map parameter if key is not
+        existed.
+
+        Args:
+            key_tensor (Tensor): The key tensor.
+            insert_default_value (bool): The flag of insert default_value.
+
+        Returns:
+            Tensor, the value tensor for the key tensor.
+        """
+        map_get = _map_tensor_ops.MapTensorGet(insert_default_value)
+        return map_get(self._map_tensor, key_tensor)
+
+    def get_keys(self):
+        """
+        Get all keys as a tensor.
+
+        Returns:
+            Tensor, the tensor contains all keys.
+        """
+        return self._map_tensor.get_keys()
+
+    def get_values(self):
+        """
+        Get all values as a tensor.
+
+        Returns:
+            Tensor, the tensor contains all values.
+        """
+        return self._map_tensor.get_values()
+
+    def get_data(self):
+        """
+        Get all keys and values as a tensor.
+
+        Returns:
+            Tensor, the tensor contains all keys and values.
+        """
+        return self._map_tensor.get_data()
+
+    def put(self, key_tensor, value_tensor):
+        """
+        Insert or update records according the given key tensor and value tensor.
+
+        Args:
+            key_tensor (Tensor): The key tensor.
+            value_tensor (Tensor): The value tensor.
+
+        Returns:
+            MapParameter, the MapParameter object itself.
+        """
+        self.map_put(self._map_tensor, key_tensor, value_tensor)
+        return self._map_tensor
+
+    def erase(self, key_tensor):
+        """
+        Remove records according the given key tensor.
+
+        Args:
+            key_tensor (Tensor): The key tensor.
+
+        Returns:
+            MapParameter, the MapParameter object itself.
+        """
+        self.map_erase(self._map_tensor, key_tensor)
+        return self._map_tensor
+
+    def export_data(self, incremental=False):
+        """
+        Export data from this map parameter.
+
+        Args:
+            incremental (bool): False for full export, otherwise for incremental export. Default: False.
+            When exporting data incrementally, the value_array does not contain unchanged data.The length
+            of the key_array and the length of the status_array are consistent.
+
+        Returns:
+            Tuple(key_array, value_array, status_array), The exported data as a tuple.
+        """
+        return self._map_tensor.export_data(incremental)
+
+    def export_bytes(self, incremental=False):
+        """
+        Export bytes from this map parameter.
+
+        Args:
+            incremental (bool): False for full export, otherwise for incremental export. Default: False.
+            When exporting data incrementally, the value_array does not contain unchanged data. The length
+            of the key_array and the length of the status_array are consistent.
+
+        Returns:
+            Tuple(bytes, bytes, bytes), The exported bytes as a tuple.
+        """
+        return self._map_tensor.export_bytes(incremental)
+
+    def import_data(self, data):
+        """
+        Import this map parameter from exported data.
+
+        Args:
+            data (Tuple): The data tuple with key_array, value_array and status_array.
+        """
+        self._map_tensor.import_data(data)
+
+    def export_slice_data(self, incremental=False):
+        """
+        Export a slice data from this map parameter.
+        When MapParameter occupies a large memory, only one slice
+        of MapParameter is exported at a time (the default slice size is 1GB).
+
+        Args:
+            incremental (bool): False for full export, otherwise for incremental export. Default: False.
+            When exporting data incrementally, the value_array does not contain unchanged data.The length
+            of the key_array and the length of the status_array are consistent.
+
+        Returns:
+            Tuple(key_array, value_array, status_array, last_slice), The exported data as a tuple, and
+            the last_slice is bool variable and means whether finish export.
+        """
+        enable_persistent = "MS_EMBEDDING_REMOTE_CACHE_MEMORY_SIZE" in os.environ
+        if not enable_persistent:
+            return self._map_tensor.export_slice_data(incremental)
+        return self._map_tensor.export_persistent_slice_data(self.key, incremental)

mindspore/experimental/optim/__init__.py

@@ -0,0 +1,40 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Optimizer.
+
+Provide common optimizers for training, such as SGD, Adam.
+The optimizer is used to calculate and update the gradients.
+"""
+from __future__ import absolute_import
+
+from mindspore.experimental.optim.optimizer import Optimizer
+from mindspore.experimental.optim.adamw import AdamW
+from mindspore.experimental.optim.sgd import SGD
+from mindspore.experimental.optim.adam import Adam
+from mindspore.experimental.optim.adadelta import Adadelta
+from mindspore.experimental.optim.adagrad import Adagrad
+from mindspore.experimental.optim.asgd import ASGD
+from mindspore.experimental.optim.adamax import Adamax
+from mindspore.experimental.optim.rmsprop import RMSprop
+from mindspore.experimental.optim.rprop import Rprop
+from mindspore.experimental.optim.radam import RAdam
+from mindspore.experimental.optim.nadam import NAdam
+
+from mindspore.experimental.optim import lr_scheduler
+
+
+__all__ = ['Optimizer', 'AdamW', 'SGD', 'Adam', 'Adadelta', 'Adagrad',
+           'ASGD', 'Adamax', 'RMSprop', 'Rprop', 'RAdam', 'NAdam']

mindspore/experimental/optim/adadelta.py

@@ -0,0 +1,161 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""adadelta"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+import mindspore.common.dtype as mstype
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
+from mindspore import _checkparam as validator
+from mindspore import jit
+
+_adadelta_opt = C.MultitypeFuncGraph("adadelta_opt")
+
+
+@_adadelta_opt.register("Function", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
+def _tensor_run_opt(opt, rho, epsilon, learning_rate, weight, accum, accum_update, gradient):
+    """Apply adadelta optimizer to the weight parameter."""
+    success = True
+    success = F.depend(success, opt(weight, accum, accum_update, learning_rate, rho, epsilon, gradient))
+    return success
+
+
+class Adadelta(Optimizer):
+    r"""
+    Implements Adadelta algorithm.
+
+    .. math::
+       \begin{aligned}
+            &\rule{150mm}{0.4pt}                                                                 \\
+            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)},
+                \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)},
+                \: \lambda \text{ (weight decay)}                                                \\
+            &\textbf{initialize} :  v_0  \leftarrow 0 \: \text{ (square avg)},
+                \: u_0 \leftarrow 0 \: \text{ (accumulate variables)}                     \\[-1.ex]
+            &\rule{110mm}{0.4pt}                                                                 \\
+            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
+            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
+            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
+            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
+            &\hspace{5mm} v_t      \leftarrow v_{t-1} \rho + g^2_t (1 - \rho)                    \\
+            &\hspace{5mm}\Delta x_t    \leftarrow   \frac{\sqrt{u_{t-1} +
+                \epsilon }}{ \sqrt{v_t + \epsilon}  }g_t \hspace{21mm}                           \\
+            &\hspace{5mm} u_t  \leftarrow   u_{t-1}  \rho +
+                 \Delta x^2_t  (1 - \rho)                                                        \\
+            &\hspace{5mm}\theta_t      \leftarrow   \theta_{t-1} - \gamma  \Delta x_t            \\
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+       \end{aligned}
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1.0``.
+        rho (float, optional): coefficient used for computing a running average
+            of squared gradients. :math:`\rho` in the formula above. Default: ``0.9``.
+        eps (float, optional): term added to the denominator to improve
+            numerical stability. :math:`\epsilon` in the formula above. Default: ``1e-6``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
+
+    Keyword Args:
+        maximize (bool, optional): maximize the params based on the objective, instead of minimizing.
+            Default: ``False``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `eps` is less than or equal to 0.0.
+        ValueError: If the `rho` is not in the range of [0, 1].
+        ValueError: If the `weight_decay` is less than 0.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.Adadelta(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1.0, rho=0.9, eps=1e-6, weight_decay=0.0, *, maximize=False):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than_without_equal(eps, "eps", self.cls_name)
+        check_not_less_than(weight_decay, "weight_decay", self.cls_name)
+        validator.check_float_range(rho, 0., 1., validator.INC_BOTH, "rho", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            rho=rho,
+            eps=eps,
+            weight_decay=weight_decay,
+            maximize=maximize,
+        )
+        super(Adadelta, self).__init__(params, defaults)
+
+        self.accum = self.parameters.clone(prefix="accum", init=0)
+        self.accum_update = self.parameters.clone(prefix="accum_update", init=0)
+        self.opt = P.ApplyAdadelta()
+        self.op_cast = P.Cast()
+
+    @jit
+    def implementation(self, lr, rho, eps, maximize, weight_decay, start_id, end_id, gradients):
+        """Extract the common computing part for acceleration"""
+        params = self.parameters[start_id: end_id]
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = self._decay_weight(weight_decay, params, grads)
+        accum = self.accum[start_id: end_id]
+        accum_update = self.accum_update[start_id: end_id]
+        self.hyper_map(F.partial(_adadelta_opt, self.opt, rho, eps, lr),
+                       params, accum, accum_update, grads)
+        return True
+
+    def construct(self, gradients):
+        for group_id, group in enumerate(self.param_groups):
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+            maximize = group.get("maximize")
+            rho = group["rho"]
+            eps = group["eps"]
+
+            start_id = self.group_start_id[group_id]
+            end_id = self.group_start_id[group_id + 1]
+            weight_decay = group["weight_decay"]
+            self.implementation(lr, rho, eps, maximize, weight_decay, start_id, end_id, gradients)
+
+        return True

mindspore/experimental/optim/adagrad.py

@@ -0,0 +1,168 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""adagrad"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common import Tensor, Parameter
+import mindspore.common.dtype as mstype
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
+from mindspore import jit
+
+_adagrad_opt = C.MultitypeFuncGraph("adagrad_opt")
+
+
+@_adagrad_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor")
+def _tensor_run_opt(opt, learning_rate, weight, accum, gradient):
+    """Apply adagrad optimizer to the weight parameter."""
+    success = True
+    success = F.depend(success, opt(weight, accum, learning_rate, gradient))
+    return success
+
+
+class Adagrad(Optimizer):
+    r"""
+    Implements Adagrad algorithm.
+
+    .. math::
+       \begin{aligned}
+            &\rule{110mm}{0.4pt}                                                                 \\
+            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)
+                \text{ (objective)}, \: \lambda \text{ (weight decay)},                          \\
+            &\hspace{12mm}    \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\
+            &\textbf{initialize} :  state\_sum_0 \leftarrow 0                             \\[-1.ex]
+            &\rule{110mm}{0.4pt}                                                                 \\
+            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
+            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
+            &\hspace{5mm} \tilde{\gamma}    \leftarrow \gamma / (1 +(t-1) \eta)                  \\
+            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
+            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
+            &\hspace{5mm}state\_sum_t  \leftarrow  state\_sum_{t-1} + g^2_t                      \\
+            &\hspace{5mm}\theta_t \leftarrow
+                \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon}            \\
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
+            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
+       \end{aligned}
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-2``.
+        lr_decay (float, optional): learning rate decay. Default: ``0.``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
+        initial_accumulator_value (float, optional): the initial accumulator value. Default: ``0.``.
+        eps (float, optional): term added to the denominator to improve
+            numerical stability. Default: ``1e-10``.
+
+    Keyword Args:
+        maximize (bool, optional): maximize the params based on the objective, instead of minimizing.
+            Default: ``False``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the learning rate decay is less than 0.
+        ValueError: If the `weight_decay` is less than 0.
+        ValueError: If the `initial_accumulator_value` is less than 0.0.
+        ValueError: If the `eps` is less than 0.0.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.Adagrad(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-2, lr_decay=0.0, weight_decay=0.0, initial_accumulator_value=0.0,
+                 eps=1e-10, *, maximize=False):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than(lr_decay, "lr_decay", self.cls_name)
+        check_not_less_than(weight_decay, "weight_decay", self.cls_name)
+        check_not_less_than(initial_accumulator_value, "initial_accumulator_value", self.cls_name)
+        check_not_less_than_without_equal(eps, "eps", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            lr_decay=lr_decay,
+            eps=eps,
+            weight_decay=weight_decay,
+            initial_accumulator_value=initial_accumulator_value,
+            maximize=maximize,
+        )
+        super(Adagrad, self).__init__(params, defaults)
+
+        self.accum = self.parameters.clone(prefix="accum", init=initial_accumulator_value)
+        self.op_cast = P.Cast()
+        self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
+        self.increase_tensor = Tensor(1, mstype.int32)
+        self.assignadd = P.AssignAdd()
+        self.assign = P.Assign()
+
+    @jit
+    def implementation(self, eps, lr, lr_decay, maximize, weight_decay, start_id, end_id, gradients):
+        """Extract the common computing part for acceleration"""
+        opt = P.ApplyAdagradV2(epsilon=eps, update_slots=True)
+        decay_lr = lr / (1 + self.step_t * lr_decay)
+        params = self.parameters[start_id: end_id]
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = self._decay_weight(weight_decay, params, grads)
+        accum = self.accum[start_id: end_id]
+        self.hyper_map(F.partial(_adagrad_opt, opt, decay_lr), params, accum, grads)
+        return True
+
+    def construct(self, gradients):
+        for group_id, group in enumerate(self.param_groups):
+
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+
+            lr_decay = group["lr_decay"]
+            maximize = group.get("maximize")
+            weight_decay = group["weight_decay"]
+            eps = group["eps"]
+
+            start_id = self.group_start_id[group_id]
+            end_id = self.group_start_id[group_id + 1]
+            self.implementation(eps, lr, lr_decay, maximize, weight_decay, start_id, end_id, gradients)
+
+        self.assignadd(self.step_t, self.increase_tensor)
+
+        return True