mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/nn/optim/ftrl.py

@@ -26,11 +26,10 @@ from mindspore.nn.optim._dist_optimizer_registry import _register_dist_optimizer
 _ftrl_opt = C.MultitypeFuncGraph("ftrl_opt")
 
 
-@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "RowTensor", "Tensor", "Tensor", "Bool", "Bool",
-                    "Function", "Bool", "Function", "Bool")
-def _tensor_run_opt_with_sparse_dist(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
-                                     gradient, weight, moment, ps_parameter, cache_enable,
+@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
+                    "RowTensor", "Tensor", "Tensor", "Bool", "Function", "Bool", "Function", "Bool")
+def _tensor_run_opt_with_sparse_dist(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
+                                     gradient, weight, moment, cache_enable,
                                      distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
@@ -38,10 +37,6 @@ def _tensor_run_opt_with_sparse_dist(opt, spars_opt, push, pull, l1, l2, lr_powe
     values = gradient.values
     if use_sparse_flag:
         success = F.depend(success, distributed_sparse_opt(weight, moment, linear, values, indices))
-    elif ps_parameter and not cache_enable:
-        op_shape = P.Shape()
-        shapes = (op_shape(weight), op_shape(moment), op_shape(linear), op_shape(values), op_shape(indices))
-        success = F.depend(success, pull(push((values, indices), shapes), weight))
     else:
         success = F.depend(success, spars_opt(weight, moment, linear, values, indices))
     return success
@@ -83,11 +78,10 @@ def _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, mome
     return success
 
 
-@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
-                    "MapTensor", "MapTensor", "MapTensor", "Bool", "Bool",
-                    "Function", "Bool", "Function", "Bool")
-def _run_map_tensor_opt_with_sparse_dist(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
-                                         gradient, weight, moment, ps_parameter, cache_enable,
+@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
+                    "MapTensor", "MapTensor", "MapTensor", "Bool", "Function", "Bool", "Function", "Bool")
+def _run_map_tensor_opt_with_sparse_dist(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
+                                         gradient, weight, moment, cache_enable,
                                          distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
@@ -103,46 +97,36 @@ def _run_map_tensor_opt_with_sparse_dist(opt, spars_opt, push, pull, l1, l2, lr_
     return success
 
 
-@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "Tensor", "Tensor", "Tensor", "Bool", "Bool",
-                    "Function", "Bool", "Function", "Bool")
-def _tensor_run_opt_dist(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
-                         gradient, weight, moment, ps_parameter, cache_enable,
+@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
+                    "Tensor", "Tensor", "Tensor", "Bool", "Function", "Bool", "Function", "Bool")
+def _tensor_run_opt_dist(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
+                         gradient, weight, moment, cache_enable,
                          distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
     """Apply ftrl optimizer to the weight parameter."""
     success = True
     if use_flag:
         success = F.depend(success, distributed_opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
-    elif ps_parameter and not cache_enable:
-        op_shape = P.Shape()
-        success = F.depend(success, pull(push((gradient, learning_rate, l1, l2, lr_power),
-                                              (op_shape(weight), op_shape(moment), op_shape(linear))), weight))
     else:
         success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
     return success
 
 
-@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "RowTensor", "Tensor", "Tensor", "Bool", "Bool")
-def _tensor_run_opt_with_sparse(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
-                                gradient, weight, moment, ps_parameter, cache_enable):
+@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
+                    "RowTensor", "Tensor", "Tensor", "Bool")
+def _tensor_run_opt_with_sparse(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
+                                gradient, weight, moment, cache_enable):
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
     indices = gradient.indices
     values = gradient.values
-    if ps_parameter and not cache_enable:
-        op_shape = P.Shape()
-        shapes = (op_shape(weight), op_shape(moment), op_shape(linear), op_shape(values), op_shape(indices))
-        success = F.depend(success, pull(push((values, indices), shapes), weight))
-    else:
-        success = F.depend(success, spars_opt(weight, moment, linear, values, indices))
+    success = F.depend(success, spars_opt(weight, moment, linear, values, indices))
     return success
 
 
-@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
-                    "MapTensor", "MapTensor", "MapTensor", "Bool", "Bool")
-def _run_map_tensor_opt_with_sparse(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
-                                    gradient, weight, moment, ps_parameter, cache_enable):
+@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
+                    "MapTensor", "MapTensor", "MapTensor", "Bool")
+def _run_map_tensor_opt_with_sparse(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
+                                    gradient, weight, moment, cache_enable):
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
     indices, values = gradient.get_data()
@@ -150,18 +134,13 @@ def _run_map_tensor_opt_with_sparse(opt, spars_opt, push, pull, l1, l2, lr_power
     return success
 
 
-@_ftrl_opt.register("Function", "Function", "Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "Tensor", "Tensor", "Tensor", "Bool", "Bool")
-def _tensor_run_opt(opt, spars_opt, push, pull, l1, l2, lr_power, learning_rate, linear,
-                    gradient, weight, moment, ps_parameter, cache_enable):
+@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
+                    "Tensor", "Tensor", "Tensor", "Bool")
+def _tensor_run_opt(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
+                    gradient, weight, moment, cache_enable):
     """Apply ftrl optimizer to the weight parameter."""
     success = True
-    if ps_parameter and not cache_enable:
-        op_shape = P.Shape()
-        success = F.depend(success, pull(push((gradient, learning_rate, l1, l2, lr_power),
-                                              (op_shape(weight), op_shape(moment), op_shape(linear))), weight))
-    else:
-        success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
+    success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
     return success
 
 
@@ -296,7 +275,7 @@ class FTRL(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.FTRL(params=net.trainable_params())
@@ -338,13 +317,6 @@ class FTRL(Optimizer):
         self.opt = P.ApplyFtrl(use_locking=use_locking)
         self.use_locking = use_locking
         self.sparse_opt = P.SparseApplyFtrl(learning_rate, l1, l2, lr_power, use_locking=use_locking)
-        self._ps_pull = P.Pull()
-        self._ps_push = P.Push("Ftrl", [0, 1, 2])
-        self._ps_push.add_prim_attr("init_accum", initial_accum)
-        self._ps_push.add_prim_attr("lr", learning_rate)
-        self._ps_push.add_prim_attr("l1", l1)
-        self._ps_push.add_prim_attr("l2", l2)
-        self._ps_push.add_prim_attr("lr_power", lr_power)
 
         self._init_distributed_opts(use_locking, learning_rate, l1, l2, lr_power)
 
@@ -362,15 +334,13 @@ class FTRL(Optimizer):
         self.assignadd(self.global_step, self.global_step_increase_tensor)
 
         if self.use_dist_optimizer:
-            success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull,
-                                          self.l1, self.l2, self.lr_power, lr),
-                                linear, grads, params, moments, self.ps_parameters, self.cache_enable,
+            success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self.l1, self.l2, self.lr_power, lr),
+                                linear, grads, params, moments, self.cache_enable,
                                 self.distributed_opts, self.use_distributed_opt_flags,
                                 self.distributed_sparse_opts, self.use_distributed_sparse_opt_flags)
         else:
-            success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull,
-                                          self.l1, self.l2, self.lr_power, lr),
-                                linear, grads, params, moments, self.ps_parameters, self.cache_enable)
+            success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self.l1, self.l2, self.lr_power, lr),
+                                linear, grads, params, moments, self.cache_enable)
         return success
 
     @Optimizer.target.setter

mindspore/nn/optim/lamb.py

@@ -132,7 +132,7 @@ class Lamb(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means. Refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/r2.2/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
@@ -183,7 +183,9 @@ class Lamb(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of current step.
 
         beta1 (float): The exponential decay rate for the 1st moment estimations. Default: ``0.9`` .
             Should be in range (0.0, 1.0).
@@ -224,7 +226,7 @@ class Lamb(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Lamb(params=net.trainable_params(), learning_rate=0.1)

mindspore/nn/optim/lars.py

@@ -92,7 +92,7 @@ class LARS(Optimizer):
         coefficient (float): Trust coefficient for calculating the local learning rate. Default: ``0.001`` .
         use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: ``False`` .
         lars_filter (Function): A function to determine which of the network parameters to use LARS algorithm. Default:
-                                lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name.
+                                ``lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name``.
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params` in the optimizer, the shape is the
@@ -109,7 +109,7 @@ class LARS(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
         >>> opt = nn.Momentum(net.trainable_params(), 0.1, 0.9)

mindspore/nn/optim/lazyadam.py

@@ -321,7 +321,7 @@ class LazyAdam(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, :class:`~.train.LearningRateScheduler`]): Default: ``1e-3`` .
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1e-3`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -333,7 +333,9 @@ class LazyAdam(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of current step.
 
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
                        Default: ``0.9`` .
@@ -371,7 +373,7 @@ class LazyAdam(Optimizer):
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable,
-            :class:`~.train.LearningRateScheduler`.
+            LearningRateSchedule.
         TypeError: If element of `parameters` is neither Parameter nor dict.
         TypeError: If `beta1`, `beta2`, `eps` or `loss_scale` is not a float.
         TypeError: If `weight_decay` is neither float nor int.
@@ -388,7 +390,7 @@ class LazyAdam(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.LazyAdam(params=net.trainable_params())

mindspore/nn/optim/momentum.py

@@ -29,33 +29,19 @@ from mindspore.nn.optim._dist_optimizer_registry import _register_dist_optimizer
 _momentum_opt = C.MultitypeFuncGraph("momentum_opt")
 
 
-@_momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
-def _tensor_run_opt_ext(opt, momentum, learning_rate, gradient, weight, moment, ps_parameter, cache_enable):
+@_momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
+def _tensor_run_opt_ext(opt, momentum, learning_rate, gradient, weight, moment):
     """Apply momentum optimizer to the weight parameter using Tensor."""
-    if ps_parameter and not cache_enable:
-        op_shape = P.Shape()
-        _ps_pull = P.Pull()
-        _ps_push = P.Push("ApplyMomentum", [])
-        shapes = (op_shape(learning_rate), op_shape(gradient), op_shape(momentum))
-        success = F.depend(True, _ps_pull(_ps_push((learning_rate, gradient, momentum), shapes), weight))
-    else:
-        success = F.depend(True, opt(weight, moment, learning_rate, gradient, momentum))
+    success = F.depend(True, opt(weight, moment, learning_rate, gradient, momentum))
     return success
 
 
-@_momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool",
-                        "Function", "Bool")
-def _tensor_run_opt_ext_dist(opt, momentum, learning_rate, gradient, weight, moment, ps_parameter, cache_enable,
+@_momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Function", "Bool")
+def _tensor_run_opt_ext_dist(opt, momentum, learning_rate, gradient, weight, moment,
                              distributed_opt, use_flag):
     """Apply momentum optimizer to the weight parameter using Tensor."""
     if use_flag:
         success = F.depend(True, distributed_opt(weight, moment, learning_rate, gradient, momentum))
-    elif ps_parameter and not cache_enable:
-        op_shape = P.Shape()
-        _ps_pull = P.Pull()
-        _ps_push = P.Push("ApplyMomentum", [])
-        shapes = (op_shape(learning_rate), op_shape(gradient), op_shape(momentum))
-        success = F.depend(True, _ps_pull(_ps_push((learning_rate, gradient, momentum), shapes), weight))
     else:
         success = F.depend(True, opt(weight, moment, learning_rate, gradient, momentum))
     return success
@@ -129,7 +115,9 @@ class Momentum(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of current step.
 
         momentum (float): Hyperparameter of type float, means momentum for the moving average.
             It must be at least 0.0.
@@ -173,7 +161,7 @@ class Momentum(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9)
@@ -224,19 +212,19 @@ class Momentum(Optimizer):
         if self.use_dist_optimizer:
             if self.is_group_lr:
                 success = self.hyper_map_reverse(F.partial(_momentum_opt, self.opt, self.momentum),
-                                                 lr, gradients, params, moments, self.ps_parameters, self.cache_enable,
+                                                 lr, gradients, params, moments,
                                                  self.distributed_opts, self.use_distributed_opt_flags)
             else:
                 success = self.hyper_map_reverse(F.partial(_momentum_opt, self.opt, self.momentum, lr),
-                                                 gradients, params, moments, self.ps_parameters, self.cache_enable,
+                                                 gradients, params, moments,
                                                  self.distributed_opts, self.use_distributed_opt_flags)
         else:
             if self.is_group_lr:
                 success = self.hyper_map_reverse(F.partial(_momentum_opt, self.opt, self.momentum),
-                                                 lr, gradients, params, moments, self.ps_parameters, self.cache_enable)
+                                                 lr, gradients, params, moments)
             else:
                 success = self.hyper_map_reverse(F.partial(_momentum_opt, self.opt, self.momentum, lr),
-                                                 gradients, params, moments, self.ps_parameters, self.cache_enable)
+                                                 gradients, params, moments)
         return success
 
 

mindspore/nn/optim/optimizer.py

@@ -95,7 +95,10 @@ class Optimizer(Cell):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate
+              of current step.
 
         parameters (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
             `parameters` is a list of `dict`, the string "params", "lr", "weight_decay", "grad_centralization" and
@@ -764,14 +767,14 @@ class Optimizer(Cell):
             param (Union[Parameter, list[Parameter]]): The `Parameter` or list of `Parameter`.
 
         Returns:
-            Parameter, single `Parameter` or `list[Parameter]` according to the input type. If learning rate is dynamic,
+            A single `Parameter` or `list[Parameter]` according to the input type. If learning rate is dynamic,
             `LearningRateSchedule` or `list[LearningRateSchedule]` that used to calculate the learning rate will be
             returned.
 
         Examples:
             >>> from mindspore import nn
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params()))
             >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params()))

mindspore/nn/optim/proximal_ada_grad.py

@@ -121,7 +121,9 @@ class ProximalAdagrad(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of the current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of the current step.
 
         l1 (float): l1 regularization strength, must be greater than or equal to zero. Default: ``0.0`` .
         l2 (float): l2 regularization strength, must be greater than or equal to zero. Default: ``0.0`` .
@@ -163,7 +165,7 @@ class ProximalAdagrad(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.ProximalAdagrad(params=net.trainable_params())

mindspore/nn/optim/rmsprop.py

@@ -29,6 +29,10 @@ _centered_rmsprop_opt = C.MultitypeFuncGraph("rmsprop_opt")
 def _rmsprop_opt_(opt, decay, epsilon, momentum, learning_rate, weight, ms, mom, grad):
     """Apply rmsprop optimizer to the weight parameter using dynamic learning rate."""
     success = True
+    learning_rate = F.cast(learning_rate, grad.dtype)
+    decay = F.cast(decay, grad.dtype)
+    momentum = F.cast(momentum, grad.dtype)
+    epsilon = F.cast(epsilon, grad.dtype)
     success = F.depend(success, opt(weight, ms, mom, learning_rate, grad, decay, momentum, epsilon))
     return success
 
@@ -132,7 +136,9 @@ class RMSProp(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of the current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of the current step.
 
         decay (float): Decay rate. Should be equal to or greater than 0. Default: ``0.9`` .
         momentum (float): Hyperparameter of type float, means momentum for the moving average. Should be equal to or
@@ -161,7 +167,7 @@ class RMSProp(Optimizer):
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
 
     Outputs:
-        Tensor[bool], the value is True.
+        Tensor[bool], the value is ``True``.
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
@@ -180,7 +186,7 @@ class RMSProp(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.RMSProp(params=net.trainable_params(), learning_rate=0.1)

mindspore/nn/optim/rprop.py

@@ -95,7 +95,9 @@ class Rprop(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of current step.
 
         etas (tuple[float, float]): The factor of multiplicative increasing or
             descreasing(etaminus, etaplus). Default: ``(0.5, 1.2)`` .
@@ -135,7 +137,7 @@ class Rprop(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Rprop(params=net.trainable_params())

mindspore/nn/optim/sgd.py

@@ -102,7 +102,9 @@ class SGD(Optimizer):
             - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
 
             - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+              `LearningRateSchedule
+              <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
+              with step as the input to get the learning rate of current step.
 
         momentum (float): A floating point value the momentum. must be at least 0.0. Default: ``0.0`` .
         dampening (float): A floating point value of dampening for momentum. must be at least 0.0. Default: ``0.0`` .
@@ -132,7 +134,7 @@ class SGD(Optimizer):
         >>> from mindspore import nn
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.SGD(params=net.trainable_params())
@@ -193,7 +195,7 @@ class SGD(Optimizer):
                             "or 'weight_decay' set in grouped 'params' must be float or int type.")
 
         if hasattr(self, "group_weight_decay") and self.group_weight_decay:
-            self.opt = tuple(P.SGD(dampening, 0.0, nesterov) for _ in self.group_weight_decay)
+            self.opt = tuple(P.SGD(dampening, 0.0, nesterov) for wd in self.group_weight_decay)
         else:
             self.opt = tuple([P.SGD(dampening, 0.0, nesterov)] * len(self._parameters))
 

mindspore/nn/optim/tft_wrapper.py

@@ -0,0 +1,127 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""OptTFTWrapper"""
+from __future__ import absolute_import
+
+import os
+from mindspore.common.tensor import Tensor
+from mindspore.nn.optim.optimizer import Optimizer
+from mindspore.ops.operations.manually_defined._inner import TensorReport
+from mindspore import ops, context
+
+class OptTFTWrapper(Optimizer):
+    r"""
+    Implements TFT optimizer wrapper, this wrapper is used to report status to MindIO TFT before optimizer updating.
+
+    Note:
+        This optimizer is depend on MindIO TFT feature. Currently only support ascend graph mode and
+        sink_size must be less than 1.
+
+    Args:
+        opt (Optimizer): Must be sub-class of Optimizer.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of opt's `params`, the shape is the same as opt's `params`.
+
+    Outputs:
+        Tensor, result of executing optimizer 'opt'.
+
+    Raises:
+        TypeError: If the parameter opt is not an subclass of Optimizer.
+        ValueError: If the platform is not Ascend graph mode, or customer doesn't switch on TFT feature.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import nn
+        >>>
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> #1) All parameters use the same learning rate and weight decay
+        >>> optim = nn.SGD(params=net.trainable_params())
+        >>> optim_wrapper = nn.OptTFTWrapper(optim)
+        >>>
+        >>> loss = nn.SoftmaxCrossEntropyWithLogits()
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
+    """
+
+    def __init__(self, opt, **kwargs):
+        super(OptTFTWrapper, self).__init__(opt.learning_rate, opt._parameters) # pylint: disable=W0212
+        if not isinstance(opt, Optimizer):
+            raise TypeError(f"For 'OptTFTWrapper', the argument 'opt' must be Optimizer type, " f"but got {type(opt)}.")
+        tft_env = os.getenv("MS_ENABLE_TFT", "")
+        if ("TTP:1" not in tft_env) and ("UCE:1" not in tft_env):
+            raise ValueError("MindIO TFT regitster need custom switch on[MS_ENABLE_TFT='{TTP:1,UCE:1}']!")
+        mode = context.get_context("mode")
+        device_target = context.get_context("device_target")
+        if device_target != "Ascend" or mode != context.GRAPH_MODE:
+            raise ValueError("MindIO adataper only support on Ascend device with GRAPH Mode!")
+        self.opt = opt
+        self.report = TensorReport()
+        self.depend = ops.Depend()
+        self.g_one = Tensor([0.1])
+        # enable consistent check by default, only disable when enable_consistent_check is False
+        self.use_allreduce = kwargs.get("enable_consistent_check", True)
+
+        if self.use_allreduce:
+            self.allreduce_sum = ops.AllReduce()
+            self.allreduce_sum.add_prim_attr("tft_report_before", True)
+
+        self.param_rank = opt.param_rank
+        self.optim_filter = opt.optim_filter
+        self.loss_scale = opt.loss_scale
+        self.dynamic_weight_decay = opt.dynamic_weight_decay
+        self.grad_centralization = opt.grad_centralization
+
+        self.dynamic_lr = opt.dynamic_lr
+        self.global_step = opt.global_step
+        self.is_group = opt.is_group
+        self.is_group_lr = opt.is_group_lr
+        self.is_group_params_ordered = opt.is_group_params_ordered
+        self.use_parallel = opt.use_parallel
+        if self.is_group:
+            self.group_params = opt.group_params
+            self.group_lr = opt.group_lr
+            self.group_weight_decay = opt.group_weight_decay
+            self.group_grad_centralization = opt.group_grad_centralization
+            self.grad_centralization_flags = opt.grad_centralization_flags
+
+        self.skip_auto_parallel_compile = opt.skip_auto_parallel_compile
+
+        self.learning_rate = opt.learning_rate
+        self.parameters = opt.parameters
+        self.decay_flags = opt.decay_flags
+        self.dynamic_decay_flags = opt.dynamic_decay_flags
+        self.weight_decay = opt.weight_decay
+        self.exec_weight_decay = opt.exec_weight_decay
+        self.ps_parameters = opt.ps_parameters
+        self.cache_enable = opt.cache_enable
+        self.reciprocal_scale = opt.reciprocal_scale
+        self.need_scale = opt.need_scale
+        self.global_step_increase_tensor = opt.global_step_increase_tensor
+        self.param_length = opt.param_length
+        self.enable_tuple_broaden = opt.enable_tuple_broaden
+
+    def construct(self, gradients):
+        g_one = self.depend(self.g_one, gradients)
+        if self.use_allreduce is True:
+            g_one_res = self.allreduce_sum(g_one)
+        else:
+            g_one_res = g_one
+        self.report("tft_report", g_one_res)
+        return self.opt(gradients)

mindspore/nn/optim/thor.py

@@ -339,10 +339,10 @@ def thor(net, learning_rate, damping, momentum, weight_decay=0.0, loss_scale=1.0
         >>> from mindspore import Tensor
         >>>
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> # Create the dataset taking MNIST as an example. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/mnist.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
         >>> dataset = create_dataset()
         >>> temp = Tensor([4e-4, 1e-4, 1e-5, 1e-5], mstype.float32)
         >>> optim = nn.thor(net, learning_rate=temp, damping=temp, momentum=0.9, loss_scale=128, frequency=4)