mindspore 2.6.0__cp310-cp310-win_amd64.whl → 2.7.0__cp310-cp310-win_amd64.whl

mindspore/mint/optim/adam.py

@@ -15,17 +15,18 @@
 """Adam"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
 from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
 from mindspore.common import dtype as mstype
 from mindspore.experimental.optim.optimizer import Optimizer
 from mindspore import _checkparam as validator
 from mindspore import mint
+from mindspore import ops
 
-_optim_adamw_opt = C.MultitypeFuncGraph("optim_adamw_opt")
-hyper_map = C.HyperMap()
-assign_add = P.AssignAdd()
+
+_optim_adamw_opt = ops.MultitypeFuncGraph("optim_adamw_opt")
+hyper_map = ops.HyperMap()
+assign_add = ops.AssignAdd()
 
 
 @_optim_adamw_opt.register("Float", "Float", "Float", "Tensor", "Tensor", "Tensor", "Tensor",
@@ -38,12 +39,12 @@ def _run_optim_adamw_amsgrad_opt(beta1, beta2, eps, neg_step_size, sqrt_bias_cor
     exp_avg_sq_tmp = mint.mul(exp_avg_sq, beta2) + mint.mul(mint.mul(grads, grads), 1 - beta2)
 
     max_exp_avg_sq = mint.maximum(max_exp_avg_sq, exp_avg_sq_tmp)
-    denom = F.cast(mint.div(mint.sqrt(max_exp_avg_sq), sqrt_bias_correction2), max_exp_avg_sq.dtype)
+    denom = ops.cast(mint.div(mint.sqrt(max_exp_avg_sq), sqrt_bias_correction2), max_exp_avg_sq.dtype)
     denom = mint.add(denom, eps)
 
-    delta_param = mint.mul(F.cast(neg_step_size, max_exp_avg_sq.dtype), mint.div(exp_avg_tmp, denom))
-    F.assign(exp_avg, exp_avg_tmp)
-    F.assign(exp_avg_sq, exp_avg_sq_tmp)
+    delta_param = mint.mul(ops.cast(neg_step_size, max_exp_avg_sq.dtype), mint.div(exp_avg_tmp, denom))
+    ops.assign(exp_avg, exp_avg_tmp)
+    ops.assign(exp_avg_sq, exp_avg_sq_tmp)
     assign_add(parameters, delta_param)
     return success
 
@@ -56,12 +57,12 @@ def _run_optim_adamw_opt(beta1, beta2, eps, neg_step_size, sqrt_bias_correction2
     exp_avg_tmp = mint.add(mint.mul(exp_avg, beta1), grads, alpha=1 - beta1)
     exp_avg_sq_tmp = mint.mul(exp_avg_sq, beta2) + mint.mul(mint.mul(grads, grads), 1 - beta2)
 
-    denom = F.cast(mint.div(mint.sqrt(exp_avg_sq_tmp), sqrt_bias_correction2), exp_avg_sq_tmp.dtype)
+    denom = ops.cast(mint.div(mint.sqrt(exp_avg_sq_tmp), sqrt_bias_correction2), exp_avg_sq_tmp.dtype)
     denom = mint.add(denom, eps)
 
-    delta_param = mint.mul(F.cast(neg_step_size, exp_avg_sq_tmp.dtype), mint.div(exp_avg_tmp, denom))
-    F.assign(exp_avg, exp_avg_tmp)
-    F.assign(exp_avg_sq, exp_avg_sq_tmp)
+    delta_param = mint.mul(ops.cast(neg_step_size, exp_avg_sq_tmp.dtype), mint.div(exp_avg_tmp, denom))
+    ops.assign(exp_avg, exp_avg_tmp)
+    ops.assign(exp_avg_sq, exp_avg_sq_tmp)
     assign_add(parameters, delta_param)
     return success
 
@@ -190,8 +191,8 @@ class Adam(Optimizer):
         self.exp_avg_sq = self.parameters.clone(prefix="exp_avg_sq", init='zeros')
         self.state_step = Parameter(Tensor([0], mstype.float32), "state_step")
         self.increase_tensor = Tensor(1, mstype.float32)
-        self.assignadd = P.AssignAdd()
-        self.pow = P.Pow()
+        self.assignadd = ops.AssignAdd()
+        self.pow = ops.Pow()
 
 
     def construct(self, gradients):
@@ -211,13 +212,13 @@ class Adam(Optimizer):
             grads = self._decay_weight(group.get("weight_decay"), self.parameters[start_id: end_id], grads)
 
             if group.get("amsgrad"):
-                self.hyper_map(F.partial(_optim_adamw_opt, beta1, beta2, group.get("eps"), neg_step_size,
-                                         sqrt_bias_correction2),
+                self.hyper_map(ops.partial(_optim_adamw_opt, beta1, beta2, group.get("eps"), neg_step_size,
+                                           sqrt_bias_correction2),
                                self.parameters[start_id: end_id], grads, self.exp_avg[start_id: end_id],
                                self.exp_avg_sq[start_id: end_id], group.get("max_exp_avg_sq"))
             else:
-                self.hyper_map(F.partial(_optim_adamw_opt, beta1, beta2, group.get("eps"), neg_step_size,
-                                         sqrt_bias_correction2),
+                self.hyper_map(ops.partial(_optim_adamw_opt, beta1, beta2, group.get("eps"), neg_step_size,
+                                           sqrt_bias_correction2),
                                self.parameters[start_id: end_id], grads, self.exp_avg[start_id: end_id],
                                self.exp_avg_sq[start_id: end_id])
         return True

mindspore/mint/optim/adamw.py

@@ -15,7 +15,7 @@
 """adamw"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C
+from mindspore import ops
 from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
 from mindspore.common import dtype as mstype
@@ -23,7 +23,7 @@ from mindspore.ops import auto_generate as gen
 from mindspore.experimental.optim.optimizer import Optimizer
 from mindspore import _checkparam as validator
 
-hyper_map = C.HyperMap()
+hyper_map = ops.HyperMap()
 
 
 def _run_optim_adamw_amsgrad_opt(opt, beta1, beta2, lr, eps, weight_decay, step, amsgrad, maximize, parameters, grads,
@@ -99,6 +99,10 @@ class AdamW(Optimizer):
             &\hline \\[-1.ex]
        \end{array}
 
+    More details of the AdamW algorithm can be found in the paper `Decoupled Weight Decay Regularization
+    <https://arxiv.org/abs/1711.05101>`_ and `On the Convergence of Adam and Beyond
+    <https://openreview.net/forum?id=ryQu7f-RZ>`_.
+
     .. warning::
         - This is an experimental optimizer API that is subject to change.
           This module must be used with lr scheduler module in `LRScheduler Class
@@ -155,6 +159,8 @@ class AdamW(Optimizer):
 
     def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
                  weight_decay=1e-2, amsgrad=False, *, maximize=False):
+        # The API definition and the validation of attributes below refers to PyTorch:
+        # https://github.com/pytorch/pytorch/blob/v2.1.0/torch/optim/adamw.py
         _check_param_value(betas, eps, weight_decay, lr, amsgrad, maximize, self.cls_name)
         if lr < 0.0:
             raise ValueError("Invalid learning rate: {}".format(lr))
@@ -190,15 +196,15 @@ class AdamW(Optimizer):
             grads = tuple(gradients[start_id: end_id])
 
             if group.get("amsgrad"):
-                self.hyper_map(F.partial(_run_optim_adamw_amsgrad_opt, self.adamw_opt, beta1, beta2, float(lr),
-                                         group.get("eps"), group.get("weight_decay"), self.state_step,
-                                         group.get("amsgrad"), maximize),
+                self.hyper_map(ops.partial(_run_optim_adamw_amsgrad_opt, self.adamw_opt, beta1, beta2, float(lr),
+                                           group.get("eps"), group.get("weight_decay"), self.state_step,
+                                           group.get("amsgrad"), maximize),
                                self.parameters[start_id: end_id], grads, self.exp_avg[start_id: end_id],
                                self.exp_avg_sq[start_id: end_id], group.get("max_exp_avg_sq"))
             else:
-                self.hyper_map(F.partial(_run_optim_adamw_opt, self.adamw_opt, beta1, beta2, float(lr),
-                                         group.get("eps"), group.get("weight_decay"), self.state_step,
-                                         group.get("amsgrad"), maximize),
+                self.hyper_map(ops.partial(_run_optim_adamw_opt, self.adamw_opt, beta1, beta2, float(lr),
+                                           group.get("eps"), group.get("weight_decay"), self.state_step,
+                                           group.get("amsgrad"), maximize),
                                self.parameters[start_id: end_id], grads, self.exp_avg[start_id: end_id],
                                self.exp_avg_sq[start_id: end_id])
 

mindspore/mint/optim/sgd.py

@@ -15,14 +15,14 @@
 """SGD"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C
+from mindspore import ops
 from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
 from mindspore.common import dtype as mstype
 from mindspore.experimental.optim.optimizer import Optimizer
 from mindspore import _checkparam as validator
 from mindspore import mint
-hyper_map = C.HyperMap()
+hyper_map = ops.HyperMap()
 
 
 def _tensor_weight_decay(weight_decay, weight, gradient):
@@ -162,10 +162,10 @@ class SGD(Optimizer):
             end_id = self.group_start_id[group_id + 1]
             grads = tuple([grad if not group.get('maximize') else -grad for grad in gradients[start_id: end_id]])
             if group.get('weight_decay') != 0.:
-                grads = self.map_(F.partial(_tensor_weight_decay, group.get('weight_decay')),
+                grads = self.map_(ops.partial(_tensor_weight_decay, group.get('weight_decay')),
                                   self.parameters[start_id: end_id], grads)
-            self.hyper_map(F.partial(_run_optim_sgd_opt, self.step_t.value(), group.get('lr_float'),
-                                     group.get('momentum'), group.get('dampening'), group.get('nesterov')),
+            self.hyper_map(ops.partial(_run_optim_sgd_opt, self.step_t.value(), group.get('lr_float'),
+                                       group.get('momentum'), group.get('dampening'), group.get('nesterov')),
                            self.parameters[start_id: end_id], self.momentum_buf[start_id: end_id], grads)
         self.step_t.add_(self.increase_tensor)
         return True