mindspore 2.7.0rc1__cp311-cp311-win_amd64.whl → 2.7.1__cp311-cp311-win_amd64.whl

mindspore/nn/optim/adam.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2021 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,7 +28,6 @@ from mindspore.common.tensor import Tensor
 from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
-from mindspore.nn.optim._dist_optimizer_registry import _register_dist_optimizer
 from mindspore.common._decorator import deprecated
 
 _adam_opt = C.MultitypeFuncGraph("adam_opt")
@@ -727,7 +726,6 @@ class Adam(Optimizer):
             self.opt = P.Adam(use_locking, use_nesterov)
             self.sparse_opt = P.FusedSparseLazyAdam(use_locking, use_nesterov)
             self.sparse_opt.set_device("CPU")
-            self._init_distributed_opts(use_locking, use_nesterov)
 
         else:
             self._is_device = True
@@ -737,7 +735,6 @@ class Adam(Optimizer):
                 self.opt = P.Adam(use_locking, use_nesterov)
             self.sparse_opt = P.FusedSparseAdam(use_locking, use_nesterov)
             self.sparse_opt.set_device("CPU")
-            self._init_distributed_opts(use_locking, use_nesterov)
 
     def _apply_adam(self, params, beta1_power, beta2_power, moment1, moment2, lr, gradients):
         """Execute Adam optimizer and its variants."""
@@ -750,83 +747,44 @@ class Adam(Optimizer):
                                                      self.beta2, self.eps, lr), gradients, params, moment1, moment2)
         # Lazy adam or normal adam
         else:
-            if self.use_dist_optimizer:
-                if self.use_dist_optimizer and self.use_amsgrad:
-                    raise ValueError(f"Adam with amsgrad is currently not supporting distributed training!"
-                                     f"Please set use_amsgrad=False for distributed training.")
-                if self.is_group_lr:
-                    if self.use_lazy:
-                        success = self.map_reverse(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt,
-                                                             self.use_locking, self.use_nesterov,
-                                                             self._is_device, beta1_power, beta2_power,
-                                                             self.beta1, self.beta2, self.eps),
-                                                   lr, gradients, self._parameters, self.moment1, self.moment2,
-                                                   self.dense_lazyadam_opts,
-                                                   self.use_dense_opt_flags, self.sparse_lazyadam_opts,
-                                                   self.use_sparse_opt_flags)
-                    # Normal Adam
-                    else:
-                        success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self.use_locking,
-                                                      self.use_nesterov, self._is_device, beta1_power, beta2_power,
-                                                      self.beta1, self.beta2, self.eps),
-                                            lr, gradients, params, moment1, moment2,
-                                            self.dense_adam_opts, self.use_dense_opt_flags,
-                                            self.sparse_adam_opts, self.use_sparse_opt_flags)
+            if self.is_group_lr:
+                if self.use_lazy:
+                    success = self.map_(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt,
+                                                  self.use_locking, self.use_nesterov,
+                                                  self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
+                                                  self.eps), lr, gradients, params, moment1, moment2)
                 else:
-                    if self.use_lazy:
-                        success = self.map_reverse(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt,
-                                                             self.use_locking, self.use_nesterov,
-                                                             self._is_device, beta1_power, beta2_power, self.beta1,
-                                                             self.beta2, self.eps, lr), gradients, self._parameters,
-                                                   self.moment1, self.moment2,
-                                                   self.dense_lazyadam_opts, self.use_dense_opt_flags,
-                                                   self.sparse_lazyadam_opts, self.use_sparse_opt_flags)
+                    if self.use_amsgrad:
+                        success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
+                                                      self.use_locking, self.use_nesterov,
+                                                      self._is_device, beta1_power, beta2_power,
+                                                      self.beta1, self.beta2, self.eps), lr, gradients, params,
+                                            moment1, moment2, self.vhat)
                     else:
                         success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
                                                       self.use_locking, self.use_nesterov,
-                                                      self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
-                                                      self.eps, lr), gradients, params, moment1, moment2,
-                                            self.dense_adam_opts,
-                                            self.use_dense_opt_flags, self.sparse_adam_opts, self.use_sparse_opt_flags)
+                                                      self._is_device, beta1_power, beta2_power,
+                                                      self.beta1, self.beta2, self.eps), lr, gradients, params,
+                                            moment1, moment2)
             else:
-                if self.is_group_lr:
-                    if self.use_lazy:
-                        success = self.map_(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt,
-                                                      self.use_locking, self.use_nesterov,
-                                                      self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
-                                                      self.eps), lr, gradients, params, moment1, moment2)
-                    else:
-                        if self.use_amsgrad:
-                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
-                                                          self.use_locking, self.use_nesterov,
-                                                          self._is_device, beta1_power, beta2_power,
-                                                          self.beta1, self.beta2, self.eps), lr, gradients, params,
-                                                moment1, moment2, self.vhat)
-                        else:
-                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
-                                                          self.use_locking, self.use_nesterov,
-                                                          self._is_device, beta1_power, beta2_power,
-                                                          self.beta1, self.beta2, self.eps), lr, gradients, params,
-                                                moment1, moment2)
+                if self.use_lazy:
+                    success = self.map_(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt,
+                                                  self.use_locking, self.use_nesterov,
+                                                  self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
+                                                  self.eps, lr), gradients, params, moment1, moment2)
                 else:
-                    if self.use_lazy:
-                        success = self.map_(F.partial(_lazy_adam_opt, self.opt, self.sparse_opt,
+                    if self.use_amsgrad:
+                        success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
                                                       self.use_locking, self.use_nesterov,
-                                                      self._is_device, beta1_power, beta2_power, self.beta1, self.beta2,
-                                                      self.eps, lr), gradients, params, moment1, moment2)
+                                                      self._is_device, beta1_power, beta2_power,
+                                                      self.beta1, self.beta2, self.eps, lr), gradients, params,
+                                            moment1, moment2, self.vhat)
                     else:
-                        if self.use_amsgrad:
-                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
-                                                          self.use_locking, self.use_nesterov,
-                                                          self._is_device, beta1_power, beta2_power,
-                                                          self.beta1, self.beta2, self.eps, lr), gradients, params,
-                                                moment1, moment2, self.vhat)
-                        else:
-                            success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
-                                                          self.use_locking, self.use_nesterov,
-                                                          self._is_device, beta1_power, beta2_power,
-                                                          self.beta1, self.beta2, self.eps, lr), gradients, params,
-                                                moment1, moment2)
+                        success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt,
+                                                      self.use_locking, self.use_nesterov,
+                                                      self._is_device, beta1_power, beta2_power,
+                                                      self.beta1, self.beta2, self.eps, lr), gradients, params,
+                                            moment1, moment2)
 
         return success
 
@@ -835,7 +793,6 @@ class Adam(Optimizer):
         params = self._parameters
         moment1 = self.moment1
         moment2 = self.moment2
-        gradients = self.flatten_gradients(gradients)
         gradients = self.decay_weight(gradients)
         if not self.use_offload:
             gradients = self.gradients_centralization(gradients)
@@ -859,13 +816,6 @@ class Adam(Optimizer):
         """
         self._set_base_target(value)
 
-    def _init_distributed_opts(self, use_locking, use_nesterov):
-        self.use_dist_optimizer = self._use_distibuted_optimizer()
-        self.dense_adam_opts, self.use_dense_opt_flags = \
-            self._get_distributed_optimizer_list("adam", use_locking, use_nesterov)
-        self.sparse_adam_opts, self.use_sparse_opt_flags = \
-            self._get_distributed_optimizer_list("fused_sparse_adam", use_locking, use_nesterov)
-
 
 class AdamWeightDecay(Optimizer):
     r"""
@@ -909,9 +859,7 @@ class AdamWeightDecay(Optimizer):
     Note:
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
-        As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means, refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html>`_ to
-        process `loss_scale` correctly.
+        As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
         'beta' or 'gamma' in their names. Users can group parameters to change the strategy of decaying weight. When
@@ -1030,11 +978,9 @@ class AdamWeightDecay(Optimizer):
 
     @jit(backend="ms_backend")
     def construct(self, gradients):
-        gradients = self.flatten_gradients(gradients)
         weight_decay = self.get_weight_decay()
         lr = self.get_lr()
         self.assignadd(self.global_step, self.global_step_increase_tensor)
-
         if self.use_fused_opt:
             if self.is_group:
                 if self.is_group_lr:
@@ -1072,19 +1018,6 @@ class AdamWeightDecay(Optimizer):
 
         return optim_result
 
-    @Optimizer.target.setter
-    def target(self, value):
-        """
-        If the input value is set to "CPU", the parameters will be updated on the host using the Fused
-        optimizer operation.
-        """
-        self._set_base_target(value)
-        if value == 'CPU':
-            self.fused_opt.set_device("CPU")
-            self.use_fused_opt = True
-        else:
-            self.use_fused_opt = False
-
 
 class AdamOffload(Optimizer):
     r"""
@@ -1253,7 +1186,6 @@ class AdamOffload(Optimizer):
         params = self._parameters
         moment1 = self.moment1
         moment2 = self.moment2
-        gradients = self.flatten_gradients(gradients)
         gradients = self.decay_weight(gradients)
         gradients = self.scale_grad(gradients)
         lr = self.get_lr()
@@ -1272,30 +1204,3 @@ class AdamOffload(Optimizer):
                                                  beta1_power, beta2_power, self.beta1, self.beta2, self.eps, lr),
                                        gradients, params, moment1, moment2)
         return success
-
-
-def create_distributed_adam(*args, **kwargs):
-    """
-    Create the distributed Adam op.
-    """
-    adam = P.Adam(*args, **kwargs)
-    adam.add_prim_attr("gradient_type", "dense_gradient")
-    adam.add_prim_attr("parameter_input_index", 0)
-    adam.add_prim_attr("gradient_input_index", 9)
-    return adam
-
-
-def create_distributed_fused_sparse_adam(*args, **kwargs):
-    """
-    Create the distributed FusedSparseAdam op.
-    """
-    sparse_adam = P.FusedSparseAdam(*args, **kwargs)
-    sparse_adam.add_prim_attr("gradient_type", "sparse_gradient")
-    sparse_adam.add_prim_attr("parameter_input_index", 0)
-    sparse_adam.add_prim_attr("gradient_input_index", 9)
-    sparse_adam.add_prim_attr("indices_input_index", 10)
-    return sparse_adam
-
-
-_register_dist_optimizer("adam", create_distributed_adam)
-_register_dist_optimizer("fused_sparse_adam", create_distributed_fused_sparse_adam)

mindspore/nn/optim/adamax.py

@@ -202,7 +202,6 @@ class AdaMax(Optimizer):
 
     @jit
     def construct(self, gradients):
-        gradients = self.flatten_gradients(gradients)
         gradients = self.decay_weight(gradients)
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)

mindspore/nn/optim/asgd.py

@@ -184,7 +184,6 @@ class ASGD(Optimizer):
 
     @jit(backend="ms_backend")
     def construct(self, gradients):
-        gradients = self.flatten_gradients(gradients)
         gradients = self.decay_weight(gradients)
         gradients = self.gradients_centralization(gradients)
         gradients = self.scale_grad(gradients)

mindspore/nn/optim/ftrl.py

@@ -21,27 +21,10 @@ from mindspore.common.api import jit
 from mindspore import _checkparam as validator
 from mindspore.nn.optim.optimizer import Optimizer
 from mindspore.nn.optim.optimizer import opt_init_args_register
-from mindspore.nn.optim._dist_optimizer_registry import _register_dist_optimizer
 
 _ftrl_opt = C.MultitypeFuncGraph("ftrl_opt")
 
 
-@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "RowTensor", "Tensor", "Tensor", "Bool", "Function", "Bool", "Function", "Bool")
-def _tensor_run_opt_with_sparse_dist(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
-                                     gradient, weight, moment, cache_enable,
-                                     distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
-    """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
-    success = True
-    indices = gradient.indices
-    values = gradient.values
-    if use_sparse_flag:
-        success = F.depend(success, distributed_sparse_opt(weight, moment, linear, values, indices))
-    else:
-        success = F.depend(success, spars_opt(weight, moment, linear, values, indices))
-    return success
-
-
 def _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, moment, indices, values):
     """Apllpy ftrl optimizer for map parameter"""
     success = True
@@ -78,43 +61,10 @@ def _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, mome
     return success
 
 
-@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
-                    "MapTensor", "MapTensor", "MapTensor", "Bool", "Function", "Bool", "Function", "Bool")
-def _run_map_tensor_opt_with_sparse_dist(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
-                                         gradient, weight, moment, cache_enable,
-                                         distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
-    """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
-    success = True
-    indices, values = gradient.get_data()
-    if use_sparse_flag:
-        # PS Mode.
-        success = F.depend(success, distributed_sparse_opt(weight, moment, linear, values, indices))
-    elif cache_enable:
-        # PS Cache mode.
-        _apply_map_tensor_ftrl(l1, l2, lr_power, learning_rate, linear, weight, moment, indices, values)
-    else:
-        raise Exception("Unexpected mode for distributed optimizer.")
-    return success
-
-
-@_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "Tensor", "Tensor", "Tensor", "Bool", "Function", "Bool", "Function", "Bool")
-def _tensor_run_opt_dist(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
-                         gradient, weight, moment, cache_enable,
-                         distributed_opt, use_flag, distributed_sparse_opt, use_sparse_flag):
-    """Apply ftrl optimizer to the weight parameter."""
-    success = True
-    if use_flag:
-        success = F.depend(success, distributed_opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
-    else:
-        success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
-    return success
-
-
 @_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "RowTensor", "Tensor", "Tensor", "Bool")
+                    "RowTensor", "Tensor", "Tensor")
 def _tensor_run_opt_with_sparse(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
-                                gradient, weight, moment, cache_enable):
+                                gradient, weight, moment):
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
     indices = gradient.indices
@@ -124,9 +74,9 @@ def _tensor_run_opt_with_sparse(opt, spars_opt, l1, l2, lr_power, learning_rate,
 
 
 @_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "MapTensor",
-                    "MapTensor", "MapTensor", "MapTensor", "Bool")
+                    "MapTensor", "MapTensor", "MapTensor")
 def _run_map_tensor_opt_with_sparse(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
-                                    gradient, weight, moment, cache_enable):
+                                    gradient, weight, moment):
     """Apply sparse ftrl optimizer to the weight parameter when the gradient is sparse."""
     success = True
     indices, values = gradient.get_data()
@@ -135,9 +85,9 @@ def _run_map_tensor_opt_with_sparse(opt, spars_opt, l1, l2, lr_power, learning_r
 
 
 @_ftrl_opt.register("Function", "Function", "Number", "Number", "Number", "Tensor", "Tensor",
-                    "Tensor", "Tensor", "Tensor", "Bool")
+                    "Tensor", "Tensor", "Tensor")
 def _tensor_run_opt(opt, spars_opt, l1, l2, lr_power, learning_rate, linear,
-                    gradient, weight, moment, cache_enable):
+                    gradient, weight, moment):
     """Apply ftrl optimizer to the weight parameter."""
     success = True
     success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power))
@@ -320,14 +270,11 @@ class FTRL(Optimizer):
         self.use_locking = use_locking
         self.sparse_opt = P.SparseApplyFtrl(learning_rate, l1, l2, lr_power, use_locking=use_locking)
 
-        self._init_distributed_opts(use_locking, learning_rate, l1, l2, lr_power)
-
     @jit
     def construct(self, grads):
         params = self._parameters
         moments = self.moments
         linear = self.linear
-        grads = self.flatten_gradients(grads)
         grads = self.decay_weight(grads)
         grads = self.gradients_centralization(grads)
         grads = self.scale_grad(grads)
@@ -335,14 +282,8 @@ class FTRL(Optimizer):
         lr = self.get_lr()
         self.assignadd(self.global_step, self.global_step_increase_tensor)
 
-        if self.use_dist_optimizer:
-            success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self.l1, self.l2, self.lr_power, lr),
-                                linear, grads, params, moments, self.cache_enable,
-                                self.distributed_opts, self.use_distributed_opt_flags,
-                                self.distributed_sparse_opts, self.use_distributed_sparse_opt_flags)
-        else:
-            success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self.l1, self.l2, self.lr_power, lr),
-                                linear, grads, params, moments, self.cache_enable)
+        success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, self.l1, self.l2, self.lr_power, lr),
+                            linear, grads, params, moments)
         return success
 
     @Optimizer.target.setter
@@ -366,38 +307,3 @@ class FTRL(Optimizer):
             self.sparse_opt = P.SparseApplyFtrl(self.lr, self.l1, self.l2, self.lr_power, self.use_locking)
 
         self._target = value
-
-    def _init_distributed_opts(self, use_locking, learning_rate, l1, l2, lr_power):
-        self.use_dist_optimizer = self._use_distibuted_optimizer()
-        self.distributed_opts, self.use_distributed_opt_flags =\
-        self._get_distributed_optimizer_list("ftrl", use_locking=use_locking)
-        self.distributed_sparse_opts, self.use_distributed_sparse_opt_flags =\
-        self._get_distributed_optimizer_list("fused_sparse_ftrl", learning_rate,
-                                             l1, l2, lr_power, use_locking=use_locking)
-
-
-def create_distributed_ftrl(*args, **kwargs):
-    """
-    Create the distributed ApplyFtrl op.
-    """
-    ftrl = P.ApplyFtrl(*args, **kwargs)
-    ftrl.add_prim_attr("gradient_type", "dense_gradient")
-    ftrl.add_prim_attr("parameter_input_index", 0)
-    ftrl.add_prim_attr("gradient_input_index", 3)
-    return ftrl
-
-
-def create_distributed_fused_sparse_ftrl(*args, **kwargs):
-    """
-    Create the distributed FusedSparseFtrl op.
-    """
-    sparse_ftrl = P.FusedSparseFtrl(*args, **kwargs)
-    sparse_ftrl.add_prim_attr("gradient_type", "sparse_gradient")
-    sparse_ftrl.add_prim_attr("parameter_input_index", 0)
-    sparse_ftrl.add_prim_attr("gradient_input_index", 3)
-    sparse_ftrl.add_prim_attr("indices_input_index", 4)
-    return sparse_ftrl
-
-
-_register_dist_optimizer("ftrl", create_distributed_ftrl)
-_register_dist_optimizer("fused_sparse_ftrl", create_distributed_fused_sparse_ftrl)

mindspore/nn/optim/lamb.py

@@ -131,9 +131,7 @@ class Lamb(Optimizer):
     Note:
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
-        As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means. Refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html>`_ to
-        process `loss_scale` correctly.
+        As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
         'beta' or 'gamma' in their names. Users can group parameters to change the strategy of decaying weight. When
@@ -271,7 +269,6 @@ class Lamb(Optimizer):
         lr = self.get_lr()
         self.assignadd(self.global_step, self.global_step_increase_tensor)
         lamb_opt = _lamb_opt
-        gradients = self.flatten_gradients(gradients)
         gradients = self.gradients_centralization(gradients)
         if self.is_group:
             if self.is_group_lr:

mindspore/nn/optim/lars.py

@@ -125,8 +125,6 @@ class LARS(Optimizer):
         self.weight_decay = optimizer.weight_decay
         self.global_step = optimizer.global_step
         self.parameters = optimizer.parameters
-        if optimizer._use_flattened_params:  # pylint: disable=W0212
-            self.opt._use_flattened_params = False  # pylint: disable=W0212
         self._user_parameters += [param.name for param in self.parameters]
         self.use_clip = use_clip
         self.lars_flag = tuple(lars_filter(x) for x in self.parameters)
@@ -173,7 +171,6 @@ class LARS(Optimizer):
     @jit
     def construct(self, gradients):
         params = self.parameters
-        gradients = self.flatten_gradients(gradients)
         if self.use_clip:
             lr = self._get_lr()
         else: