mindspore 2.7.0__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/nn/layer/embedding.py

@@ -17,17 +17,12 @@ from __future__ import absolute_import
 
 import mindspore.common.dtype as mstype
 import mindspore.ops as ops
-from mindspore import log as logger
 from mindspore.common.tensor import Tensor
 from mindspore.common.parameter import Parameter
-from mindspore.common.parameter import _get_unique_parameter_key
 from mindspore.common.initializer import initializer, Normal
-from mindspore.communication.management import get_group_size, get_rank
+from mindspore.communication.management import get_group_size
 from mindspore.context import ParallelMode
 from mindspore.parallel._utils import _get_parallel_mode, _get_full_batch
-from mindspore.parallel._ps_context import _get_ps_context, _enable_distributed_mindrt
-from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver
-from mindspore.parallel._ps_context import _insert_hash_table_size, _set_cache_enable, _set_rank_id
 from mindspore import _checkparam as Validator
 from mindspore.ops.primitive import constexpr, _primexpr
 from mindspore.nn.layer.basic import ClipByNorm
@@ -341,10 +336,6 @@ class EmbeddingLookup(Cell):
         max_norm (Union[float, None]): A maximum clipping value. The data type must be float16, float32
                                        or None. Default: ``None`` .
         sparse (bool): Using sparse mode. When 'target' is set to 'CPU', 'sparse' has to be true. Default: ``True`` .
-        vocab_cache_size (int): Cache size of the dictionary of embeddings. Default: ``0`` . It is valid only in
-            parameter server trainning mode and 'DEVICE' target. And the moment parameter of corresponding
-            optimizer will also be set to the cache size. In addition, it should be noted that it will cost the 'DEVICE'
-            memory, so suggests setting a reasonable value to avoid insufficient memory.
         dtype (:class:`mindspore.dtype`): Dtype of Parameters. Default: ``mstype.float32`` .
 
     Inputs:
@@ -358,10 +349,9 @@ class EmbeddingLookup(Cell):
         Tensor, the shape of tensor is :math:`(z_1, z_2, ..., z_N)`.
 
     Raises:
-        TypeError: If `vocab_size` or `embedding_size` or `vocab_cache_size` is not an int.
+        TypeError: If `vocab_size` or `embedding_size` is not an int.
         TypeError: If `sparse` is not a bool or `manual_shapes` is not a tuple.
         ValueError: If `vocab_size` or `embedding_size` is less than 1.
-        ValueError: If `vocab_cache_size` is less than 0.
         ValueError: If `target` is neither 'CPU' nor 'DEVICE'.
         ValueError: If `slice_mode` is not one of 'batch_slice' or 'field_slice' or
                     'table_row_slice' or 'table_column_slice'.
@@ -387,17 +377,14 @@ class EmbeddingLookup(Cell):
 
     def __init__(self, vocab_size, embedding_size, param_init='normal',
                  target='CPU', slice_mode='batch_slice', manual_shapes=None,
-                 max_norm=None, sparse=True, vocab_cache_size=0, dtype=mstype.float32):
+                 max_norm=None, sparse=True, dtype=mstype.float32):
         """Initialize EmbeddingLookup."""
         super(EmbeddingLookup, self).__init__()
         Validator.check_value_type('sparse', sparse, [bool], self.cls_name)
         self.vocab_size = Validator.check_positive_int(
             vocab_size, 'vocab_size')
-        self.vocab_cache_size = Validator.check_non_negative_int(
-            vocab_cache_size, 'vocab_cache_size')
         self.target = target
         self.sparse = sparse
-        self.cache_enable = self.vocab_cache_size > 0
         self.forward_unique = False
         Validator.check_string(
             target, ['CPU', 'DEVICE'], 'target', self.cls_name)
@@ -409,10 +396,6 @@ class EmbeddingLookup(Cell):
         else:
             self.gatherv2 = ops.Gather()
         self.embeddinglookup = ops.EmbeddingLookup().set_device('CPU')
-        self.is_ps_server = False
-        enable_ps = _get_ps_context("enable_ps")
-        if enable_ps:
-            self._process_vocab_cache(slice_mode)
         self.embedding_size = Validator.check_positive_int(
             embedding_size, 'embedding_size', self.cls_name)
         self.embedding_table = Parameter(initializer(param_init, [self.vocab_size, self.embedding_size],
@@ -427,11 +410,6 @@ class EmbeddingLookup(Cell):
         self.shape = ops.Shape()
         if is_auto_parallel:
             self.unique = ops.Unique().shard(((1,),))
-        if self.cache_enable and enable_ps:
-            self._set_voacb_cache_enable_for_ps(
-                vocab_cache_size, embedding_size, vocab_size, param_init, dtype=dtype)
-            if is_auto_parallel:
-                self.unique.add_prim_attr('cache_enable', True)
         indices_shape_size = 2
         if slice_mode == "field_slice" and is_auto_parallel:
             if not manual_shapes:
@@ -450,7 +428,7 @@ class EmbeddingLookup(Cell):
                 ((get_group_size(), 1), (1, get_group_size())))
         elif slice_mode == "table_row_slice" and is_auto_parallel:
             full_batch = _get_full_batch()
-            if (target == 'DEVICE' and not full_batch) or (self.cache_enable and enable_ps and sparse):
+            if (target == 'DEVICE' and not full_batch):
                 indices_shape_size = 1
                 self.gather_revert.shard(((1, 1), (get_group_size(),)))
                 self.forward_unique = True
@@ -479,9 +457,6 @@ class EmbeddingLookup(Cell):
                                 "table_column_slice", "batch_slice"]
                 raise ValueError(f"For '{self.cls_name}', the 'slice_mode' must be in {support_mode}, "
                                  f"but got \"{slice_mode}\".")
-        if self.cache_enable and not enable_ps:
-            raise ValueError(
-                f"For '{self.cls_name}', haven't supported cache enable for not ps mode.")
         self.embedding_table.unique = self.forward_unique
         self.max_norm = max_norm
         if self.max_norm is not None:
@@ -489,149 +464,9 @@ class EmbeddingLookup(Cell):
                 self.max_norm, 'max_norm', self.cls_name)
             self.max_norm = Tensor(self.max_norm, dtype=mstype.float32)
 
-    def _process_vocab_cache(self, slice_mode):
-        """PS embeddingLookup cache check and process."""
-        self.cache_enable = False
-        if self.vocab_cache_size > 0:
-            if self.target == 'CPU':
-                logger.warning("The configuration of 'vocab_cache_size' is valid only in 'DEVICE' target, "
-                               "current target is CPU, so it will be ignored.")
-                return
-            enable_ps = _get_ps_context("enable_ps")
-            if not enable_ps:
-                logger.warning("The configuration of 'vocab_cache_size' is valid only in parameter server training "
-                               "mode, current mode is not parameter server trainning mode, so it will be ignored.")
-                return
-            self.is_ps_server = _is_role_pserver() and _enable_distributed_mindrt()
-            parallel_mode = _get_parallel_mode()
-            is_auto_parallel = parallel_mode in (
-                ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL)
-            if is_auto_parallel:
-                rank_size = get_group_size()
-                rank_id = get_rank()
-                full_batch = _get_full_batch()
-                if rank_size > 1 and not (full_batch and slice_mode == "table_row_slice"):
-                    raise ValueError(f"For '{self.cls_name}', the cache of parameter server parallel should only be "
-                                     f"used in \"full_batch\" and the value of \"full_batch\" must be True. "
-                                     f"Meanwhile, the value of 'slice_mode' must be \"table_row_slice\"."
-                                     f"But got full_batch: {full_batch} and 'slice_mode': \"{slice_mode}\".")
-                self.vocab_cache_size = self.vocab_cache_size * rank_size
-                _set_rank_id(rank_id)
-
-            self.cache_enable = True
-            _set_cache_enable(True)
-
-            if _is_role_worker():
-                self.vocab_size = self.vocab_cache_size
-
-    def _set_voacb_cache_enable_for_ps(self, vocab_cache_size, embedding_size, vocab_size, param_init,
-                                       dtype=mstype.float32):
-        """PS embeddingLookup cache enable set."""
-        if self.sparse:
-            self.forward_unique = True
-        param_key = _get_unique_parameter_key()
-        if _is_role_worker():
-            self.embedding_table.is_param_ps = True
-            self.embedding_table.cache_enable = True
-            self.embedding_table.key = param_key
-            _insert_hash_table_size(
-                self.embedding_table.name, vocab_cache_size, embedding_size, vocab_size, param_key)
-
-        if _enable_distributed_mindrt():
-            self.rank_id = get_rank()
-            if self.is_ps_server:
-                self._slice_pserver_embeddings("zeros", dtype=dtype)
-                self._set_cache_enable_and_key_for_pserver(param_key)
-
-    def _slice_pserver_embeddings(self, param_init, dtype=mstype.float32):
-        '''
-        Method to slice embedding tables on Parameter Servers.
-        It helps to train with a large scale embedding table and is used only in Parameter Server training mode.
-        So EmbeddingLookup op is on CPU device.
-        '''
-        self.embedding_lookup_list = []
-        # The dimension of each embedding table on servers could be different according to the slicing algorithm.
-        self.embedding_table_vocab_dim_list = []
-        self.embedding_table_list = []
-        # For different servers, the offset of their embedding table should be different.
-        self.embedding_offset = []
-
-        server_num = _get_ps_context("server_num")
-        if server_num == 0:
-            raise ValueError("The Parameter Server number is zero.")
-        # Assign the embedding table dimensions.
-        for _ in range(server_num):
-            self.embedding_table_vocab_dim_list.append(
-                self.vocab_size // server_num)
-        rest_vocab_size = self.vocab_size % server_num
-        if rest_vocab_size != 0:
-            for i in range(rest_vocab_size):
-                self.embedding_table_vocab_dim_list[i] += 1
-
-        offset = 0
-        for i in range(server_num):
-            self.embedding_table_list.append(Parameter(initializer(param_init,
-                                                                   [self.embedding_table_vocab_dim_list[i],
-                                                                    self.embedding_size], dtype=dtype),
-                                                       name="embedding_table_server_" + str(i)))
-
-            self.embedding_offset.append(offset)
-            offset += self.embedding_table_vocab_dim_list[i]
-
-            # Add EmbeddingLookup ops on different servers.
-            if self.target == 'CPU':
-                embedding_lookup = ops.EmbeddingLookup().set_device('CPU')
-            else:
-                if self.sparse:
-                    embedding_lookup = ops.SparseGatherV2()
-                else:
-                    embedding_lookup = ops.Gather()
-                embedding_lookup.add_prim_attr(
-                    'offset', self.embedding_offset[i])
-            embedding_lookup.add_prim_attr('rank_id', i)
-            embedding_lookup.add_prim_attr('ms_role', 'MS_PSERVER')
-            self.embedding_lookup_list.append(embedding_lookup)
-
-        # For now unique operation is not applied,
-        # so we need to reduce the lookup results from different servers with AddN.
-        self.reduce_lookup_result = ops.AddN()
-
-    def _do_server_embedding_lookup(self, indices):
-        '''
-        Construct backbone for EmbeddingLookup operators on servers.
-        '''
-        result_from_servers = []
-        for i in range(_get_ps_context("server_num")):
-            result = self.embedding_lookup_list[i](self.embedding_table_list[i],
-                                                   indices, self.embedding_offset[i])
-            result_from_servers.append(result)
-        final_result = self.reduce_lookup_result(result_from_servers)
-        return final_result
-
-    def _set_cache_enable_and_key_for_pserver(self, param_key):
-        '''
-        Set cache enable and parameter key for embedding table on parameter servers.
-        '''
-        # Parameter The Embedding Table on the Server side will be divided according to the number of servers.
-        # The divided Embedding Table will be used instead of the complete Embedding Table.
-        self.embedding_table = self.embedding_table_list[self.rank_id]
-        self.embedding_table.cache_enable = True
-        self.embedding_table.key = param_key
-
-    def _pserver_embedding_lookup(self, indices):
-        '''
-        Construct backbone for EmbeddingLookup operators on servers for embedding cache lookup.
-        '''
-        if self.target == 'CPU':
-            return self.embedding_lookup_list[self.rank_id](self.embedding_table, indices,
-                                                            self.embedding_offset[self.rank_id])
-        return self.embedding_lookup_list[self.rank_id](self.embedding_table, indices, 0)
-
     def construct(self, indices):
         if self.target == "CPU":
             out = self.embeddinglookup(self.embedding_table, indices, 0)
-        elif self.is_ps_server:
-            out = self._pserver_embedding_lookup(indices)
         else:
             if self.forward_unique:
                 shp = self.shape(indices) + (self.embedding_size,)

mindspore/nn/layer/normalization.py

@@ -21,6 +21,7 @@ import numbers
 import hashlib
 import numpy as np
 import mindspore.ops as ops
+from mindspore.ops import operations as P
 from mindspore.ops.operations import _inner_ops as inner
 from mindspore.common.parameter import Parameter
 from mindspore.common.initializer import initializer, Initializer
@@ -917,7 +918,7 @@ class _InstanceNorm(Cell):
 
         self.shape = ops.Shape()
         self.momentum = momentum
-        self.instance_bn = ops.InstanceNorm(epsilon=self.eps, momentum=self.momentum)
+        self.instance_bn = P.InstanceNorm(epsilon=self.eps, momentum=self.momentum)
 
     def construct(self, x):
         self._check_input_dim(self.shape(x), self.cls_name)

mindspore/nn/layer/thor_layer.py

@@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
 import mindspore.log as logger
 from mindspore.common.tensor import Tensor
 from mindspore.common.initializer import initializer, Initializer
-from mindspore.communication.management import get_group_size, get_rank
+from mindspore.communication.management import get_group_size
 from mindspore.ops.operations._thor_ops import ThorIm2Col
 from mindspore.common.parameter import Parameter
 from mindspore import _checkparam as Validator
@@ -30,8 +30,6 @@ from mindspore._checkparam import twice
 from mindspore import context
 from mindspore.nn.cell import Cell
 from mindspore.nn.layer.activation import get_activation
-from mindspore.parallel._ps_context import _is_role_worker, _get_ps_context, \
-    _set_rank_id, _insert_hash_table_size, _set_cache_enable
 from mindspore.parallel._utils import _get_parallel_mode, _get_full_batch
 from mindspore.context import ParallelMode
 from mindspore.nn.layer.basic import ClipByNorm
@@ -695,10 +693,6 @@ class EmbeddingLookupThor(Cell):
                                        Default: ``None`` .
         sparse (bool): Using sparse mode. When 'target' is set to 'CPU', 'sparse' has to be ``true`` .
                        Default: ``True`` .
-        vocab_cache_size (int): Cache size of the dictionary of embeddings. Default: ``0`` . It is valid only in
-            'DEVICE' target. And the moment parameter of corresponding optimizer will also be set to the cache size.
-            In addition, it should be noted that it will cost the 'DEVICE' memory, so suggests setting a reasonable
-            value to avoid insufficient memory.
 
     Inputs:
         - **input_indices** (Tensor) - The shape of tensor is :math:`(y_1, y_2, ..., y_S)`.
@@ -712,10 +706,9 @@ class EmbeddingLookupThor(Cell):
                     'table_row_slice' or 'table_column_slice'.
         ValueError: If `sparse` is False and `target` is 'CPU'.
         ValueError: If `slice_mode` is 'field_slice' and `manual_shapes` is None.
-        TypeError: If `vocab_size` or `embedding_size` or `vocab_cache_size` is not an int.
+        TypeError: If `vocab_size` or `embedding_size` is not an int.
         TypeError: If `sparse` is not a bool or `manual_shapes` is not a tuple.
         ValueError: If `vocab_size` or `embedding_size` is less than 1.
-        ValueError: If `vocab_cache_size` is less than 0.
 
 
     Supported Platforms:
@@ -736,14 +729,12 @@ class EmbeddingLookupThor(Cell):
 
     def __init__(self, vocab_size, embedding_size, param_init='normal',
                  target='CPU', slice_mode='batch_slice', manual_shapes=None,
-                 max_norm=None, sparse=True, vocab_cache_size=0):
+                 max_norm=None, sparse=True):
         super(EmbeddingLookupThor, self).__init__()
         Validator.check_value_type('sparse', sparse, [bool], self.cls_name)
         self.vocab_size = Validator.check_positive_int(vocab_size, 'vocab_size', self.cls_name)
-        self.vocab_cache_size = Validator.check_non_negative_int(vocab_cache_size, 'vocab_cache_size', self.cls_name)
         self.target = target
         self.sparse = sparse
-        self.cache_enable = self.vocab_cache_size > 0
         self.forward_unique = False
         self.dtype = mstype.float16
         if target not in ('CPU', 'DEVICE'):
@@ -757,9 +748,6 @@ class EmbeddingLookupThor(Cell):
         else:
             self.gatherv2 = ops.Gather()
         self.embeddinglookup = ops.EmbeddingLookup().set_device('CPU')
-        enable_ps = _get_ps_context("enable_ps")
-        if enable_ps:
-            self._process_vocab_cache(slice_mode)
         self.embedding_size = Validator.check_positive_int(embedding_size, 'embedding_size', self.cls_name)
         self.embedding_table = Parameter(initializer(param_init, [self.vocab_size, self.embedding_size],
                                                      mstype.float16), name='embedding_table')
@@ -772,10 +760,6 @@ class EmbeddingLookupThor(Cell):
         self.shape = ops.Shape()
         if is_auto_parallel:
             self.unique = ops.Unique().shard(((1,),))
-        if self.cache_enable and enable_ps:
-            self._set_voacb_cache_enable_for_ps(vocab_cache_size, embedding_size, vocab_size)
-            if is_auto_parallel:
-                self.unique.add_prim_attr('cache_enable', True)
         indices_shape_size = 2
         if slice_mode == "field_slice" and is_auto_parallel:
             if not manual_shapes:
@@ -792,7 +776,7 @@ class EmbeddingLookupThor(Cell):
             self.embeddinglookup.shard(((get_group_size(), 1), (1, get_group_size())))
         elif slice_mode == "table_row_slice" and is_auto_parallel:
             full_batch = _get_full_batch()
-            if (target == 'DEVICE' and not full_batch) or (self.cache_enable and enable_ps and sparse):
+            if (target == 'DEVICE' and not full_batch):
                 indices_shape_size = 1
                 self.gather_revert.shard(((1, 1), (get_group_size(),)))
                 self.forward_unique = True
@@ -818,11 +802,6 @@ class EmbeddingLookupThor(Cell):
                 raise ValueError(f"For '{self.cls_name}', the 'slice_mode' must be one of values in "
                                  f"['field_slice', 'table_row_slice', 'table_column_slice', 'batch_slice'], "
                                  f"but got 'slice_mode': {slice_mode}")
-        if self.cache_enable and not enable_ps:
-            if parallel_mode != ParallelMode.STAND_ALONE:
-                raise ValueError(f"For '{self.cls_name}', the 'parallel_mode' must be equal to "
-                                 f"'ParallelMode.STAND_ALONE', but got {parallel_mode}.")
-            self._set_cache_enable()
         self.embedding_table.unique = self.forward_unique
         self.max_norm = max_norm
         if self.max_norm is not None:
@@ -859,66 +838,6 @@ class EmbeddingLookupThor(Cell):
         self.matrix_g = matrix_g
         return out
 
-    def _set_cache_enable(self):
-        """EmbeddingLookup cache check for not ps env, which is only support 'ascend'."""
-        if self.target != 'DEVICE':
-            raise ValueError(f"For '{self.cls_name}', the configuration of 'vocab_cache_size' is valid "
-                             f"only when 'target' is 'DEVICE', but got 'target': {self.target}.")
-        if not self.sparse:
-            raise ValueError(f"For '{self.cls_name}', the configuration of 'vocab_cache_size' is valid "
-                             f"only when 'sparse' is true, but got 'sparse': {self.sparse}.")
-        if context.get_context("device_target") != 'Ascend':
-            raise ValueError(f"For '{self.cls_name}', the configuration of 'vocab_cache_size' is valid "
-                             f"only when 'device_target' is 'Ascend', but got {context.get_context('device_target')}.")
-
-        logger.info("EmbeddingLookup cache enable takes effect.")
-        self.forward_unique = True
-        self.unique = ops.Unique().set_device('CPU')
-        self.unique.add_prim_attr('cache_enable', True)
-        self.embedding_table.cache_enable = self.cache_enable
-        self.embedding_table.cache_shape = (self.vocab_cache_size, self.embedding_size)
-        self.reshape_first = ops.Reshape().set_device('CPU')
-
-    def _process_vocab_cache(self, slice_mode):
-        """PS embeddingLookup cache check and process."""
-        self.cache_enable = False
-        if self.vocab_cache_size > 0:
-            if self.target == 'CPU':
-                logger.warning("The configuration of 'vocab_cache_size' is valid only in 'DEVICE' target, "
-                               "current target is CPU, so it will be ignored.")
-                return
-            enable_ps = _get_ps_context("enable_ps")
-            if not enable_ps:
-                logger.warning(
-                    "The configuration of 'vocab_cache_size' is valid only in parameter server trainning "
-                    "mode, current mode is not parameter server trainning mode, so it will be ignored.")
-                return
-            parallel_mode = _get_parallel_mode()
-            is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL)
-            if is_auto_parallel:
-                rank_size = get_group_size()
-                rank_id = get_rank()
-                full_batch = _get_full_batch()
-                if rank_size > 1 and not (full_batch and slice_mode == "table_row_slice"):
-                    raise ValueError(f"For '{self.cls_name}', the embeddingLookup cache of parameter server parallel "
-                                     f"only be used in 'full_batch' and 'table_row_slice' parallel strategy, but got "
-                                     f"'full_batch': {full_batch}, 'slice_mode': {slice_mode}.")
-                self.vocab_cache_size = self.vocab_cache_size * rank_size
-                _set_rank_id(rank_id)
-            self.cache_enable = True
-            if _is_role_worker():
-                self.vocab_size = self.vocab_cache_size
-
-    def _set_voacb_cache_enable_for_ps(self, vocab_cache_size, embedding_size, vocab_size):
-        """PS embeddingLookup cache enable set."""
-        self.embedding_table.cache_enable = True
-        self.embedding_table.is_param_ps = True
-        _set_cache_enable(True)
-        if self.sparse:
-            self.forward_unique = True
-        if _is_role_worker():
-            _insert_hash_table_size(self.embedding_table.name, vocab_cache_size, embedding_size, vocab_size)
-
     def construct(self, indices):
         if self.target == "CPU":
             out = self.embeddinglookup(self.embedding_table, indices, 0)

mindspore/nn/optim/ada_grad.py

@@ -204,7 +204,6 @@ class Adagrad(Optimizer):
     def construct(self, grads):
         params = self._parameters
         accum = self.accum
-        grads = self.flatten_gradients(grads)
         grads = self.decay_weight(grads)
         grads = self.gradients_centralization(grads)
         grads = self.scale_grad(grads)

mindspore/nn/optim/adafactor.py

@@ -408,7 +408,6 @@ class AdaFactor(Optimizer):
 
     @jit(backend="ms_backend")
     def construct(self, gradients):
-        gradients = self.flatten_gradients(gradients)
         lr = self.get_lr()
         self.assignadd(self.global_step, self.global_step_increase_tensor)
         step = F.assign_add(self.step, 1)