mindspore 2.3.0__cp310-cp310-win_amd64.whl → 2.4.1__cp310-cp310-win_amd64.whl

mindspore/nn/layer/embedding.py

@@ -16,13 +16,14 @@
 from __future__ import absolute_import
 
 import mindspore.common.dtype as mstype
+import mindspore.ops as ops
 from mindspore import log as logger
 from mindspore.common.tensor import Tensor
 from mindspore.ops import operations as P
 from mindspore.ops import functional as F
 from mindspore.common.parameter import Parameter
 from mindspore.common.parameter import _get_unique_parameter_key
-from mindspore.common.initializer import initializer
+from mindspore.common.initializer import initializer, Normal
 from mindspore.communication.management import get_group_size, get_rank
 from mindspore.context import ParallelMode
 from mindspore.parallel._utils import _get_parallel_mode, _get_full_batch
@@ -34,7 +35,7 @@ from mindspore.ops.primitive import constexpr, _primexpr
 from mindspore.nn.layer.basic import ClipByNorm
 from mindspore.nn.cell import Cell
 
-__all__ = ['Embedding', 'EmbeddingLookup', 'MultiFieldEmbeddingLookup']
+__all__ = ['Embedding', 'EmbeddingExt', 'EmbeddingLookup', 'MultiFieldEmbeddingLookup']
 
 
 @_primexpr
@@ -161,6 +162,139 @@ class Embedding(Cell):
             f'embedding_table={self.embedding_table}, dtype={self.dtype}, padding_idx={self.padding_idx}'
 
 
+class EmbeddingExt(Cell):
+    r"""
+    The value in `input` is used as the index, and the corresponding embedding vector is queried from `weight` .
+
+    .. warning::
+        - This is an experimental API that is subject to change or deletion.
+        - On Ascend, the behavior is unpredictable when the value of `input` is invalid.
+
+    Args:
+        num_embeddings (int): Size of the dictionary of embeddings.
+        embedding_dim (int): The size of each embedding vector.
+        padding_idx (int, optional): If the value is not None, the corresponding row of embedding vector
+            will not be updated in training. The value of embedding vector at `padding_idx` will default
+            to zeros when the Embedding layer is newly constructed. The value should be in range
+            `[-num_embeddings, num_embeddings)` if it's not ``None``. Default ``None``.
+        max_norm (float, optional): If the value is not None, firstly get the p-norm result of the embedding
+            vector specified by `input` where p is specified by `norm_type`; if the result is larger then `max_norm`,
+            update the embedding vector` with :math:`\frac{max\_norm}{result+1e^{-7}}`. Default ``None``.
+        norm_type (float, optional): Indicated the value of p in p-norm. Default ``2.0``.
+        scale_grad_by_freq (bool, optional): If ``True`` the gradients will be scaled by the inverse of frequency
+            of the index in `input`. Default ``False``.
+        sparse (bool, optional): If ``True``, gradient w.r.t. `weight` matrix will be a sparse tensor which
+            has not been supported. Default: ``False``.
+        _weight (Tensor, optional): Used to initialize the `weight` of Embedding. If ``None``, the weight will be
+            initialized from normal distribution :math:`{N}(\text{sigma=1.0}, \text{mean=0.0})`. Default ``None``.
+        _freeze(bool, optional): If `weight` , the learnable weights of this module, should be freezed.
+            Default: ``False``.
+        dtype (mindspore.dtype, optional) : Dtype of Embedding's `weight` . It is meaningless when `_weight` is
+            not None. Default: ``None``.
+
+    Variables:
+        weight (Parameter): The learnable weights of this module of shape (num_embeddings, embedding_dim), which
+            initialized from :math:`{N}(\text{sigma=1.0}, \text{mean=0.0})` or `_weight` .
+
+    Inputs:
+        - **input** (Tensor) - The indices used to lookup in the embedding vector. The data type must be
+          int32 or int64, and the value should be in range `[0, num_embeddings)`.
+
+    Outputs:
+        Tensor, has the same data type as weight, the shape is :math:`(*input.shape, embedding\_dim)`.
+
+    Raises:
+        TypeError: If `num_embeddings` is not an int.
+        TypeError: If `embedding_dim` is not an int.
+        ValueError: If `padding_idx` is out of valid range.
+        TypeError: If `max_norm` is not a float.
+        TypeError: If `norm_type` is not a float.
+        TypeError: If `scale_grad_by_freq` is not a bool.
+        ValueError: If `weight.shape` is invalid.
+        TypeError: If `dtype` is not one of mindspore.dtype.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, nn
+        >>> input = Tensor([[1, 0, 1, 1], [0, 0, 1, 0]])
+        >>> embedding = nn.EmbeddingExt(num_embeddings=10, embedding_dim=3)
+        >>> output = embedding(input)
+        >>> print(output)
+        [[[-0.0024154  -0.01203444  0.00811537]
+          [ 0.00233847 -0.00596091  0.00536799]
+          [-0.0024154  -0.01203444  0.00811537]
+          [-0.0024154  -0.01203444  0.00811537]]
+         [[ 0.00233847 -0.00596091  0.00536799]
+          [ 0.00233847 -0.00596091  0.00536799]
+          [-0.0024154  -0.01203444  0.00811537]
+          [ 0.00233847 -0.00596091  0.00536799]]]
+    """
+
+    def __init__(self, num_embeddings, embedding_dim, padding_idx=None, max_norm=None, norm_type=2.0,
+                 scale_grad_by_freq=False, sparse=False, _weight=None, _freeze=False, dtype=None):
+        """Initialize Embedding."""
+        super().__init__()
+        self.sparse = Validator.check_value_type('sparse', sparse, [bool], self.cls_name)
+        if self.sparse:
+            raise ValueError("For Embedding, the scenerio, where `sparse` is True, has not be supported.")
+        self.num_embeddings = Validator.check_value_type(
+            'num_embeddings', num_embeddings, [int], self.cls_name)
+        self.embedding_dim = Validator.check_value_type(
+            'embedding_dim', embedding_dim, [int], self.cls_name)
+        self.dtype = dtype if dtype is not None else mstype.float32
+        Validator.check_subclass(
+            "dtype", self.dtype, mstype.number_type, self.cls_name)
+        self.padding_idx = padding_idx
+        if _weight is None:
+            init_tensor = Tensor(shape=[num_embeddings, embedding_dim], dtype=self.dtype, init=Normal(1, 0))
+            init_tensor = self._zero_weight_by_index(init_tensor)
+            self.weight = Parameter(init_tensor, name='weight', requires_grad=not _freeze)
+        else:
+            if _weight.shape != (num_embeddings, embedding_dim):
+                raise ValueError(f"For Embedding, shape of weight should be match with num_embeddings "
+                                 f"and embedding_dim, but got weight.shape: {_weight.shape}, "
+                                 f"and (num_embeddings, embedding_dim): ({num_embeddings}, {embedding_dim})")
+            self.weight = Parameter(_weight, name='weight', requires_grad=not _freeze)
+
+        self.max_norm = max_norm
+        if max_norm is not None:
+            self.max_norm = Validator.check_value_type('max_norm', max_norm, [float], self.cls_name)
+
+        self.norm_type = norm_type
+        if norm_type is not None:
+            self.norm_type = Validator.check_value_type('norm_type', norm_type,
+                                                        [float], self.cls_name)
+
+        self.scale_grad_by_freq = scale_grad_by_freq
+        if scale_grad_by_freq is not None:
+            self.scale_grad_by_freq = Validator.check_value_type('scale_grad_by_freq',
+                                                                 scale_grad_by_freq,
+                                                                 [bool], self.cls_name)
+
+    def _zero_weight_by_index(self, init_tensor):
+        if self.padding_idx is not None:
+            self.padding_idx = Validator.check_int_range(self.padding_idx, -self.num_embeddings, self.num_embeddings,
+                                                         Validator.INC_LEFT, "padding_idx", self.cls_name)
+            if isinstance(init_tensor, Tensor) and init_tensor.init is not None:
+                init_tensor = init_tensor.init_data()
+            init_tensor[self.padding_idx] = 0
+
+        return init_tensor
+
+    def construct(self, input):
+        return ops.embedding(input, self.weight, self.padding_idx, self.max_norm,
+                             self.norm_type, self.scale_grad_by_freq)
+
+    def extend_repr(self):
+        return f'num_embeddings={self.num_embeddings}, embedding_dim={self.embedding_dim}, ' \
+               f'padding_idx={self.padding_idx}, max_norm={self.max_norm}, norm_type={self.norm_type}, ' \
+               f'scale_grad_by_freq={self.scale_grad_by_freq}, dtype={self.dtype}'
+
+
 @_primexpr
 def _make_axis_range(start, end):
     axis = tuple(range(start, end))
@@ -182,6 +316,7 @@ class EmbeddingLookup(Cell):
         specified 'axis = 0' to lookup table.
         In field slice mode, the manual_shapes must be given. It is a tuple ,where
         the element is vocab[i], vocab[i] is the row numbers for i-th part.
+        This module does not support the PyNative mode.
 
     Args:
         vocab_size (int): Size of the dictionary of embeddings.

mindspore/nn/layer/normalization.py

@@ -19,7 +19,8 @@ from __future__ import division
 import itertools
 import numbers
 import hashlib
-
+import numpy as np
+import mindspore.ops as ops
 from mindspore.ops import operations as P
 from mindspore.ops.operations import _inner_ops as inner
 from mindspore.common.parameter import Parameter
@@ -37,7 +38,7 @@ from mindspore.nn.cell import Cell
 from mindspore import log as logger
 from mindspore.ops import group_norm
 
-__all__ = ['BatchNorm1d', 'BatchNorm2d', 'BatchNorm3d', 'LayerNorm', 'GroupNorm',
+__all__ = ['BatchNorm1d', 'BatchNorm2d', 'BatchNorm3d', 'LayerNorm', 'LayerNormExt', 'GroupNorm',
            'SyncBatchNorm', 'InstanceNorm1d', 'InstanceNorm2d', 'InstanceNorm3d']
 
 
@@ -558,14 +559,14 @@ class SyncBatchNorm(_BatchNorm):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
-            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/docs/en/master/model_train/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with multiple devices.
 
@@ -774,6 +775,101 @@ class LayerNorm(Cell):
             self.normalized_shape, self.begin_norm_axis, self.begin_params_axis, self.gamma, self.beta)
 
 
+class LayerNormExt(Cell):
+    r"""
+    Applies Layer Normalization over a mini-batch of inputs.
+
+    Layer Normalization is widely used in recurrent neural networks. It applies
+    normalization on a mini-batch of inputs for each single training case as described
+    in the paper `Layer Normalization <https://arxiv.org/pdf/1607.06450.pdf>`_.
+
+    Unlike Batch Normalization, Layer Normalization performs exactly the same computation at training and
+    testing time. It is applied across all channels and pixel but only one batch size.
+    :math:`\gamma` is the scale value learned through training and :math:`\beta` is the shift value.
+    It can be described using the following formula:
+
+    .. math::
+        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        normalized_shape (Union(tuple[int], list[int], int)): The normalized shape of `x` for LayerNorm
+        eps (float): A value added to the denominator for numerical stability(:math:`\epsilon`). Default: ``1e-5`` .
+        elementwise_affine (bool): Whether affine transformation is required. When this parameter is set to ``True``,
+            the weight parameter is initialized to 1 and the offset is initialized to 0. Default: ``True``.
+        bias (bool): If set to ``False``, the layer will not learn an additive bias (only relevant if
+             `elementwise_affine` is ``True``). Default: ``True``.
+        dtype (:class:`mindspore.dtype`): Dtype of Parameters. Default: ``None`` .
+
+    Inputs:
+        - **x** (Tensor) - The shape is :math:`(N, *)`, where :math:`*` is equal to normalized_shape.
+
+    Outputs:
+        Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`.
+
+    Raises:
+        TypeError: If `eps` is not a float.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> import numpy as np
+        >>> x = ms.Tensor(np.ones([20, 5, 10, 10]), ms.float32)
+        >>> shape1 = x.shape[1:]
+        >>> m = ms.nn.LayerNormExt(shape1)
+        >>> output = m(x).shape
+        >>> print(output)
+        (20, 5, 10, 10)
+    """
+
+    def __init__(self,
+                 normalized_shape,
+                 eps=1e-5,
+                 elementwise_affine=True,
+                 bias=True,
+                 dtype=None
+                 ):
+        """Initialize LayerNormExt."""
+        super(LayerNormExt, self).__init__()
+        if isinstance(normalized_shape, numbers.Integral):
+            # mypy error: incompatible types in assignment
+            normalized_shape = (normalized_shape,)  # type: ignore[assignment]
+        if not isinstance(normalized_shape, (tuple, list)):
+            raise TypeError(f"For '{self.cls_name}', the type of 'normalized_shape' must be tuple[int] or list[int], "
+                            f"but got {normalized_shape} and the type is {type(normalized_shape)}.")
+        if not normalized_shape:
+            raise ValueError(
+                f"Expected normalized_shape to be at least 1-dimensional, i.e., containing at "
+                f"least one element, but got normalized_shape = {normalized_shape}"
+            )
+        self.normalized_shape = tuple(normalized_shape)
+        self.eps = eps
+        self.elementwise_affine = elementwise_affine
+        ms_dtype = mstype.float32 if dtype is None else dtype
+        if self.elementwise_affine:
+            self.weight = Parameter(Tensor(np.ones(normalized_shape), ms_dtype), name="weight")
+            if bias:
+                self.bias = Parameter(Tensor(np.zeros(normalized_shape), ms_dtype), name="bias")
+            else:
+                self.bias = None
+        else:
+            self.weight = None
+            self.bias = None
+
+    def construct(self, input):
+        y = ops.layer_norm(input, self.normalized_shape, self.weight,
+                           self.bias, self.eps)
+        return y
+
+    def extend_repr(self):
+        return 'normalized_shape={}, eps={}, elementwise_affine={}'.format(
+            self.normalized_shape, self.eps, self.elementwise_affine)
+
+
 class _InstanceNorm(Cell):
     """Instance Normalization base class."""
     @cell_attr_register

mindspore/nn/layer/padding.py

@@ -279,20 +279,6 @@ class ConstantPad1d(_ConstantPadNd):
            [0.5 1.  1.  1.  1.  0.5]]]]
         >>> print(out.shape)
         (1, 2, 3, 6)
-        >>> # padding is negative
-        >>> padding = (-1, 0)
-        >>> value = 0.5
-        >>> pad1d = ms.nn.ConstantPad1d(padding, value)
-        >>> out = pad1d(x)
-        >>> print(out)
-        [[[[1. 1. 1.]
-           [1. 1. 1.]
-           [1. 1. 1.]]
-          [[1. 1. 1.]
-           [1. 1. 1.]
-           [1. 1. 1.]]]]
-        >>> print(out.shape)
-        (1, 2, 3, 3)
     """
 
     def __init__(self, padding, value):
@@ -336,21 +322,21 @@ class ConstantPad2d(_ConstantPadNd):
         >>> import mindspore as ms
         >>> x = np.ones(shape=(1, 2, 3, 4)).astype(np.float32)
         >>> x = ms.Tensor(x)
-        >>> padding = (-1, 1, 0, 1)
+        >>> padding = (1, 1, 0, 1)
         >>> value = 0.5
         >>> pad2d = ms.nn.ConstantPad2d(padding, value)
         >>> out = pad2d(x)
         >>> print(out)
-        [[[[1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [0.5 0.5 0.5 0.5]]
-          [[1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [0.5 0.5 0.5 0.5]]]]
+        [[[[0.5  1.  1.  1.  1.  0.5]
+           [0.5  1.  1.  1.  1.  0.5]
+           [0.5  1.  1.  1.  1.  0.5]
+           [0.5  0.5 0.5 0.5 0.5 0.5]]
+          [[0.5  1.  1.  1.  1.  0.5]
+           [0.5  1.  1.  1.  1.  0.5]
+           [0.5  1.  1.  1.  1.  0.5]
+           [0.5  0.5 0.5 0.5 0.5 0.5]]]]
         >>> print(out.shape)
-        (1, 2, 4, 4)
+        (1, 2, 4, 6)
     """
 
     def __init__(self, padding, value):
@@ -396,25 +382,25 @@ class ConstantPad3d(_ConstantPadNd):
         >>> import mindspore as ms
         >>> x = np.ones(shape=(1, 2, 3, 4)).astype(np.float32)
         >>> x = ms.Tensor(x)
-        >>> padding = (-1, 1, 0, 1, 1, 0)
+        >>> padding = (1, 1, 0, 1, 1, 0)
         >>> value = 0.5
         >>> pad3d = ms.nn.ConstantPad3d(padding, value)
         >>> out = pad3d(x)
         >>> print(out)
-        [[[[0.5 0.5 0.5 0.5]
-           [0.5 0.5 0.5 0.5]
-           [0.5 0.5 0.5 0.5]
-           [0.5 0.5 0.5 0.5]]
-          [[1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [0.5 0.5 0.5 0.5]]
-          [[1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [1.  1.  1.  0.5]
-           [0.5 0.5 0.5 0.5]]]]
+        [[[[0.5 0.5 0.5 0.5 0.5 0.5]
+           [0.5 0.5 0.5 0.5 0.5 0.5]
+           [0.5 0.5 0.5 0.5 0.5 0.5]
+           [0.5 0.5 0.5 0.5 0.5 0.5]]
+          [[0.5 1.  1.  1.  1.  0.5]
+           [0.5 1.  1.  1.  1.  0.5]
+           [0.5 1.  1.  1.  1.  0.5]
+           [0.5 0.5 0.5 0.5 0.5 0.5]]
+          [[0.5 1.  1.  1.  1.  0.5]
+           [0.5 1.  1.  1.  1.  0.5]
+           [0.5 1.  1.  1.  1.  0.5]
+           [0.5 0.5 0.5 0.5 0.5 0.5]]]]
         >>> print(out.shape)
-        (1, 3, 4, 4)
+        (1, 3, 4, 6)
     """
 
     def __init__(self, padding, value):
@@ -652,20 +638,20 @@ class ZeroPad2d(_ConstantPadNd):
         >>> import mindspore as ms
         >>> x = np.ones(shape=(1, 2, 3, 4)).astype(np.float32)
         >>> x = ms.Tensor(x)
-        >>> padding = (-1, 1, 0, 1)
+        >>> padding = (1, 1, 0, 1)
         >>> pad = ms.nn.ZeroPad2d(padding)
         >>> out = pad(x)
         >>> print(out)
-        [[[[1. 1. 1. 0.]
-           [1. 1. 1. 0.]
-           [1. 1. 1. 0.]
-           [0. 0. 0. 0.]]
-          [[1. 1. 1. 0.]
-           [1. 1. 1. 0.]
-           [1. 1. 1. 0.]
-           [0. 0. 0. 0.]]]]
+        [[[[0. 1. 1. 1. 1. 0.]
+           [0. 1. 1. 1. 1. 0.]
+           [0. 1. 1. 1. 1. 0.]
+           [0. 0. 0. 0. 0. 0.]]
+          [[0. 1. 1. 1. 1. 0.]
+           [0. 1. 1. 1. 1. 0.]
+           [0. 1. 1. 1. 1. 0.]
+           [0. 0. 0. 0. 0. 0.]]]]
         >>> print(out.shape)
-        (1, 2, 4, 4)
+        (1, 2, 4, 6)
     """
 
     def __init__(self, padding):