mindspore 2.0.0a0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/nn/grad/cell_grad.py

@@ -23,6 +23,7 @@ from mindspore.ops.primitive import Primitive
 from mindspore.common import dtype as mstype
 from mindspore.common.api import jit
 from mindspore.common._decorator import deprecated
+from mindspore.common import mutable
 
 
 class _FirstGrad(Cell):
@@ -94,10 +95,10 @@ class Jvp(Cell):
 
         if self.issubclass_(self.typeof(output), mstype.tuple_):
             u = self.make_tuple()
-            for i in range(len(output)):
-                u = u + self.make_tuple(self.oneslike(output[i]))
+            for _, element in enumerate(output):
+                u = u + self.make_tuple(mutable(self.oneslike(element)))
         else:
-            u = self.oneslike(output)
+            u = mutable(self.oneslike(output))
 
         if len(jvp_input) == 1:
             second_gradient_net = self.second_grad_op(self.first_grad_single_value)
@@ -130,10 +131,10 @@ class _JvpInner(Cell):
         """Compute the jacobian-vector-product of the given fn, vector, inputs and outputs."""
         if self.issubclass_(self.typeof(output), mstype.tuple_):
             u = self.make_tuple()
-            for i in range(len(output)):
-                u = u + self.make_tuple(self.oneslike(output[i]))
+            for _, element in enumerate(output):
+                u = u + self.make_tuple(mutable(self.oneslike(element)))
         else:
-            u = self.oneslike(output)
+            u = mutable(self.oneslike(output))
 
         if len(jvp_input) == 1:
             second_gradient_net = self.second_grad_op(self.first_grad_single_value)

mindspore/nn/layer/__init__.py

@@ -20,7 +20,8 @@ The high-level components(Cells) used to construct the neural network.
 from __future__ import absolute_import
 
 from mindspore.nn.layer import activation, normalization, container, conv, basic, embedding, pooling, \
-    image, quant, math, combined, timedistributed, thor_layer, rnns, rnn_cells, padding, dense
+    image, math, combined, timedistributed, thor_layer, rnns, rnn_cells, padding, dense, transformer, \
+    channel_shuffle
 from mindspore.nn.layer.activation import *
 from mindspore.nn.layer.normalization import *
 from mindspore.nn.layer.container import *
@@ -32,14 +33,14 @@ from mindspore.nn.layer.basic import *
 from mindspore.nn.layer.embedding import *
 from mindspore.nn.layer.pooling import *
 from mindspore.nn.layer.image import *
-from mindspore.nn.layer.quant import *
 from mindspore.nn.layer.math import *
 from mindspore.nn.layer.combined import *
 from mindspore.nn.layer.timedistributed import *
+from mindspore.nn.layer.transformer import *
 from mindspore.nn.layer.channel_shuffle import ChannelShuffle
 from mindspore.nn.layer.thor_layer import DenseThor, Conv2dThor, EmbeddingThor, EmbeddingLookupThor
 from mindspore.nn.layer.padding import ConstantPad1d, ConstantPad2d, ConstantPad3d, ReflectionPad1d, \
-    ReflectionPad2d, ZeroPad2d, ReplicationPad1d, ReplicationPad2d, ReplicationPad3d
+    ReflectionPad2d, ReflectionPad3d, ZeroPad2d, ReplicationPad1d, ReplicationPad2d, ReplicationPad3d
 
 __all__ = []
 __all__.extend(activation.__all__)
@@ -53,10 +54,10 @@ __all__.extend(basic.__all__)
 __all__.extend(embedding.__all__)
 __all__.extend(pooling.__all__)
 __all__.extend(image.__all__)
-__all__.extend(quant.__all__)
 __all__.extend(math.__all__)
 __all__.extend(combined.__all__)
 __all__.extend(timedistributed.__all__)
+__all__.extend(transformer.__all__)
 __all__.extend(thor_layer.__all__)
 __all__.extend(padding.__all__)
 __all__.extend(channel_shuffle.__all__)

mindspore/nn/layer/activation.py

@@ -17,7 +17,7 @@ from __future__ import absolute_import
 
 import numpy as np
 
-from mindspore._checkparam import Rel, Validator as validator
+from mindspore import _checkparam as validator
 from mindspore._extends import cell_attr_register
 from mindspore.common import dtype as mstype
 from mindspore.common.parameter import Parameter
@@ -25,8 +25,9 @@ from mindspore.common.tensor import Tensor
 from mindspore.ops import functional as F
 from mindspore.ops import operations as P
 from mindspore.ops.operations import nn_ops as NN_OPS
-from mindspore.ops.primitive import constexpr
 from mindspore.nn.cell import Cell
+from mindspore import ops
+from mindspore.ops.primitive import _primexpr
 
 __all__ = ['Softmin',
            'Softmax',
@@ -156,19 +157,17 @@ class Softmin(Cell):
     def __init__(self, axis=-1):
         """Initialize Softmin."""
         super(Softmin, self).__init__()
-        self.softmax = P.Softmax(axis)
+        self.axis = axis
 
     def construct(self, x):
-        x = -1 * x
-        return self.softmax(x)
+        return ops.function.softmin(x, self.axis)
 
 
 class Softmax2d(Cell):
     r"""
-    Applies SoftMax over features to each spatial location.
+    Softmax function applied to 2D features data.
 
-    When given a Tensor with shape of :math:`(C, H, W)` , it will
-    apply `Softmax` to each location :math:`(c, h, w)`.
+    Applies `Softmax` to each location :math:`(c, h, w)` with an input Tensor of shape :math:`(C, H, W)` .
 
     Inputs:
         - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`.
@@ -198,7 +197,7 @@ class Softmax2d(Cell):
         self.shape = P.Shape()
 
     @staticmethod
-    @constexpr
+    @_primexpr
     def _check_input_dim(shape, cls_name):
         dim = len(shape)
         if dim not in (3, 4):
@@ -275,8 +274,6 @@ class LogSoftmax(Cell):
 
         \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right),
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
-
     Args:
         axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: -1.
 
@@ -381,7 +378,7 @@ class ReLU(Cell):
 
     Inputs:
         - **x** (Tensor) - The input of ReLU is a Tensor of any dimension. The data type is `number <https://www.mind
-          spore.cn/docs/en/r2.0.0-alpha/api_python/mindspore.html#mindspore.dtype>`_ .
+          spore.cn/docs/en/r2.0/api_python/mindspore.html#mindspore.dtype>`_ .
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
@@ -425,7 +422,6 @@ class ReLU6(Cell):
 
     Inputs:
         - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32.
-          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
 
     Outputs:
         Tensor, which has the same type as `x`.
@@ -517,8 +513,8 @@ class RReLU(Cell):
     The activation function is defined as:
 
     .. math::
-            \text{RReLU}(x_{ji}) = \begin{cases}x_{ji}, &\text{if } x_{ji} \geq 0; \cr
-            {\alpha_{ji}} * x_{ji}, &\text{otherwise.}\end{cases}
+        \text{RReLU}(x_{ji}) = \begin{cases}x_{ji}, &\text{if } x_{ji} \geq 0; \cr
+        {\alpha_{ji}} * x_{ji}, &\text{otherwise.}\end{cases}
 
     where :math:`\alpha_{ji}` ~ :math:`U(l, u)`, :math:`l \le u`.
 
@@ -565,17 +561,17 @@ class RReLU(Cell):
         if lower > upper:
             raise ValueError(f"For {self.cls_name}, the value of 'upper' must be greater than 'lower', "
                              f"but got upper: {upper}, lower: {lower}. ")
-
-        self.lower = lower
-        self.upper = upper
+        self.lower = Tensor(lower, dtype=mstype.float32)
+        self.upper = Tensor(upper, dtype=mstype.float32)
         self.sign = P.Sign()
 
     def construct(self, x):
-        size = x.shape
+        _size = x.shape
+        _dtype = x.dtype
         sign_matrix = self.sign(x)
         negative_filter = sign_matrix.clip(None, 0)
         positive_filter = sign_matrix.clip(0, None)
-        mask = P.Cast()(Tensor(np.random.uniform(self.lower, self.upper, size=size)), P.DType()(x))
+        mask = ops.uniform(_size, self.lower, self.upper).astype(_dtype)
         negative_mask = negative_filter * mask * -1
         total_mask = negative_mask + positive_filter
         out = total_mask * x
@@ -629,7 +625,7 @@ class SiLU(Cell):
     `SiLU <https://en.wikipedia.org/wiki/Activation_function#/media/File:Swish.svg>`_ .
 
     Inputs:
-        - **x** (Tensor) - Input with the data type float16 or float32. Tensor of arbitrary dimensions.
+        - **x** (Tensor) - Input with the data type float16 or float32.
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
@@ -651,10 +647,9 @@ class SiLU(Cell):
     def __init__(self):
         """Initialize SiLU."""
         super(SiLU, self).__init__()
-        self.sigmoid = P.Sigmoid()
 
     def construct(self, x):
-        return self.sigmoid(x) * x
+        return ops.function.silu(x)
 
 
 class Tanh(Cell):
@@ -713,14 +708,13 @@ class Tanhshrink(Cell):
     where :math:`x_i` is an element of the input Tensor.
 
     Inputs:
-        - **x** (Tensor) - Tensor of any dimension, input with data type of float16 or float32.
+        - **x** (Tensor) - Tensor of any dimension.
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
 
     Raises:
         TypeError: If `x` is not a Tensor.
-        TypeError: If dtype of `x` is neither float16 nor float32.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -740,17 +734,9 @@ class Tanhshrink(Cell):
     def __init__(self):
         """Initialize Tanhshrink."""
         super(Tanhshrink, self).__init__()
-        self.tanh = P.Tanh()
 
     def construct(self, x):
-        return x - self.tanh(x)
-
-
-@constexpr
-def _dtype_check(x_dtype, prim_name):
-    """Check dtype."""
-    if x_dtype not in [mstype.float32, mstype.float16]:
-        raise TypeError("For {}, the x_dtype must be float32 or float16, but got {}.".format(prim_name, x_dtype))
+        return F.tanhshrink(x)
 
 
 class Hardtanh(Cell):
@@ -785,7 +771,7 @@ class Hardtanh(Cell):
         TypeError: If dtype of `x` is neither float16 nor float32.
         TypeError: If dtype of `min_val` is neither float nor int.
         TypeError: If dtype of `max_val` is neither float nor int.
-        ValueError: If `max_val` is less than `min_val`.
+        ValueError: If `min_val` is not less than `max_val`.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -804,34 +790,14 @@ class Hardtanh(Cell):
     def __init__(self, min_val=-1.0, max_val=1.0):
         """Initialize Hardtanh."""
         super(Hardtanh, self).__init__()
-        validator.check_value_type('min_val', min_val, [float, int], self.cls_name)
-        validator.check_value_type('max_val', max_val, [float, int], self.cls_name)
-        validator.check_number("max_val", max_val, min_val, Rel.GE, self.cls_name)
-
-        self.max = P.Maximum()
-        self.min = P.Minimum()
         self.min_val = min_val
         self.max_val = max_val
-        self.dtype = P.DType()
-        self.expand = P.ExpandDims()
-        self.squeeze = P.Squeeze(0)
+        if self.min_val >= self.max_val:
+            raise ValueError(f"For Hardtanh, min_val should be less than max_val,"
+                             f"but got {self.min_val} and {self.max_val}")
 
     def construct(self, x):
-        if not isinstance(x, Tensor):
-            raise TypeError("'x' must be a Tensor")
-        _dtype_check(self.dtype(x), self.cls_name)
-        # min_val and max_val are scalars, if x is 0d, x is also a scalar.
-        # However, ops.Maximum does not support input two scalar.
-        # To solve this problem, expand x from scalar to tensor, apply Maximum, then squeeze the output back to scalar.
-        if not x.shape:
-            x = self.expand(x, 0)
-            x = self.max(x, self.min_val)
-            x = self.min(x, self.max_val)
-            x = self.squeeze(x)
-        else:
-            x = self.max(x, self.min_val)
-            x = self.min(x, self.max_val)
-        return x
+        return F.hardtanh(x, self.min_val, self.max_val)
 
 
 class GELU(Cell):
@@ -1054,14 +1020,14 @@ class PReLU(Cell):
     Activation_function#/media/File:Activation_prelu.svg>`_.
 
     Args:
-        channel (int): The elements number of parameter.
+        channel (int): The elements number of parameter `w`.
           It could be an int, and the value is 1 or the channels number of input tensor `x`. Default: 1.
         w (Union[float, list, Tensor]): The initial value of parameter. It could be a float, a float list or
           a tensor has the same dtype as the input tensor `x`. Default: 0.25.
 
     Inputs:
         - **x** (Tensor) - The input of PReLU with data type of float16 or float32.
-          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
+          The shape is :math:`(N, *)` where :math:`*` means, any number of additional dimensions.
 
     Outputs:
         Tensor, with the same dtype and shape as the `x`.
@@ -1118,15 +1084,9 @@ class PReLU(Cell):
                             f"but got {type(w).__name__}.")
         self.w = Parameter(w, name='a')
         self.prelu = P.PReLU()
-        self.relu = P.ReLU()
-        self.assign = P.Assign()
 
     def construct(self, x):
-        u = self.relu(self.w)
-        v = self.prelu(x, F.cast(u, x.dtype))
-        if self.training:
-            self.assign(self.w, u)
-        return v
+        return self.prelu(x, F.cast(self.w, x.dtype))
 
 
 class HSwish(Cell):
@@ -1138,8 +1098,6 @@ class HSwish(Cell):
     .. math::
         \text{hswish}(x_{i}) = x_{i} * \frac{ReLU6(x_{i} + 3)}{6},
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
-
     Inputs:
         - **x** (Tensor) - The input of HSwish, data type must be float16 or float32.
           The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
@@ -1179,8 +1137,6 @@ class HSigmoid(Cell):
     .. math::
         \text{hsigmoid}(x_{i}) = max(0, min(1, \frac{x_{i} + 3}{6})),
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
-
     Inputs:
         - **input_x** (Tensor) - The input of HSigmoid. Tensor of any dimension.
 
@@ -1302,7 +1258,8 @@ class SoftShrink(Cell):
         \end{cases}
 
     Args:
-        lambd: the :math:`\lambda` must be no less than zero for the SoftShrink formulation. Default: 0.5.
+        lambd (float): the :math:`\lambda` must be no less than zero for the SoftShrink formulation.
+            Default: 0.5.
 
     Inputs:
         - **input_x** (Tensor) - The input of SoftShrink with data type of float16 or float32.
@@ -1318,7 +1275,7 @@ class SoftShrink(Cell):
         ValueError: If lambd is less than 0.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> input_x = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mstype.float16)
@@ -1366,7 +1323,7 @@ class HShrink(Cell):
         TypeError: If dtype of `input_x` is neither float16 nor float32.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
@@ -1415,7 +1372,7 @@ class Threshold(Cell):
         TypeError: If `value` is not a float or an int.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
@@ -1429,19 +1386,12 @@ class Threshold(Cell):
 
     def __init__(self, threshold, value):
         """Initialize Threshold."""
-        super().__init__()
-        validator.check_value_type('threshold', threshold, [float, int], self.cls_name)
-        validator.check_value_type('value', value, [float, int], self.cls_name)
+        super(Threshold, self).__init__()
         self.threshold = threshold
         self.value = value
-        self.greater = P.Greater()
-        self.fill = P.Fill()
-        self.select = P.Select()
 
     def construct(self, input_x):
-        cond = self.greater(input_x, self.threshold)
-        value = self.fill(input_x.dtype, input_x.shape, self.value)
-        return self.select(cond, input_x, value)
+        return F.threshold(input_x, self.threshold, self.value)
 
 
 class Mish(Cell):
@@ -1473,17 +1423,17 @@ class Mish(Cell):
 
 class GLU(Cell):
     r"""
-    Applies the gated linear unit function.
+    The gated linear unit function.
 
     .. math::
         {GLU}(a, b)= a \otimes \sigma(b)
 
     where :math:`a` is the first half of the input matrices and :math:`b` is the second half.
 
-    Here :math:`\sigma` is the sigmoid function, and :math:`*` is the Hadamard product.
+    Here :math:`\sigma` is the sigmoid function, and :math:`\otimes` is the Hadamard product.
 
     Args:
-        axis (int): the dimension on which to split the input. Default: -1
+        axis (int): the axis to split the input. Default: -1, the last axis in `x`.
 
     Inputs:
         - **x** (Tensor) - :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional dimensions.
@@ -1499,7 +1449,7 @@ class GLU(Cell):
         >>> input = Tensor([[0.1,0.2,0.3,0.4],[0.5,0.6,0.7,0.8]])
         >>> output = m(input)
         >>> print(output)
-        [[0.05744425 0.11973753
+        [[0.05744425 0.11973753]
          [0.33409387 0.41398472]]
     """
 
@@ -1515,6 +1465,7 @@ class GLU(Cell):
         x2 = self.sigmoid(x2)
         return x1 * x2
 
+
 _activation = {
     'softmin': Softmin,
     'softmax': Softmax,