mindspore 1.10.0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/nn/dynamic_lr.py

@@ -17,7 +17,7 @@ from __future__ import absolute_import
 
 import math
 
-from mindspore._checkparam import Validator as validator
+from mindspore import _checkparam as validator
 
 
 def piecewise_constant_lr(milestone, learning_rates):

mindspore/nn/grad/cell_grad.py

@@ -18,10 +18,12 @@ from __future__ import absolute_import
 from mindspore.nn.cell import Cell
 from mindspore.ops import composite as C
 from mindspore.ops import operations as P
+from mindspore.ops.operations import _inner_ops as inner
 from mindspore.ops.primitive import Primitive
 from mindspore.common import dtype as mstype
-from mindspore.common.api import ms_function
+from mindspore.common.api import jit
 from mindspore.common._decorator import deprecated
+from mindspore.common import mutable
 
 
 class _FirstGrad(Cell):
@@ -80,12 +82,11 @@ class Jvp(Cell):
         self.first_grad_single_value = _FirstGradSingleValue(fn)
         self.first_grad_single_value.add_flags(enable_tuple_grad_first=True)
         self.second_grad_op = C.GradOperation(sens_param=True)
-        self.issubclass_ = P.IsSubClass()
+        self.issubclass_ = inner.IsSubClass()
         self.typeof = Primitive('typeof')
         self.make_tuple = Primitive('MakeTuple')
-        self.tuple_len = Primitive("tuple_len")
 
-    @ms_function
+    @jit
     def construct(self, *args):
         """construct for jvp."""
         jvp_input = args[0:-1]
@@ -94,12 +95,12 @@ class Jvp(Cell):
 
         if self.issubclass_(self.typeof(output), mstype.tuple_):
             u = self.make_tuple()
-            for i in range(self.tuple_len(output)):
-                u = u + self.make_tuple(self.oneslike(output[i]))
+            for _, element in enumerate(output):
+                u = u + self.make_tuple(mutable(self.oneslike(element)))
         else:
-            u = self.oneslike(output)
+            u = mutable(self.oneslike(output))
 
-        if self.tuple_len(jvp_input) == 1:
+        if len(jvp_input) == 1:
             second_gradient_net = self.second_grad_op(self.first_grad_single_value)
             gradient_output = second_gradient_net(u, jvp_input, v)
         else:
@@ -122,21 +123,20 @@ class _JvpInner(Cell):
         self.first_grad_single_value = _JvpFirstGradSingleValue()
         self.first_grad_single_value.add_flags(enable_tuple_grad_first=True)
         self.second_grad_op = C.GradOperation(sens_param=True)
-        self.issubclass_ = P.IsSubClass()
+        self.issubclass_ = inner.IsSubClass()
         self.typeof = Primitive('typeof')
         self.make_tuple = Primitive('MakeTuple')
-        self.tuple_len = Primitive("tuple_len")
 
     def compute_jvp(self, fn, v, jvp_input, output):
         """Compute the jacobian-vector-product of the given fn, vector, inputs and outputs."""
         if self.issubclass_(self.typeof(output), mstype.tuple_):
             u = self.make_tuple()
-            for i in range(self.tuple_len(output)):
-                u = u + self.make_tuple(self.oneslike(output[i]))
+            for _, element in enumerate(output):
+                u = u + self.make_tuple(mutable(self.oneslike(element)))
         else:
-            u = self.oneslike(output)
+            u = mutable(self.oneslike(output))
 
-        if self.tuple_len(jvp_input) == 1:
+        if len(jvp_input) == 1:
             second_gradient_net = self.second_grad_op(self.first_grad_single_value)
             gradient_output = second_gradient_net(u, fn, jvp_input, v)
         else:
@@ -182,40 +182,15 @@ class Vjp(Cell):
         self.fn = fn
         self.grad = C.GradOperation(get_all=True, sens_param=True)
         self.grad_single_value = C.GradOperation(sens_param=True)
-        self.issubclass_ = P.IsSubClass()
+        self.issubclass_ = inner.IsSubClass()
         self.typeof = Primitive('typeof')
-        self.tuple_len = Primitive("tuple_len")
 
-    @ms_function
+    @jit
     def construct(self, *args):
         front_input = args[0:-1]
         output = self.fn(*front_input)
-        if self.tuple_len(front_input) == 1:
+        if len(front_input) == 1:
             gradient_output = self.grad_single_value(self.fn)(*args)
         else:
             gradient_output = self.grad(self.fn)(*args)
         return output, gradient_output
-
-
-class _VjpInner(Cell):
-    """
-    Computes the dot product between a vector `v` and the Jacobian of the given network at the point
-    given by the inputs. This class implements the inner process of function vjp.
-    """
-
-    def __init__(self):
-        super(_VjpInner, self).__init__()
-        self.grad = C.GradOperation(get_all=True, sens_param=True)
-        self.grad_single_value = C.GradOperation(sens_param=True)
-        self.tuple_len = Primitive("tuple_len")
-
-    def construct(self, *args):
-        fn = args[0]
-        front_input = args[1:-1]
-        input_with_v = args[1:]
-        output = fn(*front_input)
-        if self.tuple_len(front_input) == 1:
-            gradient_output = self.grad_single_value(fn)(*input_with_v)
-        else:
-            gradient_output = self.grad(fn)(*input_with_v)
-        return output, gradient_output

mindspore/nn/layer/__init__.py

@@ -20,7 +20,8 @@ The high-level components(Cells) used to construct the neural network.
 from __future__ import absolute_import
 
 from mindspore.nn.layer import activation, normalization, container, conv, basic, embedding, pooling, \
-    image, quant, math, combined, timedistributed, thor_layer, rnns, rnn_cells, padding, dense
+    image, math, combined, timedistributed, thor_layer, rnns, rnn_cells, padding, dense, transformer, \
+    channel_shuffle
 from mindspore.nn.layer.activation import *
 from mindspore.nn.layer.normalization import *
 from mindspore.nn.layer.container import *
@@ -32,13 +33,14 @@ from mindspore.nn.layer.basic import *
 from mindspore.nn.layer.embedding import *
 from mindspore.nn.layer.pooling import *
 from mindspore.nn.layer.image import *
-from mindspore.nn.layer.quant import *
 from mindspore.nn.layer.math import *
 from mindspore.nn.layer.combined import *
 from mindspore.nn.layer.timedistributed import *
+from mindspore.nn.layer.transformer import *
+from mindspore.nn.layer.channel_shuffle import ChannelShuffle
 from mindspore.nn.layer.thor_layer import DenseThor, Conv2dThor, EmbeddingThor, EmbeddingLookupThor
 from mindspore.nn.layer.padding import ConstantPad1d, ConstantPad2d, ConstantPad3d, ReflectionPad1d, \
-    ReflectionPad2d, ZeroPad2d
+    ReflectionPad2d, ReflectionPad3d, ZeroPad2d, ReplicationPad1d, ReplicationPad2d, ReplicationPad3d
 
 __all__ = []
 __all__.extend(activation.__all__)
@@ -52,9 +54,10 @@ __all__.extend(basic.__all__)
 __all__.extend(embedding.__all__)
 __all__.extend(pooling.__all__)
 __all__.extend(image.__all__)
-__all__.extend(quant.__all__)
 __all__.extend(math.__all__)
 __all__.extend(combined.__all__)
 __all__.extend(timedistributed.__all__)
+__all__.extend(transformer.__all__)
 __all__.extend(thor_layer.__all__)
 __all__.extend(padding.__all__)
+__all__.extend(channel_shuffle.__all__)

mindspore/nn/layer/activation.py

@@ -17,7 +17,7 @@ from __future__ import absolute_import
 
 import numpy as np
 
-from mindspore._checkparam import Rel, Validator as validator
+from mindspore import _checkparam as validator
 from mindspore._extends import cell_attr_register
 from mindspore.common import dtype as mstype
 from mindspore.common.parameter import Parameter
@@ -25,11 +25,13 @@ from mindspore.common.tensor import Tensor
 from mindspore.ops import functional as F
 from mindspore.ops import operations as P
 from mindspore.ops.operations import nn_ops as NN_OPS
-from mindspore.ops.primitive import constexpr
 from mindspore.nn.cell import Cell
+from mindspore import ops
+from mindspore.ops.primitive import _primexpr
 
 __all__ = ['Softmin',
            'Softmax',
+           'Softmax2d',
            'LogSoftmax',
            'ReLU',
            'ReLU6',
@@ -55,7 +57,8 @@ __all__ = ['Softmin',
            'HShrink',
            'CELU',
            'Threshold',
-           'Mish'
+           'Mish',
+           'GLU'
            ]
 
 
@@ -69,8 +72,6 @@ class CELU(Cell):
 
         \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))
 
-    It returns element-wise :math:`\max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))`.
-
     The picture about CELU looks like this `CELU <https://arxiv.org/abs/1704.07483>`_.
 
     Args:
@@ -156,10 +157,55 @@ class Softmin(Cell):
     def __init__(self, axis=-1):
         """Initialize Softmin."""
         super(Softmin, self).__init__()
-        self.softmax = P.Softmax(axis)
+        self.axis = axis
 
     def construct(self, x):
-        x = -1 * x
+        return ops.function.softmin(x, self.axis)
+
+
+class Softmax2d(Cell):
+    r"""
+    Softmax function applied to 2D features data.
+
+    Applies `Softmax` to each location :math:`(c, h, w)` with an input Tensor of shape :math:`(C, H, W)` .
+
+    Inputs:
+        - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`.
+
+    Outputs:
+        Tensor, which has the same type and shape as `x` with values in the range[0,1].
+
+    Raises:
+        TypeError: If dtype of `x` is neither float16 nor float32.
+        ValueError: If `data_format` is neither 'NCHW' nor 'CHW'.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> x = Tensor(np.array([[[[0.1, 0.2]], [[0.3, 0.4]], [[0.6, 0.5]]]]), mindspore.float32)
+        >>> softmax2d = nn.Softmax2d()
+        >>> output = softmax2d(x)
+        >>> print(output)
+        [[[[0.258, 0.28]], [[0.316, 0.342]], [[0.426, 0.378]]]
+    """
+
+    def __init__(self):
+        """Initialize Softmax2d."""
+        super(Softmax2d, self).__init__()
+        self.softmax = P.Softmax(axis=-3)
+        self.shape = P.Shape()
+
+    @staticmethod
+    @_primexpr
+    def _check_input_dim(shape, cls_name):
+        dim = len(shape)
+        if dim not in (3, 4):
+            raise ValueError(f"For '{cls_name}', the in_shape must have 3 or 4 dims, but got {dim}.")
+
+    def construct(self, x):
+        x_shape = self.shape(x)
+        self._check_input_dim(x_shape, self.cls_name)
         return self.softmax(x)
 
 
@@ -228,8 +274,6 @@ class LogSoftmax(Cell):
 
         \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right),
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
-
     Args:
         axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: -1.
 
@@ -334,7 +378,7 @@ class ReLU(Cell):
 
     Inputs:
         - **x** (Tensor) - The input of ReLU is a Tensor of any dimension. The data type is `number <https://www.mind
-          spore.cn/docs/en/r1.10/api_python/mindspore.html#mindspore.dtype>`_ .
+          spore.cn/docs/en/r2.0/api_python/mindspore.html#mindspore.dtype>`_ .
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
@@ -378,7 +422,6 @@ class ReLU6(Cell):
 
     Inputs:
         - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32.
-          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
 
     Outputs:
         Tensor, which has the same type as `x`.
@@ -457,7 +500,7 @@ class LeakyReLU(Cell):
             self.select_op = P.Minimum()
 
     def construct(self, x):
-        alpha_array = P.Cast()(F.scalar_to_array(self.alpha), P.DType()(x))
+        alpha_array = P.Cast()(F.scalar_to_tensor(self.alpha), P.DType()(x))
         out = self.select_op(alpha_array * x, x)
         return out
 
@@ -470,8 +513,8 @@ class RReLU(Cell):
     The activation function is defined as:
 
     .. math::
-            \text{RReLU}(x_{ji}) = \begin{cases}x_{ji}, &\text{if } x_{ji} \geq 0; \cr
-            {\alpha_{ji}} * x_{ji}, &\text{otherwise.}\end{cases}
+        \text{RReLU}(x_{ji}) = \begin{cases}x_{ji}, &\text{if } x_{ji} \geq 0; \cr
+        {\alpha_{ji}} * x_{ji}, &\text{otherwise.}\end{cases}
 
     where :math:`\alpha_{ji}` ~ :math:`U(l, u)`, :math:`l \le u`.
 
@@ -518,17 +561,17 @@ class RReLU(Cell):
         if lower > upper:
             raise ValueError(f"For {self.cls_name}, the value of 'upper' must be greater than 'lower', "
                              f"but got upper: {upper}, lower: {lower}. ")
-
-        self.lower = lower
-        self.upper = upper
+        self.lower = Tensor(lower, dtype=mstype.float32)
+        self.upper = Tensor(upper, dtype=mstype.float32)
         self.sign = P.Sign()
 
     def construct(self, x):
-        size = x.shape
+        _size = x.shape
+        _dtype = x.dtype
         sign_matrix = self.sign(x)
         negative_filter = sign_matrix.clip(None, 0)
         positive_filter = sign_matrix.clip(0, None)
-        mask = P.Cast()(Tensor(np.random.uniform(self.lower, self.upper, size=size)), P.DType()(x))
+        mask = ops.uniform(_size, self.lower, self.upper).astype(_dtype)
         negative_mask = negative_filter * mask * -1
         total_mask = negative_mask + positive_filter
         out = total_mask * x
@@ -582,7 +625,7 @@ class SiLU(Cell):
     `SiLU <https://en.wikipedia.org/wiki/Activation_function#/media/File:Swish.svg>`_ .
 
     Inputs:
-        - **x** (Tensor) - Input with the data type float16 or float32. Tensor of arbitrary dimensions.
+        - **x** (Tensor) - Input with the data type float16 or float32.
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
@@ -604,10 +647,9 @@ class SiLU(Cell):
     def __init__(self):
         """Initialize SiLU."""
         super(SiLU, self).__init__()
-        self.sigmoid = P.Sigmoid()
 
     def construct(self, x):
-        return self.sigmoid(x) * x
+        return ops.function.silu(x)
 
 
 class Tanh(Cell):
@@ -666,14 +708,13 @@ class Tanhshrink(Cell):
     where :math:`x_i` is an element of the input Tensor.
 
     Inputs:
-        - **x** (Tensor) - Tensor of any dimension, input with data type of float16 or float32.
+        - **x** (Tensor) - Tensor of any dimension.
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
 
     Raises:
         TypeError: If `x` is not a Tensor.
-        TypeError: If dtype of `x` is neither float16 nor float32.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -693,17 +734,9 @@ class Tanhshrink(Cell):
     def __init__(self):
         """Initialize Tanhshrink."""
         super(Tanhshrink, self).__init__()
-        self.tanh = P.Tanh()
 
     def construct(self, x):
-        return x - self.tanh(x)
-
-
-@constexpr
-def _dtype_check(x_dtype, prim_name):
-    """Check dtype."""
-    if x_dtype not in [mstype.float32, mstype.float16]:
-        raise TypeError("For {}, the x_dtype must be float32 or float16, but got {}.".format(prim_name, x_dtype))
+        return F.tanhshrink(x)
 
 
 class Hardtanh(Cell):
@@ -738,7 +771,7 @@ class Hardtanh(Cell):
         TypeError: If dtype of `x` is neither float16 nor float32.
         TypeError: If dtype of `min_val` is neither float nor int.
         TypeError: If dtype of `max_val` is neither float nor int.
-        ValueError: If `max_val` is less than `min_val`.
+        ValueError: If `min_val` is not less than `max_val`.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -757,34 +790,14 @@ class Hardtanh(Cell):
     def __init__(self, min_val=-1.0, max_val=1.0):
         """Initialize Hardtanh."""
         super(Hardtanh, self).__init__()
-        validator.check_value_type('min_val', min_val, [float, int], self.cls_name)
-        validator.check_value_type('max_val', max_val, [float, int], self.cls_name)
-        validator.check_number("max_val", max_val, min_val, Rel.GE, self.cls_name)
-
-        self.max = P.Maximum()
-        self.min = P.Minimum()
         self.min_val = min_val
         self.max_val = max_val
-        self.dtype = P.DType()
-        self.expand = P.ExpandDims()
-        self.squeeze = P.Squeeze(0)
+        if self.min_val >= self.max_val:
+            raise ValueError(f"For Hardtanh, min_val should be less than max_val,"
+                             f"but got {self.min_val} and {self.max_val}")
 
     def construct(self, x):
-        if not isinstance(x, Tensor):
-            raise TypeError("'x' must be a Tensor")
-        _dtype_check(self.dtype(x), self.cls_name)
-        # min_val and max_val are scalars, if x is 0d, x is also a scalar.
-        # However, ops.Maximum does not support input two scalar.
-        # To solve this problem, expand x from scalar to tensor, apply Maximum, then squeeze the output back to scalar.
-        if not x.shape:
-            x = self.expand(x, 0)
-            x = self.max(x, self.min_val)
-            x = self.min(x, self.max_val)
-            x = self.squeeze(x)
-        else:
-            x = self.max(x, self.min_val)
-            x = self.min(x, self.max_val)
-        return x
+        return F.hardtanh(x, self.min_val, self.max_val)
 
 
 class GELU(Cell):
@@ -810,11 +823,11 @@ class GELU(Cell):
 
             If approximate is True, The gaussian error linear activation is:
 
-            :math:`0.5 * x * (1 + tanh(sqrt(2 / pi) * (x + 0.044715 * x^3)))`
+            :math:`0.5 * x * (1 + tanh(\sqrt(2 / \pi) * (x + 0.044715 * x^3)))`
 
             else, it is:
 
-            :math:`x * P(X <= x) = 0.5 * x * (1 + erf(x / sqrt(2)))`, where P(X) ~ N(0, 1).
+            :math:`x * P(X <= x) = 0.5 * x * (1 + erf(x / \sqrt(2)))`, where P(X) ~ N(0, 1).
 
     Inputs:
         - **x** (Tensor) - The input of GELU with data type of float16 or float32.
@@ -1007,14 +1020,14 @@ class PReLU(Cell):
     Activation_function#/media/File:Activation_prelu.svg>`_.
 
     Args:
-        channel (int): The elements number of parameter.
+        channel (int): The elements number of parameter `w`.
           It could be an int, and the value is 1 or the channels number of input tensor `x`. Default: 1.
         w (Union[float, list, Tensor]): The initial value of parameter. It could be a float, a float list or
           a tensor has the same dtype as the input tensor `x`. Default: 0.25.
 
     Inputs:
         - **x** (Tensor) - The input of PReLU with data type of float16 or float32.
-          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
+          The shape is :math:`(N, *)` where :math:`*` means, any number of additional dimensions.
 
     Outputs:
         Tensor, with the same dtype and shape as the `x`.
@@ -1027,7 +1040,7 @@ class PReLU(Cell):
         ValueError: If `channel` is less than 1.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> x = Tensor(np.array([[[[0.1, 0.6], [0.9, 0.9]]]]), mindspore.float32)
@@ -1071,15 +1084,9 @@ class PReLU(Cell):
                             f"but got {type(w).__name__}.")
         self.w = Parameter(w, name='a')
         self.prelu = P.PReLU()
-        self.relu = P.ReLU()
-        self.assign = P.Assign()
 
     def construct(self, x):
-        u = self.relu(self.w)
-        v = self.prelu(x, F.cast(u, x.dtype))
-        if self.training:
-            self.assign(self.w, u)
-        return v
+        return self.prelu(x, F.cast(self.w, x.dtype))
 
 
 class HSwish(Cell):
@@ -1091,8 +1098,6 @@ class HSwish(Cell):
     .. math::
         \text{hswish}(x_{i}) = x_{i} * \frac{ReLU6(x_{i} + 3)}{6},
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
-
     Inputs:
         - **x** (Tensor) - The input of HSwish, data type must be float16 or float32.
           The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
@@ -1132,8 +1137,6 @@ class HSigmoid(Cell):
     .. math::
         \text{hsigmoid}(x_{i}) = max(0, min(1, \frac{x_{i} + 3}{6})),
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
-
     Inputs:
         - **input_x** (Tensor) - The input of HSigmoid. Tensor of any dimension.
 
@@ -1255,7 +1258,8 @@ class SoftShrink(Cell):
         \end{cases}
 
     Args:
-        lambd: the :math:`\lambda` must be no less than zero for the SoftShrink formulation. Default: 0.5.
+        lambd (float): the :math:`\lambda` must be no less than zero for the SoftShrink formulation.
+            Default: 0.5.
 
     Inputs:
         - **input_x** (Tensor) - The input of SoftShrink with data type of float16 or float32.
@@ -1271,7 +1275,7 @@ class SoftShrink(Cell):
         ValueError: If lambd is less than 0.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> input_x = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mstype.float16)
@@ -1319,7 +1323,7 @@ class HShrink(Cell):
         TypeError: If dtype of `input_x` is neither float16 nor float32.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
@@ -1368,7 +1372,7 @@ class Threshold(Cell):
         TypeError: If `value` is not a float or an int.
 
     Supported Platforms:
-        ``Ascend`` ``CPU`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
@@ -1382,19 +1386,12 @@ class Threshold(Cell):
 
     def __init__(self, threshold, value):
         """Initialize Threshold."""
-        super().__init__()
-        validator.check_value_type('threshold', threshold, [float, int], self.cls_name)
-        validator.check_value_type('value', value, [float, int], self.cls_name)
+        super(Threshold, self).__init__()
         self.threshold = threshold
         self.value = value
-        self.greater = P.Greater()
-        self.fill = P.Fill()
-        self.select = P.Select()
 
     def construct(self, input_x):
-        cond = self.greater(input_x, self.threshold)
-        value = self.fill(input_x.dtype, input_x.shape, self.value)
-        return self.select(cond, input_x, value)
+        return F.threshold(input_x, self.threshold, self.value)
 
 
 class Mish(Cell):
@@ -1424,9 +1421,55 @@ class Mish(Cell):
         return self.mish(input_x)
 
 
+class GLU(Cell):
+    r"""
+    The gated linear unit function.
+
+    .. math::
+        {GLU}(a, b)= a \otimes \sigma(b)
+
+    where :math:`a` is the first half of the input matrices and :math:`b` is the second half.
+
+    Here :math:`\sigma` is the sigmoid function, and :math:`\otimes` is the Hadamard product.
+
+    Args:
+        axis (int): the axis to split the input. Default: -1, the last axis in `x`.
+
+    Inputs:
+        - **x** (Tensor) - :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional dimensions.
+
+    Outputs:
+        Tensor, the same dtype as the `x`, with the shape :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> m = nn.GLU()
+        >>> input = Tensor([[0.1,0.2,0.3,0.4],[0.5,0.6,0.7,0.8]])
+        >>> output = m(input)
+        >>> print(output)
+        [[0.05744425 0.11973753]
+         [0.33409387 0.41398472]]
+    """
+
+    def __init__(self, axis=-1):
+        """Initialize GLU."""
+        super().__init__("GLU")
+        self.dim = axis
+        self.spilt = P.Split(axis=axis, output_num=2)
+        self.sigmoid = P.Sigmoid()
+
+    def construct(self, x):
+        x1, x2 = self.spilt(x)
+        x2 = self.sigmoid(x2)
+        return x1 * x2
+
+
 _activation = {
     'softmin': Softmin,
     'softmax': Softmax,
+    'softmax2d': Softmax2d,
     'logsoftmax': LogSoftmax,
     'relu': ReLU,
     'relu6': ReLU6,
@@ -1448,7 +1491,7 @@ _activation = {
     'softshrink': SoftShrink,
     'hshrink': HShrink,
     'threshold': Threshold,
-    'mish': Mish
+    'mish': Mish,
 }