mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/nn/layer/activation.py

@@ -33,6 +33,7 @@ __all__ = ['Softmin',
            'Softmax',
            'Softmax2d',
            'LogSoftmax',
+           'LogSoftmaxExt',
            'ReLU',
            'ReLU6',
            'RReLU',
@@ -46,6 +47,7 @@ __all__ = ['Softmin',
            'Sigmoid',
            'Softsign',
            'PReLU',
+           'PReLUExt',
            'get_activation',
            'LeakyReLU',
            'HSigmoid',
@@ -64,7 +66,7 @@ __all__ = ['Softmin',
 
 class CELU(Cell):
     r"""
-    Continuously differentiable exponential linear units activation function.
+    CELU Activation Operator.
 
     Applies the continuously differentiable exponential linear units function element-wise.
 
@@ -72,9 +74,11 @@ class CELU(Cell):
 
         \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))
 
+    For more details, refer to `CELU <https://arxiv.org/abs/1704.07483>`_ .
+
     CELU Activation Function Graph:
 
-    .. image:: images/CELU.png
+    .. image:: ../images/CELU.png
         :align: center
 
     Args:
@@ -91,7 +95,7 @@ class CELU(Cell):
         TypeError: If `alpha` is not a float.
         ValueError: If `alpha` has the value of 0.
         TypeError: If `x` is not a Tensor.
-        TypeError: If the dtype of 'input_x' is neither float16 nor float32.
+        TypeError: If the dtype of `x` is neither float16 nor float32.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -234,25 +238,24 @@ class Softmax(Cell):
     Softmax is defined as:
 
     .. math::
-        \text{softmax}(x_{i}) =  \frac{\exp(x_i)}{\sum_{j=0}^{n-1}\exp(x_j)},
+        \text{softmax}(input_{i}) =  \frac{\exp(input_i)}{\sum_{j=0}^{n-1}\exp(input_j)},
 
-    where :math:`x_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
+    where :math:`input_{i}` is the :math:`i`-th slice in the given dimension of the input Tensor.
 
     Args:
-        axis (Union[int, tuple[int]]): The axis to apply Softmax operation, if the dimension of input `x` is x.ndim,
-            the range of axis is `[-x.ndim, x.ndim)`, -1 means the last dimension. Default: ``-1`` .
+        axis (int, optional): The axis to apply Softmax operation, if the dimension of `input` is input.ndim,
+            the range of axis is `[-input.ndim, input.ndim)`, -1 means the last dimension. Default: ``-1`` .
 
     Inputs:
-        - **x** (Tensor) - The input of Softmax with data type of float16 or float32.
+        - **input** (Tensor) - The input of Softmax.
 
     Outputs:
-        Tensor, which has the same type and shape as `x` with values in the range[0,1].
+        Tensor, which has the same type and shape as `input` with values in the range[0, 1].
 
     Raises:
         TypeError: If `axis` is neither an int nor a tuple.
-        TypeError: If dtype of `x` is neither float16 nor float32.
         ValueError: If `axis` is a tuple whose length is less than 1.
-        ValueError: If `axis` is a tuple whose elements are not all in range [-len(x), len(x)).
+        ValueError: If `axis` is a tuple whose elements are not all in range `[-input.ndim, input.ndim)`.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -262,9 +265,9 @@ class Softmax(Cell):
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
         >>> # axis = -1(default), and the sum of return value is 1.0.
-        >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
+        >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> softmax = nn.Softmax()
-        >>> output = softmax(x)
+        >>> output = softmax(input)
         >>> print(output)
         [0.03168 0.01166 0.0861  0.636   0.2341 ]
     """
@@ -274,13 +277,42 @@ class Softmax(Cell):
         super(Softmax, self).__init__()
         self.softmax = P.Softmax(axis)
 
-    def construct(self, x):
-        return self.softmax(x)
+    def construct(self, input):
+        return self.softmax(input)
+
+
+class SoftmaxExt(Cell):
+    r"""
+    Applies the Softmax function to an n-dimensional input Tensor.
+
+    For details, please refer to :func:`mindspore.mint.nn.functional.softmax`.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
+        >>> softmax = nn.SoftmaxExt()
+        >>> output = softmax(input)
+        >>> print(output)
+        [0.03168 0.01166 0.0861  0.636   0.2341 ]
+    """
+
+    def __init__(self, dim=None):
+        """Initialize Softmax."""
+        super(SoftmaxExt, self).__init__()
+        self.dim = dim
+
+    def construct(self, input):
+        return ops.function.nn_func.softmax_ext(input, self.dim)
 
 
 class LogSoftmax(Cell):
     r"""
-    Applies the LogSoftmax function to n-dimensional input tensor.
+    Applies the LogSoftmax function to n-dimensional input tensor element-wise.
 
     The input is transformed by the Softmax function and then by the log function to lie in range[-inf,0).
 
@@ -328,11 +360,55 @@ class LogSoftmax(Cell):
         return self.log_softmax(x)
 
 
-class ELU(Cell):
+class LogSoftmaxExt(Cell):
     r"""
-    Exponential Linear Unit activation function.
+    Applies the Log Softmax function to the input tensor on the specified axis.
+    Supposes a slice in the given axis, :math:`x` for each element :math:`x_i`,
+    the Log Softmax function is shown as follows:
+
+    .. math::
+        \text{output}(x_i) = \log \left(\frac{\exp(x_i)} {\sum_{j = 0}^{N-1}\exp(x_j)}\right),
+
+    where :math:`N` is the length of the Tensor.
+
+    Args:
+        dim (int, optional): The axis to perform the Log softmax operation. Default: ``None`` .
+
+    Returns:
+        Tensor, with the same shape as the input.
+
+    Raises:
+        ValueError: If `dim` is not in range [-len(input.shape), len(input.shape)).
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
+        >>> log_softmax = nn.LogSoftmaxExt(dim=-1)
+        >>> output = log_softmax(x)
+        >>> print(output)
+        [[-5.00672150e+00 -6.72150636e-03 -1.20067215e+01]
+         [-7.00091219e+00 -1.40009127e+01 -9.12250078e-04]]
+    """
 
+    def __init__(self, dim=None):
+        """Initialize LogSoftmaxExt."""
+        super(LogSoftmaxExt, self).__init__()
+        self.log_softmax = P.LogSoftmaxExt()
+        self.dim = dim
+
+    def construct(self, x):
+        return self.log_softmax(x, dim=self.dim)
+
+
+class ELU(Cell):
+    r"""
     Applies the exponential linear unit function element-wise.
+
     The activation function is defined as:
 
     .. math::
@@ -346,21 +422,22 @@ class ELU(Cell):
 
     ELU Activation Function Graph:
 
-    .. image:: images/ELU.png
+    .. image:: ../images/ELU.png
         :align: center
 
     Args:
         alpha (float): The alpha value of ELU, the data type is float. Default: ``1.0`` .
+            Only alpha equal to ``1.0`` is supported currently.
 
     Inputs:
-        - **x** (Tensor) - The input of ELU is a Tensor of any dimension with data type of float16 or float32.
+        - **input_x** (Tensor) - The input of ELU is a Tensor of any dimension with data type of float16 or float32.
 
     Outputs:
-        Tensor, with the same type and shape as the `x`.
+        Tensor, with the same type and shape as the `input_x`.
 
     Raises:
         TypeError: If `alpha` is not a float.
-        TypeError: If dtype of `x` is neither float16 nor float32.
+        TypeError: If dtype of `input_x` is neither float16 nor float32.
         ValueError: If `alpha` is not equal to 1.0.
 
     Supported Platforms:
@@ -388,13 +465,13 @@ class ELU(Cell):
 
 class ReLU(Cell):
     r"""
-    Rectified Linear Unit activation function.
+    Applies ReLU (Rectified Linear Unit activation function) element-wise.
 
     .. math::
 
-        \text{ReLU}(x) = (x)^+ = \max(0, x),
+        \text{ReLU}(input) = (input)^+ = \max(0, input),
 
-    It returns element-wise :math:`\max(0, x)`.
+    It returns element-wise :math:`\max(0, input)`.
 
     .. note::
         The neurons with the negative output
@@ -402,29 +479,28 @@ class ReLU(Cell):
 
     ReLU Activation Function Graph:
 
-    .. image:: images/ReLU.png
+    .. image:: ../images/ReLU.png
         :align: center
 
     Inputs:
-        - **x** (Tensor) - The input of ReLU is a Tensor of any dimension. The data type is `number <https://www.mind
-          spore.cn/docs/en/r2.2/api_python/mindspore.html#mindspore.dtype>`_ .
+        - **input** (Tensor) - The input of ReLU is a Tensor of any dimension.
 
     Outputs:
-        Tensor, with the same type and shape as the `x`.
+        Tensor, with the same type and shape as the `input`.
 
     Raises:
-        TypeError: If dtype of `x` is not a number.
+        TypeError: If dtype of `input` is not supported.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import numpy as np
         >>> import mindspore
         >>> from mindspore import Tensor, nn
-        >>> import numpy as np
-        >>> x = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)
+        >>> input = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)
         >>> relu = nn.ReLU()
-        >>> output = relu(x)
+        >>> output = relu(input)
         >>> print(output)
         [0. 2. 0. 2. 0.]
     """
@@ -434,13 +510,13 @@ class ReLU(Cell):
         super(ReLU, self).__init__()
         self.relu = P.ReLU()
 
-    def construct(self, x):
-        return self.relu(x)
+    def construct(self, input):
+        return self.relu(input)
 
 
 class ReLU6(Cell):
     r"""
-    Compute ReLU6 activation function.
+    Compute ReLU6 activation function element-wise.
 
     ReLU6 is similar to ReLU with a upper limit of 6, which if the inputs are greater than 6, the outputs
     will be suppressed to 6.
@@ -452,7 +528,7 @@ class ReLU6(Cell):
 
     ReLU6 Activation Function Graph:
 
-    .. image:: images/ReLU6.png
+    .. image:: ../images/ReLU6.png
         :align: center
 
     Inputs:
@@ -505,7 +581,7 @@ class LeakyReLU(Cell):
 
     LeakyReLU Activation Function Graph:
 
-    .. image:: images/LeakyReLU.png
+    .. image:: ../images/LeakyReLU.png
         :align: center
 
     Args:
@@ -547,8 +623,7 @@ class LeakyReLU(Cell):
 
 class RReLU(Cell):
     r"""
-
-    Randomized Leaky ReLU activation function.
+    Applies RReLU (Randomized Leaky ReLU activation function) element-wise.
 
     The activation function is defined as:
 
@@ -575,7 +650,7 @@ class RReLU(Cell):
         TypeError: If `lower` is not a float or an int.
         TypeError: If `upper` is not a float or an int.
         TypeError: If `x` is not a Tensor.
-        TypeError: If `x` is not a Tensor of mindspore.float16 or mindpore.float32.
+        TypeError: If `x` is not a Tensor of mindspore.float16 or mindspore.float32.
         ValueError: If `lower` is greater than upper.
 
     Supported Platforms:
@@ -621,11 +696,11 @@ class RReLU(Cell):
 
 class SeLU(Cell):
     r"""
-    Activation function SeLU (Scaled exponential Linear Unit).
+    Applies activation function SeLU (Scaled exponential Linear Unit) element-wise.
 
     SeLU Activation Function Graph:
 
-    .. image:: images/SeLU.png
+    .. image:: ../images/SeLU.png
         :align: center
 
     Refer to :func:`mindspore.ops.selu` for more details.
@@ -656,15 +731,13 @@ class SeLU(Cell):
 
 class SiLU(Cell):
     r"""
-    Sigmoid Linear Unit activation function.
-
     Applies the silu linear unit function element-wise.
 
     .. math::
 
         \text{SiLU}(x) = x * \sigma(x),
 
-    where :math:`x_i` is input, :math:`\sigma(x)` is Sigmoid function.
+    where :math:`x_i` is an element of the input, :math:`\sigma(x)` is Sigmoid function.
 
     .. math::
 
@@ -672,17 +745,18 @@ class SiLU(Cell):
 
     SiLU Activation Function Graph:
 
-    .. image:: images/SiLU.png
+    .. image:: ../images/SiLU.png
         :align: center
 
     Inputs:
-        - **x** (Tensor) - Input with the data type float16 or float32.
+        - **input** (Tensor) - `input` is :math:`x` in the preceding formula.
+          Input with the data type float16 or float32. Tensor of any dimension.
 
     Outputs:
-        Tensor, with the same type and shape as the `x`.
+        Tensor, with the same type and shape as the `input`.
 
     Raises:
-        TypeError: If dtype of `x` is neither float16 nor float32.
+        TypeError: If dtype of `input` is neither float16 nor float32.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -691,9 +765,9 @@ class SiLU(Cell):
         >>> import mindspore
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
-        >>> x = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)
+        >>> input = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)
         >>> silu = nn.SiLU()
-        >>> output = silu(x)
+        >>> output = silu(input)
         >>> print(output)
         [-0.269  1.762  -0.1423  1.762  -0.269]
     """
@@ -720,7 +794,7 @@ class Tanh(Cell):
 
     Tanh Activation Function Graph:
 
-    .. image:: images/Tanh.png
+    .. image:: ../images/Tanh.png
         :align: center
 
     Inputs:
@@ -757,9 +831,7 @@ class Tanh(Cell):
 
 class Tanhshrink(Cell):
     r"""
-    Tanhshrink activation function.
-
-    The tanhshrink function is evaluated by element and returns a new tensor.
+    Applies Tanhshrink activation function element-wise and returns a new tensor.
 
     Tanh function is defined as:
 
@@ -815,7 +887,7 @@ class Hardtanh(Cell):
 
     Hardtanh Activation Function Graph:
 
-    .. image:: images/Hardtanh.png
+    .. image:: ../images/Hardtanh.png
         :align: center
 
     Note:
@@ -868,8 +940,6 @@ class Hardtanh(Cell):
 
 class GELU(Cell):
     r"""
-    Gaussian error linear unit activation function.
-
     Applies GELU function to each element of the input. The input is a Tensor with any valid shape.
 
     GELU is defined as:
@@ -883,7 +953,7 @@ class GELU(Cell):
 
     GELU Activation Function Graph:
 
-    .. image:: images/GELU.png
+    .. image:: ../images/GELU.png
         :align: center
 
     Args:
@@ -898,14 +968,21 @@ class GELU(Cell):
             :math:`x * P(X <= x) = 0.5 * x * (1 + erf(x / \sqrt(2)))`, where P(X) ~ N(0, 1).
 
     Inputs:
-        - **x** (Tensor) - The input of GELU with data type of float16 or float32.
+        - **x** (Tensor) - The input of GELU with data type of float16, float32, or float64.
           The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
 
+    Note:
+        when calculating the input gradient of GELU with an input value of infinity, there are differences
+        in the output of the backward between ``Ascend`` and ``GPU``.
+        when x is -inf, the computation result of ``Ascend`` is 0, and the computation result of ``GPU`` is Nan.
+        when x is inf, the computation result of ``Ascend`` is dy, and the computation result of ``GPU`` is Nan.
+        In mathematical terms, the result of Ascend has higher precision.
+
     Raises:
-        TypeError: If dtype of `x` is neither float16 nor float32.
+        TypeError: If dtype of `x` is not one of float16, float32, or float64.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -942,8 +1019,6 @@ class GELU(Cell):
 
 class FastGelu(Cell):
     r"""
-    Fast Gaussian error linear unit activation function.
-
     Applies FastGelu function to each element of the input. The input is a Tensor with any valid shape.
 
     FastGelu is defined as:
@@ -956,7 +1031,7 @@ class FastGelu(Cell):
 
     FastGelu Activation Function Graph:
 
-    .. image:: images/FastGelu.png
+    .. image:: ../images/FastGelu.png
         :align: center
 
     Inputs:
@@ -995,9 +1070,7 @@ class FastGelu(Cell):
 
 class Sigmoid(Cell):
     r"""
-    Sigmoid activation function.
-
-    Applies sigmoid-type activation element-wise.
+    Applies sigmoid activation function element-wise.
 
     Sigmoid function is defined as:
 
@@ -1005,23 +1078,23 @@ class Sigmoid(Cell):
 
         \text{sigmoid}(x_i) = \frac{1}{1 + \exp(-x_i)},
 
-    where :math:`x_i` is the element of the input.
+    where :math:`x_i` is the element of `x`.
 
     Sigmoid Activation Function Graph:
 
-    .. image:: images/Sigmoid.png
+    .. image:: ../images/Sigmoid.png
         :align: center
 
     Inputs:
-        - **input_x** (Tensor) - Tensor of any dimension, the data type is
-          float16, float32, float64, complex64 or complex128.
+        - **input** (Tensor) - `input` is :math:`x` in the preceding formula. Tensor of any dimension,
+          the data type is float16, float32, float64, complex64 or complex128.
 
     Outputs:
-        Tensor, with the same type and shape as the `input_x`.
+        Tensor, with the same type and shape as the `input`.
 
     Raises:
-        TypeError: If dtype of `input_x` is not float16, float32, float64, complex64 or complex128.
-        TypeError: If `input_x` is not a Tensor.
+        TypeError: If dtype of `input` is not float16, float32, float64, complex64 or complex128.
+        TypeError: If `input` is not a Tensor.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1030,9 +1103,9 @@ class Sigmoid(Cell):
         >>> import mindspore
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
-        >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
+        >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> sigmoid = nn.Sigmoid()
-        >>> output = sigmoid(x)
+        >>> output = sigmoid(input)
         >>> print(output)
         [0.2688  0.11914 0.5     0.881   0.7305 ]
     """
@@ -1048,11 +1121,11 @@ class Sigmoid(Cell):
 
 class Softsign(Cell):
     r"""
-    Softsign activation function.
+    Applies softsign activation function element-wise.
 
     Softsign Activation Function Graph:
 
-    .. image:: images/Softsign.png
+    .. image:: ../images/Softsign.png
         :align: center
 
     Refer to :func:`mindspore.ops.softsign` for more details.
@@ -1082,9 +1155,7 @@ class Softsign(Cell):
 
 class PReLU(Cell):
     r"""
-    PReLU activation function.
-
-    Applies the PReLU function element-wise.
+    Applies PReLU activation function element-wise.
 
     PReLU is defined as:
 
@@ -1100,7 +1171,7 @@ class PReLU(Cell):
 
     PReLU Activation Function Graph:
 
-    .. image:: images/PReLU.png
+    .. image:: ../images/PReLU.png
         :align: center
 
     Args:
@@ -1176,29 +1247,100 @@ class PReLU(Cell):
         return self.prelu(x, F.cast(self.w, x.dtype))
 
 
+class PReLUExt(Cell):
+    r"""
+    Applies PReLU activation function element-wise.
+
+    PReLU is defined as:
+
+    .. math::
+
+        PReLU(x_i)= \max(0, x_i) + w * \min(0, x_i),
+
+    where :math:`x_i` is an element of an channel of the input.
+
+    Here :math:`w` is a learnable parameter with a default initial value 0.25.
+    Parameter :math:`w` has dimensionality of the argument channel. If called without argument
+    channel, a single parameter :math:`w` will be shared across all channels.
+
+    PReLU Activation Function Graph:
+
+    .. image:: ../images/PReLU2.png
+        :align: center
+
+    .. note::
+        Channel dim is the 2nd dim of input. When input has dims < 2, then there is
+        no channel dim and the number of channels = 1.
+
+    Args:
+        num_parameters (int): number of `w` to learn. Although it takes an int as input,
+            there is only two legitimate values: 1, or the number of channels at Tensor `input`. Default: ``1`` .
+        init (float): the initial value of `w`. Default: ``0.25`` .
+        dtype (mindspore.dtype, optional): the type of `w`. Default: ``None`` . Supported data type
+            is {float16, float32, bfloat16}.
+
+    Inputs:
+        - **input** (Tensor) - The input of PReLU.
+
+    Outputs:
+        Tensor, with the same dtype and shape as the `input`.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> x = Tensor(np.array([[[[0.1, 0.6], [0.9, 0.9]]]]), mindspore.float32)
+        >>> prelu = nn.PReLUExt()
+        >>> output = prelu(x)
+        >>> print(output)
+        [[[[0.1 0.6]
+           [0.9 0.9]]]]
+
+    """
+
+    def __init__(self, num_parameters=1, init=0.25, dtype=None):
+        """Initialize PReLUExt."""
+        super(PReLUExt, self).__init__()
+        tmp = np.empty((num_parameters,), dtype=np.float32)
+        tmp.fill(init)
+        w = Tensor(tmp, dtype=dtype)
+        self.weight = Parameter(w, name='weight')
+
+    def construct(self, input):
+        return ops.prelu(input, self.weight)
+
+
 class HSwish(Cell):
     r"""
-    Applies hswish-type activation element-wise. The input is a Tensor with any valid shape.
+    Applies Hard Swish activation function element-wise.
 
     Hard swish is defined as:
 
     .. math::
-        \text{hswish}(x_{i}) = x_{i} * \frac{ReLU6(x_{i} + 3)}{6},
+        \text{Hardswish}(input) =
+        \begin{cases}
+        0, & \text{ if } input \leq -3, \\
+        input, & \text{ if } input \geq +3, \\
+        input*(input + 3)/6, & \text{ otherwise }
+        \end{cases}
 
     HSwish Activation Function Graph:
 
-    .. image:: images/HSwish.png
+    .. image:: ../images/HSwish.png
         :align: center
 
     Inputs:
-        - **x** (Tensor) - The input of HSwish, data type must be float16 or float32.
-          The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
+        - **input** (Tensor) - The input of HSwish.
 
     Outputs:
-        Tensor, with the same type and shape as the `x`.
+        Tensor, with the same type and shape as the `input`.
 
     Raises:
-        TypeError: If dtype of `x` is neither float16 nor float32.
+        TypeError: If `input` is not a tensor.
+        TypeError: If `input` is neither int nor float.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1207,9 +1349,9 @@ class HSwish(Cell):
         >>> import mindspore
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
-        >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
+        >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> hswish = nn.HSwish()
-        >>> result = hswish(x)
+        >>> result = hswish(input)
         >>> print(result)
         [-0.3333 -0.3333  0.      1.667   0.6665]
     """
@@ -1219,32 +1361,38 @@ class HSwish(Cell):
         super(HSwish, self).__init__()
         self.hswish = P.HSwish()
 
-    def construct(self, x):
-        return self.hswish(x)
+    def construct(self, input):
+        return self.hswish(input)
 
 
 class HSigmoid(Cell):
     r"""
-    Hard sigmoid activation function. Calculates the output according to the input elements.
+    Applies Hard Sigmoid activation function element-wise.
 
-    Hard sigmoid is defined as:
+    Hard Sigmoid is defined as:
 
     .. math::
-        \text{hsigmoid}(x_{i}) = \max(0, \min(1, \frac{x_{i} + 3}{6})),
+        \text{Hardsigmoid}(input) =
+        \begin{cases}
+        0, & \text{ if } input \leq -3, \\
+        1, & \text{ if } input \geq +3, \\
+        input/6 + 1/2, & \text{ otherwise }
+        \end{cases}
 
     HSigmoid Activation Function Graph:
 
-    .. image:: images/HSigmoid.png
+    .. image:: ../images/HSigmoid.png
         :align: center
 
     Inputs:
-        - **input_x** (Tensor) - The input of HSigmoid. Tensor of any dimension.
+        - **input** (Tensor) - The input of HSigmoid.
 
     Outputs:
-        Tensor, with the same type and shape as the `input_x`.
+        Tensor, with the same type and shape as the `input`.
 
     Raises:
-        TypeError: If `input_x` is not a Tensor.
+        TypeError: If `input` is not a Tensor.
+        TypeError: If `input` is neither int nor float.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1253,9 +1401,9 @@ class HSigmoid(Cell):
         >>> import mindspore
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
-        >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
+        >>> input = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> hsigmoid = nn.HSigmoid()
-        >>> result = hsigmoid(x)
+        >>> result = hsigmoid(input)
         >>> print(result)
         [0.3333 0.1666 0.5    0.8335 0.6665]
     """
@@ -1265,8 +1413,8 @@ class HSigmoid(Cell):
         super(HSigmoid, self).__init__()
         self.hsigmoid = P.HSigmoid()
 
-    def construct(self, input_x):
-        return self.hsigmoid(input_x)
+    def construct(self, input):
+        return self.hsigmoid(input)
 
 
 class LogSigmoid(Cell):
@@ -1282,7 +1430,7 @@ class LogSigmoid(Cell):
 
     LogSigmoid Activation Function Graph:
 
-    .. image:: images/LogSigmoid.png
+    .. image:: ../images/LogSigmoid.png
         :align: center
 
     Inputs:
@@ -1378,25 +1526,26 @@ class SoftShrink(Cell):
 
     SoftShrink Activation Function Graph:
 
-    .. image:: images/Softshrink.png
+    .. image:: ../images/Softshrink.png
         :align: center
 
     Args:
-        lambd (float): the :math:`\lambda` must be no less than zero for the SoftShrink formulation.
-            Default: ``0.5`` .
+        lambd (number, optional): The threshold :math:`\lambda` defined by the Soft Shrink formula.
+            It should be greater than or equal to 0, default: ``0.5`` .
 
     Inputs:
-        - **input_x** (Tensor) - The input of SoftShrink with data type of float16 or float32.
-          Any number of additional dimensions.
+        - **input** (Tensor) - The input of Soft Shrink. Supported dtypes:
+
+          - Ascend: float16, float32, bfloat16.
+          - CPU/GPU: float16, float32.
 
     Outputs:
-        Tensor, has the same shape and data type as `input_x`.
+        Tensor, the same shape and data type as the input.
 
     Raises:
-        TypeError: If lambd is not a float.
-        TypeError: If input_x is not a Tensor.
-        TypeError: If dtype of input_x is neither float16 nor float32.
-        ValueError: If lambd is less than 0.
+        TypeError: If `lambd` is not a float, int or bool.
+        TypeError: If `input` is not a tensor.
+        TypeError: If dtype of `input` is not float16, float32 or bfloat16.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1405,9 +1554,9 @@ class SoftShrink(Cell):
         >>> import mindspore
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
-        >>> input_x = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mindspore.float16)
+        >>> input = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mindspore.float16)
         >>> softshrink = nn.SoftShrink()
-        >>> output = softshrink(input_x)
+        >>> output = softshrink(input)
         >>> print(output)
         [[ 0.02979  0.287    0.676  ]
          [ 0.2837   0.1216  -0.6543 ]]
@@ -1417,14 +1566,14 @@ class SoftShrink(Cell):
         super(SoftShrink, self).__init__()
         self.softshrink = P.SoftShrink(lambd)
 
-    def construct(self, input_x):
-        output = self.softshrink(input_x)
+    def construct(self, input):
+        output = self.softshrink(input)
         return output
 
 
 class HShrink(Cell):
     r"""
-    Hard Shrink activation function. Calculates the output according to the input elements.
+    Applies Hard Shrink activation function element-wise.
 
     The formula is defined as follows:
 
@@ -1438,21 +1587,25 @@ class HShrink(Cell):
 
     HShrink Activation Function Graph:
 
-    .. image:: images/HShrink.png
+    .. image:: ../images/HShrink.png
         :align: center
 
     Args:
-        lambd (float): The threshold :math:`\lambda` defined by the Hard Shrink formula. Default: ``0.5`` .
+        lambd (number, optional): The threshold :math:`\lambda` defined by the Hard Shrink formula. Default: ``0.5`` .
 
     Inputs:
-        - **input_x** (Tensor) - The input of Hard Shrink with data type of float16 or float32.
+        - **input** (Tensor) - The input of Hard Shrink. Supported dtypes:
+
+          - Ascend: float16, float32, bfloat16.
+          - CPU/GPU: float16, float32.
 
     Outputs:
         Tensor, the same shape and data type as the input.
 
     Raises:
-        TypeError: If `lambd` is not a float.
-        TypeError: If dtype of `input_x` is neither float16 nor float32.
+        TypeError: If `lambd` is not a float, int or bool.
+        TypeError: If `input` is not a tensor.
+        TypeError: If dtype of `input` is not float16, float32 or bfloat16.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1461,24 +1614,25 @@ class HShrink(Cell):
         >>> import mindspore
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
-        >>> input_x = Tensor(np.array([[ 0.5,  1,  2.0], [0.0533,0.0776,-2.1233]]), mindspore.float32)
+        >>> input = Tensor(np.array([[0.5, 1, 2.0], [0.0533, 0.0776, -2.1233]]), mindspore.float32)
         >>> hshrink = nn.HShrink()
-        >>> output = hshrink(input_x)
+        >>> output = hshrink(input)
         >>> print(output)
         [[ 0.      1.      2.    ]
-        [ 0.      0.     -2.1233]]
+         [ 0.      0.     -2.1233]]
     """
 
     def __init__(self, lambd=0.5):
         super(HShrink, self).__init__()
         self.hshrink = P.HShrink(lambd)
 
-    def construct(self, input_x):
-        return self.hshrink(input_x)
+    def construct(self, input):
+        return self.hshrink(input)
 
 
 class Threshold(Cell):
-    r"""Thresholds each element of the input Tensor.
+    r"""
+    Thresholds each element of the input Tensor.
 
     The formula is defined as follows:
 
@@ -1497,7 +1651,7 @@ class Threshold(Cell):
         - **input_x** (Tensor) - The input of Threshold with data type of float16 or float32.
 
     Outputs:
-        Tensor, the same shape and data type as the input.
+        Tensor, the same shape and data type as the `input_x`.
 
     Raises:
         TypeError: If `threshold` is not a float or an int.
@@ -1528,13 +1682,14 @@ class Threshold(Cell):
 
 class Mish(Cell):
     r"""
-    Computes MISH(A Self Regularized Non-Monotonic Neural Activation Function) of input tensors element-wise.
+    Computes MISH (A Self Regularized Non-Monotonic Neural Activation Function)
+    of input tensors element-wise.
 
     Refer to :func:`mindspore.ops.mish` for more details.
 
     Mish Activation Function Graph:
 
-    .. image:: images/Mish.png
+    .. image:: ../images/Mish.png
         :align: center
 
     Supported Platforms:
@@ -1612,6 +1767,7 @@ _activation = {
     'softmax': Softmax,
     'softmax2d': Softmax2d,
     'logsoftmax': LogSoftmax,
+    'logsoftmaxExt': LogSoftmaxExt,
     'relu': ReLU,
     'relu6': ReLU6,
     'rrelu': RReLU,
@@ -1625,6 +1781,7 @@ _activation = {
     'sigmoid': Sigmoid,
     'softsign': Softsign,
     'prelu': PReLU,
+    'preluExt': PReLUExt,
     'leakyrelu': LeakyReLU,
     'hswish': HSwish,
     'hsigmoid': HSigmoid,