mindspore 2.0.0rc1__cp38-none-any.whl → 2.2.0__cp38-none-any.whl

mindspore/nn/layer/activation.py

@@ -72,10 +72,13 @@ class CELU(Cell):
 
         \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))
 
-    The picture about CELU looks like this `CELU <https://arxiv.org/abs/1704.07483>`_.
+    CELU Activation Function Graph:
+
+    .. image:: images/CELU.png
+        :align: center
 
     Args:
-        alpha (float): The :math:`\alpha` value for the Celu formulation. Default: 1.0
+        alpha (float): The :math:`\alpha` value for the Celu formulation. Default: ``1.0`` .
 
     Inputs:
         - **x** (Tensor) - The input of CELU. The required dtype is float16 or float32.
@@ -94,6 +97,9 @@ class CELU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-2.0, -1.0, 1.0, 2.0]), mindspore.float32)
         >>> celu = nn.CELU()
         >>> output = celu(x)
@@ -127,7 +133,7 @@ class Softmin(Cell):
 
     Args:
         axis (Union[int, tuple[int]]): The axis to apply Softmin operation, if the dimension of input `x` is x.ndim,
-            the range of axis is `[-x.ndim, x.ndim)`. -1 means the last dimension. Default: -1.
+            the range of axis is `[-x.ndim, x.ndim)`. -1 means the last dimension. Default: ``-1`` .
 
     Inputs:
         - **x** (Tensor) - Tensor for computing Softmin functions with data type of float16 or float32.
@@ -145,13 +151,15 @@ class Softmin(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> # axis = -1(default), and the sum of return value is 1.0.
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> softmin = nn.Softmin()
         >>> output = softmin(x)
         >>> print(output)
         [0.2341  0.636  0.0862  0.01165  0.03168 ]
-        >>> assert(1.0 == output.sum())
     """
 
     def __init__(self, axis=-1):
@@ -171,6 +179,7 @@ class Softmax2d(Cell):
 
     Inputs:
         - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`.
+          The input of Softmax with data type of float16 or float32.
 
     Outputs:
         Tensor, which has the same type and shape as `x` with values in the range[0,1].
@@ -183,11 +192,16 @@ class Softmax2d(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([[[[0.1, 0.2]], [[0.3, 0.4]], [[0.6, 0.5]]]]), mindspore.float32)
         >>> softmax2d = nn.Softmax2d()
         >>> output = softmax2d(x)
         >>> print(output)
-        [[[[0.258, 0.28]], [[0.316, 0.342]], [[0.426, 0.378]]]
+        [[[[0.25838965 0.28001308]]
+        [[0.31559783 0.34200877]]
+        [[0.42601252 0.37797815]]]]
     """
 
     def __init__(self):
@@ -226,7 +240,7 @@ class Softmax(Cell):
 
     Args:
         axis (Union[int, tuple[int]]): The axis to apply Softmax operation, if the dimension of input `x` is x.ndim,
-            the range of axis is `[-x.ndim, x.ndim)`, -1 means the last dimension. Default: -1.
+            the range of axis is `[-x.ndim, x.ndim)`, -1 means the last dimension. Default: ``-1`` .
 
     Inputs:
         - **x** (Tensor) - The input of Softmax with data type of float16 or float32.
@@ -244,13 +258,15 @@ class Softmax(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> # axis = -1(default), and the sum of return value is 1.0.
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> softmax = nn.Softmax()
         >>> output = softmax(x)
         >>> print(output)
         [0.03168 0.01166 0.0861  0.636   0.2341 ]
-        >>> assert(1.0 == output.sum())
     """
 
     def __init__(self, axis=-1):
@@ -272,10 +288,10 @@ class LogSoftmax(Cell):
 
     .. math::
 
-        \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right),
+        \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right)
 
     Args:
-        axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: -1.
+        axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: ``-1`` .
 
     Inputs:
         - **x** (Tensor) - The input of LogSoftmax, with float16 or float32 data type.
@@ -292,6 +308,9 @@ class LogSoftmax(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
         >>> log_softmax = nn.LogSoftmax()
         >>> output = log_softmax(x)
@@ -325,11 +344,13 @@ class ELU(Cell):
 
     where :math:`x_i` represents the element of the input and :math:`\alpha` represents the `alpha` parameter.
 
-    The picture about ELU looks like this `ELU <https://en.wikipedia.org/wiki/
-    Activation_function#/media/File:Activation_elu.svg>`_.
+    ELU Activation Function Graph:
+
+    .. image:: images/ELU.png
+        :align: center
 
     Args:
-        alpha (float): The alpha value of ELU, the data type is float. Default: 1.0.
+        alpha (float): The alpha value of ELU, the data type is float. Default: ``1.0`` .
 
     Inputs:
         - **x** (Tensor) - The input of ELU is a Tensor of any dimension with data type of float16 or float32.
@@ -346,6 +367,9 @@ class ELU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float32)
         >>> elu = nn.ELU()
         >>> result = elu(x)
@@ -370,15 +394,20 @@ class ReLU(Cell):
 
         \text{ReLU}(x) = (x)^+ = \max(0, x),
 
-    It returns element-wise :math:`\max(0, x)`. Specially, the neurons with the negative output
-    will be suppressed and the active neurons will stay the same.
+    It returns element-wise :math:`\max(0, x)`.
+
+    .. note::
+        The neurons with the negative output
+        will be suppressed and the active neurons will stay the same.
+
+    ReLU Activation Function Graph:
 
-    The picture about ReLU looks like this `ReLU <https://en.wikipedia.org/wiki/
-    Activation_function#/media/File:Activation_rectified_linear.svg>`_ .
+    .. image:: images/ReLU.png
+        :align: center
 
     Inputs:
         - **x** (Tensor) - The input of ReLU is a Tensor of any dimension. The data type is `number <https://www.mind
-          spore.cn/docs/en/r2.0/api_python/mindspore.html#mindspore.dtype>`_ .
+          spore.cn/docs/en/r2.2/api_python/mindspore.html#mindspore.dtype>`_ .
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
@@ -390,6 +419,9 @@ class ReLU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)
         >>> relu = nn.ReLU()
         >>> output = relu(x)
@@ -416,12 +448,16 @@ class ReLU6(Cell):
 
     .. math::
 
-        Y = \min(\max(0, x), 6).
+        Y = \min(\max(0, x), 6)
+
+    ReLU6 Activation Function Graph:
 
-    The input is a Tensor of any valid shape.
+    .. image:: images/ReLU6.png
+        :align: center
 
     Inputs:
-        - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32.
+        - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32 and that
+          is a Tensor of any valid shape.
 
     Outputs:
         Tensor, which has the same type as `x`.
@@ -433,6 +469,9 @@ class ReLU6(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> relu6 = nn.ReLU6()
         >>> output = relu6(x)
@@ -464,8 +503,13 @@ class LeakyReLU(Cell):
     For more details, see `Rectifier Nonlinearities Improve Neural Network Acoustic Models
     <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`_.
 
+    LeakyReLU Activation Function Graph:
+
+    .. image:: images/LeakyReLU.png
+        :align: center
+
     Args:
-        alpha (Union[int, float]): Slope of the activation function at x < 0. Default: 0.2.
+        alpha (Union[int, float]): Slope of the activation function at x < 0. Default: ``0.2`` .
 
     Inputs:
         - **x** (Tensor) - The input of LeakyReLU is a Tensor of any dimension.
@@ -480,6 +524,9 @@ class LeakyReLU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
         >>> leaky_relu = nn.LeakyReLU()
         >>> output = leaky_relu(x)
@@ -491,17 +538,10 @@ class LeakyReLU(Cell):
     def __init__(self, alpha=0.2):
         """Initialize LeakyReLU."""
         super(LeakyReLU, self).__init__()
-        validator.check_value_type('alpha', alpha, [float, int], self.cls_name)
-        self.greater_equal = P.GreaterEqual()
-        self.mul = P.Mul()
         self.alpha = alpha
-        self.select_op = P.Maximum()
-        if self.alpha > 1:
-            self.select_op = P.Minimum()
 
     def construct(self, x):
-        alpha_array = P.Cast()(F.scalar_to_tensor(self.alpha), P.DType()(x))
-        out = self.select_op(alpha_array * x, x)
+        out = ops.leaky_relu(x, self.alpha)
         return out
 
 
@@ -522,8 +562,8 @@ class RReLU(Cell):
     `Empirical Evaluation of Rectified Activations in Convolution Network <https://arxiv.org/pdf/1505.00853.pdf>`_ .
 
     Args:
-        lower (Union[int, float]): Slope of the activation function at x < 0. Default: 1/8.
-        upper (Union[int, float]): Slope of the activation function at x < 0. Default: 1/3.
+        lower (Union[int, float]): Slope of the activation function at x < 0. Default: ``1 / 8`` .
+        upper (Union[int, float]): Slope of the activation function at x < 0. Default: ``1 / 3`` .
 
     Inputs:
         - **x** (Tensor) - The input of RReLU is a Tensor of any dimension.
@@ -543,8 +583,7 @@ class RReLU(Cell):
 
     Examples:
         >>> import mindspore
-        >>> import mindspore.nn as nn
-        >>> from mindspore import Tensor
+        >>> from mindspore import Tensor, nn
         >>> import numpy as np
         >>> x = Tensor(np.array([[-1.0, 4.0], [2.0, 0]]), mindspore.float32)
         >>> r_relu = nn.RReLU()
@@ -559,13 +598,15 @@ class RReLU(Cell):
         validator.check_value_type('upper', upper, [float, int], self.cls_name)
         validator.check_value_type('lower', lower, [float, int], self.cls_name)
         if lower > upper:
-            raise ValueError(f"For {self.cls_name}, the value of 'upper' must be greater than 'lower', "
+            raise ValueError(f"For {self.cls_name}, the value of 'upper' must be greater than or equal to 'lower', "
                              f"but got upper: {upper}, lower: {lower}. ")
         self.lower = Tensor(lower, dtype=mstype.float32)
         self.upper = Tensor(upper, dtype=mstype.float32)
         self.sign = P.Sign()
 
     def construct(self, x):
+        if not isinstance(x, Tensor):
+            raise TypeError(f"For 'rrelu', the input must be a Tensor, but got {type(x)}.")
         _size = x.shape
         _dtype = x.dtype
         sign_matrix = self.sign(x)
@@ -582,12 +623,20 @@ class SeLU(Cell):
     r"""
     Activation function SeLU (Scaled exponential Linear Unit).
 
+    SeLU Activation Function Graph:
+
+    .. image:: images/SeLU.png
+        :align: center
+
     Refer to :func:`mindspore.ops.selu` for more details.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> input_x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
         >>> selu = nn.SeLU()
         >>> output = selu(input_x)
@@ -609,7 +658,7 @@ class SiLU(Cell):
     r"""
     Sigmoid Linear Unit activation function.
 
-    Applies the sigmoid linear unit function element-wise.
+    Applies the silu linear unit function element-wise.
 
     .. math::
 
@@ -621,8 +670,10 @@ class SiLU(Cell):
 
         \text{sigmoid}(x_i) = \frac{1}{1 + \exp(-x_i)},
 
-    The picture about SiLU looks like this
-    `SiLU <https://en.wikipedia.org/wiki/Activation_function#/media/File:Swish.svg>`_ .
+    SiLU Activation Function Graph:
+
+    .. image:: images/SiLU.png
+        :align: center
 
     Inputs:
         - **x** (Tensor) - Input with the data type float16 or float32.
@@ -637,6 +688,9 @@ class SiLU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, 2, -3, 2, -1]), mindspore.float16)
         >>> silu = nn.SiLU()
         >>> output = silu(x)
@@ -664,6 +718,11 @@ class Tanh(Cell):
 
     where :math:`x_i` is an element of the input Tensor.
 
+    Tanh Activation Function Graph:
+
+    .. image:: images/Tanh.png
+        :align: center
+
     Inputs:
         - **x** (Tensor) - Tensor of any dimension, input with data type of float16 or float32.
 
@@ -677,6 +736,9 @@ class Tanh(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([1, 2, 3, 2, 1]), mindspore.float16)
         >>> tanh = nn.Tanh()
         >>> output = tanh(x)
@@ -711,7 +773,7 @@ class Tanhshrink(Cell):
         - **x** (Tensor) - Tensor of any dimension.
 
     Outputs:
-        Tensor, with the same type and shape as the `x`.
+        Tensor, with the same shape as the `x`.
 
     Raises:
         TypeError: If `x` is not a Tensor.
@@ -720,9 +782,8 @@ class Tanhshrink(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore as ms
-        >>> import mindspore.nn as nn
-        >>> from mindspore import Tensor
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
         >>> import numpy as np
         >>> x = Tensor(np.array([1, 2, 3, 2, 1]), ms.float16)
         >>> tanhshrink = nn.Tanhshrink()
@@ -752,12 +813,17 @@ class Hardtanh(Cell):
 
     Linear region range :math:`[-1, 1]` can be adjusted using `min_val` and `max_val`.
 
+    Hardtanh Activation Function Graph:
+
+    .. image:: images/Hardtanh.png
+        :align: center
+
     Note:
         On Ascend, data type of float16 might lead to accidental accuracy problem.
 
     Args:
-        min_val (Union[int, float]): Minimum value of the linear region range. Default: -1.0.
-        max_val (Union[int, float]): Maximum value of the linear region range. Default: 1.0.
+        min_val (Union[int, float]): Minimum value of the linear region range. Default: ``-1.0`` .
+        max_val (Union[int, float]): Maximum value of the linear region range. Default: ``1.0`` .
 
     Inputs:
         - **x** (Tensor) - Input Tensor with data type of float16 or float32.
@@ -815,13 +881,15 @@ class GELU(Cell):
     where :math:`P` is the cumulative distribution function
     of standard Gaussian distribution and :math:`x_i` is the element of the input.
 
-    The picture about GELU looks like this `GELU <https://en.wikipedia.org/wiki/
-    Activation_function#/media/File:Activation_gelu.png>`_.
+    GELU Activation Function Graph:
+
+    .. image:: images/GELU.png
+        :align: center
 
     Args:
-        approximate (bool): Whether to enable approximation. Default: True.
+        approximate (bool): Whether to enable approximation. Default: ``True`` .
 
-            If approximate is True, The gaussian error linear activation is:
+            If `approximate` is ``True``, The gaussian error linear activation is:
 
             :math:`0.5 * x * (1 + tanh(\sqrt(2 / \pi) * (x + 0.044715 * x^3)))`
 
@@ -843,6 +911,9 @@ class GELU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
         >>> gelu = nn.GELU()
         >>> output = gelu(x)
@@ -862,20 +933,13 @@ class GELU(Cell):
         super(GELU, self).__init__()
         validator.check_bool(approximate, 'approximate', self.cls_name)
         self.approximate = approximate
-        if self.approximate:
-            self.gelu = P.GeLU()
+        if approximate:
+            self.approximate = 'tanh'
         else:
-            self.erf = P.Erf()
-            self.sqrt = P.Sqrt()
-            self.const0 = Tensor(0.5, mstype.float32)
-            self.const1 = Tensor(1.0, mstype.float32)
-            self.const2 = Tensor(2.0, mstype.float32)
+            self.approximate = 'none'
 
     def construct(self, x):
-        if self.approximate:
-            return self.gelu(x)
-        return x * F.cast(self.const0, x.dtype) * (F.cast(self.const1, x.dtype) + \
-                                                   self.erf(x / self.sqrt(F.cast(self.const2, x.dtype))))
+        return ops.gelu(x, approximate=self.approximate)
 
 
 class FastGelu(Cell):
@@ -892,6 +956,11 @@ class FastGelu(Cell):
 
     where :math:`x_i` is the element of the input.
 
+    FastGelu Activation Function Graph:
+
+    .. image:: images/FastGelu.png
+        :align: center
+
     Inputs:
         - **x** (Tensor) - The input of FastGelu with data type of float16 or float32.
           The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
@@ -940,22 +1009,29 @@ class Sigmoid(Cell):
 
     where :math:`x_i` is the element of the input.
 
-    The picture about Sigmoid looks like this `Sigmoid <https://en.wikipedia.org/wiki/
-    Sigmoid_function#/media/File:Logistic-curve.svg>`_.
+    Sigmoid Activation Function Graph:
+
+    .. image:: images/Sigmoid.png
+        :align: center
 
     Inputs:
-        - **input_x** (Tensor) - The input of Sigmoid with data type of float16 or float32. Tensor of any dimension.
+        - **input_x** (Tensor) - Tensor of any dimension, the data type is
+          float16, float32, float64, complex64 or complex128.
 
     Outputs:
         Tensor, with the same type and shape as the `input_x`.
 
     Raises:
-        TypeError: If dtype of `input_x` is neither float16 nor float32.
+        TypeError: If dtype of `input_x` is not float16, float32, float64, complex64 or complex128.
+        TypeError: If `input_x` is not a Tensor.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> sigmoid = nn.Sigmoid()
         >>> output = sigmoid(x)
@@ -976,12 +1052,20 @@ class Softsign(Cell):
     r"""
     Softsign activation function.
 
+    Softsign Activation Function Graph:
+
+    .. image:: images/Softsign.png
+        :align: center
+
     Refer to :func:`mindspore.ops.softsign` for more details.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([0, -1, 2, 30, -30]), mindspore.float32)
         >>> softsign = nn.Softsign()
         >>> output = softsign(x)
@@ -1016,14 +1100,16 @@ class PReLU(Cell):
     Parameter :math:`w` has dimensionality of the argument channel. If called without argument
     channel, a single parameter :math:`w` will be shared across all channels.
 
-    The picture about PReLU looks like this `PReLU <https://en.wikipedia.org/wiki/
-    Activation_function#/media/File:Activation_prelu.svg>`_.
+    PReLU Activation Function Graph:
+
+    .. image:: images/PReLU.png
+        :align: center
 
     Args:
-        channel (int): The elements number of parameter `w`.
-          It could be an int, and the value is 1 or the channels number of input tensor `x`. Default: 1.
+        channel (int): The elements number of parameter :math:`w`.
+          It could be an int, and the value is 1 or the channels number of input tensor `x`. Default: ``1`` .
         w (Union[float, list, Tensor]): The initial value of parameter. It could be a float, a float list or
-          a tensor has the same dtype as the input tensor `x`. Default: 0.25.
+          a tensor has the same dtype as the input tensor `x`. Default: ``0.25`` .
 
     Inputs:
         - **x** (Tensor) - The input of PReLU with data type of float16 or float32.
@@ -1043,6 +1129,9 @@ class PReLU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([[[[0.1, 0.6], [0.9, 0.9]]]]), mindspore.float32)
         >>> prelu = nn.PReLU()
         >>> output = prelu(x)
@@ -1098,6 +1187,11 @@ class HSwish(Cell):
     .. math::
         \text{hswish}(x_{i}) = x_{i} * \frac{ReLU6(x_{i} + 3)}{6},
 
+    HSwish Activation Function Graph:
+
+    .. image:: images/HSwish.png
+        :align: center
+
     Inputs:
         - **x** (Tensor) - The input of HSwish, data type must be float16 or float32.
           The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
@@ -1112,6 +1206,9 @@ class HSwish(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> hswish = nn.HSwish()
         >>> result = hswish(x)
@@ -1135,7 +1232,12 @@ class HSigmoid(Cell):
     Hard sigmoid is defined as:
 
     .. math::
-        \text{hsigmoid}(x_{i}) = max(0, min(1, \frac{x_{i} + 3}{6})),
+        \text{hsigmoid}(x_{i}) = \max(0, \min(1, \frac{x_{i} + 3}{6})),
+
+    HSigmoid Activation Function Graph:
+
+    .. image:: images/HSigmoid.png
+        :align: center
 
     Inputs:
         - **input_x** (Tensor) - The input of HSigmoid. Tensor of any dimension.
@@ -1150,6 +1252,9 @@ class HSigmoid(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([-1, -2, 0, 2, 1]), mindspore.float16)
         >>> hsigmoid = nn.HSigmoid()
         >>> result = hsigmoid(x)
@@ -1173,10 +1278,15 @@ class LogSigmoid(Cell):
     Logsigmoid is defined as:
 
     .. math::
-        \text{logsigmoid}(x_{i}) = log(\frac{1}{1 + \exp(-x_i)}),
+        \text{logsigmoid}(x_{i}) = \log(\frac{1}{1 + \exp(-x_i)}),
 
     where :math:`x_{i}` is the element of the input.
 
+    LogSigmoid Activation Function Graph:
+
+    .. image:: images/LogSigmoid.png
+        :align: center
+
     Inputs:
         - **x** (Tensor) - The input of LogSigmoid with data type of float16 or float32.
           The shape is :math:`(N,*)` where :math:`*` means, any number of additional dimensions.
@@ -1188,9 +1298,12 @@ class LogSigmoid(Cell):
         TypeError: If dtype of `x` is neither float16 nor float32.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> net = nn.LogSigmoid()
         >>> x = Tensor(np.array([1.0, 2.0, 3.0]), mindspore.float32)
         >>> output = net(x)
@@ -1220,12 +1333,19 @@ class LRN(Cell):
     r"""
     Local Response Normalization.
 
+    .. warning::
+        LRN is deprecated on Ascend due to potential accuracy problem. It's recommended to use other
+        normalization methods, e.g. :class:`mindspore.nn.BatchNorm`.
+
     Refer to :func:`mindspore.ops.lrn` for more details.
 
     Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+        ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> input_x = Tensor(np.array([[[[0.1], [0.2]],
         ...                       [[0.3], [0.4]]]]), mindspore.float32)
         >>> output = nn.LRN()(input_x)
@@ -1257,9 +1377,14 @@ class SoftShrink(Cell):
         0, & \text{ otherwise }
         \end{cases}
 
+    SoftShrink Activation Function Graph:
+
+    .. image:: images/Softshrink.png
+        :align: center
+
     Args:
         lambd (float): the :math:`\lambda` must be no less than zero for the SoftShrink formulation.
-            Default: 0.5.
+            Default: ``0.5`` .
 
     Inputs:
         - **input_x** (Tensor) - The input of SoftShrink with data type of float16 or float32.
@@ -1278,7 +1403,10 @@ class SoftShrink(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> input_x = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mstype.float16)
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> input_x = Tensor(np.array([[ 0.5297,  0.7871,  1.1754], [ 0.7836,  0.6218, -1.1542]]), mindspore.float16)
         >>> softshrink = nn.SoftShrink()
         >>> output = softshrink(input_x)
         >>> print(output)
@@ -1309,8 +1437,13 @@ class HShrink(Cell):
         0, & \text{ otherwise }
         \end{cases}
 
+    HShrink Activation Function Graph:
+
+    .. image:: images/HShrink.png
+        :align: center
+
     Args:
-        lambd (float): The threshold :math:`\lambda` defined by the Hard Shrink formula. Default: 0.5.
+        lambd (float): The threshold :math:`\lambda` defined by the Hard Shrink formula. Default: ``0.5`` .
 
     Inputs:
         - **input_x** (Tensor) - The input of Hard Shrink with data type of float16 or float32.
@@ -1376,9 +1509,9 @@ class Threshold(Cell):
 
     Examples:
         >>> import mindspore
-        >>> import mindspore.nn as nn
+        >>> from mindspore import Tensor, nn
         >>> m = nn.Threshold(0.1, 20)
-        >>> inputs = mindspore.Tensor([0.1, 0.2, 0.3], mindspore.float32)
+        >>> inputs = Tensor([0.1, 0.2, 0.3], mindspore.float32)
         >>> outputs = m(inputs)
         >>> print(outputs)
         [ 20.0     0.2      0.3]
@@ -1400,16 +1533,24 @@ class Mish(Cell):
 
     Refer to :func:`mindspore.ops.mish` for more details.
 
+    Mish Activation Function Graph:
+
+    .. image:: images/Mish.png
+        :align: center
+
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
         >>> x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
         >>> mish = nn.Mish()
         >>> output = mish(x)
         >>> print(output)
-        [[-0.3034014  3.9974129 -0.0026832]
-         [ 1.9439590  -0.0033576 9.0000000]]
+        [[-3.03401530e-01 3.99741292e+00 -2.68321624e-03]
+         [ 1.94395900e+00 -3.35762873e-02 9.00000000e+00]]
     """
 
     def __init__(self):
@@ -1433,7 +1574,7 @@ class GLU(Cell):
     Here :math:`\sigma` is the sigmoid function, and :math:`\otimes` is the Hadamard product.
 
     Args:
-        axis (int): the axis to split the input. Default: -1, the last axis in `x`.
+        axis (int): the axis to split the input. Default: ``-1`` , the last axis in `x`.
 
     Inputs:
         - **x** (Tensor) - :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional dimensions.
@@ -1445,8 +1586,9 @@ class GLU(Cell):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> m = nn.GLU()
-        >>> input = Tensor([[0.1,0.2,0.3,0.4],[0.5,0.6,0.7,0.8]])
+        >>> import mindspore as ms
+        >>> m = ms.nn.GLU()
+        >>> input = ms.Tensor([[0.1,0.2,0.3,0.4],[0.5,0.6,0.7,0.8]])
         >>> output = m(input)
         >>> print(output)
         [[0.05744425 0.11973753]
@@ -1501,7 +1643,7 @@ def get_activation(name, prim_name=None):
 
     Args:
         name (str): The name of the activation function.
-        prim_name (Union[str, None]): The name of primitive. Default: None.
+        prim_name (Union[str, None]): The name of primitive. Default: ``None`` .
 
     Returns:
         Function, the activation function.
@@ -1510,6 +1652,7 @@ def get_activation(name, prim_name=None):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        >>> import mindspore.nn as nn
         >>> sigmoid = nn.get_activation('sigmoid')
         >>> print(sigmoid)
         Sigmoid<>