mindspore 2.1.0__cp38-none-any.whl → 2.2.0__cp38-none-any.whl

mindspore/ops/function/nn_func.py

@@ -27,7 +27,7 @@ from mindspore.ops.operations import nn_ops as NN_OPS
 from mindspore.ops.operations import _sequence_ops as seq
 import mindspore.common.dtype as mstype
 from mindspore.ops.function.math_func import logsumexp
-from mindspore.ops.function.random_func import _get_seed
+from mindspore.ops.function.random_func import _get_seed, _set_prim_op_user_data
 from mindspore.common.tensor import Tensor
 from mindspore._c_expression import Tensor as Tensor_
 from mindspore.ops._primitive_cache import _get_cache_prim
@@ -40,6 +40,7 @@ from mindspore.ops.operations.nn_ops import ChannelShuffle
 from mindspore.ops.operations.nn_ops import TripletMarginLoss
 from mindspore.ops.operations._inner_ops import SiLU
 from mindspore.ops.operations._sequence_ops import TupleToTensor, TensorToTuple, ListToTensor
+from mindspore.common.api import _function_forbid_reuse
 
 slice_ = P.Slice()
 fast_gelu_ = P.FastGeLU()
@@ -232,7 +233,7 @@ def adaptive_avg_pool3d(input, output_size):
 def _check_avgpool_1d_type_and_int(kernel_size, stride, ceil_mode, count_include_pad):
     """Checks the type of avgpool1d input"""
     validator.check_value_type('kernel_size', kernel_size, [int], 'avg_pool1d')
-    validator.check_value_type('stride', stride, [int], 'avg_pool1d')
+    validator.check_value_type('stride', stride, (int, tuple), 'avg_pool1d')
     validator.check_value_type('ceil_mode', ceil_mode, bool, 'avg_pool1d')
     validator.check_value_type('count_include_pad', count_include_pad, bool, 'avg_pool1d')
     validator.check_int(kernel_size, 1, validator.GE, "kernel_size", 'avg_pool1d')
@@ -263,12 +264,10 @@ def avg_pool1d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
     Args:
         input_x (Tensor): Tensor of shape :math:`(N, C_{in}, L_{in})`.
         kernel_size (int): The size of kernel window used to take the average value. Default: ``1`` .
-        stride (Union(int, tuple[int])): The distance of kernel moving, an int number that represents the height and
-            width of movement are both strides, or a tuple of two int numbers that represent height and width of
-            movement respectively. Default: ``1`` .
-        padding (Union(int, tuple[int])): The pad value to be filled. If `padding` is an integer, the paddings of left
-            and right are the same, equal to pad. If `padding` is a tuple of `2` integers, the padding of left and right
-            equal to `padding[0]` and `padding[1]` correspondingly. Default: ``0`` .
+        stride (Union(int, tuple[int])): The distance of kernel moving. `stride` can either be an int
+            number or a tuple of one int number. Default: ``1`` .
+        padding (Union(int, tuple[int])): The pad value to be filled. `padding` can either be an integer
+            or a tuple of one integer. Default: ``0`` .
         ceil_mode (bool): If True, apply ceil instead of floor to compute the output shape. Default: ``False``.
         count_include_pad (bool): If True, include the zero-padding in the averaging calculation. Default: ``True`` .
 
@@ -300,20 +299,25 @@ def avg_pool1d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
         raise TypeError("For avg_pool1d, the input input_x must be tensor")
 
     if len(input_x.shape) != 3:
-        raise ValueError("For avg_pool1d, input must have 3 dim, but got {}.".format(len(input_x.shape)))
+        raise ValueError(f"For avg_pool1d, input must have 3 dim, but got {len(input_x.shape)}.")
 
     _check_avgpool_1d_type_and_int(kernel_size, stride, ceil_mode, count_include_pad)
     if isinstance(padding, int):
         check_non_negative_int(padding, 'padding', 'avg_pool1d')
         padding = (0, 0, 0, 0, padding, padding)
     elif isinstance(padding, tuple):
-        if len(padding) != 2:
-            raise ValueError("For avg_pool1d, padding should be int or tuple of length 2.")
+        if len(padding) != 1:
+            raise ValueError("For avg_pool1d, padding should be int or tuple of length 1.")
         for item in padding:
             check_non_negative_int(item, 'padding', 'avg_pool1d')
-        padding = (0, 0, 0, 0, padding[0], padding[1])
+        padding = (0, 0, 0, 0, padding[0], padding[0])
     else:
-        raise TypeError("For avg_pool1d, padding should be int or tuple of length 2.")
+        raise TypeError("For avg_pool1d, padding should be int or tuple of length 1.")
+
+    if isinstance(stride, tuple):
+        if len(stride) != 1:
+            raise ValueError("For avg_pool1d, stride should be int or tuple of length 1.")
+        stride = stride[0]
 
     expand_op = _get_cache_prim(P.ExpandDims)()
     squeeze_op = _get_cache_prim(P.Squeeze)((2, 3))
@@ -419,7 +423,7 @@ def avg_pool2d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
         ceil_mode (bool): If True, apply ceil instead of floor to compute the output shape. Default: ``False``.
         count_include_pad (bool): If True, include the zero-padding in the averaging calculation. Default: ``True`` .
         divisor_override (int): If specified, it will be used as divisor in the averaging calculation, otherwise
-            `kernel_size` will be used. Default: ``0`` .
+            `kernel_size` will be used. Default: ``0``, which means not specified.
 
     Returns:
         Tensor, with shape :math:`(N, C_{out}, H_{out}, W_{out})`.
@@ -456,7 +460,7 @@ def avg_pool2d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
         raise TypeError("For avg_pool2d, the input input_x must be tensor")
 
     if len(input_x.shape) != 4:
-        raise ValueError("For avg_pool2d, input must have 4 dim, but got {}.".format(len(input_x.shape)))
+        raise ValueError(f"For avg_pool2d, input must have 4 dim, but got {len(input_x.shape)}.")
 
     kernel_size = _check_avgpool_2d_kernel_size(kernel_size)
     stride = _check_avgpool_2d_stride(stride)
@@ -527,7 +531,7 @@ def avg_pool3d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
         count_include_pad (bool, optional): If ``True`` , averaging calculation
             will include the zero-padding. Default: ``True`` .
         divisor_override (int, optional): If specified, it will be used as divisor in the averaging calculation,
-            otherwise `kernel_size` will be used. Default: ``0`` .
+            otherwise `kernel_size` will be used. Default: ``0`` , which means not specified.
 
     Returns:
         Tensor, with shape :math:`(N, C, D_{out}, H_{out}, W_{out})`. Has the same data type with `input_x`.
@@ -560,7 +564,7 @@ def avg_pool3d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
         raise TypeError("For avg_pool3d, the input input_x must be tensor")
 
     if len(input_x.shape) != 5:
-        raise ValueError("For avg_pool3d, input must have 5 dim, but got {}.".format(len(input_x.shape)))
+        raise ValueError(f"For avg_pool3d, input must have 5 dim, but got {len(input_x.shape)}.")
 
     _check_avg_pool3d_padding(padding)
 
@@ -637,21 +641,21 @@ def adaptive_max_pool1d(input, output_size):
     x_dtype = _get_cache_prim(P.DType)()(input)
 
     if len(x_in_shape) != 3:
-        raise ValueError("For adaptive_max_pool1d input must have 3 dim, but got {}.".format(len(x_in_shape)))
+        raise ValueError(f"For adaptive_max_pool1d input must have 3 dim, but got {len(x_in_shape)}.")
     if x_in_shape[2] < output_size:
-        raise ValueError("For adaptive_max_pool1d input's last dimension must be greater or equal to "
-                         "output size {}, but got {}.".format(output_size, x_in_shape[2]))
+        raise ValueError(f"For adaptive_max_pool1d input's last dimension must be greater or equal to "
+                         f"output size {output_size}, but got {x_in_shape[2]}.")
     if x_in_shape[2] % output_size != 0:
-        raise ValueError("For adaptive_max_pool1d input's last dimension must be divisible by "
-                         "output size {}, but got {}.".format(output_size, x_in_shape[2]))
+        raise ValueError(f"For adaptive_max_pool1d input's last dimension must be divisible by "
+                         f"output size {output_size}, but got {x_in_shape[2]}.")
     if is_ascend_backend():
         if x_dtype not in [mstype.float16]:
-            raise TypeError("For adaptive_max_pool1d in Ascend platform, the input dtype must be float16, "
-                            "but got {}.".format(x_dtype))
+            raise TypeError(f"For adaptive_max_pool1d in Ascend platform, the input dtype must be float16, "
+                            f"but got {x_dtype}.")
     else:
         if x_dtype not in [mstype.float16, mstype.float32]:
-            raise TypeError("For adaptive_max_pool1d, the input dtype must be float16 or float32, "
-                            "but got {}.".format(x_dtype))
+            raise TypeError(f"For adaptive_max_pool1d, the input dtype must be float16 or float32, "
+                            f"but got {x_dtype}.")
 
     expand_ = _get_cache_prim(P.ExpandDims)()
     squeeze_ = _get_cache_prim(P.Squeeze)(2)
@@ -1147,7 +1151,7 @@ def max_unpool3d(x, indices, kernel_size, stride=None, padding=0, output_size=No
     return out
 
 
-def binary_cross_entropy_with_logits(logits, label, weight, pos_weight, reduction='mean'):
+def binary_cross_entropy_with_logits(logits, label, weight=None, pos_weight=None, reduction='mean'):
     r"""
     Adds sigmoid activation function to input `logits`, and uses the given logits to compute binary cross entropy
     between the logits and the label.
@@ -1177,7 +1181,7 @@ def binary_cross_entropy_with_logits(logits, label, weight, pos_weight, reductio
 
     This operator will multiply the output by the corresponding weight.
     The tensor :math:`weight` assigns different weights to each piece of data in the batch,
-    and the tensor :math:`pos_weight` adds corresponding weights to the positive examples of each category.
+    and the tensor :math:`pos\_weight` adds corresponding weights to the positive examples of each category.
 
     In addition, it can trade off recall and precision by adding weights to positive examples.
     In the case of multi-label classification the loss can be described as:
@@ -1196,17 +1200,21 @@ def binary_cross_entropy_with_logits(logits, label, weight, pos_weight, reductio
         logits (Tensor): Input logits. Data type must be float16 or float32.
         label (Tensor): Ground truth label, has the same shape as `logits`.
           Data type must be float16 or float32.
-        weight (Tensor): A rescaling weight applied to the loss of each batch element. It can be
+        weight (Tensor, optional): A rescaling weight applied to the loss of each batch element. It can be
           broadcast to a tensor with shape of `logits`. Data type must be float16 or float32.
-        pos_weight (Tensor): A weight of positive examples. Must be a vector with length equal to the
+          Default: ``None``, `weight` is a Tensor whose value is ``1``.
+        pos_weight (Tensor, optional): A weight of positive examples. Must be a vector with length equal to the
           number of classes. It can be broadcast to a tensor with shape of `logits`.
-          Data type must be float16 or float32.
-        reduction (str): Type of reduction to be applied to loss. The optional values
-             are ``'mean'`` , ``'sum'`` , and ``'none'`` ,
-             not case sensitive. If ``'none'`` , do not perform reduction. Default: ``'mean'`` .
+          Data type must be float16 or float32. Default: ``None``, `pos_weight` is a Tensor whose value is ``1``.
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor or Scalar, if `reduction` is 'none', it's a tensor with the same shape and type as input `logits`.
+        Tensor or Scalar, if `reduction` is ``'none'``, it's a tensor with the same shape and type as input `logits`.
         Otherwise, the output is a scalar.
 
     Raises:
@@ -1214,7 +1222,7 @@ def binary_cross_entropy_with_logits(logits, label, weight, pos_weight, reductio
         TypeError: If data type of input `logits`, `label`, `weight`, `pos_weight` is neither float16 nor float32.
         TypeError: If data type of input `reduction` is not string.
         ValueError: If `weight` or `pos_weight` can not be broadcast to a tensor with shape of `logits`.
-        ValueError: If `reduction` is not one of 'none', 'mean' or 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` or ``'sum'``.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1232,10 +1240,15 @@ def binary_cross_entropy_with_logits(logits, label, weight, pos_weight, reductio
         0.3463612
     """
 
+    if weight is None:
+        weight = ops.ones_like(logits)
+    if pos_weight is None:
+        pos_weight = ops.ones_like(logits)
     bce_with_logits_loss_op = _get_cache_prim(NN_OPS.BCEWithLogitsLoss)(reduction)
     return bce_with_logits_loss_op(logits, label, weight, pos_weight)
 
 
+@_function_forbid_reuse
 def dropout(input, p=0.5, training=True, seed=None):
     r"""
     During training, randomly zeroes some of the elements of the input tensor
@@ -1275,7 +1288,9 @@ def dropout(input, p=0.5, training=True, seed=None):
         return input
     keep_prob = 1 - p
     seed0, seed1 = _get_seed(seed, "dropout")
-    out, _ = P.Dropout(keep_prob=keep_prob, Seed0=seed0, Seed1=seed1)(input)
+    dropout_op = P.Dropout(keep_prob=keep_prob, Seed0=seed0, Seed1=seed1)
+    dropout_op = _set_prim_op_user_data(dropout_op, "random_cache", False)
+    out, _ = dropout_op(input)
     return out
 
 
@@ -1820,7 +1835,7 @@ def kl_div(logits, labels, reduction='mean'):
             Its value must be one of ``'none'`` , ``'mean'`` , ``'batchmean'`` or ``'sum'`` . Default: ``'mean'`` .
 
     Returns:
-        Tensor or Scalar, if `reduction` is 'none', then output is a tensor and has the same shape as `logits`.
+        Tensor or Scalar, if `reduction` is ``'none'``, then output is a tensor and has the same shape as `logits`.
         Otherwise, it is a scalar.
 
     Raises:
@@ -2220,7 +2235,9 @@ def interpolate(input,
             One and only one of size and scale_factor can be set to None. Default: ``None`` .
         mode (str): The sampling algorithm.
             One of 'nearest', 'linear' (3D only), 'bilinear' (4D only), 'trilinear' (5D only), 'bicubic' (4D only),
-            'area', 'nearest-exact'(3D and 4D). Default: ``"nearest"`` .
+            'area', 'nearest-exact'(matches Scikit-Image and PIL nearest neighbours interpolation algorithms and fixes
+            knows issues with `nearest`, 3D and 4D). Default: ``"nearest"`` .
+
         align_corners (bool): If True, rescale input by :math:`(new\_height - 1) / (height - 1)`, which exactly
             aligns the corners of data and resized data. If False, rescale by :math:`new\_height / height`.
             Default: ``None`` .
@@ -2568,10 +2585,12 @@ def soft_margin_loss(input, target, reduction='mean'):
     Args:
         input (Tensor): Predict data. Data type must be float16 or float32.
         target (Tensor): Ground truth data, with the same type and shape as `logits`.
-        reduction (str, optional): Implements the reduction method to the output with ``'none'`` , ``'mean'`` ,
-            or ``'sum'`` ,
-            respectively indicate that no calculation is specified, that the mean is used, and that is calculated
-            using summation. Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Outputs:
         Tensor or Scalar. If `reduction` is ``'none'``, its shape is the same as `logits`.
@@ -2751,6 +2770,55 @@ def soft_shrink(input, lambd=0.5):
     return soft_shrink_op(input)
 
 
+def softplus(input, beta=1, threshold=20): # pylint:disable=redefined-outer-name
+    r"""
+    Applies softplus function to `input` element-wise.
+
+    The softplus function is shown as follows, x is the element of `input` :
+
+    .. math::
+
+        \text{output} = \frac{1}{beta}\log(1 + \exp(\text{beta * x}))
+
+    When :math:`input * beta > threshold`, the implementation converts to the linear function
+    to ensure numerical stability.
+
+    Args:
+        input (Tensor) - Tensor of any dimension.
+            Supported dtypes:
+
+            - GPU/CPU: float16, float32, float64.
+            - Ascend: float16, float32.
+
+        beta (int, optional) - The :math:`\beta` value in softplus function. Default: ``1`` .
+        threshold (int, optional) - When :math:`input * beta > threshold`, converting softplus to a linear function.
+            Default: ``20`` .
+
+    Returns:
+        Tensor, with the same type and shape as the `input` .
+
+    Raises:
+        TypeError: If `input` is not a Tensor.
+        TypeError: If the dtype of `input` is not float16, float32 or float64.
+
+    Supported Platforms:
+        ``Ascend``  ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> input = Tensor(np.array([0.1, 0.2, 30, 25]), mindspore.float32)
+        >>> output = ops.softplus(input)
+        >>> print(output)
+        [0.7443967 0.79813886 30. 25.]
+    """
+    softplus_op = _get_cache_prim(P.Softplus)()
+    scaling_input = beta * input
+    op_output = (1 / beta) * softplus_op(scaling_input)
+    return ops.select(input * beta > threshold, input, op_output)
+
+
 def silu(x):
     r"""
     Computes Sigmoid Linear Unit of input element-wise. The SiLU function is defined as:
@@ -2860,7 +2928,7 @@ def sigmoid(input):
         >>> print(output)
         [0.7310586  0.880797   0.95257413 0.98201376 0.9933072 ]
     """
-    return sigmoid_(input)
+    return _get_cache_prim(NN_OPS.Sigmoid)()(input)
 
 
 def logsigmoid(x):
@@ -2946,11 +3014,19 @@ def dense(input, weight, bias=None):
     _check_is_tensor("bias", bias, "dense")
     weight = ops.t(weight)
     input = ops.matmul(input, weight)
+    input_shape = input.shape
     if bias is not None:
         input = input + bias
+        _check_dense_add_bias_shape(input_shape, input.shape, bias.shape)
     return input
 
 
+def _check_dense_add_bias_shape(input_shape, output_shape, bias_shape):
+    """Check that the output has the correct shape after adding bias."""
+    if input_shape != output_shape:
+        raise ValueError(f"For dense, the bias shape {bias_shape} does not match the input shape {input_shape}.")
+
+
 @_primexpr
 def check_dense_inputs_same_shape(input1_shape, input2_shape, prim_name=None):
     """check bidense input Tensors' shape"""
@@ -2965,7 +3041,10 @@ def bidense(input1, input2, weight, bias=None):
     Applies bilinear dense connected layer for `input1` and `input2`. The bilinear dense function is defined as:
 
     .. math::
-        output = input1^{T} weight input2 + bias
+        output = x_{1}^{T}Ax_{2} + b
+
+    :math:`x_{1}` represents `input1` , :math:`x_{2}` represents `input2` , :math:`A` represents `weight` ,
+    :math:`b` represents `bias` .
 
     .. warning::
         This is an experimental API that is subject to change or deletion.
@@ -3391,7 +3470,9 @@ def relu6(x):
     It returns :math:`\min(\max(0,x), 6)` element-wise.
 
     Args:
-        x (Tensor): Input Tensor of float16 or float32 data type.
+        x (Tensor): Tensor of shape :math:`(N, *)`,
+            where :math:`*` means any number of additional dimensions.
+            Data type must be float16, float32.
 
     Returns:
         Tensor, with the same dtype and shape as the `x`.
@@ -3528,6 +3609,9 @@ def rrelu(input, lower=1.0 / 8, upper=1.0 / 3):
     _lower = Tensor(lower, mstype.float32)
     _upper = Tensor(upper, mstype.float32)
     _size = input.shape
+    if ops.is_sequence_value_unknown(_size):
+        dyn_shape = _get_cache_prim(P.TensorShape)()
+        _size = dyn_shape(input)
     sign_matrix = _get_cache_prim(P.Sign)()(input)
     negative_filter = sign_matrix.clip(None, 0)
     positive_filter = sign_matrix.clip(0, None)
@@ -3615,11 +3699,10 @@ def cross_entropy(input, target, weight=None, ignore_index=-100, reduction='mean
           l_n = - w_{y_n} \log \frac{\exp(x_{n,y_n})}{\sum_{c=1}^C \exp(x_{n,c})}
           \cdot \mathbb{1}\{y_n \not= \text{ignore_index}\}
 
-      where :math:`x` is the inputs, :math:`y` is the target, :math:`w` is the weight,
-      N is the batch size, :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of
-      classes.
+      where :math:`x` is the inputs, :math:`y` is the target, :math:`w` is the weight, N is the batch size,
+      :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of classes.
 
-      If reduction is not 'none' (default 'mean'), then
+      If `reduction` is not ``None`` (default ``'mean'`` ), then
 
       .. math::
 
@@ -3638,11 +3721,10 @@ def cross_entropy(input, target, weight=None, ignore_index=-100, reduction='mean
           \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
           l_n = - \sum_{c=1}^C w_c \log \frac{\exp(x_{n,c})}{\sum_{i=1}^C \exp(x_{n,i})} y_{n,c}
 
-      where :math:`x` is the inputs, :math:`y` is the target, :math:`w` is the weight,
-      N is the batch size, :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of
-      classes.
+      where :math:`x` is the inputs, :math:`y` is the target, :math:`w` is the weight, N is the batch size,
+      :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of classes.
 
-      If reduction is not 'none' (default 'mean'), then
+      If `reduction` is not ``None`` (default ``'mean'`` ), then
 
       .. math::
 
@@ -3658,16 +3740,19 @@ def cross_entropy(input, target, weight=None, ignore_index=-100, reduction='mean
             in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)`.
             `input` is expected to be log-probabilities, data type must be float16 or float32.
         target (Tensor): For class indices, tensor of shape :math:`()`, :math:`(N)` or
-            :math:`(N, d_1, d_2, ..., d_K)` , data type must be int32.
-            For probabilities, tensor of shape :math:`(C,)` :math:`(N, C)` or :math:`(N, C, d_1, d_2, ..., d_K)` ,
-            data type must be float16 or float32.
+            :math:`(N, d_1, d_2, ..., d_K)` , data type must be int32. For probabilities, tensor of shape :math:`(C,)` ,
+            :math:`(N, C)` or :math:`(N, C, d_1, d_2, ..., d_K)` , data type must be float16 or float32.
         weight (Tensor): A rescaling weight applied to the loss of each batch element.
-            If not None, the shape is :math:`(C,)`,
-            data type must be float16 or float32. Default: ``None`` .
+            If not None, the shape is :math:`(C,)`, data type must be float16 or float32. Default: ``None`` .
         ignore_index (int): Specifies a target value that is ignored
             and does not contribute to the input gradient. Default: ``-100`` .
-        reduction (str):  Apply specific reduction method to the output: ``'none'`` , ``'mean'`` , or ``'sum'`` .
-            Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
         label_smoothing (float): Label smoothing values, a regularization tool used to prevent the model
             from overfitting when calculating Loss. The value range is [0.0, 1.0]. Default value: ``0.0`` .
 
@@ -3678,17 +3763,16 @@ def cross_entropy(input, target, weight=None, ignore_index=-100, reduction='mean
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore
+        >>> import mindspore as ms
         >>> import numpy as np
-        >>> from mindspore import Tensor, ops
         >>> # Case 1: Indices labels
-        >>> inputs = mindspore.Tensor(np.random.randn(3, 5), mindspore.float32)
-        >>> target = mindspore.Tensor(np.array([1, 0, 4]), mindspore.int32)
-        >>> output = ops.cross_entropy(inputs, target)
+        >>> inputs = ms.Tensor(np.random.randn(3, 5), ms.float32)
+        >>> target = ms.Tensor(np.array([1, 0, 4]), ms.int32)
+        >>> output = ms.ops.cross_entropy(inputs, target)
         >>> # Case 2: Probability labels
-        >>> inputs = mindspore.Tensor(np.random.randn(3, 5), mindspore.float32)
-        >>> target = mindspore.Tensor(np.random.randn(3, 5), mindspore.float32)
-        >>> output = ops.cross_entropy(inputs, target)
+        >>> inputs = ms.Tensor(np.random.randn(3, 5), ms.float32)
+        >>> target = ms.Tensor(np.random.randn(3, 5), ms.float32)
+        >>> output = ms.ops.cross_entropy(inputs, target)
     """
     _check_is_tensor('input', input, "cross_entropy_loss")
     _check_is_tensor('target', target, "cross_entropy_loss")
@@ -3743,7 +3827,7 @@ def nll_loss(inputs, target, weight=None, ignore_index=-100, reduction='mean', l
     N is the batch size, :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of
     classes.
 
-    If reduction is not 'none' (default 'mean'), then
+    If `reduction` is not ``None`` (default 'mean'), then
 
     .. math::
 
@@ -3763,8 +3847,13 @@ def nll_loss(inputs, target, weight=None, ignore_index=-100, reduction='mean', l
             The data type must be float16 or float32. Default: ``None`` .
         ignore_index (int): Specifies a target value that is ignored
             and does not contribute to the input gradient. Default: ``-100`` .
-        reduction (str):  Apply specific reduction method to the output: ``'none'`` , ``'mean'`` , or ``'sum'`` .
-            Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
         label_smoothing (float): Label smoothing values, a regularization tool used to prevent the model
             from overfitting when calculating Loss. The value range is [0.0, 1.0]. Default value: ``0.0`` .
 
@@ -3858,7 +3947,7 @@ def l1_loss(input, target, reduction='mean'):
     r"""
     Calculate the mean absolute error between the `input` value and the `target` value.
 
-    Assuming that the :math:`x` and :math:`y` are 1-D Tensor, length :math:`N`, `reduction` is set to "none" ,
+    Assuming that the :math:`x` and :math:`y` are 1-D Tensor, length :math:`N`, `reduction` is set to ``"none"``,
     then calculate the loss of :math:`x` and :math:`y` without dimensionality reduction.
 
     The formula is as follows:
@@ -3881,18 +3970,21 @@ def l1_loss(input, target, reduction='mean'):
         input (Tensor): Predicted value, Tensor of any dimension.
         target (Tensor): Target value, usually has the same shape as the `input`.
             If `input` and `target` have different shape, make sure they can broadcast to each other.
-        reduction (str, optional): Type of reduction to be applied to loss.
-            The optional value is ``"mean"`` , ``"sum"`` or
-            ``"none"`` . Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor or Scalar, if `reduction` is "none", return a Tensor with same shape and dtype as `input`.
+        Tensor or Scalar, if `reduction` is ``"none"``, return a Tensor with same shape and dtype as `input`.
         Otherwise, a scalar value will be returned.
 
     Raises:
         TypeError: If `input` is not a Tensor.
         TypeError: If `target` is not a Tensor.
-        ValueError: If `reduction` is not one of "none", "mean" or "sum".
+        ValueError: If `reduction` is not one of ``"none"``, ``"mean"`` or ``"sum"``.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -3948,16 +4040,20 @@ def smooth_l1_loss(input, target, beta=1.0, reduction='none'):
         target (Tensor): Ground truth data, tensor of shape :math:`(N, *)`, same shape and dtype as the `input`.
         beta (float): A parameter used to control the point where the function will change between
             L1 to L2 loss. The value should be greater than zero. Default: ``1.0`` .
-        reduction (str): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` or ``'sum'`` .
-            Default: ``'none'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'none'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor, if `reduction` is 'none', then output is a tensor with the same shape as `input`.
+        Tensor, if `reduction` is ``'none'``, then output is a tensor with the same shape as `input`.
         Otherwise, the shape of output tensor is :math:`(1,)`.
 
     Raises:
         TypeError: If `beta` is not a float.
-        ValueError: If `reduction` is not one of 'none', 'mean', 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
         TypeError: If dtype of `input` or `target` is not one of float16, float32, float64.
         ValueError: If `beta` is less than or equal to 0.
         ValueError: If shape of `input` is not the same as `target`.
@@ -4072,6 +4168,7 @@ def leaky_relu(input, alpha=0.2):
     select_op = _get_cache_prim(P.Maximum)()
     if alpha > 1:
         select_op = _get_cache_prim(P.Minimum)()
+    alpha = _get_cache_prim(P.Cast)()(F.scalar_to_tensor(alpha), input.dtype)
     return select_op(alpha * input, input)
 
 
@@ -4158,6 +4255,10 @@ def lrn(x, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5, norm_region="ACROSS_CH
     r"""
     Local Response Normalization.
 
+    .. warning::
+        lrn is deprecated on Ascend due to potential accuracy problem. It's recommended to use other
+        normalization methods, e.g. :class:`mindspore.ops.batch_norm`.
+
     .. math::
 
         b_{c} = a_{c}\left(k + \frac{\alpha}{n}
@@ -4186,7 +4287,7 @@ def lrn(x, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5, norm_region="ACROSS_CH
         TypeError: If `x` is not a Tensor.
 
     Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+        ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
@@ -4219,7 +4320,11 @@ def mish(x):
     <https://arxiv.org/abs/1908.08681>`_.
 
     Args:
-        x (Tensor): The input Tensor with float16, float32 or float64 data type.
+        x (Tensor): The input Tensor.
+            Supported dtypes:
+
+            - GPU/CPU: float16, float32, float64.
+            - Ascend: float16, float32.
 
     Returns:
         Tensor, with the same type and shape as the `x`.
@@ -4320,10 +4425,40 @@ def _check_type_and_shape_same(param_name1, input_data1, param_name2, input_data
 
 
 def margin_ranking_loss(input1, input2, target, margin=0.0, reduction='mean'):
-    """
+    r"""
     MarginRankingLoss creates a criterion that measures the loss.
 
-    For details, please refer to :class:`mindspore.nn.MarginRankingLoss`.
+    Given two tensors :math:`input1`, :math:`input2` and a Tensor label :math:`target` with values 1 or -1,
+    the operation is as follows:
+
+    .. math::
+        \text{loss}(input1, input2, target) = \max(0, -target * (input1 - input2) + \text{margin})
+
+    Args:
+        input1 (Tensor): Tensor of shape :math:`(N, *)` where :math:`*` means, any number of additional dimensions.
+        input2 (Tensor): Tensor of shape :math:`(N, *)`, same shape and dtype as `input1`.
+        target (Tensor): Contains value 1 or -1. Suppose the shape of `input1` is
+          :math:`(x_1, x_2, x_3, ..., x_R)`, then the shape of `target` must be :math:`(x_1, x_2, x_3, ..., x_R)`.
+        margin (float, optional): Specify the adjustment factor of the operation. Default: ``0.0`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
+    Returns:
+        Tensor or Scalar. if `reduction` is ``"none"``, its shape is the same as `labels`.
+        Otherwise, a scalar value will be returned.
+
+    Raises:
+        TypeError: If `margin` is not a float.
+        TypeError: If `input1`, `input2` or `target` is not a Tensor.
+        TypeError: If the types of `input1` and `input2` are inconsistent.
+        TypeError: If the types of `input1` and `target` are inconsistent.
+        ValueError: If the shape of `input1` and `input2` are inconsistent.
+        ValueError: If the shape of `input1` and `target` are inconsistent.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` , ``'sum'``.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -4334,7 +4469,7 @@ def margin_ranking_loss(input1, input2, target, margin=0.0, reduction='mean'):
         >>> import numpy as np
         >>> input1 = Tensor(np.array([0.3864, -2.4093, -1.4076]), ms.float32)
         >>> input2 = Tensor(np.array([-0.6012, -1.6681, 1.2928]), ms.float32)
-        >>> target = sign(Tensor(np.array([-2, -2, 3]), ms.float32))
+        >>> target = ops.Sign()(Tensor(np.array([-2, -2, 3]), ms.float32))
         >>> output = ops.margin_ranking_loss(input1, input2, target)
         >>> print(output)
         1.2293333
@@ -4375,17 +4510,20 @@ def cosine_embedding_loss(input1, input2, target, margin=0.0, reduction="mean"):
         target (Tensor): Contains value 1 or -1. Suppose the shape of `input1` is
           :math:`(x_1, x_2, x_3, ..., x_R)`, then the shape of `target` must be :math:`(x_1, x_3, x_4, ..., x_R)`.
         margin (float, optional): Should be in [-1.0, 1.0]. Default: 0.0.
-        reduction (str, optional): Specifies which reduction to be applied to the output. It must be one of
-          ``"none"`` , ``"mean"`` , and ``"sum"`` ,
-          meaning no reduction, reduce mean and sum on output, respectively. Default: ``"mean"`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor or Scalar, if `reduction` is "none", its shape is the same as `target`.
+        Tensor or Scalar, if `reduction` is ``"none"``, its shape is the same as `target`.
         Otherwise, a scalar value will be returned.
 
     Raises:
         TypeError: If `margin` is not a float.
-        ValueError: If `reduction` is not one of 'none', 'mean', 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
         ValueError: If `margin` is not in range [-1, 1].
 
     Supported Platforms:
@@ -4529,14 +4667,24 @@ def grid_sample(input, grid, mode='bilinear', padding_mode='zeros', align_corner
 
     Args:
         input (Tensor): input with shape of :math:`(N, C, H_{in}, W_{in})` (4-D case) or :math:`(N, C, D_{in},
-            H_{in}, W_{in})` (5-D case) and dtype of float16, float32 or float64.
+            H_{in}, W_{in})` (5-D case) and dtype of float32 or float64.
         grid (Tensor): flow-field with shape of :math:`(N, H_{out}, W_{out}, 2)` (4-D case) or :math:`(N, D_{out},
             H_{out}, W_{out}, 3)` (5-D case) and same dtype as `input`.
         mode (str): An optional string specifying the interpolation method. The optional values are
-            'bilinear', 'nearest'. Default: ``'bilinear'`` . Note: `bicubic` is not supported yet. When
+            ``'bilinear'``, ``'nearest'``. Default: ``'bilinear'`` . Note: `bicubic` is not supported yet. When
             `mode="bilinear"` and the input is 5-D, the interpolation mode used internally will actually
             be trilinear. However, when the input is 4-D, the interpolation mode will legistimately be bilinear.
             Default: ``'bilinear'`` .
+
+            - ``'nearest'``: Nearest neighbor interpolation. Each output pixel is assigned the value of the
+              nearest input pixel. This method is simple and fast but can result in blocky or pixelated outputs.
+            - ``'bilinear'``: Bilinear interpolation. Each output pixel is a weighted average of the four nearest input
+              pixels, computed using bilinear interpolation. This method produces smoother results compared
+              to nearest neighbor interpolation.
+            - ``'trilinear'``: Trilinear interpolation. This is an extension of bilinear interpolation to 3D data.
+              It performs bilinear interpolation in the two spatial dimensions and linear interpolation along
+              the third dimension. It is commonly used for volume or 3D image interpolation.
+
         padding_mode (str): An optional string specifying the pad method. The optional values are "zeros", "border" or
             "reflection". Default: ``'zeros'`` .
         align_corners (bool): An optional bool. If set to `True`, the extrema (-1 and 1) are considered as referring to
@@ -4617,10 +4765,13 @@ def ctc_loss(log_probs, targets, input_lengths, target_lengths, blank=0, reducti
         input_lengths (Union(tuple, Tensor)): Lengths of the input. A tuple or Tensor of shape(N).
         target_lengths (Union(tuple, Tensor)): Lengths of the target. A tuple or Tensor of shape(N).
         blank (int, optional): The blank label. Default: ``0`` .
-        reduction (str, optional): Implements the reduction method to the output with
-            ``'none'`` , ``'mean'`` , or ``'sum'`` ,
-            respectively indicate that no calculation is specified, that the mean is used, and that is calculated
-            using summation. Default: ``"mean"`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
         zero_infinity (bool, optional): Whether to set infinite loss and correlation gradient to 0. Default: ``False`` .
 
     Returns:
@@ -4704,8 +4855,12 @@ def gaussian_nll_loss(x, target, var, full=False, eps=1e-6, reduction='mean'):
         full (bool, optional): Include the constant term in the loss calculation. When :math:`full=True`,
             the constant term will be :math:`const = 0.5*log(2\pi)`. Default: ``False``.
         eps (float, optional): Used to improve the stability of log function must be greater than 0. Default: ``1e-6`` .
-        reduction (str, optional): Apply specific reduction method to the
-            output: ``"none"``, ``"mean"``, or ``"sum"``. Default: ``'mean'``.
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
         Tensor or Tensor scalar, the computed loss depending on :math:`reduction`.
@@ -4722,8 +4877,7 @@ def gaussian_nll_loss(x, target, var, full=False, eps=1e-6, reduction='mean'):
 
     Examples:
         >>> import numpy as np
-        >>> from mindspore import Tensor
-        >>> import mindspore.ops as ops
+        >>> from mindspore import Tensor, ops
         >>> import mindspore.common.dtype as mstype
         >>> arr1 = np.arange(8).reshape((4, 2))
         >>> arr2 = np.array([2, 3, 1, 4, 6, 4, 4, 9]).reshape((4, 2))
@@ -4831,9 +4985,12 @@ def hinge_embedding_loss(inputs, targets, margin=1.0, reduction='mean'):
             Has the same shape as `inputs`, contains -1 or 1.
         margin (float, int): Threshold defined by Hinge Embedding Loss :math:`margin`.
             Represented as :math:`\Delta` in the formula. Default: ``1.0`` .
-        reduction (str): Specify the computing method to be applied to the outputs:
-            ``'none'`` , ``'mean'`` , or ``'sum'`` .
-            Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
         Tensor or Tensor scalar, the computed loss depending on :math:`reduction`.
@@ -4843,7 +5000,7 @@ def hinge_embedding_loss(inputs, targets, margin=1.0, reduction='mean'):
         TypeError: If `targets` is not a Tensor.
         TypeError: If `margin` is not a float or int.
         ValueError: If `targets` does not have the same shape as `inputs` or they could not broadcast to each other.
-        ValueError: If `reduction` is not one of 'none', 'mean', 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -4889,6 +5046,9 @@ def ctc_greedy_decoder(inputs, sequence_length, merge_repeated=True):
     r"""
     Performs greedy decoding on the logits given in inputs.
 
+    Note:
+        On Ascend, 'merge_repeated' can not be set to false.
+
     Args:
         inputs (Tensor): The input Tensor must be a 3-D tensor whose shape is
             :math:`(max\_time, batch\_size, num\_classes)`. `num_classes` must be `num_labels + 1` classes,
@@ -5068,74 +5228,87 @@ def _check_conv_iterable_lengths(iterable, dim, iter_name):
 
 def conv1d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dilation=1, groups=1):
     r"""
-    Applies a 1D convolution over an input tensor.
-    The input tensor is typically of shape :math:`(N, C_{in}, W_{in})`,
-    where :math:`N` is batch size, :math:`C_{in}` is channel number, :math:`W` is width, :math:`X_i` is
-    the :math:`i^{th}` input value and :math:`b_i` indicates the deviation value of the :math:`i^{th}` input value.
-    For each batch of shape :math:`(C_{in}, W_{in})`, the formula is defined as:
+    Applies a 1D convolution over an input tensor. The input Tensor is typically
+    of shape :math:`(N, C_{in}, L_{in})`,
+    where :math:`N` is batch size, :math:`C` is channel number, :math:`L` is input sequence width.
+
+    The output is calculated based on formula:
 
     .. math::
 
-        out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{j}, X_i) + b_j,
+        \text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
+        \sum_{k = 0}^{C_{in} - 1} \text{ccor}({\text{weight}(C_{\text{out}_j}, k), \text{X}(N_i, k)})
+
+    where :math:`bias` is the output channel bias, :math:`ccor` is
+    the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_,
+    , :math:`weight` is the convolution kernel value and :math:`X` represents the input feature map.
+
+    Here are the indices' meanings:
+    - :math:`i` corresponds to the batch number, ranging from 0 to N-1, where N is the batch size of the input.
+
+    - :math:`j` corresponds to the output channel, ranging from 0 to C_{out}-1, where C_{out} is the number of
+      output channels, which is also equal to the number of kernels.
 
-    where :math:`ccor` is the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_  operator,
-    :math:`C_{in}` is the input channel number, :math:`j` ranges
-    from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to the :math:`i`-th channel of the :math:`j`-th
-    filter and :math:`out_{j}` corresponds to the :math:`j`-th channel of the output. :math:`W_{j}` is a slice
-    of kernel, and it has shape :math:`(\text{kernal_size})`, where :math:`\text{kernel_size}` is the width of
-    the convolution kernel. The full kernel has shape :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size})`,
-    where `groups` is the group number to split the input in the channel dimension.
+    - :math:`k` corresponds to the input channel, ranging from 0 to C_{in}-1, where C_{in} is the number of
+      input channels, which is also equal to the number of channels in the convolutional kernels.
 
-    If the `pad_mode` is set to be "valid", the output width will be :math:`\left \lfloor{
-    1 + \frac{W_{in} + \text{padding[0]} - \text{kernel_size} - (\text{kernel_size} - 1) \times(\text{dilation} - 1)}
-    {\text { stride }}} \right \rfloor`.
+    Therefore, in the above formula, :math:`{bias}(C_{out_j})` represents the bias of the :math:`j`-th
+    output channel, :math:`{weight}(C_{out_j}, k)` represents the slice of the :math:`j`-th convolutional
+    kernel in the :math:`k`-th channel, and :math:`{X}(N_i, k)` represents the slice of the :math:`k`-th input
+    channel in the :math:`i`-th batch of the input feature map.
 
-    where :math:`dilation` is spacing between kernel elements, :math:`stride` is The step length of each step,
-    :math:`padding` is zero-padding added to both sides of the input.
-    For output width on other `pad_mode`, please refer to formula on `mindspore.nn.Conv1d
-    <https://www.mindspore.cn/docs/en/r2.1/api_python/nn/mindspore.nn.Conv2d.html>`_.
+    The shape of the convolutional kernel is given by :math:`(kernel\_size)`,
+    where :math:`kernel\_size` is the width of the kernel.
+    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{group}, \text{kernel_size})`,
+    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
 
-    The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition
-    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_. More detailed introduction can be found here:
-    `ConvNets <http://cs231n.github.io/convolutional-networks/>`_ .
+    For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
+    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_
+    and `ConvNets <http://cs231n.github.io/convolutional-networks/>`_ .
 
     Note:
         On Ascend platform, only group convolution in depthwise convolution scenarios is supported.
         That is, when `groups>1`, condition `C_{in}` = `C_{out}` = `groups` must be satisfied.
 
     Args:
-        input (Tensor): Tensor of shape :math:`(N, C_{in}, W_{in})`.
-        weight (Tensor): Tensor of shape
-            :math:`(N, C_{in} / \text{groups}, \text{kernel_size})`, then the size of kernel is
-            :math:`(\text{kernel_size})`.
-        bias (Tensor): Bias Tensor with shape :math:`(C_{out})`.
+        input (Tensor): Input Tensor of shape :math:`(N, C_{in}, L_{in})`.
+        weight (Tensor): The convolutional kernel value, it should has shape
+            :math:`(N, C_{in} / \text{groups}, \text{kernel_size})`.
+        bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
             When bias is None, zeros will be used. Default: ``None`` .
         stride (Union(int, tuple[int]), optional): The distance of kernel moving, an int number or a tuple of one int
-            that represents width of movement. Default: 1.
+            that represents width of movement. Default: ``1``.
         pad_mode (str, optional): Specifies padding mode. The optional values are
             ``"same"`` , ``"valid"`` and ``"pad"`` . Default: ``"valid"`` .
 
-            - same: Adopts the way of completion. The height and width of the output will be equal to
+            - ``"same"``: Adopts the way of completion. The height and width of the output will be equal to
               the input `x` divided by stride. The padding will be evenly calculated in left and right possiblily.
               Otherwise, the last extra padding will be calculated from the right side.
               If this mode is set, `padding` must be 0.
 
-            - valid: Adopts the way of discarding. The possible largest width of output will be returned
+            - ``"valid"``: Adopts the way of discarding. The possible largest width of output will be returned
               without padding. Extra pixels will be discarded. If this mode is set, `padding` must be 0.
 
-            - pad: Implicit paddings on both sides of the input `x`. The number of `padding` will be padded to the input
+            - ``"pad"``: Implicit paddings on both sides of the input `x`.
+              The number of `padding` will be padded to the input
               Tensor borders. `padding` must be greater than or equal to 0.
-        padding (Union(int, tuple[int], list[int]), optional): Implicit paddings on both sides of `input`, meaning the
+        padding (Union(int, tuple[int], list[int]), optional):  Specifies the amount of padding to apply on
+            both side of `input` when `pad_mode` is set to ``"pad"``. The
             paddings of left and right are the same, equal to padding or padding[0] when padding is a tuple of
             1 integer. Default: ``0`` .
-        dilation (Union(int, tuple[int]), optional): Gaps between kernel elements. The data type is int or a tuple of
-            1 integer. Specifies the dilation rate to use for dilated convolution. If set to be :math:`k > 1`,
-            there will be :math:`k - 1` pixels skipped for each sampling location. Its value must be greater than or
-            equal to 1 and bounded by the width of `input`. Default: ``1`` .
+        dilation (Union(int, tuple[int]), optional): Specifies the dilation rate to use for dilated convolution.
+            It can be a single int or a tuple of 1 integer.
+            Assuming :math:`dilation=(d0,)`, the convolutional kernel samples the input with a
+            spacing of :math:`d0-1` elements in the width direction.
+            The value should be in the ranges [1, L].
+            Default: ``1`` .
         groups (int, optional): Splits `input` into groups. Default: ``1`` .
 
     Returns:
-        Tensor, the value that applied 1D convolution. The shape is :math:`(N, C_{out}, W_{out})`.
+        Tensor, the value that applied 1D convolution. The shape is :math:`(N, C_{out}, L_{out})`.
+        To see how different pad modes affect the output shape, please refer to
+        :class:`mindspore.nn.Conv1d` for more details.
 
     Raises:
         TypeError: If `stride`, `padding` or `dilation` is neither an int nor a tuple.
@@ -5204,40 +5377,44 @@ def conv1d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
 
 def conv2d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dilation=1, groups=1):
     r"""
-    Applies a 2D convolution over an input tensor.
-    The input tensor is typically of shape :math:`(N, C_{in}, H_{in}, W_{in})`,
-    where :math:`N` is batch size, :math:`C` is channel number, :math:`H` is height, :math:`W` is width, :math:`X_i` is
-    the :math:`i^{th}` input value and :math:`b_i` indicates the deviation value of the :math:`i^{th}` input value.
-    For each batch of shape :math:`(C_{in}, H_{in}, W_{in})`, the formula is defined as:
+    Applies a 2D convolution over an input tensor. The input tenor is typically of
+    shape :math:`(N, C_{in}, H_{in}, W_{in})`, where :math:`N` is batch size, :math:`C` is
+    channel number, :math:`H` is feature height, :math:`W` is feature width.
+
+    The output is calculated based on formula:
 
     .. math::
 
-        out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{ij}, X_i) + b_j,
+        \text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
+        \sum_{k = 0}^{C_{in} - 1} \text{ccor}({\text{weight}(C_{\text{out}_j}, k), \text{X}(N_i, k)})
 
-    where :math:`ccor` is the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_  operator,
-    :math:`C_{in}` is the input channel number, :math:`j` ranges
-    from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to the :math:`i`-th channel of the :math:`j`-th
-    filter and :math:`out_{j}` corresponds to the :math:`j`-th channel of the output. :math:`W_{ij}` is a slice
-    of kernel, and it has shape :math:`(\text{kernel_size[0]}, \text{kernel_size[1]})`, where :math:`\text{
-    kernel_size[0]}` and :math:`\text{kernel_size[1]}` are the height and width of the convolution kernel.
-    The full kernel has shape :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size[0]}, \text{kernel_size[1]})`,
-    where `groups` is the group number to split the input in the channel dimension.
+    where :math:`bias` is the output channel bias, :math:`ccor` is
+    the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_,
+    , :math:`weight` is the convolution kernel value and :math:`X` represents the input feature map.
 
-    If the `pad_mode` is set to be "valid", the output height and width will be :math:`\left \lfloor{
-    1 + \frac{H_{in} + \text{padding[0]} + \text{padding[1]} - \text{kernel_size[0]} -
-    (\text{kernel_size[0]} - 1) \times(\text{dilation[0]} - 1)} {\text { stride[0] }}} \right \rfloor` and
+    Here are the indices' meanings:
+    - :math:`i` corresponds to the batch number, ranging from 0 to N-1, where N is the batch size of the input.
 
-    :math:`\left \lfloor{1 + \frac{W_{in} + \text{padding[2]} + \text{padding[3]} - \text{kernel_size[1]} -
-    (\text{kernel_size[1]} - 1) \times(\text{dilation[1]} - 1)} {\text { stride[1] }}} \right \rfloor` respectively.
+    - :math:`j` corresponds to the output channel, ranging from 0 to C_{out}-1, where C_{out} is the number of
+      output channels, which is also equal to the number of kernels.
 
-    where :math:`dilation` is spacing between kernel elements, :math:`stride` is The step length of each step,
-    :math:`padding` is zero-padding added to both sides of the input.
-    For output height and width on other `pad_mode`, please refer to formula on `mindspore.nn.Conv2d
-    <https://www.mindspore.cn/docs/en/r2.1/api_python/nn/mindspore.nn.Conv2d.html>`_.
+    - :math:`k` corresponds to the input channel, ranging from 0 to C_{in}-1, where C_{in} is the number of
+      input channels, which is also equal to the number of channels in the convolutional kernels.
 
-    The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition
-    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_. More detailed introduction can be found here:
-    `ConvNets <http://cs231n.github.io/convolutional-networks/>`_ .
+    Therefore, in the above formula, :math:`{bias}(C_{out_j})` represents the bias of the :math:`j`-th
+    output channel, :math:`{weight}(C_{out_j}, k)` represents the slice of the :math:`j`-th convolutional
+    kernel in the :math:`k`-th channel, and :math:`{X}(N_i, k)` represents the slice of the :math:`k`-th input
+    channel in the :math:`i`-th batch of the input feature map.
+
+    The shape of the convolutional kernel is given by :math:`(kernel\_size[0], kernel\_size[1])`,
+    where :math:`kernel\_size[0]` and :math:`kernel\_size[1]` are the height and width of the kernel, respectively.
+    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{group}, \text{kernel_size[0]}, \text{kernel_size[1]})`,
+    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
+
+    For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
+    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_ and
+    `ConvNets <http://cs231n.github.io/convolutional-networks/>`_.
 
     Note:
         On Ascend platform, only group convolution in depthwise convolution scenarios is supported.
@@ -5248,7 +5425,7 @@ def conv2d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
         weight (Tensor): Tensor of shape
             :math:`(N, C_{in} / \text{groups}, \text{kernel_size[0]}, \text{kernel_size[1]})`, then the size of kernel
             is :math:`(\text{kernel_size[0]}, \text{kernel_size[1]})`.
-        bias (Tensor): Bias Tensor with shape :math:`(C_{out})`.
+        bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
             When bias is ``None`` , zeros will be used. Default: ``None`` .
         stride (Union(int, tuple[int]), optional): The distance of kernel moving, an int number that represents
             the height and width of movement are both strides, or a tuple of two int numbers that
@@ -5278,6 +5455,9 @@ def conv2d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
 
     Returns:
         Tensor, the value that applied 2D convolution. The shape is :math:`(N, C_{out}, H_{out}, W_{out})`.
+        To see how different pad modes affect the output shape, please refer to
+        :class:`mindspore.nn.Conv2d` for more details.
+
 
     Raises:
         TypeError: If `stride`, `padding` or `dilation` is neither an int nor a tuple.
@@ -5421,8 +5601,9 @@ def huber_loss(input, target, reduction='mean', delta=1.0):
     Calculates the error between the predicted value and the target value,
     which has the best of both the loss of l1 and the loss of mse.
 
-    Assuming that the :math:`x` and :math:`y` are 1-D Tensor, length :math:`N`, the reduction parameter is set to "none"
-    then calculate the loss of :math:`x` and :math:`y` without dimensionality reduction. The formula is as follows:
+    Assuming that the :math:`x` and :math:`y` are 1-D Tensor, length :math:`N`, the `reduction` parameter
+    is set to ``"none"`` then calculate the loss of :math:`x` and :math:`y` without dimensionality reduction.
+    The formula is as follows:
 
     .. math::
         \ell(x, y) = L = \{l_1,\dots,l_N\}^\top
@@ -5451,21 +5632,25 @@ def huber_loss(input, target, reduction='mean', delta=1.0):
         target (Tensor): Target value, has same dtype and shape as the `input` in common cases.
             However, when the shape of `target` is different from the shape of `input`,
             and they should be broadcasted to each other.
-        reduction (str): Type of reduction to be applied to loss.
-            The optional values are ``'mean'`` , ``'sum'`` and ``'none'`` .
-            Default: ``'mean'``.
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
         delta (Union[int, float]): The threshold to change between two type of loss.
             The value must be greater than zero. Default: ``1.0`` .
 
     Returns:
-        Tensor or Scalar, if `reduction` is "none", return a Tensor with same shape and dtype as `input`.
+        Tensor or Scalar, if `reduction` is ``"none"``, return a Tensor with same shape and dtype as `input`.
         Otherwise, a scalar value will be returned.
 
     Raises:
         TypeError: If `input` or `target` is not a Tensor.
         TypeError: If dtype of `delta` is neither float nor int.
         ValueError: If `delta` is less than or equal to 0.
-        ValueError: If `reduction` is not one of "none", "mean", "sum".
+        ValueError: If `reduction` is not one of ``"none"``, ``"mean"``, ``"sum"``.
         ValueError: If `input` and `target` have different shapes and cannot be broadcasted to each other.
 
     Supported Platforms:
@@ -5655,15 +5840,20 @@ def bias_add(input_x, bias):
     consistent with the shape of the `input_x` Tensor.
 
     Args:
-        input_x (Tensor): The input tensor. The shape can be 2-5 dimensions.
-        bias (Tensor): The bias tensor, with shape :math:`(C)`. C must be the same as channel dimension C of `input_x`.
+        input_x (Tensor): The input tensor. The shape can be 2-5 dimensions. Supported dtypes:
+
+            - Ascend/CPU: all Number type.
+            - GPU: float16, float32, int8.
+
+        bias (Tensor): The bias tensor, with shape :math:`(C)`. C must be the same as channel dimension C of
+            `input_x`. It has the same type as `input_x`.
 
     Returns:
         Tensor, with the same shape and data type as `input_x`.
 
     Raises:
         TypeError: If `input_x` or `bias` is not a Tensor.
-        TypeError: If dtype of `input_x` or `bias` is inconsistent.
+        TypeError: If dtype of `input_x` and `bias` is inconsistent.
         TypeError: If dimension of `input_x` is not in the range [2, 5].
 
     Supported Platforms:
@@ -5718,11 +5908,12 @@ def binary_cross_entropy(logits, labels, weight=None, reduction='mean'):
             the loss function
             will not consider any sample weights, and each sample will be treated as having equal importance
             when calculating the loss.
-        reduction (str, optional): Specify the protocol calculation method used to output the results.
-            Its value must be one of ``'none'`` , ``'mean'`` or ``'sum'`` ,
-            respectively indicate that no calculation method is
-            specified, using the average value for calculation, and using summation for calculation, not case-sensitive.
-            Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
         Tensor or Scalar. Returns Tensor that has the same dtype and shape as `logits` if `reduction` is 'none'.
@@ -5731,7 +5922,7 @@ def binary_cross_entropy(logits, labels, weight=None, reduction='mean'):
     Raises:
         TypeError: If `logits`, `labels` or `weight` is not a Tensor.
         TypeError: If dtype of `logits`, `labels` or `weight` (if given) is neither float16 nor float32.
-        ValueError: If `reduction` is not one of 'none', 'mean' or 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` or ``'sum'``.
         ValueError: If shape of `labels` is not the same as `logits` or `weight` (if given).
 
     Supported Platforms:
@@ -5754,32 +5945,46 @@ def binary_cross_entropy(logits, labels, weight=None, reduction='mean'):
 
 def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dilation=1, groups=1):
     r"""
-    Applies a 3D convolution over an input tensor. The input tensor is typically of shape
-    :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})` and output shape
-    :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`, where :math:`N` is batch size, :math:`C` is channel number,
-    :math:`D` is depth, :math:`H, W` is feature height and width respectively.
-    the output value of a layer is calculated as:
+    Applies a 3D convolution over an input tensor. The input tensor is typically of
+    shape :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`, where :math:`N` is batch size, :math:`C`
+    is channel number, :math:`D` is feature depth, :math:`H` is feature height, :math:`W` is feature width.
+
+    The output is calculated based on formula:
 
     .. math::
-        \operatorname{out}\left(N_{i}, C_{\text {out}_j}\right)=\operatorname{bias}\left(C_{\text {out}_j}\right)+
-        \sum_{k=0}^{C_{in}-1} ccor(\text {weight}\left(C_{\text {out}_j}, k\right),
-        \operatorname{input}\left(N_{i}, k\right))
-
-    where :math:`k` is kernel,
-    :math:`ccor` is the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_ ,
-    :math:`C_{in}` is the channel number of the input, :math:`out_{j}` corresponds to the jth channel of
-    the output and :math:`j` is in the range of :math:`[0, C_{out}-1]`. :math:`\text{weight}(C_{\text{out}_j}, k)`
-    is a convolution kernel slice with shape
-    :math:`(\text{kernel_size[0]}, \text{kernel_size[1]}, \text{kernel_size[2]})`,
-    where :math:`\text{kernel_size[0]}`, :math:`\text{kernel_size[1]}` and :math:`\text{kernel_size[2]}` are
-    the depth, height and width of the convolution kernel respectively. :math:`\text{bias}` is the bias parameter
-    and :math:`\text{X}` is the input tensor.
-    The shape of full convolution kernel is
-    :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size[0]}, \text{kernel_size[1]}, \text{kernel_size[2]})`,
-    where `groups` is the number of groups to split `input` in the channel dimension.
-
-    For more details, please refer to the paper `Gradient Based Learning Applied to Document
-    Recognition <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_ .
+
+        \text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
+        \sum_{k = 0}^{C_{in} - 1} \text{ccor}({\text{weight}(C_{\text{out}_j}, k), \text{X}(N_i, k)})
+
+    where :math:`bias` is the output channel bias, :math:`ccor` is
+    the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_,
+    , :math:`weight` is the convolution kernel value and :math:`X` represents the input feature map.
+
+    Here are the indices' meanings:
+    - :math:`i` corresponds to the batch number, ranging from 0 to N-1, where N is the batch size of the input.
+
+    - :math:`j` corresponds to the output channel, ranging from 0 to C_{out}-1, where C_{out} is the number of
+      output channels, which is also equal to the number of kernels.
+
+    - :math:`k` corresponds to the input channel, ranging from 0 to C_{in}-1, where C_{in} is the number of
+      input channels, which is also equal to the number of channels in the convolutional kernels.
+
+    Therefore, in the above formula, :math:`{bias}(C_{out_j})` represents the bias of the :math:`j`-th
+    output channel, :math:`{weight}(C_{out_j}, k)` represents the slice of the :math:`j`-th convolutional
+    kernel in the :math:`k`-th channel, and :math:`{X}(N_i, k)` represents the slice of the :math:`k`-th input
+    channel in the :math:`i`-th batch of the input feature map.
+
+    The shape of the convolutional kernel is given by
+    :math:`(\text{kernel_size[0]}, \text{kernel_size[1]}, \text{kernel_size[2]})`
+    where :math:`kernel\_size[0]` , :math:`kernel\_size[1]` and :math:`kernel\_size[2]` are the depth,
+    height and width of the kernel, respectively.
+    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{group}, \text{kernel_size[0]},
+    \text{kernel_size[1]}, \text{kernel_size[2]})`,
+    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
+
+    For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
+    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_.
 
     Note:
         1. On Ascend platform, `groups = 1` must be satisfied.
@@ -5790,8 +5995,8 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
         weight (Tensor): Set size of kernel is :math:`(\text{kernel_size[0]}, \text{kernel_size[1]},
             \text{kernel_size[2]})`, then the shape is :math:`(C_{out}, C_{in}, \text{kernel_size[0]},
             \text{kernel_size[1]}, \text{kernel_size[1]})`.
-        bias (Tensor): Bias Tensor with shape :math:`(C_{out})`. When bias is None, zeros will be used. Default:
-            ``None`` .
+        bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
+            When bias is None, zeros will be used. Default: ``None`` .
         stride (Union[int, tuple[int]], optional): The distance of kernel moving,
             it can be an int number that represents
             the depth, height and width of movement or a tuple of three int numbers that
@@ -5799,18 +6004,18 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
         pad_mode (str, optional): Specifies padding mode. The optional values are
             ``"same"`` , ``"valid"`` and ``"pad"`` . Default: ``"valid"`` .
 
-            - same: Adopts the way of completion. The depth, height and width of the output will be equal to
+            - ``"same"``: Adopts the way of completion. The depth, height and width of the output will be equal to
               the input `x` divided by stride. The padding will be evenly calculated in head and tail, top and bottom,
               left and right directions possiblily.
               Otherwise, the last extra padding will be calculated from the tail, bottom and the right side.
               If this mode is set, `pad` must be 0.
 
-            - valid: Adopts the way of discarding. The possible largest depth, height and width of output
+            - ``"valid"``: Adopts the way of discarding. The possible largest depth, height and width of output
               will be returned without padding. Extra pixels will be discarded. If this mode is set, `pad`
               must be 0.
 
-            - pad: Implicit paddings on both sides of the input in depth, height and width. The number of `pad` will
-              be padded to the input Tensor borders. `pad` must be greater than or equal to 0.
+            - ``"pad"``: Implicit paddings on both sides of the input in depth, height and width.
+              The number of `pad` will be padded to the input Tensor borders. `pad` must be greater than or equal to 0.
 
         padding (Union[int, tuple[int], list[int]], optional): The pad value to be filled. If `pad` is an integer,
             the paddings of head, tail, top, bottom, left and right are the same, equal to pad.
@@ -5828,7 +6033,7 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
     Returns:
         Tensor, the value that applied 3D convolution. The shape is :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`.
 
-        `pad_mode` is 'same':
+        `pad_mode` is ``"same"``:
 
         .. math::
             \begin{array}{ll} \\
@@ -5837,7 +6042,7 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
                 W_{out} ＝ \left \lceil{\frac{W_{in}}{\text{stride[2]}}} \right \rceil \\
             \end{array}
 
-        `pad_mode` is 'valid':
+        `pad_mode` is ``"valid"``:
 
         .. math::
             \begin{array}{ll} \\
@@ -5849,7 +6054,7 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
                 {\text{stride[2]}} + 1} \right \rfloor \\
             \end{array}
 
-        `pad_mode` is 'pad':
+        `pad_mode` is ``"pad"``:
 
         .. math::
             \begin{array}{ll} \\
@@ -6082,7 +6287,7 @@ def glu(x, axis=-1):
         TypeError: If `x` is not a Tensor.
 
     Supported Platforms:
-        ``Ascend`` ``CPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> from mindspore import Tensor, ops
@@ -6128,12 +6333,12 @@ def multi_margin_loss(input, target, p=1, margin=1, weight=None, reduction='mean
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'mean'`` .
 
-            - ``'none'`` : no reduction will be applied.
-            - ``'mean'`` : the sum of the output will be divided by the number of elements in the output.
-            - ``'sum'`` : the output will be summed.
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor. If `reduction` is 'none', returns a Tensor with the same shape as `target`.
+        Tensor. If `reduction` is ``'none'``, returns a Tensor with the same shape as `target`.
         Otherwise, it is a scalar.
 
     Raises:
@@ -6200,13 +6405,14 @@ def multilabel_margin_loss(input, target, reduction='mean'):
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'mean'`` .
 
-            - ``'none'`` : no reduction will be applied.
-            - ``'mean'`` : the sum of the output will be divided by the number of elements in the output.
-            - ``'sum'`` : the output will be summed.
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        - **outputs** (Union[Tensor, Scalar]) - The loss of MultilabelMarginLoss. If `reduction` is "none", its shape
-          is :math:`(N)`. Otherwise, a scalar value will be returned.
+        - **outputs** (Union[Tensor, Scalar]) - The loss of MultilabelMarginLoss.
+          If `reduction` is ``"none"``, its shape is :math:`(N)`.
+          Otherwise, a scalar value will be returned.
 
     Raises:
         TypeError: If `input` or `target` is not a Tensor.
@@ -6214,7 +6420,7 @@ def multilabel_margin_loss(input, target, reduction='mean'):
         TypeError: If dtype of `target` is not int32.
         ValueError: If length of shape of `input` is neither 1 nor 2.
         ValueError: If shape of `input` is not the same as `target`.
-        ValueError: If `reduction` is not one of 'none', 'mean', 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
 
     Supported Platforms:
         ``Ascend`` ``GPU``
@@ -6260,12 +6466,15 @@ def multilabel_soft_margin_loss(input, target, weight=None, reduction='mean'):
         input (Tensor): A tensor of shape :math:`(N, C)` , where N is batch size and C is number of classes.
         target (Tensor): The label target Tensor which has the same shape as `input`.
         weight (Union[Tensor, int, float]): The manual rescaling weight given to each class. Default: ``None``.
-        reduction (str): Specifies which reduction to be applied to the output. It must be one of
-            ``'none'`` , ``'mean'`` , and ``'sum'`` , meaning no reduction, reduce mean and sum on output, respectively.
-            Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor, the data type is the same as input, if the reduction is 'none',
+        Tensor, the data type is the same as input, if the `reduction` is ``'none'``,
         its shape is :math:`(N)` , otherwise it is zero.
 
     Raises:
@@ -6409,15 +6618,15 @@ def gelu(input_x, approximate='none'):
 
     x_dtype = _get_cache_prim(P.DType)()(input_x)
     if x_dtype not in [mstype.float16, mstype.float32, mstype.float64]:
-        raise TypeError("For gelu, the input dtype must be float16, float32 or float64, "
-                        "but got {}.".format(x_dtype))
+        raise TypeError(f"For gelu, the input dtype must be float16, float32 or float64, "
+                        f"but got {x_dtype}.")
     if approximate == 'tanh':
         output = _get_cache_prim(P.GeLU)()(input_x)
     else:
-        output = _get_cache_prim(P.Sqrt)()(Tensor(2.0))
+        output = _get_cache_prim(P.Sqrt)()(Tensor(2.0, x_dtype))
         output = _get_cache_prim(P.Div)()(input_x, output)
-        output = _get_cache_prim(P.Erf)()(output) + Tensor(1.0)
-        output = input_x * output * Tensor(0.5)
+        output = _get_cache_prim(P.Erf)()(output) + Tensor(1.0, x_dtype)
+        output = input_x * output * Tensor(0.5, x_dtype)
 
     return output
 
@@ -6655,8 +6864,12 @@ def mse_loss(input, target, reduction='mean'):
         target (Tensor): The input label. Tensor of any dimension, same shape as the `input` in common cases.
             However, it supports that the shape of `input` is different from the shape of `target`
             and they should be broadcasted to each other.
-        reduction (str, optional): Type of reduction to be applied to loss.
-            The optional values are ``"mean"`` , ``"none"`` and ``"sum"`` . Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
         Tensor, loss of type float, the shape is zero if `reduction` is ``'mean'`` or ``'sum'`` ,
@@ -6759,11 +6972,15 @@ def triplet_margin_loss(anchor, positive, negative, margin=1.0, p=2, eps=1e-06,
         eps (float, optional): Add small value to avoid division by zero. Default: ``1e-06``.
         swap (bool, optional): The distance swap change the negative distance to the distance between positive
             sample and negative sample. Default: ``False`` .
-        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` , ``'sum'`` .
-            Default: ``'mean'`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor. If `reduction` is "none", its shape is :math:`(N)`. Otherwise, a scalar value will be returned.
+        Tensor. If `reduction` is ``"none"``, its shape is :math:`(N)`. Otherwise, a scalar value will be returned.
 
     Raises:
         TypeError: If `anchor` or `positive` or 'negative' is not a Tensor.
@@ -6776,7 +6993,7 @@ def triplet_margin_loss(anchor, positive, negative, margin=1.0, p=2, eps=1e-06,
             same time.
         ValueError: If the dimension of input `anchor` or `positive` or `negative` is bigger than or equal to 8.
         ValueError: If shape of `anchor`, `positive` and `negative` cannot broadcast.
-        ValueError: If `reduction` is not one of 'none', 'mean', 'sum'.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
 
     Supported Platforms:
         ``GPU``
@@ -6811,7 +7028,7 @@ def linear(x, w, b):
 def _inner_dropout(x, p, training):
     """inner dropout"""
     _dropout = _get_cache_prim(P.Dropout)(1 - p)
-    if p > 0. and training:
+    if 0. < p <= 1. and training:
         return _dropout(x)[0]
     return x
 
@@ -6864,10 +7081,11 @@ def _in_projection_packed(q, k, v, w, b, k_is_v, q_is_k):
     return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
 
 
-def _scaled_dot_product_attention(query, key, value, attn_mask, dropout_p, is_causal, is_training):
+def _scaled_dot_product_attention(query, key, value, attn_mask, dropout_p, is_causal, is_training, dtype):
     """scaled dot product attention"""
     embed_size = query.shape[-1]
-    scaling_factor = Tensor(embed_size, mstype.float32).sqrt().sqrt()
+    embed_size_tensor = scalar_to_tensor_(embed_size, dtype)
+    scaling_factor = embed_size_tensor.sqrt().sqrt()
     query = query / scaling_factor
 
     if is_causal:
@@ -6960,7 +7178,7 @@ def multi_head_attention_forward(query, key, value, embed_dim_to_check, num_head
                                  out_proj_bias, training=True, key_padding_mask=None, attn_mask=None,
                                  use_separate_proj_weight=False, q_proj_weight=None, k_proj_weight=None,
                                  v_proj_weight=None, static_k=None, static_v=None, average_attn_weights=True,
-                                 is_causal=False, k_is_v=False, q_is_k=False):
+                                 is_causal=False, k_is_v=False, q_is_k=False, dtype=mstype.float32):
     """multi head attetion forward function"""
     is_batched = _check_qkv_shape(query.ndim, key.ndim, value.ndim)
     if key_padding_mask is not None:
@@ -7117,7 +7335,7 @@ def multi_head_attention_forward(query, key, value, embed_dim_to_check, num_head
     v = v.view((bsz, num_heads, src_len, head_dim))
 
     attn_output, attn_output_weights = _scaled_dot_product_attention(
-        q, k, v, attn_mask, dropout_p, is_causal, training)
+        q, k, v, attn_mask, dropout_p, is_causal, training, dtype)
     attn_output = attn_output.transpose(2, 0, 1, 3).view((bsz * tgt_len, embed_dim))
 
     attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
@@ -7260,6 +7478,7 @@ __all__ = [
     'softsign',
     'softshrink',
     'soft_shrink',
+    'softplus',
     'selu',
     'silu',
     'soft_margin_loss',