mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.6.0rc1__cp311-cp311-win_amd64.whl

mindspore/ops/function/nn_func.py

@@ -26,10 +26,10 @@ from mindspore.ops import functional as F
 from mindspore.ops.operations import nn_ops as NN_OPS
 from mindspore.ops.operations import _sequence_ops as seq
 import mindspore.common.dtype as mstype
-from mindspore.ops.function.math_func import logsumexp
+from mindspore.ops.function.math_func import logsumexp, div
 from mindspore.ops.function.random_func import _get_seed, _set_prim_op_user_data
 from mindspore.common.tensor import Tensor
-from mindspore._c_expression import Tensor as Tensor_
+from mindspore._c_expression import TensorPy as Tensor_
 from mindspore.ops._primitive_cache import _get_cache_prim
 from mindspore import _checkparam as validator
 from mindspore.ops.composite.multitype_ops._constexpr_utils import raise_value_error
@@ -40,22 +40,72 @@ from mindspore.ops.operations.nn_ops import ChannelShuffle
 from mindspore.ops.operations.nn_ops import TripletMarginLoss
 from mindspore.ops.operations._sequence_ops import TupleToTensor, TensorToTuple, ListToTensor
 from mindspore.common.api import _function_forbid_reuse
-from mindspore.ops.auto_generate import log_softmax, dense, prelu, celu, relu, fast_gelu, silu, elu, sigmoid, relu6, \
-    softmax_impl, swiglu, logsigmoid_op
-from mindspore.ops.auto_generate import group_norm_op, rms_norm, layer_norm_ext_op, batch_norm_ext_op, mse_loss_ext
+from mindspore.ops.auto_generate import log_softmax, dense, prelu, celu, fast_gelu, silu, elu, sigmoid, relu6, \
+    softmax_impl, swiglu, logsigmoid_op, kl_div_op, divs_op
+from mindspore.ops.auto_generate import relu_op, inplace_relu_op
+from mindspore.ops.auto_generate import group_norm_op, rms_norm, add_rms_norm, layer_norm_ext_op, batch_norm_ext_op,\
+    mse_loss_ext
+# 1
 from mindspore.ops.auto_generate import (reflection_pad_1d_op, reflection_pad_2d_op, add_layernorm_v2_op,
                                          reflection_pad_3d_op,  # pylint: disable=W0611
                                          replication_pad_1d_op, replication_pad_2d_op, replication_pad_3d_op,
                                          constant_pad_nd_op, dropout_ext_op, reverse_v2_impl, avg_pool2d_op,
                                          upsample_nearest1d_op, upsample_nearest2d_op, upsample_nearest3d_op,
                                          upsample_linear1d_op, upsample_bilinear2d_op, upsample_bicubic2d_op,
-                                         upsample_trilinear3d_impl, fill_scalar_op, floor_op)
-from mindspore.ops.auto_generate.gen_ops_prim import embedding_op, Convolution, ConstantPadND, MaxPoolWithIndices, \
-    MaxPoolWithMask
+                                         upsample_trilinear3d_impl, fill_scalar_op, floor_op, nllloss_2d_op,
+                                         masked_fill_op, masked_select, ones, flatten_ext, conv_transpose2d)
+# 2
+
+# 3
+
+# 4
+
+# 5
+
+# 6
+
+# 7
+
+# 8
+
+# 9
+
+# 10
+
+# 11
+
+# 12
+
+# 13
+
+# 14
+
+# 15
+from mindspore.ops.auto_generate import avg_pool3d_ext_op
+# 16
+
+# 17
+
+# 18
+
+# 19
+
+# 20
+
+from mindspore.ops.auto_generate.gen_ops_prim import embedding_op, MaxPoolWithIndices, \
+    PromptFlashAttention, MaxPoolWithMask
+from mindspore.ops.auto_generate.gen_ops_prim import conv3d_ext_op, conv3d_padding_op, conv2d_ext_op, \
+    conv2d_padding_op, conv1d_ext_op, conv1d_padding_op, speed_fusion_attention_op
 from mindspore.common.generator import default_generator
 from mindspore.ops.auto_generate import hardshrink, hardsigmoid, hardswish
 from mindspore.ops.auto_generate import softshrink
+from mindspore.ops.auto_generate import soft_margin_loss
+from mindspore.ops.auto_generate import moe_token_permute, moe_token_unpermute
 from mindspore.ops.auto_generate import adaptive_avg_pool2d_ext_op
+from mindspore.ops.auto_generate.pyboost_inner_prim import nllloss_impl
+from mindspore.ops.auto_generate.pyboost_inner_prim import adaptive_max_pool2d_impl
+from mindspore.ops.function.array_func import gather_ext
+from mindspore.ops.operations.manually_defined import flash_attention_score, fused_infer_attention_score
 
 abs_ = P.Abs()
 add_ = P.Add()
@@ -111,7 +161,7 @@ check_int_const = validator.check_is_int
 check_non_negative_float_const = validator.check_non_negative_float
 check_string_const = constexpr(validator.check_string)
 
-generator_step_ = Tensor(1, mstype.int64)
+generator_step_ = Tensor(12, mstype.int64)
 
 
 def adaptive_avg_pool2d(input, output_size):
@@ -153,11 +203,11 @@ def adaptive_avg_pool2d(input, output_size):
     .. math::
 
         out\_shape = \begin{cases}
-        input\_shape[-2] + output\_size[1], & \text{if } output\_size text{ is (None, w);}\\
-        output\_size[0] + input\_shape[-1], & \text{if } output\_size text{ is (h, None);}\\
-        input\_shape[-2:], & \text{if } output\_size text{ is (None, None);}\\
-        (h, h), & \text{if } output\_size text{ is h;}\\
-        (h, w), & \text{if } output\_size text{ is (h, w)}
+        input\_shape[-2] + output\_size[1], & \text{if } output\_size \text{ is (None, w);}\\
+        output\_size[0] + input\_shape[-1], & \text{if } output\_size \text{ is (h, None);}\\
+        input\_shape[-2:], & \text{if } output\_size \text{ is (None, None);}\\
+        (h, h), & \text{if } output\_size \text{ is h;}\\
+        (h, w), & \text{if } output\_size \text{ is (h, w)}
         \end{cases}
 
     Raises:
@@ -247,11 +297,11 @@ def adaptive_avg_pool2d_ext(input, output_size):
     .. math::
 
         out\_shape = \begin{cases}
-        input\_shape[-2] + output\_size[1], & \text{if } output\_size text{ is (None, w);}\\
-        output\_size[0] + input\_shape[-1], & \text{if } output\_size text{ is (h, None);}\\
-        input\_shape[-2:], & \text{if } output\_size text{ is (None, None);}\\
-        (h, h), & \text{if } output\_size text{ is h;}\\
-        (h, w), & \text{if } output\_size text{ is (h, w)}
+        input\_shape[-2] + output\_size[1], & \text{if } output\_size \text{ is (None, w);}\\
+        output\_size[0] + input\_shape[-1], & \text{if } output\_size \text{ is (h, None);}\\
+        input\_shape[-2:], & \text{if } output\_size \text{ is (None, None);}\\
+        (h, h), & \text{if } output\_size \text{ is h;}\\
+        (h, w), & \text{if } output\_size \text{ is (h, w)}
         \end{cases}
 
     Raises:
@@ -399,13 +449,15 @@ def avg_pool1d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
 
     Args:
         input_x (Tensor): Tensor of shape :math:`(N, C_{in}, L_{in})`.
-        kernel_size (int): The size of kernel window used to take the average value. Default: ``1`` .
-        stride (Union(int, tuple[int])): The distance of kernel moving. `stride` can either be an int
+        kernel_size (int, optional): The size of kernel window used to take the average value. Default: ``1`` .
+        stride (Union(int, tuple[int]), optional): The distance of kernel moving. `stride` can either be an int
             number or a tuple of one int number. Default: ``1`` .
-        padding (Union(int, tuple[int])): The pad value to be filled. `padding` can either be an integer
+        padding (Union(int, tuple[int]), optional): The pad value to be filled. `padding` can either be an integer
             or a tuple of one integer. Default: ``0`` .
-        ceil_mode (bool): If True, apply ceil instead of floor to compute the output shape. Default: ``False``.
-        count_include_pad (bool): If True, include the zero-padding in the averaging calculation. Default: ``True`` .
+        ceil_mode (bool, optional): If True, apply ceil instead of floor to compute the output shape.
+            Default: ``False``.
+        count_include_pad (bool, optional): If True, include the zero-padding in the averaging calculation.
+            Default: ``True`` .
 
     Returns:
         Tensor of shape :math:`(N, C_{out}, L_{out})`.
@@ -620,13 +672,13 @@ def avg_pool2d_ext(input, kernel_size, stride=None, padding=0, ceil_mode=False,
         outputs regional average in the :math:`(H_{in}, W_{in})` -dimension.
         Given kernel size :math:`(kH, kW)` and `stride` , the operation is as follows.
 
-        .. note::
-            On the Atlas platform, when calculating the input, the precision is degraded from float32 to float16.
-
         .. math::
             \text{output}(N_i, C_j, h, w) = \frac{1}{kH * kW} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1}
             \text{input}(N_i, C_j, stride[0] \times h + m, stride[1] \times w + n)
 
+        .. note::
+            On the Atlas platform, when calculating the input, the precision is degraded from float32 to float16.
+
         Args:
             input (Tensor): Tensor of shape :math:`(N, C, H_{in}, W_{in})` or :math:`(C, H_{in}, W_{in})`.
             kernel_size (Union[int, tuple[int], list[int]]): The size of kernel used to take the average value.
@@ -779,6 +831,77 @@ def avg_pool3d(input_x, kernel_size=1, stride=1, padding=0, ceil_mode=False, cou
     return avg_pool_op(input_x)
 
 
+def avg_pool3d_ext(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True,
+                   divisor_override=None):
+    r"""
+    Applies a 3D average pooling over an input Tensor which can be regarded as a composition of
+    3D input planes. Typically the input is of shape :math:`(N, C, D_{in}, H_{in}, W_{in})` ,
+    outputs regional average in the :math:`(D_{in}, H_{in}, W_{in})` -dimension.
+    Given kernel size :math:`(kD, kH, kW)` and `stride` , the operation is as follows.
+
+    .. math::
+        \text{output}(N_i, C_j, d, h, w) = \frac{1}{kD * kH * kW} \sum_{l=0}^{kD-1} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1}
+
+        \text{input}(N_i, C_j, stride[0] \times d + l, stride[1] \times h + m, stride[2] \times w + n)
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Note:
+        This interface currently does not support Atlas A2 training series products.
+
+    Args:
+        input (Tensor): Tensor of shape :math:`(N, C, D_{in}, H_{in}, W_{in})` or :math:`(C, D_{in}, H_{in}, W_{in})`.
+        kernel_size (Union[int, tuple[int], list[int]]): The size of kernel used to take the average value.
+            Can be a single number or a tuple :math:`(kD, kH, kW)` .
+        stride (Union[int, tuple[int], list[int]], optional): The distance of kernel moving.
+            Can be a single number or a tuple :math:`(sD, sH, sW)` . Default: ``None``,
+            where its value is equal to `kernel_size`.
+        padding (Union[int, tuple[int], list[int]], optional): Implicit zero padding to be added on both sides.
+            Can be a single number or a tuple :math:`(padD, padH, padW)` . Default: ``0``.
+        ceil_mode (bool, optional): If True, apply ceil instead of floor to compute the output shape.
+            Default: ``False``.
+        count_include_pad (bool, optional): If True, include the zero-padding in the averaging calculation.
+            Default: ``True`` .
+        divisor_override (int, optional): If specified, it will be used as divisor in the averaging calculation,
+            otherwise size of pooling region will be used. Default: ``None``.
+
+    Returns:
+        Tensor, with shape :math:`(N, C, D_{out}, H_{out}, W_{out})` or :math:`(C, D_{out}, H_{out}, W_{out})`.
+
+        .. math::
+            \begin{array}{ll} \\
+                D_{out} = \frac{D_{in} + 2 \times padding[0] - kernel\_size[0]}{stride[0]} + 1 \\
+                H_{out} = \frac{H_{in} + 2 \times padding[1] - kernel\_size[0]}{stride[1]} + 1 \\
+                W_{out} = \frac{W_{in} + 2 \times padding[2] - kernel\_size[1]}{stride[2]} + 1
+            \end{array}
+
+    Raises:
+        TypeError: If `input` is not a Tensor.
+        TypeError: If `kernel_size` or `stride` is neither int nor tuple.
+        TypeError: If `ceil_mode` or `count_include_pad` is not a bool.
+        TypeError: If `divisor_override` is not an int or None.
+        ValueError: If the dimension of `input` is not equal to `4` or `5`.
+        ValueError: If `kernel_size` or `stride` is less than 1.
+        ValueError: If value of `padding` is less than `0`.
+        ValueError: If `kernel_size`, `padding` or `stride` is a tuple whose length is not equal to `1` or `3`.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> input_x = Tensor(np.arange(1 * 2 * 2 * 2 * 3).reshape((1, 2, 2, 2, 3)), mindspore.float16)
+        >>> output = ops.avg_pool3d_ext(input_x, kernel_size=2, stride=1)
+        >>> print(output)
+        [[[[[ 5.  6.]]]
+          [[[17. 18.]]]]]
+    """
+    return avg_pool3d_ext_op(input, kernel_size, stride, padding, ceil_mode, count_include_pad, divisor_override)
+
+
 @constexpr
 def is_ascend_backend():
     """Check if the Ascend is used"""
@@ -898,7 +1021,7 @@ def adaptive_max_pool2d(input, output_size, return_indices=False):
         \end{align}
 
     Note:
-        Ascend platform only supports float16 type for input.
+        In KBK mode, `output_size` does not support mutable.
 
     Args:
         input (Tensor): A 3D or 4D tensor,
@@ -907,7 +1030,7 @@ def adaptive_max_pool2d(input, output_size, return_indices=False):
             or an int H for :math:`(H, H)`. :math:`H` and :math:`W` can be int or None.
             If it is None, it means the output size is the same as the input size.
 
-        return_indices (bool): If `return_indices` is ``True`` , the indices of max value would be output.
+        return_indices (bool, optional): If `return_indices` is ``True`` , the indices of max value would be output.
             Default: ``False`` .
 
     Returns:
@@ -959,11 +1082,17 @@ def adaptive_max_pool2d(input, output_size, return_indices=False):
           [[8. 9.]]
           [[8. 9.]]]]
     """
+    output_size_ = None
     _check_adaptive_max_pool2d(return_indices)
-    _adaptive_max_pool2d = _get_cache_prim(NN_OPS.AdaptiveMaxPool2D)(output_size)
-    out = _adaptive_max_pool2d(input)
-    output = out if return_indices else out[0]
-    return output
+
+    if isinstance(output_size, int):
+        output_size_ = (output_size, output_size)
+    else:
+        output_size_ = tuple(-1 if val is None else val for val in output_size)
+
+    if return_indices:
+        return adaptive_max_pool2d_impl(input, output_size_)
+    return adaptive_max_pool2d_impl(input, output_size_)[0]
 
 
 def adaptive_max_pool3d(input, output_size, return_indices=False):
@@ -1438,7 +1567,7 @@ def dropout(input, p=0.5, training=True, seed=None):
         input (Tensor): The input Tensor of shape :math:`(*, N)`, with data type of float16, float32 or float64.
         p (float, optional): The dropping rate, between 0 and 1, e.g. p = 0.1,
             means dropping out 10% of input units. Default: ``0.5`` .
-        training (bool): Apply dropout if is True. Default: ``True``.
+        training (bool, optional): Apply dropout if is True. Default: ``True``.
         seed (int, optional): Seed is used as entropy source for Random number engines generating pseudo-random numbers.
             Default: ``None`` , which will be treated as ``0`` .
 
@@ -1473,7 +1602,7 @@ def dropout(input, p=0.5, training=True, seed=None):
 
 
 @_function_forbid_reuse
-def dropout_ext(input, p=0.5, training=True):
+def dropout_ext(input, p=0.5, training=True, inplace=False):
     r"""
     During training, randomly zeroes some of the elements of the input tensor
     with probability `p` from a Bernoulli distribution. It plays the role of reducing neuron correlation and
@@ -1482,10 +1611,12 @@ def dropout_ext(input, p=0.5, training=True):
 
     Args:
         input (Tensor): The input Tensor of shape :math:`(*, N)`.
-        p (float): The dropping rate of input neurons, between 0 and 1, e.g. `p` = 0.1,
+        p (float, optional): The dropping rate of input neurons, between 0 and 1, e.g. `p` = 0.1,
             means dropping out 10% of input neurons. Default: ``0.5`` .
-        training (bool): Apply dropout if it is ``True`` , if it is ``False`` , the input is returned directly,
-            and `p` is invalid. Default: ``True``.
+        training (bool, optional): Apply dropout if it is ``True`` ,
+            if it is ``False`` , the input is returned directly,
+            and `p` is invalid. Default: ``True`` .
+        inplace (bool, optional): If set to ``True`` , will do this operation in-place. Default: ``False`` .
 
     Returns:
         - **output** (Tensor) - Zeroed tensor, with the same shape and data type as `input`.
@@ -1506,10 +1637,14 @@ def dropout_ext(input, p=0.5, training=True):
         (2, 2)
     """
     check_bool_const(training, "training", "dropout_ext")
-    if training is False:
+    check_bool_const(inplace, "inplace", "dropout_ext")
+    if not training:
         return input
     seed, offset = default_generator._step(generator_step_)  # pylint: disable=protected-access
     out, _ = dropout_ext_op(input, p, seed, offset)
+    if inplace:
+        input.copy_(out)
+        return input
     return out
 
 
@@ -1610,7 +1745,7 @@ def dropout2d(input, p=0.5, training=True):
         input (Tensor): A `4D` tensor with shape :math:`(N, C, H, W)`, where `N` is the batch size, `C` is the number
             of channels, `H` is the feature height, and `W` is the feature width. The data type must be int8,
             int16, int32, int64, float16, float32 or float64.
-        p (float): The dropping probability of a channel, between 0 and 1, e.g. `p` = 0.8,
+        p (float): The dropping probability of a channel. The range is [0.0, 1.0], e.g. `p` = 0.8,
             which means dropping out 80% of channels. Default: ``0.5`` .
         training(bool): If `training` is True, applying dropout, otherwise, not applying. Default: ``True`` .
 
@@ -2022,6 +2157,75 @@ def kl_div(logits, labels, reduction='mean'):
     return _get_cache_prim(P.KLDivLoss)(reduction=reduction)(logits, labels)
 
 
+def kl_div_ext(input, target, reduction='mean', log_target=False):
+    r"""
+    Computes the Kullback-Leibler divergence between the `input` and the `target`.
+
+    For tensors of the same shape :math:`x` and :math:`y`,
+    the updating formulas of KLDivLoss algorithm are as follows,
+
+    .. math::
+        L(x, y) = y \cdot (\log y - x)
+
+    Then,
+
+    .. math::
+        \ell(x, y) = \begin{cases}
+        L(x, y), & \text{if reduction} = \text{'none';}\\
+        \operatorname{mean}(L(x, y)), & \text{if reduction} = \text{'mean';}\\
+        \operatorname{sum}(L(x, y)) / x.\operatorname{shape}[0], & \text{if reduction} = \text{'batchmean';}\\
+        \operatorname{sum}(L(x, y)),  & \text{if reduction} = \text{'sum'.}
+        \end{cases}
+
+    where :math:`x` represents `input`, :math:`y` represents `target`, and :math:`\ell(x, y)` represents the output.
+
+    Note:
+        The output aligns with the mathematical definition of Kullback-Leibler divergence
+        only when `reduction` is set to ``'batchmean'``.
+
+    Args:
+        input (Tensor): The input Tensor. The data type must be float16, float32 or bfloat16(only supported by Atlas A2
+            training series products).
+        target (Tensor): The target Tensor which has the same type as `input`. The shapes of `target` and `input`
+            should be broadcastable.
+        reduction (str, optional): Specifies the reduction to be applied to the output. Default: ``'mean'``.
+        log_target (bool, optional): Specifies whether `target` is passed in the log space. Default: ``False``.
+
+    Returns:
+        Tensor, has the same dtype as `input`. If `reduction` is ``'none'``, then output has the shape as broadcast
+        result of the `input` and `target`. Otherwise, it is a scalar Tensor.
+
+    Raises:
+        TypeError: If neither `input` nor `target` is a Tensor.
+        TypeError: If dtype of `input` or `target` is not float16, float32 or bfloat16.
+        TypeError: If dtype of `target` is not the same as `input`.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``, ``'batchmean'``.
+        ValueError: If shapes of `target` and `input` can not be broadcastable.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import ops
+        >>> import numpy as np
+        >>> input = ms.Tensor(np.array([[0.5, 0.5], [0.4, 0.6]]), ms.float32)
+        >>> target = ms.Tensor(np.array([[0., 1.], [1., 0.]]), ms.float32)
+        >>> output = ops.kl_div_ext(input, target, reduction='mean', log_target=False)
+        >>> print(output)
+        -0.225
+    """
+    if reduction == 'batchmean':
+        reduced = kl_div_op(input, target, 'sum', log_target)
+    else:
+        reduced = kl_div_op(input, target, reduction, log_target)
+
+    if reduction == 'batchmean' and input.ndim != 0:
+        reduced = divs_op(reduced, input.shape[0])
+
+    return reduced
+
+
 @constexpr
 def _check_axis_in_range(axis, ndim):
     """Checks axes are with the bounds of ndim"""
@@ -2081,33 +2285,22 @@ def _check_input_tensor(arg_name, *tensors):
 
 def flip(input, dims):
     """
-    Reverses the order of elements in a tensor along the given axis.
-
-    The shape of the tensor is preserved, but the elements are reordered.
+    Reverses elements in a tensor along the given dims.
 
     Args:
-        input (Tensor): Input tensor.
-        dims (Union[list[int], tuple[int]]): Axis or axes along which to flip over.
-            Flipping is performed on all of the axes specified in the tuple,
-            If `dims` is a tuple of integers contains negative, it counts from the last to the first axis.
+        input (Tensor): The input tensor.
+        dims (Union[list[int], tuple[int]]): The dimension to flip.
 
     Returns:
-        Tensor, with the entries of `dims` reversed.
-
-    Raises:
-        TypeError: If the input is not a tensor.
-        ValueError: If `dims` is None.
-        ValueError: If `dims` is not a list/tuple of ints.
+        Tensor
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
-        >>> from mindspore import ops
-        >>> import numpy as np
-        >>> input = mindspore.Tensor(np.arange(1, 9).reshape((2, 2, 2)))
-        >>> output = ops.flip(input, (0, 2))
+        >>> input = mindspore.tensor(mindspore.ops.arange(1, 9).reshape((2, 2, 2)))
+        >>> output = mindspore.ops.flip(input, (0, 2))
         >>> print(output)
         [[[6 5]
           [8 7]]
@@ -2120,26 +2313,21 @@ def flip(input, dims):
 
 def flipud(input):
     """
-    Flips the elements of each column in the up/down direction, while preserving the rows of the input tensor.
+    Flip the input tensor in up/down direction.
 
     Args:
-        input (Tensor): Input array.
+        input (Tensor): The input tensor, the dimension must be at least 2.
 
     Returns:
-        Tensor after the flip.
-
-    Raises:
-        TypeError: If the input is not a tensor.
+        Tensor
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore as ms
-        >>> from mindspore import ops
-        >>> import numpy as np
-        >>> input = ms.Tensor(np.arange(1, 9).reshape((2, 2, 2)))
-        >>> output = ops.flipud(input)
+        >>> import mindspore
+        >>> input = mindspore.tensor(mindspore.ops.arange(1, 9).reshape((2, 2, 2)))
+        >>> output = mindspore.ops.flipud(input)
         >>> print(output)
         [[[5 6]
           [7 8]]
@@ -2151,26 +2339,21 @@ def flipud(input):
 
 def fliplr(input):
     """
-    Flips the elements of each row in the left/right direction, while preserving the columns of the input tensor.
+    Flip the input tensor in left/right direction.
 
     Args:
-        input (Tensor): Input tensor.
+        input (Tensor): The input tensor, the dimension must be at least 2.
 
     Returns:
-        Tensor after the flip.
-
-    Raises:
-        TypeError: If the input is not a tensor.
+        Tensor
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore as ms
-        >>> from mindspore import ops
-        >>> import numpy as np
-        >>> input = ms.Tensor(np.arange(1, 9).reshape((2, 2, 2)))
-        >>> output = ops.fliplr(input)
+        >>> import mindspore
+        >>> input = mindspore.tensor(mindspore.ops.arange(1, 9).reshape((2, 2, 2)))
+        >>> output = mindspore.ops.fliplr(input)
         >>> print(output)
         [[[3 4]
           [1 2]]
@@ -2182,29 +2365,33 @@ def fliplr(input):
 
 def is_floating_point(input):
     """
-    Judge whether the data type of `input` is a floating point data type i.e., one of mindspore.float64,
-    mindspore.float32, mindspore.float16.
+    If the data type of the tensor is a floating point data type, return True. Otherwise return False.
 
     Args:
         input (Tensor): The input Tensor.
 
     Returns:
-        Bool. If the dtype of `input` is a floating point data type, return ``True`` . Otherwise, return ``False`` .
+        Bool
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore as ms
-        >>> from mindspore import ops
-        >>> from mindspore import Tensor
-        >>> x = ms.Tensor([1, 2, 3], ms.float32)
-        >>> y = ms.Tensor([1, 2, 3], ms.int64)
-        >>> output = ops.is_floating_point(x)
-        >>> output2 = ops.is_floating_point(y)
-        >>> print(output)
+        >>> import mindspore
+        >>> input = mindspore.tensor([False, 0j, 1, 2.1, 1+2j], mindspore.float64)
+        >>> mindspore.ops.is_floating_point(input)
         True
-        >>> print(output2)
+        >>>
+        >>> input = mindspore.tensor([False, 0j, 1, 2.1, 1+2j], mindspore.float32)
+        >>> mindspore.ops.is_floating_point(input)
+        True
+        >>>
+        >>> input = mindspore.tensor([False, 0j, 1, 2.1, 1+2j], mindspore.float16)
+        >>> mindspore.ops.is_floating_point(input)
+        True
+        >>>
+        >>> input = mindspore.tensor([False, 0j, 1, 2.1, 1+2j], mindspore.int32)
+        >>> mindspore.ops.is_floating_point(input)
         False
     """
     return input.dtype in [mstype.float32, mstype.bfloat16, mstype.float16, mstype.float64]
@@ -2326,12 +2513,20 @@ def interpolate(input,
             If scale_factor is a tuple or list, its length should be the same as the number of dimensions in input
             after removing the first two dimensions N, C.
             One and only one of size and scale_factor can be set to None. Default: ``None`` .
-        mode (str): The sampling algorithm.
-            One of 'nearest', 'linear' (3D only), 'bilinear' (4D only), 'trilinear' (5D only), 'bicubic' (4D only),
-            'area', 'nearest-exact'(matches Scikit-Image and PIL nearest neighbours interpolation algorithms and fixes
-            knows issues with `nearest`, 3D and 4D). Default: ``"nearest"`` .
-
-        align_corners (bool): Whether to use corner alignment for coordinate mapping. Assuming a transformation is
+        mode (str, optional): The sampling algorithm. Default: ``"nearest"`` .
+            One of the following sampling methods can be used:
+
+            - 'nearest': the nearest neighbours interpolation.
+            - 'linear': Linear interpolation, 3D only.
+            - 'bilinear': Bilinear interpolation, 4D only.
+            - 'trilinear': Trilinear interpolation, 5D only.
+            - 'bicubic': Double trilinear interpolation, 4D only.
+            - 'area': area interpolation.
+            - 'nearest-exact': matches Scikit-Image and PIL nearest neighbours interpolation algorithms and fixes
+              knows issues with `nearest`, for 3D and 4D.
+
+        align_corners (bool, optional): Whether to use corner alignment for coordinate mapping.
+            Assuming a transformation is
             applied to the input Tensor along the x-axis, the specific calculation formula is as follows:
 
             .. code-block::
@@ -2348,9 +2543,10 @@ def interpolate(input,
 
             This is only valid for ``'linear'``, ``'bilinear'``, or ``'bicubic'`` modes. Default: ``False`` .
         recompute_scale_factor (bool, optional): Recalculate `scale_factor`.
-            If True, the parameter `size` will be calculated using the value of the `scale_factor`,
-            and finally scaled using the value of `size`.
-            If False, the value of `size` or `scale_factor` will be used for direct interpolation. Default: ``None`` .
+
+            - If True, the parameter `size` will be calculated using the value of the `scale_factor`,
+              and finally scaled using the value of `size`.
+            - If False, the value of `size` or `scale_factor` will be used for direct interpolation. Default: ``None`` .
 
     .. note::
         The 'nearest-exact' mode is the same as the nearest-neighbor interpolation algorithm used in
@@ -2415,8 +2611,8 @@ def interpolate(input,
         >>> input = Tensor([[[1, 2, 3], [4, 5, 6]]], mindspore.float32)
         >>> output = ops.interpolate(input, size=(6,), mode='nearest')
         >>> print(output)
-            [[[1. 1. 2. 2. 3. 3.]
-              [4. 4. 5. 5. 6. 6.]]]
+        [[[1. 1. 2. 2. 3. 3.]
+          [4. 4. 5. 5. 6. 6.]]]
     """
 
     def run_nearest(x, size, align_corners=None, scale_factor=None):
@@ -2667,7 +2863,7 @@ def interpolate_ext(input,
     r"""
     Samples the input Tensor to the given size or scale_factor by using one of the interpolate algorithms.
 
-    .. warnings:
+    .. warning::
         This is an experimental API that is subject to change or deletion.
 
     .. note::
@@ -2675,7 +2871,7 @@ def interpolate_ext(input,
           is not supported.
         - In 'nearest' mode, there may exist precision problem in the scenarios, where input is 3-D/4-D Tensor
           and the image is scaled by scale_factor.
-        - `mode` and `scale_factor` should be constants.
+        - `mode` and `recompute_scale_factor` should be constants.
 
     Args:
         input (Tensor): Tensor to be resized.
@@ -2690,9 +2886,11 @@ def interpolate_ext(input,
             after removing the first two dimensions N, C.
             One and only one of size and scale_factor can be set to None. Default: ``None`` .
         mode (str): The sampling algorithm.
-            One of 'nearest', 'linear' (3D only), 'bilinear' (4D only), 'trilinear' (5D only), and 'bicubic' (4D only).
+            One of 'nearest', 'linear' (3D only),
+            'bilinear' (4D only), 'trilinear' (5D only), and 'bicubic' (4D only).
             Default: ``"nearest"`` .
-        align_corners (bool): Whether to use corner alignment for coordinate mapping. Assuming a transformation is
+        align_corners (bool, optional): Whether to use corner alignment for coordinate mapping.
+            Assuming a transformation is
             applied to the input Tensor along the x-axis, the specific calculation formula is as follows:
 
             .. code-block::
@@ -2707,7 +2905,7 @@ def interpolate_ext(input,
             the corresponding coordinate of the original
             data along the x-axis.
 
-            This is only valid for ``'linear'``, ``'bilinear'``, or ``'bicubic'`` modes. Default: ``False`` .
+            This is only valid for ``'linear'``, ``'bilinear'``, or ``'bicubic'`` modes. Default: ``None`` .
         recompute_scale_factor (bool, optional): Recalculate `scale_factor`.
             If True, the parameter `size` will be calculated using the value of the `scale_factor`,
             and finally scaled using the value of `size`.
@@ -2740,20 +2938,6 @@ def interpolate_ext(input,
     Returns:
         Tensor, sampled, whose dimensions and dtype are the same as `input`.
 
-    Shape:
-        - Input: :math:`(N, C, W_{in})`, :math:`(N, C, H_{in}, W_{in})` or :math:`(N, C, D_{in}, H_{in}, W_{in})`
-        - Output: :math:`(N, C, W_{out})`, :math:`(N, C, H_{out}, W_{out})`
-          or :math:`(N, C, D_{out}, H_{out}, W_{out})`, where
-
-    .. math::
-        D_{out} = \left\lfloor D_{in} \times \text{scale\_factor} \right\rfloor
-
-    .. math::
-        H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor
-
-    .. math::
-        W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor
-
     Raises:
         TypeError: `input` is not a Tensor.
         ValueError: Both `size` and `scale_factor` are not empty.
@@ -2771,12 +2955,12 @@ def interpolate_ext(input,
 
     Examples:
         >>> import mindspore
-        >>> from mindspore import Tensor, mint
+        >>> from mindspore import Tensor, ops
         >>> input = Tensor([[[1, 2, 3], [4, 5, 6]]], mindspore.float32)
-        >>> output = mint.interpolate(input, size=(6,), mode='nearest')
+        >>> output = ops.interpolate_ext(input, size=(6,), mode='nearest')
         >>> print(output)
-            [[[1. 1. 2. 2. 3. 3.]
-              [4. 4. 5. 5. 6. 6.]]]
+        [[[1. 1. 2. 2. 3. 3.]
+          [4. 4. 5. 5. 6. 6.]]]
     """
 
     def run_nearest(x, size, align_corners=None, scale_factor=None):
@@ -2914,58 +3098,6 @@ def softsign(x):
     return softsign_(x)
 
 
-def soft_margin_loss(input, target, reduction='mean'):
-    r"""
-    Calculate the soft margin loss of input and target.
-
-    Creates a criterion that optimizes a two-class classification
-    logistic loss between input tensor :math:`x` and target tensor :math:`y`
-    (containing 1 or -1).
-
-    .. math::
-        \text{loss}(x, y) = \sum_i \frac{\log(1 + \exp(-y[i]*x[i]))}{\text{x.nelement}()}
-
-    where :math:`x.nelement()` is the number of elements of :math:`x`.
-
-    .. warning::
-        This is an experimental API that is subject to change or deletion.
-
-    Args:
-        input (Tensor): Predict data. Data type must be float16 or float32.
-        target (Tensor): Ground truth data, with the same type and shape as `input`.
-        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
-            ``'sum'`` . Default: ``'mean'`` .
-
-            - ``'none'``: no reduction will be applied.
-            - ``'mean'``: compute and return the mean of elements in the output.
-            - ``'sum'``: the output elements will be summed.
-
-    Outputs:
-        Tensor or Scalar. If `reduction` is ``'none'``, its shape is the same as `input`.
-        Otherwise, a scalar value will be returned.
-
-    Raises:
-        TypeError: If `input` or `target` is not a Tensor.
-        TypeError: If dtype of `input` or `target` is neither float16 nor float32.
-        ValueError: If shape of `input` is not the same as that of `target`.
-        ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` or ``'sum'``.
-
-    Supported Platforms:
-        ``Ascend`` ``GPU``
-
-    Examples:
-        >>> import mindspore
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> logits = Tensor(np.array([[0.3, 0.7], [0.5, 0.5]]), mindspore.float32)
-        >>> labels = Tensor(np.array([[-1, 1], [1, -1]]), mindspore.float32)
-        >>> output = ops.soft_margin_loss(logits, labels)
-        >>> print(output)
-        0.6764238
-    """
-    soft_margin_loss_op = _get_cache_prim(P.SoftMarginLoss)(reduction=reduction)
-    output = soft_margin_loss_op(input, target)
-    return output
 
 
 def softmax(input, axis=-1, *, dtype=None):
@@ -3030,8 +3162,6 @@ def softmax_ext(input, dim=None, dtype=None):
         input (Tensor): Tensor of shape :math:`(N, *)`, where :math:`*` means, any number of
             additional dimensions.
         dim (int, optional): The dim to perform the Softmax operation. Default: ``None`` .
-
-    Keyword Args:
         dtype (:class:`mindspore.dtype`, optional): When set, `input` will be converted to the specified type,
             `dtype`, before execution, and dtype of returned Tensor will also be `dtype`. Default: ``None`` .
 
@@ -3264,7 +3394,7 @@ def selu(input_x):
 
 def logsigmoid(x):
     r"""
-    Applies logsigmoid activation element-wise. The input is a Tensor with any valid shape.
+    Applies LogSigmoid activation element-wise. The input is a Tensor with any valid shape.
 
     Logsigmoid is defined as:
 
@@ -3631,7 +3761,7 @@ def _replication_pad(input, pad):
     return out
 
 
-def pad_ext(input, pad, mode='constant', value=0.0):
+def pad_ext(input, pad, mode='constant', value=None):
     r"""
     Pads the input tensor according to the pad.
 
@@ -3679,7 +3809,7 @@ def pad_ext(input, pad, mode='constant', value=0.0):
 
         value (Union[int, float, None], optional): Valid only in ``'constant'`` mode.
             Set the padding value in ``'constant'`` mode. If the value is None, 0 is used as the default padding value.
-            Default: ``0.0`` .
+            Default: ``None`` .
 
     Returns:
         Tensor, the tensor after padding.
@@ -3689,7 +3819,7 @@ def pad_ext(input, pad, mode='constant', value=0.0):
         TypeError: If `input` is not a Tensor.
         ValueError: If length of `pad` is not even.
         ValueError: If length of `pad` is greater than 6.
-        ValueError: If `mode` is not ``'constant'`` and `value` not ``None``.
+        ValueError: If `mode` is not ``'constant'`` and `value` is neither ``None`` nor 0.
 
     Supported Platforms:
         ``Ascend``
@@ -3717,7 +3847,7 @@ def pad_ext(input, pad, mode='constant', value=0.0):
         value = 0 if value is None else value
         out = constant_pad_nd_op(input, pad, value)
     else:
-        if value != 0.0:
+        if value is not None and value != 0:
             raise ValueError(f"Padding mode {mode} doesn\'t take in value argument.")
         if mode == "circular":
             out = _circular_pad(input, pad)
@@ -3897,9 +4027,11 @@ def rrelu(input, lower=1.0 / 8, upper=1.0 / 3):
     `Empirical Evaluation of Rectified Activations in Convolution Network <https://arxiv.org/pdf/1505.00853.pdf>`_ .
 
     Args:
-        input  (Tensor): The input of rrelu is a Tensor of any dimension.
-        lower (Union[int, float]): Slope of the activation function at x < 0. Default: ``1.0 / 8`` .
-        upper (Union[int, float]): Slope of the activation function at x < 0. Default: ``1.0 / 3`` .
+        input (Tensor): The input of rrelu is a Tensor of any dimension.
+        lower (Union[int, float]): Slope of the activation function at data of `input` is less than 0.
+            Default: ``1.0 / 8`` .
+        upper (Union[int, float]): Slope of the activation function at data of `input` is less than 0.
+            Default: ``1.0 / 3`` .
 
     Returns:
         Tensor, after rrelu, has the same type and shape as the `input`.
@@ -4161,7 +4293,7 @@ def nll_loss(inputs, target, weight=None, ignore_index=-100, reduction='mean', l
     N is the batch size, :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of
     classes.
 
-    If `reduction` is not ``None`` (default ``'mean'``), then
+    If `reduction` is not ``'None'`` (default ``'mean'``), then
 
     .. math::
 
@@ -4271,67 +4403,364 @@ def _nll_loss(inputs, target, target_dim=-1, weight=None, ignore_index=None, red
     return loss
 
 
-def l1_loss(input, target, reduction='mean'):
+def nll_loss_ext(input, target, weight=None, ignore_index=-100, reduction='mean'):
     r"""
-    Calculate the mean absolute error between the `input` value and the `target` value.
-
-    Assuming that the :math:`x` and :math:`y` (predicted and target value) are 1-D Tensor,
-    length :math:`N`, `reduction` is set to ``'none'``, then calculate the loss of
-    :math:`x` and :math:`y` without dimensionality reduction.
+    Gets the negative log likelihood loss between input and target.
 
-    The formula is as follows:
+    The nll loss with reduction=none can be described as:
 
     .. math::
-        \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad \text{with } l_n = \left| x_n - y_n \right|,
 
-    where :math:`N` is the batch size.
+        \ell(x, t)=L=\left\{l_{1}, \ldots, l_{N}\right\}^{\top},
+        \quad l_{n}=-w_{t_{n}} x_{n, t_{n}},
+        \quad w_{c}=\text { weight }[c] \cdot \mathbb{1}
+        \{c \not= \text{ignore_index}\},
 
-    If `reduction` is ``'mean'`` or ``'sum'`` , then:
+    where :math:`x` is the input, :math:`t` is the target, :math:`w` is the weight,
+    :math:`N` is the batch size, :math:`c` belonging to :math:`[0, C-1]` is class index,
+    where :math:`C` is the number of classes.
+
+    If `reduction` is not ``'None'`` (default ``'mean'``), then
 
     .. math::
-        \ell(x, y) =
-        \begin{cases}
-            \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
-            \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
-        \end{cases}
+
+        \ell(x, t)=\left\{\begin{array}{ll}
+        \sum_{n=1}^{N} \frac{1}{\sum_{n=1}^{N} w_{t n}} l_{n}, & \text { if reduction }=\text { 'mean', } \\
+        \sum_{n=1}^{N} l_{n}, & \text { if reduction }=\text { 'sum' }
+        \end{array}\right.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
 
     Args:
-        input (Tensor): Predicted value, Tensor of any dimension.
-        target (Tensor): Target value, usually has the same shape as the `input`.
-            If `input` and `target` have different shape, make sure they can broadcast to each other.
+        input (Tensor): :math:`(N)` or :math:`(N, C)` where `C = number of classes` , `N = batch size` ,
+            or :math:`(N, C, d_1, d_2, ..., d_K)` (for high-dimensional data).
+            `input` is expected to be log-probabilities.
+            Data type only supports float32 or float16 or bfloat16(only supported by
+            Atlas A2 training series products).
+        target (Tensor): :math:`()` or :math:`(N)` ,
+          where the value range is :math:`[0, C-1]`, or :math:`(N, d_1, d_2, ..., d_K)` for
+          high-dimensional loss, data type must be int32 or int64 or uint8.
+        weight (Tensor, optional): A rescaling weight applied to the loss of each batch element.
+            If not None, the shape is :math:`(C,)`.
+            The data type must be float16 or float32 or bfloat16(only supported by Atlas A2 training series products).
+            It should have the same data type as `input` . Default: ``'None'`` .
+        ignore_index (int, optional): Specifies a target value that is ignored
+            and does not contribute to the input gradient. Default: ``-100`` .
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'mean'`` .
 
             - ``'none'``: no reduction will be applied.
-            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
             - ``'sum'``: the output elements will be summed.
 
     Returns:
-        Tensor or Scalar, if `reduction` is ``'none'``, return a Tensor with same shape and dtype as `input`.
-        Otherwise, a scalar value will be returned.
-
-    Raises:
-        TypeError: If `input` is not a Tensor.
-        TypeError: If `target` is not a Tensor.
-        ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` or ``'sum'``.
+        Tensor. The data type is the same as that of `input`.
 
     Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+        ``Ascend``
 
     Examples:
-        >>> from mindspore import Tensor, ops
-        >>> from mindspore import dtype as mstype
-        >>> x = Tensor([[1, 2, 3], [4, 5, 6]], mstype.float32)
-        >>> target = Tensor([[6, 5, 4], [3, 2, 1]], mstype.float32)
-        >>> output = ops.l1_loss(x, target, reduction="mean")
-        >>> print(output)
-        3.0
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, mint
+        >>> input = mindspore.Tensor(np.random.randn(3, 5), mindspore.float32)
+        >>> target = mindspore.Tensor(np.array([1, 0, 4]), mindspore.int32)
+        >>> output = mint.nn.functional.nll_loss(input, target)
+
     """
-    _check_is_tensor('input', input, "l1_loss")
-    _check_is_tensor('target', target, "l1_loss")
-    if reduction not in ('mean', 'sum', 'none'):
-        raise ValueError(f"For l1_loss, the 'reduction' must be in ['mean', 'sum', 'none'], but got {reduction}.")
-    loss = abs_(input - target)
+    return _nllloss_nd(input, target, weight, ignore_index, reduction)
+
+
+def _nllloss_nd(input, target, weight=None, ignore_index=-100, reduction='mean'):
+    """nllloss_nd inner function"""
+    input_dim = input.ndim
+    class_dim = 0 if input_dim == 1 else 1
+    n_classes = input.shape[class_dim]
+    if weight is None:
+        weight = ones(n_classes, input.dtype)
+    if input_dim < 1:
+        raise ValueError(f"input dim should be less than 1, but got {input_dim}")
+    if input_dim != 1 and input.shape[0] != target.shape[0]:
+        raise ValueError(f"input bacth_size should be equal to target batch_size, but got {input.shape[0]} and "
+                         f"{target.shape[0]}")
+    if input_dim == 1 or input_dim == 2:
+        return nllloss_impl(input, target, weight, reduction, ignore_index)[0]
+    if input_dim == 4:
+        return nllloss_2d_op(input, target, weight, reduction, ignore_index)[0]
+    # input_dim==3 or input_dim>4
+    n = input.shape[0]
+    c = input.shape[1]
+    out_size = (n,) + input.shape[2:]
+    if input.size > 0:
+        input = input.view((n, c, 1, -1))
+    else:
+        input = input.view((n, c, 0, 0))
+    if target.size > 0:
+        target = target.view((n, 1, -1))
+    else:
+        target = target.view((n, 0, 0))
+    if reduction != 'none':
+        return nllloss_2d_op(input, target, weight, reduction, ignore_index)[0]
+    ret = nllloss_2d_op(input, target, weight, reduction, ignore_index)[0]
+    return ret.view(out_size)
+
+
+def _cross_entropy_for_probabilities(input, target, weight, reduction, label_smoothing, class_dim, n_classes):
+    """cross_entropy inner function for class probabilities"""
+    if input.shape != target.shape:
+        raise ValueError("For cross_entropy that target is probabilities, input shape should equal to target shape.")
+    if label_smoothing > 0.0:
+        target = target * (1 - label_smoothing) + label_smoothing / n_classes
+    loss = input * target
+    if weight is not None:
+        weight_ = weight
+        ori_shape = loss.shape
+        if input.ndim > 2:
+            loss = loss.view(ori_shape[:2] + (-1,))
+            weight_ = weight_.view(1, -1, 1)
+        loss = loss * weight_
+        loss = loss.view(ori_shape)
+    if reduction == "mean":
+        return -div(loss.sum(), (input.size / n_classes))
+    if reduction == "sum":
+        return -loss.sum()
+    if reduction == "none":
+        return -loss.sum(class_dim)
+    raise ValueError(f"redution value {reduction} not valid.")
+
+
+def _cross_entropy_for_class_indices(input, target, weight, ignore_index, reduction, label_smoothing, class_dim,
+                                     n_classes):
+    """cross_entropy inner function for class indices"""
+    nllloss = _nllloss_nd(input, target, weight, ignore_index, reduction)
+    if label_smoothing > 0.0:
+        if weight is not None:
+            weight_ = weight
+            input_ = input
+            ori_shape = input.shape
+            if input.ndim > 2:
+                input_ = input.view(ori_shape[:2] + (-1,))
+                weight_ = weight_.view(1, -1, 1)
+            loss = input_ * weight_
+            loss = loss.view(ori_shape)
+            smooth_loss = -loss.sum(class_dim)
+        else:
+            smooth_loss = -input.sum(class_dim)
+        ignore_mask = ops.eq(target, ignore_index)
+        smooth_loss = masked_fill_op(smooth_loss, ignore_mask, 0)
+        if reduction == "mean":
+            true_mask = ~ignore_mask
+            if weight is not None:
+                weight_sum = gather_ext(weight, 0, flatten_ext(masked_select(target, true_mask))).sum()
+                if weight_sum == 0:
+                    ret = smooth_loss.sum()
+                else:
+                    ret = smooth_loss.sum() / weight_sum
+            else:
+                weight_sum = true_mask.sum()
+                if weight_sum == 0:
+                    ret = smooth_loss.sum()
+                else:
+                    ret = smooth_loss.sum() / weight_sum
+        elif reduction == "sum":
+            ret = smooth_loss.sum()
+        elif reduction == "none":
+            ret = smooth_loss
+        else:
+            raise ValueError(f"redution value {reduction} not valid.")
+        return (1 - label_smoothing) * nllloss + ret * (label_smoothing / n_classes)
+    return nllloss
+
+
+def cross_entropy_ext(input, target, weight=None, ignore_index=-100, reduction='mean', label_smoothing=0.0):
+    r"""
+    The cross entropy loss between input and target.
+
+    The cross entropy supports two kind of targets:
+
+    - Class indices (int) in the range :math:`[0, C)` where :math:`C` is the number of classes,
+      the loss with reduction=none can be described as:
+
+      .. math::
+
+          \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
+          l_n = - w_{y_n} \log \frac{\exp(x_{n,y_n})}{\sum_{c=1}^C \exp(x_{n,c})}
+          \cdot \mathbb{1}\{y_n \not= \text{ignore_index}\}
+
+      where :math:`x` is the inputs, :math:`y` is the target, :math:`w` is the weight, :math:`N` is the batch size,
+      :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of classes.
+
+      If `reduction` is not ``None`` (default ``'mean'`` ), then
+
+      .. math::
+
+          \ell(x, y) = \begin{cases}
+              \sum_{n=1}^N \frac{1}{\sum_{n=1}^N w_{y_n} \cdot \mathbb{1}\{y_n \not= \text{ignore_index}\}} l_n, &
+              \text{if reduction} = \text{'mean',}\\
+              \sum_{n=1}^N l_n,  &
+              \text{if reduction} = \text{'sum'.}
+              \end{cases}
+
+    - Probabilities (float) for each class, useful when labels beyond a single class per minibatch item
+      are required, the loss with reduction=none can be described as:
+
+      .. math::
+
+          \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
+          l_n = - \sum_{c=1}^C w_c \log \frac{\exp(x_{n,c})}{\sum_{i=1}^C \exp(x_{n,i})} y_{n,c}
+
+      where :math:`x` is the inputs, :math:`y` is the target, :math:`w` is the weight, N is the batch size,
+      :math:`c` belonging to :math:`[0, C-1]` is class index, where :math:`C` is the number of classes.
+
+      If `reduction` is not ``None`` (default ``'mean'`` ), then
+
+      .. math::
+
+          \ell(x, y) = \begin{cases}
+              \frac{\sum_{n=1}^N l_n}{N}, &
+              \text{if reduction} = \text{'mean',}\\
+              \sum_{n=1}^N l_n,  &
+              \text{if reduction} = \text{'sum'.}
+              \end{cases}
+
+    .. warning::
+            This is an experimental API that is subject to change or deletion.
+
+    Note:
+        Dynamic shape, dynamic rank and variable constant input are not supported in `strict graph mode
+        (jit_syntax_level=mindspore.STRICT)
+        <https://www.mindspore.cn/tutorials/en/master/compile/static_graph.html>`_.
+
+    Args:
+        input (Tensor): :math:`(N)` or :math:`(N, C)` where `C = number of classes` or :math:`(N, C, H, W)`
+            in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)`.
+            `input` is expected to be log-probabilities, data type must be float16 or float32 or bfloat16
+            (only supported by Atlas A2 training series products).
+        target (Tensor): For class indices, tensor of shape :math:`()`, :math:`(N)` or
+            :math:`(N, d_1, d_2, ..., d_K)` , data type must be int32 or int64. For probabilities, tensor of shape
+            :math:`(N,)` , :math:`(N, C)` or :math:`(N, C, d_1, d_2, ..., d_K)` , data type must be float16 or float32
+            or bfloat16(only supported by Atlas A2 training series products).
+        weight (Tensor, optional): A rescaling weight applied to the loss of each batch element.
+            If not None, the shape is :math:`(C,)`, data type must be float16 or float32 or bfloat16(only supported by
+            Atlas A2 training series products). Default: ``None`` .
+        ignore_index (int, optional): Specifies a target value that is ignored and does not contribute to the input
+            gradient. Only valid in class indices, please set it to a negative number in probabilities.
+            Default: ``-100`` .
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the weighted mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
+        label_smoothing (float, optional): Label smoothing values, a regularization tool used to prevent the model
+            from overfitting when calculating Loss. The value range is [0.0, 1.0]. Default: ``0.0`` .
+
+    Returns:
+        Tensor, the data type is the same as `input` .
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import ops, Tensor
+        >>> import numpy as np
+        >>> # Case 1: Indices labels
+        >>> inputs = Tensor(np.random.randn(3, 5), ms.float32)
+        >>> target = Tensor(np.array([1, 0, 4]), ms.int32)
+        >>> output = ops.cross_entropy_ext(inputs, target)
+        >>> # Case 2: Probability labels
+        >>> inputs = Tensor(np.random.randn(3, 5), ms.float32)
+        >>> target = Tensor(np.random.randn(3, 5), ms.float32)
+        >>> output = ops.cross_entropy_ext(inputs, target)
+    """
+    if not isinstance(input, Tensor) or not isinstance(target, Tensor):
+        raise TypeError(
+            f"For cross_entropy, input and target must be Tensor, but got input:{type(input)}, target:{type(target)}.")
+    if weight is not None and not isinstance(weight, Tensor):
+        raise TypeError(f"For cross_entropy, weight must be Tensor or None, but got {type(weight)}.")
+    if label_smoothing < 0.0 or label_smoothing > 1.0:
+        raise ValueError(f"For cross_entropy, label_smoothing must in [0, 1]")
+    if input.ndim == 0 or input.shape[0] == 0:
+        raise ValueError(f"For cross_entropy, input don't support 0-dim and shape[0].")
+    class_dim = 0 if input.ndim == 1 else 1
+    n_classes = input.shape[class_dim]
+    input = log_softmax_ext(input, class_dim, dtype=input.dtype)
+    # for probabilities
+    target_dtype = target.dtype
+    if isinstance(target_dtype, type(mstype.tensor_type)):
+        target_dtype = target_dtype.element_type()
+    if target_dtype in mstype.float_type:
+        return _cross_entropy_for_probabilities(input, target, weight, reduction, label_smoothing, class_dim,
+                                                n_classes)
+    # for class indices
+    return _cross_entropy_for_class_indices(input, target, weight, ignore_index, reduction, label_smoothing,
+                                            class_dim, n_classes)
+
+
+def l1_loss(input, target, reduction='mean'):
+    r"""
+    Calculate the mean absolute error between the `input` value and the `target` value.
+
+    Assuming that the :math:`x` and :math:`y` (predicted and target value) are 1-D Tensor,
+    length :math:`N`, `reduction` is set to ``'none'``, then calculate the loss of
+    :math:`x` and :math:`y` without dimensionality reduction.
+
+    The formula is as follows:
+
+    .. math::
+        \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad \text{with } l_n = \left| x_n - y_n \right|,
+
+    where :math:`N` is the batch size.
+
+    If `reduction` is set to ``'mean'`` or ``'sum'`` , then:
+
+    .. math::
+        \ell(x, y) =
+        \begin{cases}
+            \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
+            \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
+        \end{cases}
+
+    Args:
+        input (Tensor): Predicted value, Tensor of any dimension.
+        target (Tensor): Target value, usually has the same shape as the `input`.
+            If `input` and `target` have different shape, make sure they can broadcast to each other.
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
+    Returns:
+        Tensor or Scalar, if `reduction` is ``'none'``, return a Tensor with same shape and dtype as `input`.
+        Otherwise, a scalar value will be returned.
+
+    Raises:
+        TypeError: If `input` is not a Tensor.
+        TypeError: If `target` is not a Tensor.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` or ``'sum'``.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import Tensor, ops
+        >>> from mindspore import dtype as mstype
+        >>> x = Tensor([[1, 2, 3], [4, 5, 6]], mstype.float32)
+        >>> target = Tensor([[6, 5, 4], [3, 2, 1]], mstype.float32)
+        >>> output = ops.l1_loss(x, target, reduction="mean")
+        >>> print(output)
+        3.0
+    """
+    _check_is_tensor('input', input, "l1_loss")
+    _check_is_tensor('target', target, "l1_loss")
+    if reduction not in ('mean', 'sum', 'none'):
+        raise ValueError(f"For l1_loss, the 'reduction' must be in ['mean', 'sum', 'none'], but got {reduction}.")
+    loss = abs_(input - target)
     return _get_loss(loss, reduction, "l1_loss")
 
 
@@ -4348,8 +4777,8 @@ def smooth_l1_loss(input, target, beta=1.0, reduction='none'):
     .. math::
         L_{i} =
         \begin{cases}
-        \frac{0.5 (x_i - y_i)^{2}}{\beta}, & \text{if } |x_i - y_i| < \beta \\
-        |x_i - y_i| - 0.5 * \beta, & \text{otherwise. }
+        \frac{0.5 (x_i - y_i)^{2}}{\text{beta}}, & \text{if } |x_i - y_i| < \text{beta} \\
+        |x_i - y_i| - 0.5 * \text{beta}, & \text{otherwise. }
         \end{cases}
 
     If `reduction` is not `none`, then:
@@ -4364,12 +4793,26 @@ def smooth_l1_loss(input, target, beta=1.0, reduction='none'):
     Here :math:`\text{beta}` controls the point where the loss function changes from quadratic to linear.
     :math:`\text{beta}>0` , its default value is ``1.0`` . :math:`N` is the batch size.
 
+    .. warning::
+        This API has poor performance on CPU and it is recommended to run it on the Ascend/GPU.
+
     Args:
-        input (Tensor): Tensor of shape :math:`(N, *)` where :math:`*` means, any number of additional dimensions.
-            Data type is float16, float32 or float64.
-        target (Tensor): Ground truth data, tensor of shape :math:`(N, *)`, same shape and dtype as the `input`.
-        beta (float): A parameter used to control the point where the function will change between
-            L1 to L2 loss. The value should be greater than zero. Default: ``1.0`` .
+        input (Tensor): Tensor of shape :math:`(N, *)` where :math:`*` means,
+            any number of additional dimensions.Supported dtypes:
+
+            - Ascend: float16, float32, bfloat16.
+            - CPU/GPU: float16, float32, float64.
+        target (Tensor): Ground truth data, tensor of shape :math:`(N, *)`.
+
+            - CPU/Ascend: has the same shape as the `input`, `target` and `input`
+              comply with the implicit type conversion rules to make the data types consistent.
+            - GPU: has the same shape and dtype as the `input`.
+
+        beta (number, optional): A parameter used to control the point where the function will change between
+            L1 to L2 loss. Default: ``1.0`` .
+
+            - Ascend: The value should be equal to or greater than zero.
+            - CPU/GPU: The value should be greater than zero.
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'none'`` .
 
@@ -4379,14 +4822,15 @@ def smooth_l1_loss(input, target, beta=1.0, reduction='none'):
 
     Returns:
         Tensor, if `reduction` is ``'none'``, then output is a tensor with the same shape as `input`.
-        Otherwise, the shape of output tensor is :math:`(1,)`.
+        Otherwise, the shape of output tensor is :math:`()`.
 
     Raises:
-        TypeError: If `beta` is not a float.
-        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
-        TypeError: If dtype of `input` or `target` is not one of float16, float32, float64.
-        ValueError: If `beta` is less than or equal to 0.
+        TypeError: If input `input`, `target` is not Tensor.
+        RuntimeError: If dtype of `input` or `target` is not one of float16, float32, float64, bfloat16.
         ValueError: If shape of `input` is not the same as `target`.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
+        TypeError: If `beta` is not a float, int or bool.
+        RuntimeError: If `beta` is less than or equal to 0.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -4476,8 +4920,8 @@ def leaky_relu(input, alpha=0.2):
 
     Args:
         input (Tensor): The input of leaky_relu is a Tensor of any dimension.
-        alpha (Union[int, float]): Slope of the activation function when the element of `input` is less than 0.
-          Default: ``0.2`` .
+        alpha (Union[int, float], optional): Slope of the activation function when
+            the element of `input` is less than 0. Default: ``0.2`` .
 
     Returns:
         Tensor, has the same type and shape as the `input`.
@@ -4509,37 +4953,25 @@ def leaky_relu(input, alpha=0.2):
 
 def intopk(x1, x2, k):
     r"""
-    Determines whether the targets are in the top `k` predictions.
+    Return whether the elements in second input tensor exist among the top `k` elements of the first input tensor.
 
     Args:
-        x1 (Tensor): A 2D Tensor defines the predictions of a batch of samples with float16 or float32
-          data type.
-        x2 (Tensor): A 1D Tensor defines the labels of a batch of samples with int32 data type. The size of `x2`
-          must be equal to the first dimension of `x1`. The values of `x2` can not be negative and
-          must be equal to or less than index of x1's second dimension.
-        k (int): Specifies the number of top elements to be used for computing precision along the last dimension.
+        x1 (Tensor): The 2-D input tensor.
+        x2 (Tensor): The 1-D input tensor, should satisfy :math:`x2.shape[0] = x1.shape[0]` .
+        k (int): Top `k` elements.
 
     Returns:
-        Tensor has 1 dimension of type bool and the same shape with `x2`. For labeling sample `i` in `x2`,
-        if the label in the first `k` predictions for sample `i` is in `x1`, then the value is True, otherwise False.
-
-    Raises:
-        TypeError: If `k` is not an int.
-        TypeError: If `x1` or `x2` is not a Tensor.
-        TypeError: If dtype of `x1` is neither float16 nor float32.
+        A 1-D tensor whose data type is bool, has the same shape with `x2`.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> x1 = Tensor(np.array([[1, 8, 5, 2, 7], [4, 9, 1, 3, 5]]), mindspore.float32)
-        >>> x2 = Tensor(np.array([1, 3]), mindspore.int32)
-        >>> output = ops.intopk(x1, x2, 3)
-        >>> print(output)
-        [ True  False]
+        >>> x1 = mindspore.tensor([[1, 8, 5, 2, 7], [4, 9, 1, 3, 5]], mindspore.float32)
+        >>> x2 = mindspore.tensor([1, 3], mindspore.int32)
+        >>> mindspore.ops.intopk(x1, x2, 3)
+        Tensor(shape=[2], dtype=Bool, value= [ True, False])
     """
     _in_topk = _get_cache_prim(P.InTopK)(k)
     return _in_topk(x1, x2)
@@ -4961,7 +5393,7 @@ def grid_sample(input, grid, mode='bilinear', padding_mode='zeros', align_corner
             H_{in}, W_{in})` (5-D case) and dtype of float32 or float64.
         grid (Tensor): flow-field with shape of :math:`(N, H_{out}, W_{out}, 2)` (4-D case) or :math:`(N, D_{out},
             H_{out}, W_{out}, 3)` (5-D case) and same dtype as `input`.
-        mode (str): An optional string specifying the interpolation method. The optional values are
+        mode (str, optional): An optional string specifying the interpolation method. The optional values are
             ``'bilinear'``, ``'nearest'``. Default: ``'bilinear'`` . Note: `bicubic` is not supported yet. When
             `mode="bilinear"` and the input is 5-D, the interpolation mode used internally will actually
             be trilinear. However, when the input is 4-D, the interpolation mode will legistimately be bilinear.
@@ -4976,9 +5408,10 @@ def grid_sample(input, grid, mode='bilinear', padding_mode='zeros', align_corner
               It performs bilinear interpolation in the two spatial dimensions and linear interpolation along
               the third dimension. It is commonly used for volume or 3D image interpolation.
 
-        padding_mode (str): An optional string specifying the pad method. The optional values are "zeros", "border" or
+        padding_mode (str, optional): An optional string specifying the pad method.
+            The optional values are "zeros", "border" or
             "reflection". Default: ``'zeros'`` .
-        align_corners (bool): If set to `True`, the extrema (-1 and 1) are considered as referring to
+        align_corners (bool, optional): If set to `True`, the extrema (-1 and 1) are considered as referring to
             the center points of the input's corner pixels. If set to `False`, they are instead considered as referring
             to the corner points of the input's corner pixels, making the sampling more resolution agnostic. Default:
             ``False`` .
@@ -5389,12 +5822,15 @@ def conv3d_transpose(inputs, weight, pad_mode='valid', padding=0, stride=1, dila
     Args:
         inputs (Tensor): The gradients with respect to the output of the convolution.
            The shape conforms to the default.
-           data_format :math:`(N, C_{in}, D_{out}, H_{out}, W_{out})`. Currently dout data type only supports float16
-           and float32.
+           data_format :math:`(N, C_{in}, D_{out}, H_{out}, W_{out})`.
+           Supported dtypes:
+
+           - Ascend: float16.
+           - GPU/CPU: float16, float32.
         weight (Tensor): Set size of kernel is :math:`(K_d, K_h, K_w)`, then the shape is
            :math:`(C_{in}, C_{out}//group, K_d, K_h, K_w)`. Where :math:`group` is the Args parameter,
            :math:`//` is the symbol for integer division.
-           Currently weight data type only supports float16 and float32.
+           It has the same dtype as `dout`.
         pad_mode (str): Specifies padding mode. The optional values are
             "same", "valid", "pad". Default: "valid".
 
@@ -5538,9 +5974,9 @@ def conv1d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
 
     The shape of the convolutional kernel is given by :math:`(\text{kernel_size})`,
     where :math:`\text{kernel_size}` is the width of the kernel.
-    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
-    will be :math:`(C_{out}, C_{in} / \text{group}, \text{kernel_size})`,
-    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
+    If we consider the input and output channels as well as the `groups` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size})`,
+    where `groups` is the number of groups dividing `x`'s input channel when applying group convolution.
 
     For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
     <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_
@@ -5655,7 +6091,7 @@ def conv1d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
 
 def conv2d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dilation=1, groups=1):
     r"""
-    Applies a 2D convolution over an input tensor. The input tenor is typically of
+    Applies a 2D convolution over an input tensor. The input tensor is typically of
     shape :math:`(N, C_{in}, H_{in}, W_{in})`, where :math:`N` is batch size, :math:`C` is
     channel number, :math:`H` is feature height, :math:`W` is feature width.
 
@@ -5690,9 +6126,9 @@ def conv2d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
     The shape of the convolutional kernel is given by :math:`(\text{kernel_size[0]}, \text{kernel_size[1]})`,
     where :math:`\text{kernel_size[0]}` and :math:`\text{kernel_size[1]}` are the height and width of the kernel,
     respectively.
-    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
-    will be :math:`(C_{out}, C_{in} / \text{group}, \text{kernel_size[0]}, \text{kernel_size[1]})`,
-    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
+    If we consider the input and output channels as well as the `groups` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size[0]}, \text{kernel_size[1]})`,
+    where `groups` is the number of groups dividing `x`'s input channel when applying group convolution.
 
     For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
     <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_ and
@@ -5792,6 +6228,127 @@ def conv2d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
     return output
 
 
+def conv1d_ext(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1):
+    r"""
+    Applies a 1D convolution over an input tensor. The input tenor is typically
+    of shape :math:`(N, C_{in}, L_{in})`,
+    where :math:`N` is batch size, :math:`C` is channel number, :math:`L` is sequence length.
+
+    The output is calculated based on formula:
+
+    .. math::
+
+        \text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
+        \sum_{k = 0}^{C_{in} - 1} \text{ccor}({\text{weight}(C_{\text{out}_j}, k), \text{X}(N_i, k)})
+
+    where :math:`bias` is the output channel bias, :math:`ccor` is
+    the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_,
+    :math:`weight` is the convolution kernel value and :math:`X` represents the input feature map.
+
+    - :math:`i` corresponds to the batch number, the range is :math:`[0, N-1]`,
+      where :math:`N` is the batch size of the input.
+
+    - :math:`j` corresponds to the output channel, the range is :math:`[0, C_{out}-1]`,
+      where :math:`C_{out}` is the number of
+      output channels, which is also equal to the number of kernels.
+
+    - :math:`k` corresponds to the input channel, the range is :math:`[0, C_{in}-1]`,
+      where :math:`C_{in}` is the number of
+      input channels, which is also equal to the number of channels in the convolutional kernels.
+
+    Therefore, in the above formula, :math:`{bias}(C_{\text{out}_j})` represents the bias of the :math:`j`-th
+    output channel, :math:`{weight}(C_{\text{out}_j}, k)` represents the slice of the :math:`j`-th convolutional
+    kernel in the :math:`k`-th channel, and :math:`{X}(N_i, k)` represents the slice of the :math:`k`-th input
+    channel in the :math:`i`-th batch of the input feature map.
+
+    The shape of the convolutional kernel is given by :math:`(\text{kernel_size})`,
+    where :math:`\text{kernel_size}` is the length of the kernel.
+    If we consider the input and output channels as well as the `groups` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size})`,
+    where `groups` is the number of groups dividing `x`'s input channel when applying groups convolution.
+
+    For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
+    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        input (Tensor): Tensor of shape :math:`(N, C_{in}, L_{in})` or :math:`(C_{in}, L_{in})`.
+        weight (Tensor): Tensor of shape
+            :math:`(N, C_{in} / \text{groups}, \text{kernel_size})`, then the size of kernel
+            is :math:`(\text{kernel_size})`.
+        bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
+            When bias is ``None`` , zeros will be used. Default: ``None`` .
+        stride (Union[int, tuple[int], list[int]], optional): The movement stride of the 1D convolution kernel.
+            The data type is an integer or a tuple of one integer. Default: ``1`` .` .
+        padding (Union[int, tuple[int], list[int], str], optional): The number of padding
+            on the input.
+            The data type is an integer or a tuple of one integer or string {`valid`, `same`}.
+            The value should be greater than or equal to 0. Default: ``0`` .
+
+            - ``"same"``: Pad the input around its edges so that the shape of input and output
+              are the same when `stride` is set to ``1``.
+              The amount of padding to is calculated by the operator internally, If the amount is even, it is
+              uniformly distributed around the input, if it is odd, the excess amount goes to the right side.
+              If this mode is set, `stride` must be 1.
+
+            - ``"valid"``: No padding is applied to the input, and the output returns the maximum
+              possible length. Extra sequence that could not complete a full stride will
+              be discarded.
+
+        dilation (Union[int, tuple[int], list[int]], optional): Specifies the dilation rate to use for
+            dilated convolution. It can be a single int or a tuple of 1 integer.
+            Assuming :math:`dilation=(d)`, the convolutional kernel samples the input with a
+            spacing of :math:`d-1` elements in the length direction.
+            Default: ``1`` .
+        groups (int, optional): Splits filter into groups, `in_channels` and `out_channels` must be
+            divisible by `groups`. If the groups is equal to `in_channels` and `out_channels`,
+            this 1D convolution layer also can be called 1D depthwise convolution layer. Default: ``1`` .
+
+            - :math:`(C_{in} \text{ % } \text{groups} == 0)` , :math:`(C_{out} \text{ % } \text{groups} == 0)` ,
+              :math:`(C_{out} >= \text{groups})` , :math:`(\text{kernel_size[1]} = C_{in} / \text{groups})`。
+
+    Returns:
+        Tensor, the value that applied 1D convolution. The shape is :math:`(N, C_{out}, L_{out})`.
+        To see how different pad modes affect the output shape, please refer to
+        :class:`mindspore.mint.nn.Conv1d` for more details.
+
+    Raises:
+        ValueError: Args and size of the input feature map should satisfy the output formula to ensure that the size of
+            the output feature map is positive; otherwise, an error will be reported.
+        RuntimeError: On Ascend, due to the limitation of the L1 cache size of different NPU chip, if input size or
+            kernel size is too large, it may trigger an error.
+        TypeError: If `in_channels`, `out_channels` or `groups` is not an int.
+        TypeError: If `kernel_size`, `stride` or `dilation` is neither an int not a tuple.
+        ValueError: If `in_channels`, `out_channels`, `kernel_size`, `stride` or `dilation` is less than 1.
+        ValueError: If `padding` is less than 0.
+        ValueError: If `padding` is `same` , `stride` is not equal to 1.
+        ValueError: The input parameters do not satisfy the convolution output formula.
+        ValueError: The KernelSize cannot exceed the size of the input feature map.
+        ValueError: The value of padding cannot cause the calculation area to exceed the input size.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops, mint
+        >>> x = Tensor(np.ones([10, 32, 32]), mindspore.float32)
+        >>> weight = Tensor(np.ones([32, 32, 3]), mindspore.float32)
+        >>> output = mint.nn.functional.conv1d(x, weight)
+        >>> print(output.shape)
+        (10, 32, 30)
+    """
+    if isinstance(padding, (int, tuple, list)):
+        return conv1d_ext_op(input, weight, bias, stride, padding, dilation, groups)
+    if isinstance(padding, str):
+        return conv1d_padding_op(input, weight, bias, stride, padding, dilation, groups)
+    raise TypeError(f"For conv1d, the parameter 'padding' must be a tuple/list " \
+                    f"or a string, but got {type(padding)}")
+
+
 def _check_stride_when_same_mode(stride):
     """ stride must be 1 when pad mode is same """
     if isinstance(stride, int):
@@ -5840,9 +6397,9 @@ def _get_pad_nd_info(pad_l, pad_r):
 
 def conv2d_ext(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1):
     r"""
-    Applies a 2D convolution over an input tensor. The input tenor is typically of
-    shape :math:`(N, C_{in}, H_{in}, W_{in})`, where :math:`N` is batch size, :math:`C` is
-    channel number, :math:`H` is feature height, :math:`W` is feature width.
+    Applies a 2D convolution over an input tensor. The input tensor is typically of
+    shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`,
+    where :math:`N` is batch size, :math:`C` is channel number, :math:`H` is feature height, :math:`W` is feature width.
 
     The output is calculated based on formula:
 
@@ -5855,8 +6412,6 @@ def conv2d_ext(input, weight, bias=None, stride=1, padding=0, dilation=1, groups
     the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_,
     , :math:`weight` is the convolution kernel value and :math:`X` represents the input feature map.
 
-    Here are the indices' meanings:
-
     - :math:`i` corresponds to the batch number, the range is :math:`[0, N-1]`,
       where :math:`N` is the batch size of the input.
 
@@ -5883,55 +6438,67 @@ def conv2d_ext(input, weight, bias=None, stride=1, padding=0, dilation=1, groups
     <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_ and
     `ConvNets <http://cs231n.github.io/convolutional-networks/>`_.
 
-    Note:
-        On Ascend platform, only group convolution in depthwise convolution scenarios is supported.
-        That is, when `groups>1`, condition :math:`C_{in}` = :math:`C_{out}` = `groups` must be satisfied.
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
 
     Args:
-        input (Tensor): Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`.
+        input (Tensor): Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`.
         weight (Tensor): Tensor of shape
             :math:`(N, C_{in} / \text{groups}, \text{kernel_size[0]}, \text{kernel_size[1]})`, then the size of kernel
             is :math:`(\text{kernel_size[0]}, \text{kernel_size[1]})`.
         bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
             When bias is ``None`` , zeros will be used. Default: ``None`` .
-        stride (Union(int, tuple[int]), optional): The distance of kernel moving, an int number that represents
-            the height and width of movement are both strides, or a tuple of two int numbers that
+        stride (Union(int, tuple[int], list[int]), optional): The distance of kernel moving, an int number that
+            represents the height and width of movement are both strides, or a tuple of two int numbers that
             represent height and width of movement respectively. Default: ``1`` .
-        padding (Union(int, tuple[int], list[int], str), optional): Implicit paddings on both sides of the input `x`.
-            Can be a string, one integer or a tuple/list with 2 integers.
-            If `padding` is a string, the optional values are ``"same"`` , ``"valid"``.
-
-            - same: Adopts the way of completion. The height and width of the output will be equal to
-              the input `x` divided by stride. The padding will be evenly calculated in top and bottom,
-              left and right possiblily. Otherwise, the last extra padding will be calculated from the bottom
-              and the right side. If this mode is set, `padding` must be 0.
-
-            - valid: Adopts the way of discarding. The possible largest height and width of output will be returned
-              without padding. Extra pixels will be discarded. If this mode is set, `padding` must be 0.
-
-            If `padding` is one integer, the paddings of top, bottom, left and right are the same, equal to padding.
-            If `padding` is a tuple/list with 2 integers, the padding of top adn bottom is padding[0],
-            and the padding of left and right is padding[1]. Default: ``0`` .
-        dilation (Union(int, tuple[int]), optional): Gaps between kernel elements.The data type is int or a tuple of
-            2 integers. Specifies the dilation rate to use for dilated convolution. If set to be :math:`k > 1`,
+        padding (Union[int, tuple[int], list[int], str], optional): The number of padding
+            on the height and width directions of the input.
+            The data type is an integer or a tuple of two integers or string {`valid`, `same`}. If `padding` is an
+            integer, then `padding_{H}` and `padding_{W}` are all equal to `padding`.
+            If `padding` is a tuple of 2 integers, then `padding_{H}` and `padding_{W}`
+            is equal to `padding[0]` and `padding[1]` respectively.
+            The value should be greater than or equal to 0. Default: ``0`` .
+
+            - ``"same"``: Pad the input around its edges so that the shape of input and output
+              are the same when `stride` is set to ``1``.
+              The amount of padding to is calculated by the operator internally, If the amount is even, it is
+              uniformly distributed around the input, if it is odd, the excess amount goes to the right/bottom side.
+              If this mode is set, `stride` must be 1.
+
+            - ``"valid"``: No padding is applied to the input, and the output returns the maximum
+              possible height and width. Extra pixels that could not complete a full stride will
+              be discarded.
+
+        dilation (Union(int, tuple[int], list[int]), optional): Gaps between kernel elements.The data type
+            is int or a tuple of 2 integers. Specifies the dilation rate to use for dilated convolution.
+            If set to be :math:`k > 1`,
             there will be :math:`k - 1` pixels skipped for each sampling location. Its value must
             be greater than or equal to 1 and bounded by the height and width of the input `x`. Default: ``1`` .
         groups (int, optional): Splits `input` into groups. Default: ``1`` .
 
+            - :math:`(C_{in} \text{ % } \text{groups} == 0)` , :math:`(C_{out} \text{ % } \text{groups} == 0)` ,
+              :math:`(C_{out} >= \text{groups})` , :math:`(\text{kernel_size[1]} = C_{in} / \text{groups})`
+
     Returns:
         Tensor, the value that applied 2D convolution. The shape is :math:`(N, C_{out}, H_{out}, W_{out})`.
         To see how different pad modes affect the output shape, please refer to
-        :class:`mindspore.nn.Conv2d` for more details.
-
+        :class:`mindspore.mint.nn.Conv2d` for more details.
 
     Raises:
-        TypeError: If `stride`, `padding` or `dilation` is neither an int nor a tuple.
-        TypeError: `groups` is not an int.
+        ValueError: Args and size of the input feature map should satisfy the output formula to ensure that the size of
+            the output feature map is positive; otherwise, an error will be reported. For more details on the output
+            formula, please refer to :class:`mindspore.mint.nn.Conv2d`.
+        RuntimeError: On Ascend, due to the limitation of the L1 cache size of different NPU chip, if input size or
+            kernel size is too large, it may trigger an error.
+        TypeError: If `in_channels` , `out_channels` or `groups` is not an int.
+        TypeError: If `kernel_size` , `stride` or `dilation` is neither an int nor a tuple.
         TypeError: If `bias` is not a Tensor.
         ValueError: If  the shape of `bias` is not :math:`(C_{out})` .
         ValueError: If `stride` or `dilation` is less than 1.
-        ValueError: If `pad_mode` is not one of 'same', 'valid' or 'pad'.
-        ValueError: If `padding` is a tuple/list whose length is not equal to 2.
+        ValueError: If `padding` is `same` , `stride` is not equal to 1.
+        ValueError: The input parameters do not satisfy the convolution output formula.
+        ValueError: The KernelSize cannot exceed the size of the input feature map.
+        ValueError: The value of padding cannot cause the calculation area to exceed the input size.
 
     Supported Platforms:
         ``Ascend``
@@ -5939,123 +6506,21 @@ def conv2d_ext(input, weight, bias=None, stride=1, padding=0, dilation=1, groups
     Examples:
         >>> import mindspore
         >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> from mindspore.ops.function.nn_func import conv2d_ext
+        >>> from mindspore import Tensor, ops, mint
         >>> x = Tensor(np.ones([10, 32, 32, 32]), mindspore.float32)
         >>> weight = Tensor(np.ones([32, 32, 3, 3]), mindspore.float32)
-        >>> output = conv2d_ext(x, weight)
+        >>> output = mint.nn.functional.conv2d(x, weight)
         >>> print(output.shape)
         (10, 32, 30, 30)
     """
-
-    def _convolution_same(input, weight, bias, dilation, groups):
-        """ convolution when mode is 'same' """
-        if isinstance(dilation, int):
-            dilation = (dilation,) * 2
-        validator.check_int(len(weight.shape), 4, validator.EQ, "weight.shape", 'conv2d')
-        validator.check_int(len(dilation), 2, validator.EQ, "dilation", 'conv2d')
-
-        # Calc padding info
-        need_pad_nd, pad_l, pad_r = _get_pad_info(dilation, weight)
-        if not need_pad_nd:
-            conv = _get_cache_prim(Convolution)(stride, pad_l, dilation, False, (0, 0), groups)
-            return conv(input, weight, bias)
-
-        # Calc pad nd info
-        pad_nd, pad_l = _get_pad_nd_info(pad_l, pad_r)
-        pad_nd_op = _get_cache_prim(ConstantPadND)()
-        padded_input = pad_nd_op(input, pad_nd, 0)
-        conv = _get_cache_prim(Convolution)(stride, pad_l, dilation, False, (0, 0), groups)
-        return conv(padded_input, weight, bias)
-
-    if isinstance(padding, int):
-        padding = (padding,) * 2
-
-    if isinstance(padding, (tuple, list)):
-        conv = _get_cache_prim(Convolution)(stride, padding, dilation, False, (0, 0), groups)
-        return conv(input, weight, bias)
+    if isinstance(padding, (int, tuple, list)):
+        return conv2d_ext_op(input, weight, bias, stride, padding, dilation, groups)
     if isinstance(padding, str):
-        if padding == 'valid':
-            conv = _get_cache_prim(Convolution)(stride, (0, 0), dilation, False, (0, 0), groups)
-            return conv(input, weight, bias)
-        if padding == 'same':
-            _check_stride_when_same_mode(stride)
-            return _convolution_same(input, weight, bias, dilation, groups)
-        raise ValueError(f"For conv2d, the parameter 'padding' must be 'same' or 'valid' when " \
-                         f"the type of 'padding' is string.")
+        return conv2d_padding_op(input, weight, bias, stride, padding, dilation, groups)
     raise TypeError(f"For conv2d, the parameter 'padding' must be a tuple/list " \
                     f"or a string, but got {type(padding)}")
 
 
-def conv_transpose2d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1):
-    r"""
-    Calculates a 2D transposed convolution, which can be regarded as Conv2d for the gradient of the input,
-    also called deconvolution (although it is not an actual deconvolution).
-
-    The input is typically of shape :math:`(N, C_{in}, H_{in}, W_{in})`,
-    where :math:`N` is batch size, :math:`C_{in}` is space dimension,
-    :math:`H_{in}, W_{in}` are the height and width of the feature layer respectively.
-
-    When Conv2d and Conv2dTranspose are initialized with the same parameters, and `pad_mode` is set to 'pad',
-    :math:`dilation * (kernel\_size - 1) - padding` amount of zero will be paded to the height and width
-    directions of the input, they are inverses of each other in regard to the input and output shapes in this case.
-    However, when `stride` > 1, Conv2d maps multiple input shapes to the same output shape. Deconvolutional network
-    can refer to `Deconvolutional Networks <https://www.matthewzeiler.com/mattzeiler/deconvolutionalnetworks.pdf>`_.
-
-    Args:
-        input (Tensor): Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`.
-        weight (Tensor): Tensor of shape
-            :math:`(N, C_{in} / \text{groups}, \text{kernel_size[0]}, \text{kernel_size[1]})`, then the size of kernel
-            is :math:`(\text{kernel_size[0]}, \text{kernel_size[1]})`.
-        bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
-            When bias is ``None`` , zeros will be used. Default: ``None`` .
-        stride (Union(int, tuple[int]), optional): The distance of kernel moving, an int number that represents
-            the height and width of movement are both strides, or a tuple of two int numbers that
-            represent height and width of movement respectively. Default: ``1`` .
-        padding (Union(int, tuple[int], list[int]), optional): Implicit paddings on both sides of the input `x`.
-            Can be an integer or a tuple/list with 2 integers.
-        output_padding (Union[int, tuple[int]]): The number of padding on the height and width directions of the output.
-            The data type is an integer or a tuple of two integers. If `output_padding` is an integer,
-            then the bottom and right padding are all equal to `output_padding`. If `output_padding` is a tuple of
-            2 integers, then the bottom and right padding is equal to `output_padding[0]`, `output_padding[1]`
-            respectively.
-        groups (int, optional): Splits `input` into groups. Default: ``1`` .
-        dilation (Union(int, tuple[int]), optional): Gaps between kernel elements.The data type is int or a tuple of
-            2 integers. Specifies the dilation rate to use for dilated convolution. If set to be :math:`k > 1`,
-            there will be :math:`k - 1` pixels skipped for each sampling location. Its value must
-            be greater than or equal to 1 and bounded by the height and width of the input `x`. Default: ``1`` .
-
-    Returns:
-        Tensor, the value that applied 2D convolution. The shape is :math:`(N, C_{out}, H_{out}, W_{out})`.
-        To see how different pad modes affect the output shape, please refer to
-        :class:`mindspore.nn.Conv2dTranspose` for more details.
-
-
-    Raises:
-        TypeError: If `stride`, `padding` or `dilation` is neither an int nor a tuple.
-        TypeError: `groups` is not an int.
-        TypeError: If `bias` is not a Tensor.
-        ValueError: If  the shape of `bias` is not :math:`(C_{out})` .
-        ValueError: If `stride` or `dilation` is less than 1.
-        ValueError: If `padding` is a tuple/list whose length is not equal to 2.
-
-    Supported Platforms:
-        ``Ascend``
-
-    Examples:
-        >>> import mindspore
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> x = Tensor(np.ones([1, 6, 32, 32]), mindspore.float32)
-        >>> weight = Tensor(np.ones([6, 3, 5, 5]), mindspore.float32)
-        >>> output = ops.conv_transpose2d(x, weight)
-        >>> print(output.shape)
-        (1, 3, 36, 36)
-    """
-    conv = _get_cache_prim(Convolution)(stride, padding, dilation, True, output_padding, groups)
-    return conv(input, weight, bias)
-
-
 def hardtanh(input, min_val=-1.0, max_val=1.0):
     r"""
     Applies the hardtanh activation function element-wise. The activation function is defined as:
@@ -6742,10 +7207,10 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
     :math:`(\text{kernel_size[0]}, \text{kernel_size[1]}, \text{kernel_size[2]})`
     where :math:`\text{kernel_size[0]}` , :math:`\text{kernel_size[1]}` and :math:`\text{kernel_size[2]}` are the depth,
     height and width of the kernel, respectively.
-    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
-    will be :math:`(C_{out}, C_{in} / \text{group}, \text{kernel_size[0]},
+    If we consider the input and output channels as well as the `groups` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{groups}, \text{kernel_size[0]},
     \text{kernel_size[1]}, \text{kernel_size[2]})`,
-    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
+    where `groups` is the number of groups dividing `x`'s input channel when applying group convolution.
 
     For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
     <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_.
@@ -6817,69 +7282,211 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
                 {\text{stride[2]}} + 1} \right \rfloor \\
             \end{array}
 
-        `pad_mode` is ``"pad"``:
+        `pad_mode` is ``"pad"``:
+
+        .. math::
+            \begin{array}{ll} \\
+                D_{out} = \left \lfloor{\frac{D_{in} + padding[0] + padding[1] - (\text{dilation[0]} - 1) \times
+                \text{kernel_size[0]} - 1 }{\text{stride[0]}} + 1} \right \rfloor \\
+                H_{out} = \left \lfloor{\frac{H_{in} + padding[2] + padding[3] - (\text{dilation[1]} - 1) \times
+                \text{kernel_size[1]} - 1 }{\text{stride[1]}} + 1} \right \rfloor \\
+                W_{out} = \left \lfloor{\frac{W_{in} + padding[4] + padding[5] - (\text{dilation[2]} - 1) \times
+                \text{kernel_size[2]} - 1 }{\text{stride[2]}} + 1} \right \rfloor \\
+            \end{array}
+
+    Raises:
+        TypeError: If `out_channel` or `groups` is not an int.
+        TypeError: If `stride`, `padding` or `dilation` is neither an int nor a tuple.
+        TypeError: If `bias` is not a Tensor.
+        ValueError: If the shape of `bias` is not :math:`(C_{out})`.
+        ValueError: If `stride` or `dilation` is less than 1.
+        ValueError: If `pad_mode` is not one of 'same', 'valid' or 'pad'.
+        ValueError: If `padding` is a tuple or list whose length is not equal to 3.
+        ValueError: If `pad_mode` is not equal to 'pad' and `pad` is greater than 0.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> x = Tensor(np.ones([16, 3, 10, 32, 32]), mindspore.float16)
+        >>> weight = Tensor(np.ones([32, 3, 4, 3, 3]), mindspore.float16)
+        >>> output = ops.conv3d(x, weight, pad_mode="same", padding=0, stride=1, dilation=1, groups=1)
+        >>> print(output.shape)
+        (16, 32, 10, 32, 32)
+        >>> output = ops.conv3d(x, weight, pad_mode="valid", padding=0, stride=1, dilation=1, groups=1)
+        >>> print(output.shape)
+        (16, 32, 7, 30, 30)
+        >>> output = ops.conv3d(x, weight, pad_mode="pad", padding=(2, 1, 1), stride=1, dilation=1, groups=1)
+        >>> print(output.shape)
+        (16, 32, 11, 32, 32)
+    """
+    weight_shape = weight.shape
+    out_channel = weight_shape[0]
+    kernel_size = weight_shape[2:5]
+    if isinstance(stride, (tuple, list)):
+        _check_conv_iterable_lengths(stride, dim=3, iter_name='stride')
+    if isinstance(dilation, (tuple, list)):
+        _check_conv_iterable_lengths(dilation, dim=3, iter_name='dilation')
+    input_shape = input.shape
+    in_channel = input_shape[1]
+    if not (in_channel % groups == 0 and out_channel % groups == 0):
+        raise ValueError("The argument 'groups' should be divisible by 'in_channel' " \
+                         "and 'out_channel'")
+    if isinstance(padding, (list, tuple)):
+        padding = _manipulate_padding(padding, dim=3)
+    conv = _get_cache_prim(P.Conv3D)(out_channel, kernel_size, 1, pad_mode, padding, stride, dilation, groups, "NCDHW")
+    if bias is None:
+        return conv(input, weight)
+    if not isinstance(bias, Tensor):
+        raise TypeError(f"For 'conv3d', the 'bias' must be a Tensor, but got {type(bias)}.")
+    conv_result = conv(input, weight)
+    output = bias_add(conv_result, bias)
+    return output
+
+
+def conv3d_ext(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1):
+    r"""
+    Applies a 3D convolution over an input tensor. The input tensor is typically of
+    shape :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})` or :math:`(C_{in}, D_{in}, H_{in}, W_{in})`,
+    where :math:`N` is batch size, :math:`C` is channel number, :math:`D, H, W` are the depth,
+    height and width of the feature graph, respectively.
+
+    The output is calculated based on formula:
+
+    .. math::
+
+        \text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
+        \sum_{k = 0}^{C_{in} - 1} \text{ccor}({\text{weight}(C_{\text{out}_j}, k), \text{X}(N_i, k)})
+
+    where :math:`bias` is the output channel bias, :math:`ccor` is
+    the `cross-correlation <https://en.wikipedia.org/wiki/Cross-correlation>`_
+    , :math:`weight` is the convolution kernel value and :math:`X` represents the input feature map.
+
+    Here are the indices' meanings:
+
+    - :math:`i` corresponds to the batch number, the range is :math:`[0, N-1]`,
+      where :math:`N` is the batch size of the input.
+
+    - :math:`j` corresponds to the output channel, the range is :math:`[0, C_{out}-1]`,
+      where :math:`C_{out}` is the number of
+      output channels, which is also equal to the number of kernels.
+
+    - :math:`k` corresponds to the input channel, the range is :math:`[0, C_{in}-1]`,
+      where :math:`C_{in}` is the number of
+      input channels, which is also equal to the number of channels in the convolutional kernels.
+
+    Therefore, in the above formula, :math:`{bias}(C_{\text{out}_j})` represents the bias of the :math:`j`-th
+    output channel, :math:`{weight}(C_{\text{out}_j}, k)` represents the slice of the :math:`j`-th convolutional
+    kernel in the :math:`k`-th channel, and :math:`{X}(N_i, k)` represents the slice of the :math:`k`-th input
+    channel in the :math:`i`-th batch of the input feature map.
+
+    The shape of the convolutional kernel is given by :math:`(kd, kh, kw)` where :math:`kd` , :math:`kd` and\
+    :math:`kw` are the depth, height and width of the kernel, respectively.
+    If we consider the input and output channels as well as the `group` parameter, the complete kernel shape
+    will be :math:`(C_{out}, C_{in} / \text{group}, kd, kh, kw)`,
+    where `group` is the number of groups dividing `x`'s input channel when applying group convolution.
+
+    For more details about convolution layer, please refer to `Gradient Based Learning Applied to Document Recognition
+    <http://vision.stanford.edu/cs598_spring07/papers/Lecun98.pdf>`_.
+
+    The following lists some of the limitations of the parameters.
+
+    - input -- The input to the conv3d. The input must have each dimension size within the range [1, int32_max].
+    - weight -- Filters of shape :math:`(C_{out}, C_{in} / groups, kd, kh, kw)`. The value of :math:`kh`
+      and :math:`kw` is in the range [1, 511]. The remaining values are in the range [1, int32_max].
+      And :math:`kh*kw*k0` is less 65536 (k0 is 16. If data type is float32, k0 is 8).
+    - bias -- Bias Tensor with shape :math:`(C_{out})`. The shape must equal the first dimension of the weight.
+    - stride -- The distance of kernel moving. It can be an int number or
+      tuple (noted by :math:`(stride_d, stride_h, stride_w)`). stride_h and stride_w are in the range [1, 63].
+      stride_d is in the range [1, 255].
+    - padding -- If padding is an int number, it is in the range [0, 255].
+    - dilation -- The value is in the range [1, 255].
+    - groups -- The value is in the range [1, 65535].
+    - :math:`C_{in} \% \text{groups} == 0 \quad \text{and} \quad C_{out} \% \text{groups} == 0` .
+    - :math:`weight[1] == C_{in} / groups` .
+    - :math:`H_{in} + PadUp + PadDown >= (kh - 1) * DilationH + 1` .
+    - :math:`W_{in} + PadLeft + PadRight >= (kw - 1) * DilationW + 1` .
+    - :math:`D_{in} + PadFront + PadBack >= (kd - 1) * DilationD + 1` .
+    - :math:`H_{out} = (H_{in} + PadUp + PadDown - ((kh - 1) * DilationH + 1)) / StrideH + 1` .
+    - :math:`W_{out} = (W_{in} + PadLeft + PadRight - ((kw - 1) * DilationW + 1)) / StrideW + 1` .
+    - :math:`D_{out} = (D_{in} + PadFront + PadBack - ((kd - 1) * DilationD + 1)) / StrideD + 1` .
+    - :math:`(D_{in}+PadFront+PadBack - ((kd-1)*DilationD+1)) /% StrideD <= PadBack` .
+    - :math:`(H_{in}+PadUp+PadDown - ((kh-1)*Dilationh+1)) /% StrideH <= PadDown` .
+    - :math:`stride_d <= kernel_d` .
+    - :math:`PadUp < kh` and :math:`PadDown < kh` . When `padding` = ``'valid'``, both PadUp and PadDown are zeros.
+      When `padding` = ``'same'``, pad can be calculated by
+      :math:`floor(((H_{out}-1) * strideH + (kh - 1) * DilationH + 1 - H_{in}) / 2)` for high dimension.
+      It is similar way to calculate the padding for depth and width dimension. And the depth and width
+      dimensions also have the same constraints.
+    - :math:`((kh - 1) * DilationH - PadUp)` should be in [0, 255]. It is the same constraint for depth
+      and width dimension.
+    - If `padding` is ``'same'``, `stride` must be 1.
+
+    .. warning::
+        This API does not support Atlas series products.
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        input (Tensor): Tensor of shape :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`.
+        weight (Tensor): Set size of kernel is :math:`(kd, kh,
+            kw)`, then the shape is :math:`(C_{out}, C_{in} / groups, kd, kh, kw)`.
+        bias (Tensor, optional): Bias Tensor with shape :math:`(C_{out})`.
+            When bias is ``None`` , zeros will be used. Default: ``None`` .
+        stride (Union(int, tuple[int], list[int]), optional): The distance of kernel moving, an int
+            number that represents the depth, the height and width of movement are both strides, or a
+            tuple of triple int numbers that
+            represent the depth, height and width of movement respectively. Default: ``1`` .
+        padding (Union(int, tuple[int], list[int], str), optional): Implicit paddings on both sides of the input `x`.
+            Can be a string, one integer or a tuple/list with 3 integers.
+            If `padding` is a string, the optional values are ``"same"`` , ``"valid"``.
+
+            - same: Adopts the way of completion. The height and width of the output will be equal to
+              the input `x` divided by stride. The padding will be evenly calculated in top and bottom,
+              left and right possiblily. Otherwise, the last extra padding will be calculated from the bottom
+              and the right side. If this mode is set, `stride` must be 1.
+
+            - valid: Adopts the way of discarding. The possible largest height and width of output will be returned
+              without padding. Extra pixels will be discarded.
+
+            If `padding` is one integer, the paddings of top, bottom, left and right are the same, equal to padding.
+            If `padding` is a tuple/list with 3 integers, the padding of head, tail, top, bottom,
+            left and right equal to pad[0], pad[0], pad[1], pad[1], pad[2] and pad[2] correspondingly. Default: ``0`` .
+        dilation (Union[int, tuple[int], list[int]], optional): Controlling the space between the kernel points.
+            Default: ``1`` .
+        groups (int, optional): Splits `input` into groups. Default: ``1`` .
 
-        .. math::
-            \begin{array}{ll} \\
-                D_{out} = \left \lfloor{\frac{D_{in} + padding[0] + padding[1] - (\text{dilation[0]} - 1) \times
-                \text{kernel_size[0]} - 1 }{\text{stride[0]}} + 1} \right \rfloor \\
-                H_{out} = \left \lfloor{\frac{H_{in} + padding[2] + padding[3] - (\text{dilation[1]} - 1) \times
-                \text{kernel_size[1]} - 1 }{\text{stride[1]}} + 1} \right \rfloor \\
-                W_{out} = \left \lfloor{\frac{W_{in} + padding[4] + padding[5] - (\text{dilation[2]} - 1) \times
-                \text{kernel_size[2]} - 1 }{\text{stride[2]}} + 1} \right \rfloor \\
-            \end{array}
+    Returns:
+        Tensor, the same dtype as the `input`, with the shape :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`
+        or :math:`(C_{out}, D_{out}, H_{out}, W_{out})`.
 
     Raises:
-        TypeError: If `out_channel` or `groups` is not an int.
         TypeError: If `stride`, `padding` or `dilation` is neither an int nor a tuple.
+        TypeError: `groups` is not an int.
         TypeError: If `bias` is not a Tensor.
-        ValueError: If the shape of `bias` is not :math:`(C_{out})`.
-        ValueError: If `stride` or `dilation` is less than 1.
-        ValueError: If `pad_mode` is not one of 'same', 'valid' or 'pad'.
-        ValueError: If `padding` is a tuple or list whose length is not equal to 3.
-        ValueError: If `pad_mode` is not equal to 'pad' and `pad` is greater than 0.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``
+        ``Ascend``
 
     Examples:
         >>> import mindspore
         >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> x = Tensor(np.ones([16, 3, 10, 32, 32]), mindspore.float16)
-        >>> weight = Tensor(np.ones([32, 3, 4, 3, 3]), mindspore.float16)
-        >>> output = ops.conv3d(x, weight, pad_mode="same", padding=0, stride=1, dilation=1, groups=1)
-        >>> print(output.shape)
-        (16, 32, 10, 32, 32)
-        >>> output = ops.conv3d(x, weight, pad_mode="valid", padding=0, stride=1, dilation=1, groups=1)
-        >>> print(output.shape)
-        (16, 32, 7, 30, 30)
-        >>> output = ops.conv3d(x, weight, pad_mode="pad", padding=(2, 1, 1), stride=1, dilation=1, groups=1)
-        >>> print(output.shape)
-        (16, 32, 11, 32, 32)
+        >>> from mindspore import mint
+        >>> x = mindspore.Tensor(np.random.randn(12, 1, 60, 50, 8), mindspore.float16)
+        >>> w = mindspore.Tensor(np.random.randn(26, 1, 2, 4, 4), mindspore.float16)
+        >>> out = mint.nn.functional.conv3d(x, w)
+        >>> print(out.shape)
+        (12, 26, 59, 47, 5)
     """
-    weight_shape = weight.shape
-    out_channel = weight_shape[0]
-    kernel_size = weight_shape[2:5]
-    if isinstance(stride, (tuple, list)):
-        _check_conv_iterable_lengths(stride, dim=3, iter_name='stride')
-    if isinstance(dilation, (tuple, list)):
-        _check_conv_iterable_lengths(dilation, dim=3, iter_name='dilation')
-    input_shape = input.shape
-    in_channel = input_shape[1]
-    if not (in_channel % groups == 0 and out_channel % groups == 0):
-        raise ValueError("The argument 'groups' should be divisible by 'in_channel' " \
-                         "and 'out_channel'")
-    if isinstance(padding, (list, tuple)):
-        padding = _manipulate_padding(padding, dim=3)
-    conv = _get_cache_prim(P.Conv3D)(out_channel, kernel_size, 1, pad_mode, padding, stride, dilation, groups, "NCDHW")
-    if bias is None:
-        return conv(input, weight)
-    if not isinstance(bias, Tensor):
-        raise TypeError(f"For 'conv3d', the 'bias' must be a Tensor, but got {type(bias)}.")
-    conv_result = conv(input, weight)
-    output = bias_add(conv_result, bias)
-    return output
+
+    if isinstance(padding, (tuple, list, int)):
+        return conv3d_ext_op(input, weight, bias, stride, padding, dilation, groups)
+    if isinstance(padding, str):
+        return conv3d_padding_op(input, weight, bias, stride, padding, dilation, groups)
+    raise TypeError(f"For conv3d, the parameter 'padding' must be a tuple/list " \
+                    f"or a string, but got {type(padding)}")
 
 
 @_primexpr
@@ -7062,6 +7669,50 @@ def glu(x, axis=-1):
     return x * y
 
 
+def glu_ext(input, dim=-1):
+    r"""
+    Computes GLU (Gated Linear Unit activation function) of the input tensor.
+
+    .. math::
+        {GLU}(a, b)= a \otimes \sigma(b)
+
+    where :math:`a` is the first half of the `input` Tensor after `input` is split and :math:`b` is the second half.
+
+    Here :math:`\sigma` is the sigmoid function, and :math:`\otimes` is the Hadamard product.
+    See `Language Modeling with Gated Convluational Networks <https://arxiv.org/abs/1612.08083>`_.
+
+    Args:
+        input (Tensor): Tensor to be calculated. Dtype is floating point and the shape
+            is :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional dimensions. :math:`N`
+            is required to be an even number, where :math:`N` is the size of `input` on the dimension
+            selected by `dim`.
+        dim (int, optional): The dimension to split the input `input`. The value range is `[-r, r)` where `r`
+            is the number of dimensions of `input`. Default: ``-1`` , the last dimension in `input`.
+
+    Returns:
+        Tensor, the same dtype as the input `input`. The shape is :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`.
+
+    Raises:
+        TypeError: If `input` is not a Tensor or `dim` is not an int.
+        IndexError: If the value of `dim` is out of the range of `[-r, r)`, where `r` is the number
+            of dimensions of `input`.
+        RuntimeError: If dtype of `input` is not supported.
+        RuntimeError: If the length of `input` in the dimension selected by `dim` is not even.
+
+    Supported Platforms:
+        ``Ascend`` ``CPU``
+
+    Examples:
+        >>> from mindspore import Tensor, ops
+        >>> input = Tensor([[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]])
+        >>> output = ops.function.nn_func.glu_ext(input)
+        >>> print(output)
+        [[0.05744425 0.11973753]
+         [0.33409387 0.41398472]]
+    """
+    return _get_cache_prim(P.GLU)(axis=dim)(input)
+
+
 def multi_margin_loss(input, target, p=1, margin=1, weight=None, reduction='mean'):
     r"""
     Hinge loss for optimizing a multi-class classification.
@@ -7222,7 +7873,8 @@ def multilabel_soft_margin_loss(input, target, weight=None, reduction='mean'):
     Args:
         input (Tensor): A tensor of shape :math:`(N, C)` , where N is batch size and C is number of classes.
         target (Tensor): The label target Tensor which has the same shape as `input`.
-        weight (Union[Tensor, int, float]): The manual rescaling weight given to each class. Default: ``None``.
+        weight (Union[Tensor, int, float], optional): The manual rescaling weight given to each class.
+            Default: ``None``.
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'mean'`` .
 
@@ -7291,7 +7943,8 @@ def gelu(input, approximate='none'):
 
     Args:
         input (Tensor): The input of the activation function GeLU, the data type is float16, float32 or float64.
-        approximate (str): the gelu approximation algorithm to use. Acceptable vaslues are ``'none'`` and ``'tanh'`` .
+        approximate (str, optional): the gelu approximation algorithm to use.
+            Acceptable vaslues are ``'none'`` and ``'tanh'`` .
             Default: ``'none'`` .
 
     Returns:
@@ -7309,7 +7962,7 @@ def gelu(input, approximate='none'):
         >>> import mindspore
         >>> from mindspore import Tensor, ops
         >>> x = Tensor([1.0, 2.0, 3.0], mindspore.float32)
-        >>> result = ops.gelu(x)
+        >>> result = ops.gelu(x, approximate='none')
         >>> print(result)
         [0.8413447 1.9544997 2.9959505]
     """
@@ -7334,33 +7987,23 @@ def gelu(input, approximate='none'):
 def channel_shuffle(x, groups):
     r"""
     Divide the channels in a tensor of shape :math:`(*, C, H, W)` into :math:`g` groups and
-    rearrange them as :math:`(*, \frac{C}{g}, g, H*W)`, while keeping the original tensor shapes.
+    rearrange them as :math:`(*, \frac{C}{g}, g, H*W)`, while retaining the original tensor
+    shape in the final output.
 
     Args:
-        x (Tensor): Tensor to be divided, it has shape :math:`(*, C, H, W)`,
-          with float16, float32, int8, int16, int32, int64, uint8, uint16, uint32, uint64 data type.
+        x (Tensor): The input tensor.
         groups (int): Number of groups to divide channels in.
 
     Returns:
-        A Tensor, has the same type as the `x`, and has the shape :math:`(*, C, H, W)`.
-
-    Raises:
-        TypeError: If data type of `x` is not one of the following:
-                   float16, float32, int8, int16, int32, int64, uint8, uint16, uint32, uint64.
-        TypeError: If dim of `x` is < 4.
-        TypeError: If `groups` is not a positive number.
-        ValueError: If channel number of `x` is not divisible by `groups`.
+        Tensor
 
     Supported Platforms:
         ``Ascend`` ``CPU``
 
     Examples:
         >>> import mindspore
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> group = 2
-        >>> x = Tensor(np.arange(1* 4 * 2 * 2).reshape(1, 4, 2, 2).astype(np.int16))
-        >>> y = mindspore.ops.channel_shuffle(x, group)
+        >>> x = mindspore.tensor(mindspore.ops.arange(0, 16, dtype=mindspore.int16).reshape(1, 4, 2, 2))
+        >>> y = mindspore.ops.channel_shuffle(x, groups=2)
         >>> print(y)
         [[[[ 0  1]
            [ 2  3]]
@@ -7550,6 +8193,96 @@ def lp_pool2d(x, norm_type, kernel_size, stride=None, ceil_mode=False):
     return ((sign(out) * ops.relu(ops.abs(out))) * (kernel_size[0] * kernel_size[1])).pow(1.0 / norm_type)
 
 
+def relu(input, inplace=False):
+    r"""
+    Computes ReLU (Rectified Linear Unit activation function) of input tensors element-wise.
+
+    It returns :math:`\max(input,\  0)` element-wise. Specially, the neurons with the negative output
+    will be suppressed and the active neurons will stay the same.
+
+    .. math::
+
+        ReLU(input) = (input)^+ = \max(0, input)
+
+    ReLU Activation Function Graph:
+
+    .. image:: ../images/ReLU.png
+        :align: center
+
+    Args:
+        input (Tensor): The input Tensor.
+        inplace (bool, optional): Whether to use inplace mode, Defaults to ``False``.
+
+    Returns:
+        Tensor, with the same dtype and shape as the `input`.
+
+    Raises:
+        TypeError: If dtype of `input` is not Number type.
+        TypeError: If `input` is not a Tensor.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> input = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
+        >>> output = ops.relu(input)
+        >>> print(output)
+        [[0. 4. 0.]
+         [2. 0. 9.]]
+    """
+    if inplace:
+        return inplace_relu_op(input)
+    return relu_op(input)
+
+
+def relu_(input):
+    r"""
+    ReLuComputes ReLU (Rectified Linear Unit activation function) inplace of input tensors element-wise.
+
+    It returns :math:`\max(input,\  0)` element-wise. Specially, the neurons with the negative output
+    will be suppressed and the active neurons will stay the same.
+
+    .. math::
+
+        ReLU(input) = (input)^+ = \max(0, input)
+
+    ReLU Activation Function Graph:
+
+    .. image:: ../images/ReLU.png
+        :align: center
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        input (Tensor): The input Tensor.
+
+    Returns:
+        Tensor, with the same dtype and shape as the `input`.
+
+    Raises:
+        TypeError: If dtype of `input` is not Number type.
+        TypeError: If `input` is not a Tensor.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> input = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
+        >>> ops.relu_(input)
+        >>> print(input)
+        [[0. 4. 0.]
+         [2. 0. 9.]]
+    """
+    return inplace_relu_op(input)
+
+
 def mse_loss(input, target, reduction='mean'):
     r"""
     Calculates the mean squared error between the predicted value and the label value.
@@ -7623,35 +8356,36 @@ def mse_loss(input, target, reduction='mean'):
 
 def msort(input):
     r"""
-    Sorts the elements in Tensor in ascending order of value along its first dimension.
-
-    ops.msort(t) is equivalent to ops.Sort(axis=0)(t)[0]. See also :class:`mindspore.ops.Sort()`.
+    Return a tensor obtained by sorting the input tensor in ascending order along its first dimension.
 
-    .. Note::
-        The Ascend backend only supports sorting the 1D input.
+    `ops.msort(input)` is equivalent to `ops.sort(axis=0)(input)[0]`. See also :class:`mindspore.ops.Sort()` for more
+    details.
 
     Args:
-        input (Tensor): The input to sort, with float16 or float32 data type.
+        input (Tensor): The input tensor to sort.
 
     Returns:
-        A tensor whose values are the sorted values, with the same shape and data type as input.
-
-    Raises:
-        TypeError: If dtype of `input` is neither float16 nor float32.
+        Tensor
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> import mindspore as ms
-        >>> from mindspore import ops
-        >>> import numpy as np
-        >>> input = ms.Tensor(np.array([[8, 2, 1], [5, 9, 3], [4, 6, 7]]), ms.float16)
-        >>> output = ops.msort(input)
-        >>> print(output)
-        [[4. 2. 1.]
-         [5. 6. 3.]
-         [8. 9. 7.]]
+        >>> import mindspore
+        >>> input = mindspore.tensor([[8, 2, 1],
+        ...                           [5, 9, 3],
+        ...                           [4, 6, 7]])
+        >>> mindspore.ops.msort(input)
+        Tensor(shape=[3, 3], dtype=Int64, value=
+        [[4, 2, 1],
+         [5, 6, 3],
+         [8, 9, 7]])
+        >>> # is equivalent to `ops.sort(axis=0)(input)[0]`
+        >>> mindspore.ops.sort(input, axis=0)[0]
+        Tensor(shape=[3, 3], dtype=Int64, value=
+        [[4, 2, 1],
+         [5, 6, 3],
+         [8, 9, 7]])
     """
     return ops.Sort(axis=0)(input)[0]
 
@@ -7667,7 +8401,8 @@ def triplet_margin_loss(anchor, positive, negative, margin=1.0, p=2, eps=1e-06,
             as `anchor`.
         negative (Tensor): A sample belonging to the different class from `anchor`, with the same type and shape
             as `anchor`.
-        margin (float, optional): Make a margin between the positive pair and the negative pair. Default: ``1.0`` .
+        margin (float, optional): Make a margin between the positive pair and the negative pair. The shape of margin
+            must be 0. Default: ``1.0`` .
         p (int, optional): The degree of norm for pairwise distance. Default: ``2`` .
         eps (float, optional): Add small value to avoid division by zero. Default: ``1e-06``.
         swap (bool, optional): The distance swap change the negative distance to the distance between positive
@@ -8131,7 +8866,7 @@ def max_pool2d(x, kernel_size, stride=None, padding=0, dilation=1, return_indice
     return out
 
 
-def max_pool2d_ext(input, kernel_size, stride=None, padding=0, dilation=1, *, ceil_mode=False, return_indices=False):
+def max_pool2d_ext(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False):
     r"""
     Performs a 2D max pooling on the input Tensor.
 
@@ -8153,21 +8888,23 @@ def max_pool2d_ext(input, kernel_size, stride=None, padding=0, dilation=1, *, ce
         kernel_size (Union[int, tuple[int]]): The size of kernel used to take the maximum value and arg
             value, is an int number that represents height and width of the kernel, or a tuple of
             two int numbers that represent height and width respectively.
-        stride (Union[int, tuple[int], None]): The distance of kernel moving, an int number that represents
+        stride (Union[int, tuple[int], None], optional): The distance of kernel moving, an int number that represents
             the height and width of movement are both stride, or a tuple of two int numbers that
             represent height and width of movement respectively.
             Default: ``None`` , which indicates the moving step is `kernel_size` .
-        padding (Union[int, tuple[int]]): An int number that represents the height and width of movement are both
+        padding (Union[int, tuple[int]], optional):
+            An int number that represents the height and width of movement are both
             strides, or a tuple of two int numbers that represent height and width of movement respectively.
             Default: ``0`` .
-        dilation (Union[int, tuple[int]]): Control the stride of elements in the kernel. Default: ``1`` .
-        ceil_mode (bool): Whether to use ceil instead of floor to calculate output shape. Default: ``False`` .
-        return_indices (bool): Whether to output the indices of max value. Default: ``False`` .
+        dilation (Union[int, tuple[int]], optional): Control the stride of elements in the kernel. Default: ``1`` .
+        ceil_mode (bool, optional): Whether to use ceil instead of floor to calculate output shape. Default: ``False`` .
+        return_indices (bool, optional): Whether to output the indices of max value. Default: ``False`` .
 
     Returns:
         If `return_indices` is ``False`` , return a Tensor `output`, else return a tuple (`output`, `argmax`).
 
-        - **output** (Tensor) - Maxpooling result, with shape :math:`(N_{out}, C_{out}, H_{out}, W_{out})`.
+        - **output** (Tensor) - Maxpooling result,
+          with shape :math:`(N_{out}, C_{out}, H_{out}, W_{out})`.
           It has the same data type as `input`.
 
         .. math::
@@ -8198,10 +8935,9 @@ def max_pool2d_ext(input, kernel_size, stride=None, padding=0, dilation=1, *, ce
         >>> import mindspore
         >>> import numpy as np
         >>> from mindspore import Tensor, ops
-        >>> from mindspore.ops.function.nn_func import max_pool2d_ext
         >>> input = Tensor(np.arange(20 * 16 * 50 * 32).reshape((20, 16, 50, 32)), mindspore.float32)
-        >>> output_tensor, argmax = max_pool2d_ext(input, kernel_size=(3, 2), stride=(2, 1),
-                                                          ceil_mode=False, return_indices=True)
+        >>> output_tensor, argmax = ops.function.nn_func.max_pool2d_ext(input, kernel_size=(3, 2), stride=(2, 1),
+        ...                                                             ceil_mode=False, return_indices=True)
         >>> print(output_tensor.shape)
         (20, 16, 24, 31)
         >>> print(argmax.shape)
@@ -8219,62 +8955,199 @@ def max_pool2d_ext(input, kernel_size, stride=None, padding=0, dilation=1, *, ce
     return out
 
 
-def prompt_flash_attention(query, key, value, attn_mask, actual_seq_lengths, actual_seq_lengths_kv, pse_shift,
-                           deq_scale1, quant_scale1, deq_scale2, quant_scale2, quant_offset2, num_heads,
-                           scale_value=1.0, pre_tokens=2147483547, next_tokens=0, input_layout='BSH',
-                           num_key_value_heads=0, sparse_mode=0, inner_precise=1):
+def prompt_flash_attention(query, key, value, attn_mask=None, actual_seq_lengths=None, actual_seq_lengths_kv=None,
+                           pse_shift=None, deq_scale1=None, quant_scale1=None, deq_scale2=None, quant_scale2=None,
+                           quant_offset2=None, num_heads=1, scale_value=1.0, pre_tokens=2147483647, next_tokens=0,
+                           input_layout='BSH', num_key_value_heads=0, sparse_mode=0, inner_precise=1):
     r"""
     The interface for fully inference.
-    B -- Batch size
-    S -- Sequence length
-    H -- Hidden size
+
+    - B: Batch size
+    - N: Num of attention heads
+    - S: Sequence length
+    - D: Head dim
+    - H: Hidden layer size
+
+    Self attention constructs an attention model based on the relationship between input samples themselves. The
+    principle is to assume that there is an input sample sequence :math:`x` of length :math:`n`, and each
+    element of :math:`x` is a :math:`d` dimensional vector, which can be viewed as a token embedding. This sequence
+    can be transformed through 3 weight matrices to obtain 3 matrices with dimensions of :math:`n\times d`.
+
+    The self attention calculation formula is defined as:
+
+    .. math::
+        Attention(Q,K,V)=Softmax(\frac{QK^{T} }{\sqrt{d} } )V
+
+    where the product of :math:`Q` and :math:`K^{T}` represents the attention of input :math:`x`. To avoid the value
+    becoming too large, it is usually scaled by dividing it by the square root of :math:`d` and perform softmax
+    normalization on each row, yields a matrix of :math:`n\times d` after multiplying :math:`V`.
+
+    .. warning::
+        - Support dtype of float16 for `attn_mask` will be deprecated in the future.
+        - When `sparse_mode` is 2, 3 or 4, the shape of `attn_mask` must be :math:`(2048, 2048)` /
+          :math:`(B, 1, 2048, 2048)` / :math:`(1, 1, 2048, 2048)`.
 
     Note:
-    experiment ops
+        - Maximum Support for each axis
+
+          - Supports B-axis values less than or equal to 65536 (64k).
+            When the input type includes int8 with D-axis not aligned to 32, or the input type is
+            float16 or bfloat16 with D-axis not aligned to 16, the B-axis supports up to 128 only.
+          - Supports N-axis values less than or equal to 256.
+          - Supports S-axis values less than or equal to 20971520 (20M).
+          - Supports D-axis values less than or equal to 512.
+
+        - Quantization
+
+          - int8 Input, int8 Output: Parameters `deq_scale1`, `quant_scale1`, `deq_scale2`, and `quant_scale2`
+            must all be provided. `quant_offset2` is optional (default is 0 if not provided).
+          - int8 Input, float16 Output: Parameters `deq_scale1`, `quant_scale1`, and `deq_scale2` must all be provided.
+            If `quant_offset2` or `quant_scale2` is provided (i.e., not null), it will result in an error.
+          - float16 or bfloat16 Input, int8 Output: Parameter `quant_scale2` must be provided. `quant_offset2` is
+            optional (default is 0 if not provided). If `deq_scale1`, `quant_scale1`, or `deq_scale2` is
+            provided (i.e., not null), it will result in an error.
+          - int8 Output:
+
+            - `quant_scale2` and `quant_offset2` in per-channel format do not support scenarios with
+              left padding, Ring Attention, or non-32-byte aligned D-axis.
+            - In GE mode: `quant_scale2` and `quant_offset2` in per-tensor format do not support scenarios
+              with non-32-byte aligned D-axis.
+            - Does not support sparse as band and `pre_tokens`/`next_tokens` being negative.
+
+          - `quant_scale2` and `quant_offset2` can be bfloat16 only when `query` is bfloat16.
+
+
+        - Other Usage Caveats:
+
+          - :math:`N` of parameter `query` must be equal to `num_heads`. :math:`N` of parameter `key` and parameter
+            `value` must be equal to `num_key_value_heads`.
+          - `num_heads` must be divisible by `num_key_value_heads` and `num_heads` divided by `num_key_value_heads`
+            can not be greater than 64.
+          - When `query` dtype is bfloat16, D axis should align with 16.
+          - Each element of `actual_seq_lengths` must not exceed q_S and element
+            of `actual_seq_lengths_kv` must not exceed kv_S.
 
     .. warning::
-        This is an experimental API that is subject to change or deletion.
+        Only support on Atlas A2 training series.
+
+    Args:
+        query (Tensor): The query tensor with data type of int8, float16 or bfloat16.
+            The shape is :math:`(B, q_S, q_H)` / `(B, q_N, q_S, q_D)`.
+        key (Tensor): The key tensor with the same dtype as `query`.
+            The shape is :math:`(B, kv_S, kv_H)` / `(B, kv_N, kv_S, kv_D)`.
+        value (Tensor): The value tensor with the same dtype as `query`.
+            The shape is :math:`(B, kv_S, kv_H)` / `(B, kv_N, kv_S, kv_D)`.
+        attn_mask (Tensor, optional) - The attention mask tensor with data type of bool, int8, uint8 or float16.
+            For each element, 0/False indicates retention and 1/True indicates discard.
+            If `sparse_mode` is 0 or 1: the shape is :math:`(q_S, kv_S)` / :math:`(B, q_S, kv_S)` /
+            :math:`(1, q_S, kv_S)` / :math:`(B, 1, q_S, kv_S)` / :math:`(1, 1, q_S, kv_S)`.
+            If `sparse_mode` is 2, 3 or 4, the shape is :math:`(2048, 2048)` / :math:`(1, 2048, 2048)` /
+            :math:`(1, 1, 2048, 2048)`.
+            Default: ``None``.
+        actual_seq_lengths (Union[Tensor, tuple[int], list[int]], optional): Describe actual sequence length of each
+            batch of `query` with data type of int64. The shape is :math:`(B, )` and every element should be
+            positive integer.
+            Default: ``None``.
+        actual_seq_lengths_kv (Union[Tensor, tuple[int], list[int]], optional): Describe actual sequence length of each
+            batch of `key` or `value` with data type of int64. The shape is :math:`(B, )` and every element should be
+            positive integer.
+            Default: ``None``.
+        pse_shift (Tensor, optional): The position encoding tensor with data type of float16 or bfloat16.
+            Input tensor of shape :math:`(B, N, q_S, kv_S)` / :math:`(1, N, q_S, kv_S)`.
+            Default: ``None``.
 
-    Inputs:
-        query (Tensor) - The query tensor with data type of float16 or float32.
-          Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
-        key (Tensor) - The key tensor with data type of float16 or float32.
-          Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
-        value (Tensor) - The value tensor with data type of float16 or float32.
-          Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
-        attn_mask (Tensor) - The attention mask tensor with data type of float16 or float32.
-          For each element, 0 indicates retention and 1 indicates discard. Input tensor of shape :math:`(B, 1, S, S)`.
-        actual_seq_lengths (Tensor): Describe actual sequence length of each input with data type of int64.
-        actual_seq_lengths_kv (Tensor): Describe actual sequence length of each input with data type of int64.
-        pse_shift (Tensor) - The position encoding tensor with data type of float16 or float32.
-        dep_scale1 (Tensor)
-        quant_scale1 (Tensor)
-        deq_scale2 (Tensor)
-        quant_scale2 (Tensor)
-        quant_offset2 (Tensor)
-        num_heads (int): The number of heads.
-        scale_value (float): The scale value indicating the scale coefficient, which is used as the scalar of
-          Muls in the calculation. Default: 1.0.
-        pre_tokens (int): Previous tokens. Default: 2147483547.
-        next_tokens (int): next tokens.  Default: 0.
-          indicate the upper triangle, Indicate the number of data blocks involved in the calculation. The value 0
-          indicates that the data blocks in the upper triangle are not involved in the calculation
-        input_layout (str): the data layout of the input qkv, support `(BSH)` and `(BNSD)`, Default `BSH`.
-        num_key_value_heads (int): head numbers of key/value which are used in GQA algorithm.
-          The value o indicates if the key and value have the same head nums, use numHeads.  Default: 0.
-        sparse_mode (int): Default: 0
-        inner_precise (int): 0, float16 high precision. 1, high performance. default 1
+            - q_S must be greater than or equal to the query's S length, and kv_S must be greater than or
+              equal to the key's S length.'
+            - If `pse_shift` has dtype float16, `query` should have dtype float16 or int8, in which case high
+              precision mode is enabled automatically.
+            - If `pse_shift` has dtype bfloat16, `query` should have dtype bfloat16.
+
+        deq_scale1 (Tensor, optional): Quantitative parametor, the tensor with data type of uint64 or float32.
+            Input Tensor of shape :math:`(1,)`.
+            Default: ``None``.
+        quant_scale1 (Tensor, optional): Quantitative parametor, the tensor with data type of float32.
+            Input Tensor of shape :math:`(1,)`.
+            Default: ``None``.
+        deq_scale2 (Tensor, optional): Quantitative parametor, input Tensor of shape :math:`(1,)` and it has
+            the same dtype as `deq_scale1`.
+            Default: ``None``.
+        quant_scale2 (Tensor, optional): Quantitative parametor, the tensor with data type of float32 or bfloat16.
+            The suggested shape is :math:`(1,)` / :math:`(1, 1, q_H)` / :math:`(q_H, )` when output layout is BSH,
+            :math:`(1,)` / :math:`(1, q_N, 1, D)` / :math:`(q_N, D) when layout is BNSD.
+            Default: ``None``.
+        quant_offset2 (Tensor, optional): Quantitative parametor, the tensor with data type of float32 or bfloat16.
+            It has the same dtype and shape as `quant_scale2`.
+            Default: ``None``.
+        num_heads (int, optional): The number of heads. It is an integer in range [0, 256].
+            Default: ``1``.
+        scale_value (double, optional): The scale value indicating the scale coefficient, which is used as the scalar of
+            Muls in the calculation.
+            Default: ``1.0``.
+        pre_tokens (int, optional): For sparse cumputing, indicating the number of previous tokens Attention needs to
+            associated with.
+            Default: ``2147483647``.
+        next_tokens (int, optional): For sparse cumputing, indicating the number of next tokens Attention needs to
+            associated with.
+            Default: ``0``.
+        input_layout (str, optional): the data layout of the input qkv, support `(BSH)` and `(BNSD)`.
+            Default ``BSH``.
+        num_key_value_heads (int, optional): An int indicates head numbers of ``key``/``value`` which are used
+            in GQA algorithm. The value 0 indicates if the key and value have the same head nums, use `num_heads`.
+            It it is specified(not 0), it must be a factor of `num_heads` and it must be equal to kv_n.
+            Default: ``0``.
+        sparse_mode (int, optional): An int specifies sparse mode, can be int from {0, 1, 2, 3, 4}.
+            Default: ``0``.
+
+            - sparseMode = 0: If `attn_mask` is a null pointer, `pre_tokens` and `next_tokens` inputs are ignored
+              (internally set to INT_MAX).
+            - sparseMode = 2, 3, 4: `attn_mask` shape must be :math:`(S, S)` or :math:`(1, S, S)` or
+              :math:`(1, 1, S, S)`, with S fixed at 2048. User must ensure that `attn_mask` is lower triangular.
+              If not provided or incorrect shape, it will result in an error.
+            - sparseMode = 1, 2, 3: Ignores `pre_tokens`, `next_tokens` inputs and sets values according
+              to specific rules.
+            - sparseMode = 4: `pre_tokens` and `next_tokens` must be non-negative.
+
+        inner_precise (int, optional): An int number from {0, 1} indicates computing mode.
+            ``0`` for high precision mode for float16 dtype. ``1`` for high performance mode.
+            Default: ``1``.
 
+    Returns:
+        attention_out (Tensor) - Output tensor, has the same shape as `query` of
+        :math:`(B, q_S, q_H)` / :math:`(B, q_N, q_S, q_D)`.
+        Output dtype is determined by multiple factors, please refer to Note above for details.
 
-    Outputs:
-        attention_out (Tensor) - Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
+    Raises:
+        TypeError: Dtype of `query` is not int8, float16 or bfloat16.
+        TypeError: `query`, `key` and `value` don't have the same dtype.
+        TypeError: Dtype of `attn_mask` is not bool, int8 or uint8.
+        TypeError: Dtype of `pse_shift` is not bfloat16 or float16.
+        TypeError: `scale_value` is not a double number.
+        TypeError: `input_layout` is not a string.
+        TypeError: `num_key_value_heads` is not an int.
+        TypeError: `sparse_mode` is not an int.
+        TypeError: `sparse_inner_precisemode` is not an int.
+        TypeError: `quant_scale1` is not Tensor of type float32.
+        TypeError: `deq_scale1` is not Tensor of type uint64 or float32.
+        TypeError: `quant_scale2` is not Tensor of type float32.
+        TypeError: `deq_scale2` is not Tensor of type uint64 or float32.
+        TypeError: `quant_offset2` is not Tensor of type float32.
+        ValueError: `input_layout` is a string but of `(BSH)` or `(BNSD)`.
+        RuntimeError: `num_heads` is not divisible by `num_key_value_heads`.
+        RuntimeError: `num_heads` is not greater than 0.
+        RuntimeError: `num_key_value_heads` is not greater than or equal to 0.
+        RuntimeError: kv_n is not equal to `num_key_value_heads`.
+        RuntimeError: `attn_mask` shape is not valid.
+        RuntimeError: `sparse_mode` is specified but is not 0, 1, 2, 3 or 4.
+        RuntimeError: `query` dtype is bfloat16 and D axis is not aligned with 16.
+        RuntimeError: `input_layout` is BSH and kv_h is not divisible by `num_key_value_heads`.
+        RuntimeError: D-axis of `query`, `key` and `value` is not the same.
+        RuntimeError: In post quant per-channel scenario, D-axis is not 32 Byte aligned.
 
-        Supported Platforms:
+    Supported Platforms:
         ``Ascend``
 
     Examples:
-        >>> from mindspore.ops.function.nn_func import prompt_flash_attention
-        >>> from mindspore import Tensor
+        >>> from mindspore import Tensor, ops
         >>> import numpy as np
         >>> B = 1
         >>> N = 16
@@ -8284,13 +9157,13 @@ def prompt_flash_attention(query, key, value, attn_mask, actual_seq_lengths, act
         >>> key = Tensor(np.ones((B, N, S, D), dtype=np.float16))
         >>> value = Tensor(np.ones((B, N, S, D), dtype=np.float16))
         >>> out = ops.prompt_flash_attention(query, key, value, None, None, None, None, None, None, None, None,
-                                             None, N, input_layout='BNSD')
+        ...                                  None, N, input_layout='BNSD')
         >>> print(out.shape)
         (1, 16, 256, 16)
     """
 
-    pfa = _get_cache_prim(NN_OPS.PromptFlashAttention)(num_heads, scale_value, pre_tokens, next_tokens, input_layout,
-                                                       num_key_value_heads, sparse_mode, inner_precise)
+    pfa = _get_cache_prim(PromptFlashAttention)(num_heads, scale_value, pre_tokens, next_tokens, input_layout,
+                                                num_key_value_heads, sparse_mode, inner_precise)
     return pfa(query, key, value, attn_mask, actual_seq_lengths, actual_seq_lengths_kv, pse_shift, deq_scale1,
                quant_scale1, deq_scale2, quant_scale2, quant_offset2)
 
@@ -8301,21 +9174,17 @@ def incre_flash_attention(query, key, value, attn_mask=None, actual_seq_lengths=
                           num_heads=1, input_layout='BSH', scale_value=1.0, num_key_value_heads=0,
                           block_size=0, inner_precise=1, kv_padding_size=None):
     r"""
-    B -- Batch size
-
-    N -- Num heads
-
-    kvN -- Num key value heads
-
-    S -- Sequence length
+    The interface for incremental inference.
 
-    D -- Head dim
+    - B: Batch size
+    - N: Num of attention heads
+    - kvN: Num of `key` / `value` heads
+    - S: Sequence length
+    - D: Head dim
+    - H: Hidden layer size
+    - kvH: Hidden size of `key` / `value`
 
-    H -- Hidden size
-
-    kvH -- Hidden size of key value
-
-    where :math:`H=N\times D`, :math:`kvH=kvN\times D`
+    where :math:`H=N\times D`, :math:`kvH=kvN\times D`.
 
     Self attention constructs an attention model based on the relationship between input samples themselves. The
     principle is to assume that there is a length of the input sample sequence :math:`x` of :math:`n`, and each
@@ -8330,62 +9199,62 @@ def incre_flash_attention(query, key, value, attn_mask=None, actual_seq_lengths=
     becoming too large, it is usually scaled by dividing it by the square root of :math:`d` and perform softmax
     normalization on each row, yields a matrix of :math:`n\times d` after multiplying :math:`V`.
 
-    .. warning::
-        This is an experimental API that is subject to change or deletion.
-
     Note:
-      - If there is no input parameter and no default value, None needs to be passed.
-      - The shape of the tensor corresponding to the key and value parameters needs to be completely consistent.
-      - :math:`N` of parameter query is equal with num_heads. :math:`N` of parameter key and parameter value is equal
-        with num_key_value_heads. num_heads is a multiple of num_key_value_heads.
+      - If there is no input parameter and no default value, ``None`` needs to be passed.
+      - The shape of the tensor corresponding to the `key` and `value` parameters needs to be completely consistent.
+      - :math:`N` of parameter `query` is equal with `num_heads`. :math:`N` of parameter `key` and parameter `value`
+        is equal with `num_key_value_heads`. `num_heads` is a multiple of `num_key_value_heads`.
       - Quantization
 
-        - When the data type of query, key, and value is float16 and the data type of output is int8, the input
-          parameter quant_scale2 is required and quant_offset2 is optional.
-        - When antiquant_scale exists, key and value need to be passed by int8. antiquant_offset is optional.
-        - The data type of antiquant_scale and antiquant_offset should be consistency with that of query.
-      - pse_shift
+        - When the data type of `query`, `key`, and `value` is float16 and the data type of output is int8, the input
+          parameter `quant_scale2` is required and `quant_offset2` is optional.
+        - When `antiquant_scale` exists, `key` and `value` need to be passed by int8. `antiquant_offset` is optional.
+        - The data type of `antiquant_scale` and `antiquant_offset` should be consistenct with that of `query`.
+      - `pse_shift`
 
-        - The pse_shift data type needs to be consistent with the query data type, and only supports D-axis alignment,
+        - The `pse_shift` data type needs to be consistent with `query`, and only supports D-axis alignment,
           which means that the D-axis can be divided by 16.
       - Page attention:
 
-        - The necessary condition for enabling page attention is that the block_table exists, and the key
-          and value are arranged in a contiguous memory according to the index in the block_table. The support for
-          key and value dtypes is float16/bfloat16/int8.
-        - In the enabling scenario of page attention, 16 alignment is required when input types of key and value are
-          float16/bfloat16, and 32 alignment is required when input types of key and value are int8. It is
-          recommended to use 128.
+        - The necessary condition for enabling page attention is that the `block_table` exists, and the `key`
+          and `value` are arranged in a contiguous memory according to the index in the `block_table`. The support
+          dtype for `key` and `value` is float16/bfloat16/int8.
+        - In the enabling scenario of page attention, 16 alignment is required when input types of `key`
+          and `value` are float16/bfloat16, and 32 alignment is required when input dtype of `key` and `value`
+          is int8. It is recommended to use 128.
         - The maximum max_block_num_per_seq currently supported by blocktable is 16k, and exceeding 16k will result
           in interception and error messages; If you encounter :math:`S` being too large and causing
-          max_block_num_per_seq to exceed 16k, you can increase the block_size to solve the problem.
-        - The multiplication of all dimensions of the shape of the parameters key and value in the page attention
+          max_block_num_per_seq to exceed 16k, you can increase the `block_size` to solve the problem.
+        - The multiplication of all dimensions of the shape of the parameters `key` and `value` in the page attention
           scenario cannot exceed the representation range of int32.
         - When performing per-channel post quantization, page attention cannot be enabled simultaneously.
-      - kv_padding_size:
+      - `kv_padding_size`:
 
         - The calculation formula for the starting point of KV cache transfer is
           :math:`S-kv\_padding\_size-actual\_seq\_lengths`. The calculation formula for the transfer endpoint of KV
           cache is :math:`S-kv\_padding\_size`. When the starting or ending point of the KV cache transfer is less
           than 0, the returned data result is all 0.
-        - When kv_padding_size is less than 0, it will be set to 0.
-        - kv_padding_size needs to be enabled together with the actual_seq_lengths parameter, otherwise it is
+        - When `kv_padding_size` is less than 0, it will be set to 0.
+        - `kv_padding_size` needs to be enabled together with the `actual_seq_lengths` parameter, otherwise it is
           considered as the KV right padding scene.
         - It needs to be enabled together with the atten_mask parameter and ensure that the meaning of atten_mask is
           correct, that is, it can correctly hide invalid data. Otherwise, it will introduce accuracy issues.
-        - kv_padding_size does not support page attention scenarios
+        - `kv_padding_size` does not support page attention scenarios.
+
+    .. warning::
+        Only support on Atlas A2 training series.
 
     Args:
         query (Tensor): The query tensor with data type of float16 or bfloat16.
             The shape is :math:`(B, 1, H)` / :math:`(B, N, 1, D)`.
-        key (TensorList): The key tensor with data type of float16 or bfloat16 or int8.
+        key (Union[tuple, list]): The key tensor with data type of float16 or bfloat16 or int8.
             The shape is :math:`(B, S, kvH)` / :math:`(B, kvN, S, D)`.
-        value (TensorList): The value tensor with data type of float16 or bfloat16 or int8.
+        value (Union[tuple, list]): The value tensor with data type of float16 or bfloat16 or int8.
             The shape is :math:`(B, S, kvH)` / :math:`(B, kvN, S, D)`.
         attn_mask (Tensor, optional): The attention mask tensor with data type of bool or int8 or uint8.
             The shape is :math:`(B, S)` / :math:`(B, 1, S)` / :math:`(B, 1, 1, S)`. Default: ``None``.
         actual_seq_lengths (Union[Tensor, tuple[int], list[int]], optional): Describe actual sequence length of each
-            input with data type of int32 or int64. The shape is :math:`(B, )`. Default: ``None``.
+            input with data type of int64. The shape is :math:`(B, )`. Default: ``None``.
         pse_shift (Tensor, optional): The position encoding tensor with data type of float16 or bfloat16. Input tensor
             of shape :math:`(1, N, 1, S)` / :math:`(B, N, 1, S)`. Default: ``None``.
         dequant_scale1 (Tensor, optional): Quantitative parametor, the tensor with data type of uint64 or float32. It
@@ -8400,22 +9269,25 @@ def incre_flash_attention(query, key, value, attn_mask=None, actual_seq_lengths=
             The shape is :math:`(1,)`. Default: ``None``.
         antiquant_scale (Tensor, optional): Pseudo Quantitative parametor, the tensor with data type of float16 or
             bfloat16. The shape is :math:`(2, kvN, 1, D)` when input_layout is 'BNSD' or :math:`(2, kvH)` when
-          input_layout is 'BSH'. Default: ``None``.
+            input_layout is 'BSH'. Default: ``None``.
         antiquant_offset (Tensor, optional): Pseudo Quantitative parametor, the tensor with data type of float16 or
             bfloat16. The shape is :math:`(2, kvN, 1, D)` when input_layout is 'BNSD' or :math:`(2, kvH)` when
-          input_layout is 'BSH'. Default: ``None``.
+            input_layout is 'BSH'. Default: ``None``.
         block_table (Tensor, optional): The tensor with data type of int32. The shape is
             :math:`(B, max\_block\_num\_per\_seq)`,
             where :math:`max\_block\_num\_per\_seq = ceil(\frac{max(actual\_seq\_length)}{block\_size} )`.
             Default: ``None``.
-        num_heads (int): The number of heads.
-        input_layout (str): The data layout of the input qkv, support 'BSH' and 'BNSD'. Default ``'BSH'``.
-        scale_value (double): The scale value indicating the scale coefficient, which is used as the scalar of
-            Muls in the calculation. Default: ``1.0``.
-        num_key_value_heads (int): Head numbers of key/value which are used in GQA algorithm.
-            The value 0 indicates if the key and value have the same head nums, use numHeads.  Default: ``0``.
-        block_size (int): The maximum number of tokens stored in each block of KV in page attention. Default: ``0``.
-        inner_precise (int): Default: ``1``.
+        num_heads (int, optional): The number of heads.  Default: ``1``.
+        input_layout (str, optional): The data layout of the input qkv, support 'BSH' and 'BNSD'. Default ``'BSH'``.
+        scale_value (double, optional): The scale value indicating the scale coefficient, which is used as
+            the scalar of Muls in the calculation. Default: ``1.0``.
+        num_key_value_heads (int, optional): Head numbers of `key`/`value` which are used in GQA algorithm.
+            The value 0 indicates if the `key` and `value` have the same head nums, use numHeads.  Default: ``0``.
+        block_size (int, optional): The maximum number of tokens stored in each block of KV in page attention.
+            Default: ``0``.
+        inner_precise (int, optional): An int number from {0, 1} indicates computing mode.
+            ``0`` for high precision mode for float16 dtype. ``1`` for high performance mode.
+            Default: ``1``.
         kv_padding_size (Tensor, optional): The tensor with data type of int64. The range of values is
             :math:`0\le kv\_padding\_size \le  S-max(actual\_seq\_length)`. The shape is :math:`()` or :math:`(1,)`.
             Default: ``None``.
@@ -8423,6 +9295,25 @@ def incre_flash_attention(query, key, value, attn_mask=None, actual_seq_lengths=
     Returns:
         attention_out (Tensor), the shape is :math:`(B, 1, H)` / :math:`(B, N, 1, D)`.
 
+    Raises:
+        TypeError: dtype of `query` is not float16 or bfloat16.
+        TypeError: `key` and `value` don't have the same dtype.
+        TypeError: dtype of `attn_mask` is not bool, int8 or uint8.
+        TypeError: dtype of `pse_shift` is not bfloat16 or float16.
+        TypeError: `scale_value` is not a double number.
+        TypeError: `input_layout` is not a string.
+        TypeError: `num_key_value_heads` or `num_heads` is not an int.
+        TypeError: `inner_precise` is not an int.
+        TypeError: `quant_scale1` is not Tensor of type float32.
+        TypeError: `quant_scale2` is not Tensor of type float32.
+        TypeError: `quant_offset2` is not Tensor of type float32.
+        ValueError: size of `actual_seq_lengths` is not 1 or B.
+        ValueError: `input_layout` is a string but of `(BSH)` or `(BNSD)`.
+        ValueError: `num_heads` is not divisible by Q_H.
+        ValueError: `num_heads` is not divisible by `num_key_value_heads`.
+        RuntimeError: `num_heads` is not greater than 0.
+        RuntimeError: `attn_mask` shape is not valid.
+
     Supported Platforms:
         ``Ascend``
 
@@ -8435,7 +9326,7 @@ def incre_flash_attention(query, key, value, attn_mask=None, actual_seq_lengths=
         >>> query = Tensor(np.random.randn(B, 1, N * D), mstype.float16)
         >>> key = [Tensor(np.random.randn(B, S, kvN * D), mstype.float16)]
         >>> value = [Tensor(np.random.randn(B, S, kvN * D), mstype.float16)]
-        >>> ifa_ms = ops.functional.incre_flash_attention
+        >>> ifa_ms = ops.incre_flash_attention
         >>> attn_out = ifa_ms(query, key, value, num_heads=N, num_key_value_heads=kvN)
         >>> attn_out
         Tensor(shape=[1, 1, 512], dtype=Float16, value=
@@ -8458,7 +9349,7 @@ def embedding(input, weight, padding_idx=None, max_norm=None, norm_type=2.0, sca
     Args:
         input (Tensor): The indices used to lookup in the `weight`. The data type must be mindspore.int32 or
             mindspore.int64, and the value should be in range `[0, weight.shape[0])`.
-        weight (Parameter): The matrix where to lookup from. The shape must be 2D.
+        weight (Union[Parameter, Tensor]): The matrix where to lookup from. The shape must be 2D.
         padding_idx (int, optional): If the value is not None, the corresponding row of `weight` will not be updated
             in training. The value should be in range `[-weight.shape[0], weight.shape[0])` if it's not ``None``.
             Default ``None``.
@@ -8475,7 +9366,6 @@ def embedding(input, weight, padding_idx=None, max_norm=None, norm_type=2.0, sca
     Raises:
         ValueError: If `padding_idx` is out of valid range.
         ValueError: If the shape of `weight` is invalid.
-        TypeError: `weight` is not a :class:`mindspore.Parameter`.
 
     Supported Platforms:
         ``Ascend``
@@ -8500,6 +9390,215 @@ def embedding(input, weight, padding_idx=None, max_norm=None, norm_type=2.0, sca
     return embedding_op(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq)
 
 
+def speed_fusion_attention(query, key, value, head_num, input_layout, *, pse=None, padding_mask=None, atten_mask=None,
+                           scale=1.0, keep_prob=1.0, pre_tokens=2147483647, next_tokens=2147483647, inner_precise=0,
+                           prefix=None, actual_seq_qlen=None, actual_seq_kvlen=None, sparse_mode=0,
+                           gen_mask_parallel=True, sync=False, pse_type=1, q_start_idx=None, kv_start_idx=None):
+    r"""
+    The interface is used for self-attention fusion computing.
+    If `pse_type` is ``1`` , calculation formula is:
+
+    .. math::
+        attention\_out = Dropout(Softmax(Mask(scale * (pse + query * key^{T}), atten\_mask)), keep\_prob) * value
+
+    If `pse_type` is other valid value, calculation formula is:
+
+    .. math::
+        attention\_out = Dropout(Softmax(Mask(scale * (query * key^{T}) + pse, atten\_mask)), keep\_prob) * value
+
+    - B: Batch size. Value range 1 to 2k.
+    - S1: Sequence length of query. Value range 1 to 512k.
+    - S2: Sequence length of key and value. Value range 1 to 512k.
+    - N1: Num heads of query. Value range 1 to 256.
+    - N2: Num heads of key and value, and N2 must be a factor of N1.
+    - D: Head size. The value ranges is a multiple of 16, with the max value of 512.
+    - H1: Hidden size of query, which equals to N1 * D.
+    - H2: Hidden size of key and value, which equals to N2 * D.
+
+    .. warning::
+        - This is an experimental API that is subject to change or deletion.
+        - Only support on Atlas A2 training series.
+
+    Note:
+        This interface is not supported in `graph mode (mode=mindspore.GRAPH_MODE)
+        <https://www.mindspore.cn/tutorials/en/master/compile/static_graph.html>`_.
+
+    Args:
+        query (Tensor): The query tensor. Input tensor of shape :math:`(B, S1, H1)`,
+            :math:`(B, N1, S1, D)`, :math:`(S1, B, H1)`, :math:`(B, S1, N1, D)` or :math:`(T1, N1, D)`.
+        key (Tensor): The key tensor. Input tensor of shape :math:`(B, S2, H2)`,
+            :math:`(B, N2, S2, D)`, :math:`(S2, B, H2)`, :math:`(B, S2, N2, D)` or :math:`(T2, N2, D)`.
+        value (Tensor): The value tensor. Input tensor of shape :math:`(B, S2, H2)`,
+            :math:`(B, N2, S2, D)`, :math:`(S2, B, H2)`, :math:`(B, S2, N2, D)` or :math:`(T2, N2, D)`.
+            The `key` and `value` should have the same shape.
+        head_num (int): The head num of query, equal to N1.
+        input_layout (str): Specifies the layout of input `query`, `key` and `value`. The value can be ``"BSH"`` ,
+            ``"BNSD"`` , ``"SBH"`` , ``"BSND"`` or ``"TND"`` . ``"TND"`` is an experimental format.
+            When `input_layout` is ``"TND"`` , the following restrictions must be met.
+            There are two lists that represent the length of the input sequence: list_seq_q and list_seq_k. Each
+            value in the list indicates the length of the sequence in the batch. For example, list_seq_q = [4, 2, 6],
+            list_seq_k = [10, 3, 9]. The element of list indicate S. T1 is sum(list_seq_q) = 12, T2 is
+            sum(list_seq_k) = 22.
+            max_seqlen_q = max(list_seq_q), max_seqlen_k = max(list_seq_k).
+            qk_pointer = sum(list_seq_q * list_seq_k), which is the sum of the element multiplication.
+
+            - The lengths of two lists are the same, and size of list is batch. batch is less than or equal to 1024.
+            - When `input_layout` is ``"TND"`` , `actual_seq_qlen` and `actual_seq_kvlen` must be not ``None`` .
+              Otherwise, they are ``None`` .
+            - The `actual_seq_qlen` and `actual_seq_kvlen` are the cumulative sum of sequence of key/value, so they must
+              be non-decreasing.
+            - If `pse` is not ``None`` , list_seq_q and list_seq_k must be same. The maximum value of list_seq_q and
+              list_seq_k is greater than 1024. `pse` should be :math:`(B, N1, 1024, S2)` and
+              :math:`(1, N1, 1024, S2)`, and S2 is equal to max_seqlen_k.
+            - `atten_mask` must be a lower trianglar matrix, so `sparse_mode` should be 2 or 3. The shape of
+              `atten_mask` should be :math:`(2048, 2048)`.
+            - Prefix is ``None`` .
+            - `next_tokens` is 0, and `pre_tokens` is not less than max_seqlen_q.
+            - When `sparse_mode` is 3, S1 of each batch should be less than or equal to S2.
+            - 0 should not exist in list_seq_k.
+
+    Keyword Args:
+        pse (Tensor, optional): The position embedding code, dtype is same as `query`.  Default: ``None`` .
+            If S is greater than 1024 and the mask of the lower triangle is used, enter only the inverse 1024 lines of
+            the lower triangle for memory optimization. Input tensor of shape :math:`(B, N1, S1, S2)`,
+            :math:`(1, N1, S1, S2)`, :math:`(B, N1, 1024, S2)`, :math:`(1, N1, 1024, S2)`.
+
+            - ALiBi scenario: `pse` must meet the ALiBi rule, and `sparse_mode` is 2 or 3 for the lower triangle.
+              In this scenario, `pse` is :math:`(B, N1, 1024, S2)`, :math:`(1, N1, 1024, S2)`.
+            - Non-ALiBi scenario: `pse` is :math:`(B, N1, S1, S2)`, :math:`(1, N1, S1, S2)`.
+            - The shape of `pse` should be :math:`(B, N1, 1024, S2)` and :math:`(1, N1, 1024, S2)` when `input_layout`
+              is ``"TND"`` .
+            - If `pse_type` is 2 or 3, dtype of `pse` must be float32, and shape of `pse` should be :math:`(B, N1)` or
+              :math:`(N1,)`.
+
+        padding_mask (Tensor, optional): Reserved parameter. Not implemented yet. Default: ``None`` .
+        atten_mask (Tensor, optional): The attention mask tensor. For each element, 0/False indicates retention and
+            1/True indicates discard. Input tensor of shape :math:`(B, N1, S1, S2)`, :math:`(B, 1, S1, S2)`,
+            :math:`(S1, S2)` or :math:`(2048, 2048)`. Default: ``None`` .
+
+            - In compression scenario, `sparse_mode` is 2, 3, or 4, `atten_mask` must be :math:`(2048, 2048)`.
+            - When `sparse_mode` is 5, `atten_mask` must be :math:`(B, N1, S1, S2)`, :math:`(B, 1, S1, S2)`.
+            - When `sparse_mode` is 0 and 1, `atten_mask` should be :math:`(B, N1, S1, S2)`, :math:`(B, 1, S1, S2)`,
+              :math:`(S1, S2)`.
+
+        scale (float, optional): The scale factor of score. Generally, the value is 1.0 / (D ** 0.5). Default: ``1.0`` .
+        keep_prob (float, optional): The keep probability of dropout. Value range is (0.0, 1.0]. Default: ``1.0`` .
+        pre_tokens (int, optional): Parameter for sparse computation, represents how many tokens are counted forward.
+            When `sparse_mode` is set to 1, 2, 3, or 5, this parameter does not take effect. Default: ``2147483647`` .
+        next_tokens (int, optional): Parameter for sparse computation, represents how many tokens are counted backward.
+            When `sparse_mode` is set to 1, 2, 3, or 5, this parameter does not take effect. Default: ``2147483647`` .
+            The value of pre_tokens corresponds to S1, and the value of next_tokens corresponds to S2. They define the
+            valid area on the `atten_mask` matrix. It must ensure that the band is not empty.
+            The following values are not allowed:
+
+            - pre_tokens < 0 and next_tokens < 0.
+            - (pre_tokens < 0 and next_tokens >= 0) and (next_tokens < abs(pre_tokens) or abs(pre_tokens) >= S2).
+            - (pre_tokens >= 0 and next_tokens < 0) and (abs(next_tokens) > pre_tokens or abs(next_tokens) >= S1).
+
+        inner_precise (int, optional): The parameter is reserved and not implemented yet. Default: ``0`` .
+        prefix (Union[tuple[int], list[int]], optional): N value of each Batch in the prefix sparse calculation
+            scenario. Input tensor of shape :math:`(B,)`. B max value 32. Not none only when sparse_mode is 5.
+            If S1 > S2, N ranges from 0 to S2. If S1 <= S2, N ranges from S2 - S1 to S2. Default: ``None`` .
+        actual_seq_qlen (Union[tuple[int], list[int]], optional): Size of query corresponding to each batch, array
+            with increasing values and the last value equal to T1. Default: ``None`` .
+        actual_seq_kvlen (Union[tuple[int], list[int]], optional): Size of key and value corresponding to each batch,
+            array with increasing values and the last value equal to T2. Default: ``None`` .
+        sparse_mode (int, optional): Indicates sparse mode. Default ``0`` .
+
+            - 0: Indicates the defaultMask mode. If `atten_mask` is not passed, the mask operation is not performed,
+              and preTokens and nextTokens(internally assigned as INT_MAX) are ignored. If passed in, the full
+              `atten_mask` matrix (S1 * S2) needs to be passed in, indicating that the part between preTokens and
+              nextTokens needs to be calculated.
+            - 1: Represents allMask, that is, passing in the complete `atten_mask` matrix.
+            - 2: Representing the leftUpCausal mode corresponds to the lower triangle scenario divided by the left
+              vertex, and the optimized `atten_mask` matrix (2048*2048) is required.
+            - 3: Representing the rightDownCausal model corresponds to the lower triangle scene divided by the lower
+              right vertex, and the optimized `atten_mask` matrix (2048*2048) is required.
+            - 4: Represents the band scenario, that is, the part between counting preTokens and nextTokens, and the
+              optimized `atten_mask` matrix (2048*2048) is required.
+            - 5: Represents the prefix scenario, that is, on the basis of rightDownCasual, a matrix with length S1 and
+              width N is added to the left side. The value of N is obtained by the new input prefix, and the N value
+              of each Batch axis is different. Currently not enabled.
+            - 6: Represents the global scenario. Currently not enabled.
+            - 7: Represents the dilated scenario. Currently not enabled.
+            - 8: Represents the block_local scenario. Currently not enabled.
+
+        gen_mask_parallel (bool, optional): Debug parameter, a switch to control dropout_gen_mask execution method.
+            If ``True`` , dropout_gen_mask is executed in parallel. If ``False`` , execution is serial.
+            Not implemented yet. Default: ``True`` .
+        sync (bool, optional): Debug parameter, a switch to control dropout_gen_mask execution method.
+            If ``True`` , dropout_gen_mask is executed synchronously. If ``False`` , execution is asynchronous.
+            Not implemented yet. Default: ``False`` .
+        pse_type (int, optional): Indicates how to use `pse`. Default ``1`` .
+
+            - 0: `pse` is passed from outside, and the calculation process is to first mul `scale` and then add `pse`.
+            - 1: `pse` is passed from outside, and the calculation process is to add `pse` first and then mul `scale`.
+            - 2: `pse` is generated internally and generates standard alibi position information. The internally
+              generated alibi matrix 0 line is aligned with the upper left corner of :math:`query * key^{T}`.
+            - 3: `pse` is generated internally, and the generated alibi position information is based on the standard
+              and then the square root of sqrt is done. The internally generated alibi matrix 0 line is aligned with
+              the upper left corner of :math:`query * key^{T}`.
+
+        q_start_idx (Union[tuple[int], list[int]], optional): Int array with length 1. Default: ``None`` .
+            When pse_type is configured as ``2`` or ``3`` , it indicates the number of cells that the internally
+            generated alibi code is offset in the S1 direction. A positive number indicates that 0 moves diagonally
+            upward.
+        kv_start_idx (Union[tuple[int], list[int]], optional): Int array with length 1. Default: ``None`` .
+            When pse_type is configured as ``2`` or ``3`` , it indicates the number of cells that the internally
+            generated alibi code is offset in the S2 direction. A positive number indicates that 0 moves diagonally
+            upward.
+
+    Returns:
+        A tuple of tensors containing `attention_out`, `softmax_max`, `softmax_sum`, `softmax_out`, `seed`, `offset`
+        and `numels` .
+
+        - `attention_out` is the output of attention, it's shape, and data type are the same as the query.
+        - `softmax_max` is the max intermediate result calculated by Softmax, used for grad calculation.
+        - `softmax_sum` is the sum intermediate result calculated by Softmax, used for grad calculation.
+        - `softmax_out` is a reserved parameter.
+        - `seed` is generated seed, used for Dropout.
+        - `offset` is generated offset, used for Dropout.
+        - `numels` is the length of generated dropout_mask.
+
+    Raises:
+        TypeError: `query`, `key` and `value` don't have the same dtype.
+        TypeError: Dtype of `atten_mask` is not bool or uint8.
+        TypeError: `scale` or `keep_prob` is not a float number.
+        TypeError: `input_layout` is not a string.
+        TypeError: `head_num` is not an int.
+        TypeError: `sparse_mode` is not an int.
+        TypeError: `pse` is not Tensor type.
+        TypeError: `padding_mask` is not Tensor type.
+        TypeError: `atten_mask` is not Tensor type.
+        TypeError: `pse_type` is not an int.
+        ValueError: `input_layout` is a string but not valid.
+        ValueError: The specified value of `sparse_mode` is invalid.
+        ValueError: The specified value of `pse_type` is invalid.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import mindspore.common.dtype as mstype
+        >>> import numpy as np
+        >>> from mindspore import ops, Tensor
+        >>> query = Tensor(np.ones([2, 4, 64]), dtype=mstype.float16)
+        >>> key = Tensor(np.ones([2, 4, 64]), dtype=mstype.float16)
+        >>> value = Tensor(np.ones([2, 4, 64]), dtype=mstype.float16)
+        >>> head_num = 4
+        >>> input_layout = "BSH"
+        >>> output = ops.speed_fusion_attention(query, key, value, head_num, input_layout)
+        >>> print(output[0].shape)
+        (2, 4, 64)
+    """
+    seed, offset = default_generator._step(generator_step_)  # pylint: disable=protected-access
+    return speed_fusion_attention_op(query, key, value, head_num, input_layout, seed, offset, pse, padding_mask,
+                                     atten_mask, scale, keep_prob, pre_tokens, next_tokens, inner_precise, prefix,
+                                     actual_seq_qlen, actual_seq_kvlen, sparse_mode, gen_mask_parallel, sync, pse_type,
+                                     q_start_idx, kv_start_idx)
+
+
 __all__ = [
     'adaptive_avg_pool1d',
     'adaptive_avg_pool2d',
@@ -8530,11 +9629,15 @@ __all__ = [
     'fast_gelu',
     'fractional_max_pool2d',
     'fractional_max_pool3d',
+    'speed_fusion_attention',
     'pixel_shuffle',
     'pixel_unshuffle',
     'hardshrink',
     'is_floating_point',
     'incre_flash_attention',
+    'prompt_flash_attention',
+    'flash_attention_score',
+    'fused_infer_attention_score',
     'flip',
     'fliplr',
     'flipud',
@@ -8555,7 +9658,6 @@ __all__ = [
     'softplus',
     'selu',
     'silu',
-    'soft_margin_loss',
     'softmax',
     'softmin',
     'pdist',
@@ -8577,6 +9679,7 @@ __all__ = [
     'conv2d',
     'conv_transpose2d',
     'sigmoid',
+    'soft_margin_loss',
     'logsigmoid',
     'relu',
     'relu6',
@@ -8594,6 +9697,8 @@ __all__ = [
     'gaussian_nll_loss',
     'lp_pool1d',
     'lp_pool2d',
+    'moe_token_permute',
+    'moe_token_unpermute',
     'max_unpool1d',
     'max_unpool2d',
     'max_unpool3d',
@@ -8605,5 +9710,6 @@ __all__ = [
     'add_layer_norm',
     'group_norm',
     'rms_norm',
+    'add_rms_norm',
 ]
 __all__.sort()