mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.5.0__cp311-cp311-win_amd64.whl

mindspore/ops/operations/manually_defined/ops_def.py

@@ -28,7 +28,7 @@ from mindspore.common import Tensor, CSRTensor, COOTensor
 from mindspore.common._stub_tensor import _convert_stub
 from mindspore._c_expression import typing
 from mindspore._c_expression import Tensor as Tensor_
-from mindspore._c_expression import pyboost_cast, pyboost_tile, pyboost_zeros, pyboost_ones
+from mindspore._c_expression import pyboost_cast, pyboost_tile, pyboost_zeros, pyboost_ones, pyboost_type_as
 from mindspore.common import dtype as mstype
 from mindspore.common._utils import is_shape_unknown
 from mindspore import _checkparam as validator
@@ -580,21 +580,15 @@ class BatchNorm(Primitive):
             is only supported in GPU target. Default: ``"NCHW"`` .
 
     Inputs:
-        If `is_training` is ``False`` , inputs are Tensors.
-
-        - **input_x** (Tensor) - Tensor of shape :math:`(N, C)`, with float16 or float32 data type.
-        - **scale** (Tensor) - Tensor of shape :math:`(C,)`, with float16 or float32 data type.
-        - **bias** (Tensor) - Tensor of shape :math:`(C,)`, has the same data type with `scale`.
-        - **mean** (Tensor) - Tensor of shape :math:`(C,)`, has the same data type with `scale`.
-        - **variance** (Tensor) - Tensor of shape :math:`(C,)`, has the same data type with `scale`.
-
-        If `is_training` is ``True`` , `scale`, `bias`, `mean` and `variance` are Parameters.
-
         - **input_x** (Tensor) - Tensor of shape :math:`(N, C)`, with float16 or float32 data type.
-        - **scale** (Parameter) - Parameter of shape :math:`(C,)`, with float16 or float32 data type.
-        - **bias** (Parameter) - Parameter of shape :math:`(C,)`, has the same data type with `scale`.
-        - **mean** (Parameter) - Parameter of shape :math:`(C,)`, has the same data type with `scale`.
-        - **variance** (Parameter) - Parameter of shape :math:`(C,)`, has the same data type with `scale`.
+        - **scale** (Union[Parameter, Tensor]) - Tensor or Parameter of shape :math:`(C,)`,
+          with float16 or float32 data type.
+        - **bias** (Union[Parameter, Tensor]) - Tensor or Parameter of shape :math:`(C,)`,
+          has the same data type with `scale`.
+        - **mean** (Union[Parameter, Tensor]) - Tensor or Parameter of shape :math:`(C,)`,
+          has the same data type with `scale`.
+        - **variance** (Union[Parameter, Tensor]) - Tensor or Parameter of shape :math:`(C,)`,
+          has the same data type with `scale`.
 
     Outputs:
         Tuple of 5 Tensors, the normalized inputs and the updated parameters.
@@ -1026,9 +1020,9 @@ class Tile(Primitive):
 
 def tile(input, dims):
     r"""
-    Creates a new tensor by replicating `input` `dims` times. The i'th dimension of
+    Creates a new tensor by repeating `input` `dims` times. The i'th dimension of
     output tensor has `input.shape[i] * dims[i]` elements, and the values of `input`
-    are replicated `dims[i]` times along the i'th dimension.
+    are repeated `dims[i]` times along the i'th dimension.
 
     Note:
         On Ascend, the number of `dims` should not exceed 8, and currently does not support scenarios
@@ -1176,7 +1170,6 @@ class Cast(Primitive):
             if data.dtype == dtype:
                 return (True, x)
         if isinstance(x, Tensor) and x.dtype == dtype:
-            x.set_cast_dtype()
             return (True, x)
         if isinstance(x, numbers.Number):
             return (True, Tensor(x, dtype=dtype))
@@ -1189,6 +1182,59 @@ class Cast(Primitive):
         return _convert_stub(pyboost_cast(self, [input_x, dtype_to_type_id('Cast', 'dtype', dtype)]))
 
 
+class TypeAs(Primitive):
+    """
+    Returns first input tensor cast to the type of the with the second input tensor.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Note:
+        When converting complex numbers to boolean type, the imaginary part of the complex number is not
+        taken into account. As long as the real part is non-zero, it returns True; otherwise, it returns False.
+
+    Inputs:
+        - **input** (Tensor) -  The shape of tensor is :math:`(x_0, x_1, ..., x_R)`.
+          The tensor whose data type is to be converted.
+        - **other ** (Tensor) - The shape of tensor is :math:`(x_0, x_1, ..., x_R)`.
+          The tensor whose data type is specified.
+
+    Outputs:
+        Tensor, the shape of tensor is the same as `input`, :math:`(x_0, x_1, ..., x_R)`.
+
+    Raises:
+        TypeError: If `input` is not a Tensor.
+        TypeError: If `other` is not a Tensor.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> input_np = np.random.randn(2, 3, 4, 5).astype(np.float32)
+        >>> input = Tensor(input_np)
+        >>> other_np = np.random.randn(2, 3, 4).astype(np.int32)
+        >>> other = Tensor(other_np)
+        >>> type_as = ops.TypeAs()
+        >>> output = type_as(input, other)
+        >>> print(output.dtype)
+        Int32
+        >>> print(output.shape)
+        (2, 3, 4, 5)
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        pass
+
+    def __call__(self, input, other):
+        if input.dtype == other.dtype:
+            return input
+        return _convert_stub(pyboost_type_as(self, [input, other]))
+
+
 def to_sequence(val):
     """
     to_sequence
@@ -1791,7 +1837,7 @@ class Ones(Primitive):
         Tensor, whose dtype and size are defined by input.
 
     Raises:
-        TypeError: If `shape` is neither an int nor an tuple/list/Tensor of int.
+        TypeError: If `shape` is neither an int nor a tuple/list/Tensor of int.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1827,7 +1873,7 @@ class Ones(Primitive):
 
 class Zeros(Primitive):
     r"""
-    Zeros will be deprecated in the future. Please use class `mindspore.ops.zeros` instead.
+    Zeros will be deprecated in the future. Please use class :func:`mindspore.ops.zeros` instead.
 
     Creates a tensor filled with value zeros.
 
@@ -1845,7 +1891,7 @@ class Zeros(Primitive):
         Tensor, whose dtype and size are defined by input.
 
     Raises:
-        TypeError: If `shape` is neither an int nor an tuple/list/Tensor of int.
+        TypeError: If `shape` is neither an int nor a tuple/list/Tensor of int.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1880,116 +1926,132 @@ def flash_attention_score(query, key, value, head_num, real_shift=None, drop_mas
                           scalar_value=1.0, pre_tokens=2147483647, next_tokens=2147483647, inner_precise=0,
                           input_layout='BSH', sparse_mode=0):
     r"""
-    The interface is not open to the public, just for internal use,
+    Implement self-attention calculations in training scenarios.
+
+    - B: Batch size. Value range 1 to 2k.
+    - S1: Sequence length of `query`. Value range 1 to 512k.
+    - S2: Sequence length of `key` and `value`. Value range 1 to 512k.
+    - N1: Num heads of `query`. Value range 1 to 256.
+    - N2: Num heads of `key` and `value`, and N2 must be a factor of N1.
+    - D: Head size. The value ranges is a multiple of 16, with the max value of 512.
+    - H1: Hidden size of `query`, which equals to N1 * D.
+    - H2: Hidden size of `key` and `value`, which equals to N2 * D.
+
+    The self attention calculation formula is defined as:
 
     .. math::
         \begin{array}{ll} \\
-            y = Dropout(Softmax(Mask(scale_value \mul (real_shift + query * key), attn_mask), -1), keep\_prob) \\
-            \mul value \\
+            \text { attention_out }=\operatorname{Dropout}\left(\operatorname{Softmax}\left(\text
+            { Mask(scale } *\left(\text { query } * \mathrm{key}^{\top}\right)+\text { pse }\right)\text
+            {, atten_mask), keep_prob) } *\right. \text { value }
         \end{array}
 
-    B -- Batch size. Value range 1 to 2k.
-    S1 -- Sequence length of query. Value range 1 to 512k.
-    S2 -- Sequence length of key and value. Value range 1 to 512k.
-    N1 -- Num heads of query. Value range 1 to 256.
-    N2 -- Num heads of key and value, and N2 must be a factor of N1.
-    D -- Head size. The value ranges is a multiple of 16, with the max value of 512.
-    H1 -- Hidden size of query, which equals to N1 * D.
-    H2 -- Hidden size of key and value, which equals to N2 * D.
-
     .. warning::
-        This is an experimental API that is subject to change or deletion. Only support on Atlas A2 training series.
+        - This is an experimental API that is subject to change or deletion.
+        - Only support on Atlas A2 training series.
 
     Args:
-        query (Tensor[float16, bfloat16]): The query tensor. Input tensor of shape :math:`(B, S1, H1)`,
-            `(B, N1, S1, D)`, `(S1, B, H1)`, `(B, S1, N1, D)` or `(T1, N1, D)`.
-        key (Tensor[float16, bfloat16]): The key tensor. Input tensor of shape :math:`(B, S2, H2)`,
-            `(B, N2, S2, D)`, `(S2, B, H2)`, `(B, S2, N2, D)` or `(T2, N2, D)`.
-        value (Tensor[float16, bfloat16]): The value tensor. Input tensor of shape :math:`(B, S2, H2)`,
-            `(B, N2, S2, D)`, `(S2, B, H2)`, `(B, S2, N2, D)` or `(T2, N2, D)`. The key and value have the same shape.
-        head_num (int): The head num of query, equal to N1.
-        real_shift (Union[Tensor[float16, bfloat16], None]): Also known as pse. The position embedding code. If S
-            is greater than 1024 and the mask of the lower triangle is used, enter only the inverse 1024 lines of
-            the lower triangle for memory optimization. Input tensor of shape :math:`(B, N1, S1, S2)`,
-            `(1, N1, S1, S2)`, `(B, N1, 1024, S2)`, `(1, N1, 1024, S2)`.
-
-            - ALiBi scenario: real_shift must meet the ALiBi rule, and sparse_mode is 2 or 3 for the lower triangle.
-              In this scenario, real_shift is `(B, N1, 1024, S2)`, `(1, N1, 1024, S2)`.
-            - Non-ALiBi scenario: real_shift is `(B, N1, S1, S2)`, `(1, N1, S1, S2)`.
-
-            The shape of `real_shift` should be `(B, N1, 1024, S2)` and `(1, N1, 1024, S2)` when input_layout is
-            `TND`.
-        drop_mask (Union[Tensor[uint8], None]): The dropout mask tensor. Input tensor of shape :math:
-            `(B, N1, S1, S2 // 8) or None`. S2 is a multiple of 8 when not None.
-        padding_mask (None): Reserved parameter. Not implemented yet.
-        attn_mask (Union[Tensor[uint8], Tensor[bool], None]): The attention mask tensor. For each element, 0
-            indicates retention and 1 indicates discard. Input tensor of shape :math:`(B, N1, S1, S2)`,
-            `(B, 1, S1, S2)`, `(S1, S2)` or `(2048, 2048)`. In compression scenario, sparse_mode is 2, 3, or 4,
-            attn_mask must be `(2048, 2048)`. When sparse_mode is 5, attn_mask must be `(B, N1, S1, S2)`,
-            `(B, 1, S1, S2)`. When sparse_mode is 0 and 1, attn_mask should be `(B, N1, S1, S2)`, `(B, 1, S1, S2)`,
-            `(S1, S2)`.
-        prefix (Union[List[int64], Tuple[int64] None]): N value of each Batch in the prefix sparse calculation
-            scenario. Input tensor of shape :math:`(B,)`. B max value 32. Not none only when sparse_mode is 5.
+        query (Tensor): The query tensor. Input tensor of shape :math:`(B, S1, H1)`,
+            :math:`(B, N1, S1, D)`, :math:`(S1, B, H1)`, :math:`(B, S1, N1, D)` or :math:`(T1, N1, D)`.
+            The supported dtype is float16 and bfloat16.
+        key (Tensor): The key tensor with the same dtype as `query`. Supported shape: :math:`(B, S2, H2)`,
+            :math:`(B, N2, S2, D)`, :math:`(S2, B, H2)`, :math:`(B, S2, N2, D)` or :math:`(T2, N2, D)`.
+        value (Tensor): The value tensor with the same dtype and shape as `key`.
+        head_num (int): The head num of `query`, equal to N1.
+        real_shift (Tensor, optional): The position embedding code which is also known as pse, it has the same
+            dtype as `query`.
+            Default: ``None``.
+            If S is greater than 1024 and the mask of the lower triangle is used, only the inverse 1024 lines of
+            the lower triangle is used for memory optimization. Input tensor of shape :math:`(B, N1, S1, S2)`,
+            :math:`(1, N1, S1, S2)`, :math:`(B, N1, 1024, S2)`, :math:`(1, N1, 1024, S2)`.
+
+            - ALiBi scenario: `real_shift` must meet the ALiBi rule, and sparse_mode is 2 or 3 for the lower triangle.
+              In this scenario, `real_shift` is :math:`(B, N1, 1024, S2)`, :math:`(1, N1, 1024, S2)`.
+            - Non-ALiBi scenario: `real_shift` is :math:`(B, N1, S1, S2)`, :math:`(1, N1, S1, S2)`.
+            - input_layout is TND: shape should be :math:`(B, N1, 1024, S2)` and :math:`(1, N1, 1024, S2)`.
+
+        drop_mask (Tensor, optional): The dropout mask tensor of uint8. Input tensor of shape
+            :math:`(B, N1, S1, S2 // 8) or None`. `S2` is a multiple of 8 when not None. Default: ``None``.
+        padding_mask (Tensor, optional): Reserved parameter. Not implemented yet. Default: ``None``.
+        attn_mask (Tensor, optional): The attention mask tensor of bool or uint8. For each element, 0/False
+            indicates retention and 1/True indicates discard. Input tensor of shape :math:`(B, N1, S1, S2)`,
+            :math:`(B, 1, S1, S2)`, :math:`(S1, S2)` or :math:`(2048, 2048)`.
+            Default: ``None``.
+
+            - In compression scenario, `sparse_mode` is 2, 3, or 4, `attn_mask` must be :math:`(2048, 2048)`.
+            - When `sparse_mode` is 5, `attn_mask` should be :math:`(B, N1, S1, S2)`, :math:`(B, 1, S1, S2)`.
+            - When `sparse_mode` is 0 and 1, `attn_mask` should be :math:`(B, N1, S1, S2)`, :math:`(B, 1, S1, S2)`,
+              :math:`(S1, S2)`.
+
+        prefix (Union[Tensor, tuple[int], list[int]], optional): N value of each Batch in the prefix sparse calculation
+            scenario. Input tensor of shape :math:`(B,)`. B max value 32. Not none only when `sparse_mode` is 5.
+            Default: ``None``.
             If S1 > S2, N ranges from 0 to S2. If S1 <= S2, N ranges from S2 - S1 to S2.
-        actual_seq_qlen (Union[List[int64], Tuple[int64], None]): Size of query corresponding to each batch, array
-            with increasing values and the last value equal to T1.
-        actual_seq_kvlen (Union[List[int64], Tuple[int64], None]): Size of key and value corresponding to each batch,
-            array with increasing values and the last value equal to T2.
-        keep_prob (float): The keep probability of dropout. Value range is (0.0, 1.0]. Default: 1.0. when keep_prob
-            is 1.0, drop_mask should be none.
-        scale_value (float): The scale factor of score. Generally, the value is 1.0 / (D ** 0.5). Default: 1.0.
-        pre_tokens (int): Parameter for sparse computation, represents how many tokens are counted forward.
-            When sparse_mode is set to 1, 2, 3, or 5, this parameter does not take effect. Default: 2147483647.
-        next_tokens (int): Parameter for sparse computation, represents how many tokens are counted backward.
-            When sparse_mode is set to 1, 2, 3, or 5, this parameter does not take effect. Default: 2147483647.
-            The value of pre_tokens corresponds to S1, and the value of next_tokens corresponds to S2. They define the
-            valid area on the attn_mask matrix. It must ensure that the band is not empty.
+        actual_seq_qlen (Union[Tensor, tuple[int], list[int]], optional): Size of query corresponding to each batch,
+            array with increasing values and the last value equal to T1.
+            Default: ``None``.
+        actual_seq_kvlen (Union[Tensor, tuple[int], list[int]], optional): Size of key and value corresponding
+            to each batch, array with increasing values and the last value equal to T2.
+            Default: ``None``.
+        keep_prob (double, optional): The keep probability of dropout. Value range is (0.0, 1.0]. When `keep_prob`
+            is 1.0, `drop_mask` should be None.
+            Default: ``1.0``.
+        scale_value (double, optional): The scale factor of score. Generally, the value is 1.0 / (D ** 0.5).
+            Default: ``1.0``.
+        pre_tokens (int, optional): Parameter for sparse computation, represents how many tokens are counted forward.
+            When `sparse_mode` is set to 1, 2, 3, or 5, this parameter does not take effect.
+            Default: ``2147483647``.
+        next_tokens (int, optional): Parameter for sparse computation, represents how many tokens are counted backward.
+            When `sparse_mode` is set to 1, 2, 3, or 5, this parameter does not take effect. Default: ``2147483647``.
+            The value of `pre_tokens` corresponds to S1, and the value of `next_tokens` corresponds to S2.
+            They define the valid area on the `attn_mask` matrix. It must ensure that the band is not empty.
             The following values are not allowed:
 
             - pre_tokens < 0 and next_tokens < 0.
             - (pre_tokens < 0 and next_tokens >= 0) and (next_tokens < abs(pre_tokens) or abs(pre_tokens) >= S2).
             - (pre_tokens >= 0 and next_tokens < 0) and (abs(next_tokens) > pre_tokens or abs(next_tokens) >= S1).
 
-        inner_precise (int): The parameter is reserved and not implemented yet. Default: 0.
-        input_layout (str): Specifies the layout of input `query`, key and value. The value can be "BSH", "BNSD",
-            "SBH", "BSND" or "TND". "TND" is an experimental format. Default: "BSH".
+        inner_precise (int, optional): The parameter is reserved and not implemented yet. Default:``0``.
+        input_layout (str, optional): Specifies the layout of input `query`, `key` and `value`. The value can be
+            "BSH", "BNSD", "SBH", "BSND" or "TND". "TND" is an experimental format. Default: ``"BSH"``.
             When input_layout is "TND", the following restrictions must be met.
-            There are two lists that represent the length of the input sequence: list_seq_q and list_seq_k. Each
+            Assume there are two lists that represent the length of the input sequence: list_seq_q and list_seq_k. Each
             value in the list indicates the length of the sequence in the batch. For example, list_seq_q = [4, 2, 6],
             list_seq_k = [10, 3, 9]. The element of list indicate S. T1 is sum(list_seq_q) = 12, T2 is
             sum(list_seq_k) = 22.
             max_seqlen_q = max(list_seq_q), max_seqlen_k = max(list_seq_k).
             qk_pointer = sum(list_seq_q * list_seq_k), which is the sum of the element multiplication.
 
-            - The lengths of two lists are the same, and size of list is batch. batch is less than or equal to 1024.
-            - When input_layout is "TND", actual_seq_qlen and actual_seq_kvlen must be not none.
+            - The lengths of two lists must be the same, and size of list is batch. batch is less than or equal to
+              1024.
+            - When `input_layout` is "TND", `actual_seq_qlen` and `actual_seq_kvlen` must be not none.
               Otherwise, they are none.
-            - The actual_seq_qlen and actual_seq_kvlen are the cumulative sum of sequence of key/value, so they must
+            - The `actual_seq_qlen` and `actual_seq_kvlen` are the cumulative sum of sequence of key/value, so they must
               be non-decreasing.
-            - If real_shift is not none, list_seq_q and list_seq_k must be same. The maximum value of list_seq_q and
-              list_seq_k is greater than 1024. Real_shift should be `(B, N1, 1024, S2)` and `(1, N1, 1024, S2)`, and
-              S2 is equal to max_seqlen_k.
-            - Attn mask must be a lower trianglar matrix, so sparse_mode should be 2 or 3. The shape of attn_mask
-              should be `(2048, 2048)`.
-            - The shape of drop_mask is (qk_pointer * N1 // 8,).
-            - Prefix is none.
-            - Next_tokens is 0, and pre_tokens is not less than max_seqlen_q.
-            - When sparse_mode is 3, S1 of each batch should be less than or equal to S2.
+            - If `real_shift` is not none, list_seq_q and list_seq_k must be same. The maximum value of list_seq_q and
+              list_seq_k is greater than 1024. `real_shift` should be :math:`(B, N1, 1024, S2)` and
+              :math:`(1, N1, 1024, S2)`, and S2 is equal to max_seqlen_k.
+            - `attn_mask` must be a lower trianglar matrix, so `sparse_mode` should be 2 or 3. The shape of `attn_mask`
+              should be :math:`(2048, 2048)`.
+            - The shape of `drop_mask` is :math:`(qk\_pointer * N1 // 8,)`.
+            - `prefix` is none.
+            - `next_tokens` is 0, and `pre_tokens` is not less than max_seqlen_q.
+            - When `sparse_mode` is 3, S1 of each batch should be less than or equal to S2.
             - 0 should not exist in list_seq_k.
 
-        sparse_mode (int): Indicates sparse mode. Default 0.
+        sparse_mode (int, optional): Indicates sparse mode. Default: ``0``.
 
-            - 0: Indicates the defaultMask mode. If attn_mask is not passed, the mask operation is not performed,
-              and preTokens and nextTokens(internally assigned as INT_MAX) are ignored. If passed in, the full
-              attn_mask matrix (S1 * S2) needs to be passed in, indicating that the part between preTokens and
-              nextTokens needs to be calculated.
-            - 1: Represents allMask, that is, passing in the complete attn_mask matrix.
+            - 0: Indicates the defaultMask mode. If `attn_mask` is not passed, the mask operation is not performed,
+              `next_tokens` and `pre_tokens` (internally assigned as INT_MAX) are ignored. If passed in, the full
+              `attn_mask` matrix (S1 * S2) needs to be passed in, indicating that the part between `next_tokens` and
+              `pre_tokens` needs to be calculated.
+            - 1: Represents allMask, that is, passing in the complete `attn_mask` matrix.
             - 2: Representing the leftUpCausal mode corresponds to the lower triangle scenario divided by the left
-              vertex, and the optimized attn_mask matrix (2048*2048) is required.
+              vertex, and the optimized `attn_mask` matrix (2048*2048) is required.
             - 3: Representing the rightDownCausal model corresponds to the lower triangle scene divided by the lower
-              right vertex, and the optimized attn_mask matrix (2048*2048) is required.
-            - 4: Represents the band scenario, that is, the part between counting preTokens and nextTokens, and the
-              optimized attn_mask matrix (2048*2048) is required.
+              right vertex, and the optimized `attn_mask` matrix (2048*2048) is required.
+            - 4: Represents the band scenario, that is, the part between counting `next_tokens` and `pre_tokens`,
+              and the optimized `attn_mask` matrix (2048*2048) is required.
             - 5: Represents the prefix scenario, that is, on the basis of rightDownCasual, a matrix with length S1 and
               width N is added to the left side. The value of N is obtained by the new input prefix, and the N value
               of each Batch axis is different, not implemented yet.
@@ -1998,8 +2060,27 @@ def flash_attention_score(query, key, value, head_num, real_shift=None, drop_mas
             - 8: Represents the block_local scenario, not implemented yet.
 
     Returns:
-        attention_out (Tensor[float16, bfloat16]), The output of attention, its shape, and data type are the same
-        as the query.
+        attention_out (Tensor) - The output of attention, it has the same shape and dtype as `query`.
+
+    Raises:
+        TypeError: Dtype of `query` is not float16 or bfloat16.
+        TypeError: `query`, `key` and `value` don't have the same dtype.
+        TypeError: Dtype of `attn_mask` is not bool or uint8.
+        TypeError: Dtype of `real_shift` has a different dtype as `query`.
+        TypeError: `scale_value` or `keep_prob` is not a double number.
+        TypeError: `input_layout` is not a string.
+        TypeError: `num_key_value_heads` is not an int.
+        TypeError: `sparse_mode` is not an int.
+        TypeError: `real_shift` is not Tensor type.
+        TypeError: `drop_mask` is not Tensor type.
+        TypeError: `padding_mask` is not Tensor type.
+        TypeError: `attn_mask` is not Tensor type.
+        ValueError: `input_layout` is a string but not valid.
+        RuntimeError: `head_num` is not divisible by `N2`.
+        RuntimeError: `head_num` is not greater than 0.
+        RuntimeError: `attn_mask` shape is not valid.
+        RuntimeError: The specified value of `sparse_mode` is invalid.
+        RuntimeError: D-axis of `query`, `key` and `value` is not the same.
 
     Supported Platforms:
         ``Ascend``