mindspore 2.2.0__cp37-cp37m-manylinux1_x86_64.whl → 2.2.11__cp37-cp37m-manylinux1_x86_64.whl

mindspore/ops/function/array_func.py

@@ -660,14 +660,14 @@ def one_hot(indices, depth, on_value=1, off_value=0, axis=-1):
 
     Note:
         If the input indices is rank `N`, the output will have rank `N+1`. The new axis is created at dimension `axis`.
+        On Ascend, if `on_value` is Int64 dtype, `indices` must be Int64 dtype.
 
     Args:
         indices(Tensor): A tensor of indices. Tensor of shape :math:`(X_0, \ldots, X_n)`.
             Data type must be int32 or int64.
         depth(int): A scalar defining the depth of the one-hot dimension.
         on_value(Union[Tensor, int, float], optional): A value to fill in output when `indices[j] = i`.
-            Support uint8, uint16, uint32, uint64, int8, int16, int32, int64, float16, float32, float64,
-            bool, complex64, complex128. Default: ``1`` .
+            Data type must be int32, int64, float16 or float32. Default: ``1`` .
         off_value(Union[Tensor, int, float], optional): A value to fill in output when `indices[j] != i`.
             Has the same data type as `on_value`. Default: ``0`` .
         axis(int, optional): Position to insert the value. e.g. If shape of `self` is :math:`(N, C)`, and `axis` is -1,
@@ -676,7 +676,8 @@ def one_hot(indices, depth, on_value=1, off_value=0, axis=-1):
             Default: ``-1`` .
 
     Returns:
-        Tensor, one-hot tensor. Tensor of shape :math:`(X_0, \ldots, X_{axis}, \text{depth} ,X_{axis+1}, \ldots, X_n)`.
+        Tensor, one-hot tensor. Tensor of shape :math:`(X_0, \ldots, X_{axis}, \text{depth} ,X_{axis+1}, \ldots, X_n)`,
+        and it has the same data type as `on_value`.
 
     Raises:
         TypeError: If `axis` or `depth` is not an int.
@@ -1734,7 +1735,11 @@ def flatten(input, order='C', *, start_dim=1, end_dim=-1):
         raise TypeError(f"For 'flatten', both 'start_dim' and 'end_dim' must be int.")
     check_flatten_order_const(order)
     if order == 'F':
-        perm = ops.make_range(0, ops.rank(input))
+        x_rank = rank_(input)
+        # If input is a 0-dimensional Tensor, a 1-dimensional Tensor will be returned.
+        if x_rank in (0, 1):
+            return reshape_(input, (-1,))
+        perm = ops.make_range(0, x_rank)
         new_order = ops.tuple_reversed(perm)
         input = _get_cache_prim(P.Transpose)()(input, new_order)
 
@@ -2161,8 +2166,6 @@ def concat(tensors, axis=0):
 
     Tutorial Examples:
         - `Tensor - Tensor Operation <https://mindspore.cn/tutorials/en/r2.2/beginner/tensor.html#tensor-operation>`_
-        - `FGSM Network Adversarial Attack - Implementing FGSM
-          <https://mindspore.cn/tutorials/application/en/r2.2/cv/fgsm.html#implementing-fgsm>`_
         - `Vision Transformer Image Classification - Building ViT as a whole
           <https://mindspore.cn/tutorials/application/en/r2.2/cv/vit.html#building-vit-as-a-whole>`_
         - `Sentiment Classification Implemented by RNN - Dense
@@ -6828,7 +6831,7 @@ def diagonal(input, offset=0, dim1=0, dim2=1):
     """
     x_ndim = input.ndim
     if x_ndim < 2:
-        raise ValueError(f"ops.diagonal requires an array of at least two dimensions")
+        raise ValueError(f"For 'ops.diagonal', the original tensor requires at least two dimensions, but got {x_ndim}")
     _check_attr_dtype("dim1", dim1, [int], "diagonal")
     _check_attr_dtype("dim2", dim2, [int], "diagonal")
     dtype = input.dtype

mindspore/ops/function/grad/grad_func.py

@@ -37,7 +37,6 @@ oneslike = P.OnesLike()
 def _check_has_aux_type(inputs):
     if not isinstance(inputs, bool):
         raise TypeError("The 'has_aux' must be bool type.")
-    return True
 
 
 @constexpr

mindspore/ops/function/nn_func.py

@@ -4609,6 +4609,19 @@ def max_pool3d(x, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=Fal
 
         - **output** (Tensor) - Maxpooling result, with shape :math:`(N_{out}, C_{out}, D_{out}, H_{out}, W_{out})`.
           It has the same data type as `x`.
+
+        .. math::
+            D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - \text{dilation}[0] \times
+            (\text{kernel_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor
+
+        .. math::
+            H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - \text{dilation}[1] \times
+            (\text{kernel_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor
+
+        .. math::
+            W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - \text{dilation}[2] \times
+            (\text{kernel_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor
+
         - **argmax** (Tensor) - Index corresponding to the maximum value. Data type is int64. It will be return
           only when `return_indices` is ``True`` .
 
@@ -6037,20 +6050,20 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
 
         .. math::
             \begin{array}{ll} \\
-                D_{out} ＝ \left \lceil{\frac{D_{in}}{\text{stride[0]}}} \right \rceil \\
-                H_{out} ＝ \left \lceil{\frac{H_{in}}{\text{stride[1]}}} \right \rceil \\
-                W_{out} ＝ \left \lceil{\frac{W_{in}}{\text{stride[2]}}} \right \rceil \\
+                D_{out} = \left \lceil{\frac{D_{in}}{\text{stride[0]}}} \right \rceil \\
+                H_{out} = \left \lceil{\frac{H_{in}}{\text{stride[1]}}} \right \rceil \\
+                W_{out} = \left \lceil{\frac{W_{in}}{\text{stride[2]}}} \right \rceil \\
             \end{array}
 
         `pad_mode` is ``"valid"``:
 
         .. math::
             \begin{array}{ll} \\
-                D_{out} ＝ \left \lfloor{\frac{D_{in} - \text{dilation[0]} \times (\text{kernel_size[0]} - 1) }
+                D_{out} = \left \lfloor{\frac{D_{in} - \text{dilation[0]} \times (\text{kernel_size[0]} - 1) }
                 {\text{stride[0]}} + 1} \right \rfloor \\
-                H_{out} ＝ \left \lfloor{\frac{H_{in} - \text{dilation[1]} \times (\text{kernel_size[1]} - 1) }
+                H_{out} = \left \lfloor{\frac{H_{in} - \text{dilation[1]} \times (\text{kernel_size[1]} - 1) }
                 {\text{stride[1]}} + 1} \right \rfloor \\
-                W_{out} ＝ \left \lfloor{\frac{W_{in} - \text{dilation[2]} \times (\text{kernel_size[2]} - 1) }
+                W_{out} = \left \lfloor{\frac{W_{in} - \text{dilation[2]} \times (\text{kernel_size[2]} - 1) }
                 {\text{stride[2]}} + 1} \right \rfloor \\
             \end{array}
 
@@ -6058,11 +6071,11 @@ def conv3d(input, weight, bias=None, stride=1, pad_mode="valid", padding=0, dila
 
         .. math::
             \begin{array}{ll} \\
-                D_{out} ＝ \left \lfloor{\frac{D_{in} + padding[0] + padding[1] - (\text{dilation[0]} - 1) \times
+                D_{out} = \left \lfloor{\frac{D_{in} + padding[0] + padding[1] - (\text{dilation[0]} - 1) \times
                 \text{kernel_size[0]} - 1 }{\text{stride[0]}} + 1} \right \rfloor \\
-                H_{out} ＝ \left \lfloor{\frac{H_{in} + padding[2] + padding[3] - (\text{dilation[1]} - 1) \times
+                H_{out} = \left \lfloor{\frac{H_{in} + padding[2] + padding[3] - (\text{dilation[1]} - 1) \times
                 \text{kernel_size[1]} - 1 }{\text{stride[1]}} + 1} \right \rfloor \\
-                W_{out} ＝ \left \lfloor{\frac{W_{in} + padding[4] + padding[5] - (\text{dilation[2]} - 1) \times
+                W_{out} = \left \lfloor{\frac{W_{in} + padding[4] + padding[5] - (\text{dilation[2]} - 1) \times
                 \text{kernel_size[2]} - 1 }{\text{stride[2]}} + 1} \right \rfloor \\
             \end{array}
 
@@ -7431,6 +7444,82 @@ def max_pool2d(x, kernel_size, stride=None, padding=0, dilation=1, return_indice
     return out
 
 
+def prompt_flash_attention(query, key, value, padding_mask, attn_mask, actual_seq_lengths,
+                           actual_seq_lengths_kv, deq_scale1, quant_scale1,
+                           deq_scale2, quant_scale2, quant_offset2, num_heads, scale_value=1.0, pre_tokens=2147483547,
+                           next_tokens=0, input_layout='BSH',
+                           num_key_value_heads=0, sparse_mode=0):
+    r"""
+    The interface for fully inference.
+    B -- Batch size
+    S -- Sequence length
+    H -- Hidden size
+
+    Note:
+    is only supported on ascend910B
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Inputs:
+        query (Tensor) - The query tensor with data type of float16 or float32.
+          Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
+        key (Tensor) - The key tensor with data type of float16 or float32.
+          Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
+        value (Tensor) - The value tensor with data type of float16 or float32.
+          Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
+        padding_mask (Tensor) - The padding mask tensor with data type of float16 or float32
+        attn_mask (Tensor) - The attention mask tensor with data type of float16 or float32.
+          For each element, 0 indicates retention and 1 indicates discard. Input tensor of shape :math:`(B, 1, S, S)`.
+        actual_seq_lengths (list[int]): Describe actual sequence length of each input with data type of int.
+        actual_seq_lengths_kv (list[int]): Describe actual sequence length of each input with data type of int.
+        dep_scale1 (Tensor)
+        quant_scale1 (Tensor)
+        deq_scale2 (Tensor)
+        quant_scale2 (Tensor)
+        quant_offset2 (Tensor)
+        num_heads (int): The number of heads.
+        scale_value (float): The scale value indicating the scale coefficient, which is used as the scalar of
+          Muls in the calculation. Default: 1.0.
+        pre_tokens (int): Previous tokens. Default: 2147483547.
+        next_tokens (int): next tokens.  Default: 0.
+          indicate the upper triangle, Indicate the number of data blocks involved in the calculation. The value 0
+          indicates that the data blocks in the upper triangle are not involved in the calculation
+        input_layout (str): the data layout of the input qkv, support `(BSH)` and `(BNSD)`, Default `BSH`.
+        num_key_value_heads (int): head numbers of key/value which are used in GQA algorithm.
+          The value o indicates if the key and value have the same head nums, use numHeads.  Default: 0.
+        sparse_mode (int): Default: 0
+
+
+    Outputs:
+        attention_out (Tensor) - Input tensor of shape :math:`(B, S, H)` / `(B, N, S, D)`.
+
+        Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.ops.function.nn_func import prompt_flash_attention
+        >>> from mindspore import Tensor
+        >>> import numpy as np
+        >>> B = 1
+        >>> N = 16
+        >>> S = 256
+        >>> D = 16
+        >>> query = Tensor(np.ones((B, N, S, D), dtype=np.float16))
+        >>> key = Tensor(np.ones((B, N, S, D), dtype=np.float16))
+        >>> value = Tensor(np.ones((B, N, S, D), dtype=np.float16))
+        >>> out = ops.prompt_flash_attention(query, key, value, None, None, None, None, None, None, None, None,
+                                             None, N, input_layout='BNSD')
+        >>> print(out[0].shape)
+        (1, 16, 256, 16)
+    """
+
+    pfa = _get_cache_prim(NN_OPS.PromptFlashAttention)(num_heads, scale_value, pre_tokens, next_tokens, input_layout,
+                                                       num_key_value_heads, sparse_mode)
+    return pfa(query, key, value, padding_mask, attn_mask, actual_seq_lengths, actual_seq_lengths_kv, deq_scale1,
+               quant_scale1, deq_scale2, quant_scale2, quant_offset2)
+
+
 __all__ = [
     'adaptive_avg_pool1d',
     'adaptive_avg_pool2d',

mindspore/ops/function/random_func.py

@@ -1180,7 +1180,8 @@ def randint_like(input, low, high, seed=None, *, dtype=None):
     cast_ = P.Cast()
     low_ = Tensor(low, mstype.int32)
     high_ = Tensor(high, mstype.int32)
-    output = rand_op(size, low_, high_)
+    size_ = Tensor(size, mstype.int32)
+    output = rand_op(size_, low_, high_)
     return cast_(output, dtype)
 
 

mindspore/ops/op_info_register.py

@@ -36,13 +36,16 @@ if platform.system() == "Linux":
 BUILT_IN_OPS_REGISTER_PATH = "mindspore/ops/_op_impl"
 BUILT_IN_CUSTOM_OPS_REGISTER_PATH = "mindspore/ops/_op_impl/_custom_op"
 
+KEY_NAME = "name"
+ASCEND_CUSTOM_OPP_PATH = "ASCEND_CUSTOM_OPP_PATH"
 
-def _get_reg_info_attr(op_info, attr_name):
+
+def _get_reg_info_attr(op_info, attr_name, default_value=None):
     """get attr value"""
     for _, item in enumerate(op_info.get("attr", [])):
-        if item.get("name") == attr_name:
+        if item.get(KEY_NAME) == attr_name:
             return item.get("defaultValue")
-    return None
+    return default_value
 
 
 class _CustomInstaller:
@@ -66,12 +69,12 @@ class _CustomInstaller:
     @staticmethod
     def _set_env(custom_opp_path):
         """set custom file path to env"""
-        if not os.environ.get("ASCEND_CUSTOM_OPP_PATH"):
-            os.environ["ASCEND_CUSTOM_OPP_PATH"] = custom_opp_path
+        if not os.environ.get(ASCEND_CUSTOM_OPP_PATH):
+            os.environ[ASCEND_CUSTOM_OPP_PATH] = custom_opp_path
         else:
-            paths = os.environ["ASCEND_CUSTOM_OPP_PATH"].split(':')
+            paths = os.environ[ASCEND_CUSTOM_OPP_PATH].split(':')
             if custom_opp_path not in paths:
-                os.environ["ASCEND_CUSTOM_OPP_PATH"] = custom_opp_path + ':' + os.environ["ASCEND_CUSTOM_OPP_PATH"]
+                os.environ[ASCEND_CUSTOM_OPP_PATH] = custom_opp_path + ':' + os.environ[ASCEND_CUSTOM_OPP_PATH]
 
     @staticmethod
     def _create_dir(*dir_names):
@@ -94,11 +97,11 @@ class _CustomInstaller:
         _CustomInstaller.copied_paths.append(src_path)
         if os.path.isfile(src_path):
             lock_file = os.path.join(dst_dir, "file.lock")
-            with open(lock_file, "w") as f:
+            with os.fdopen(os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
                 fcntl.flock(f.fileno(), fcntl.LOCK_EX)
                 shutil.copy(src_path, dst_dir)
 
-    def _check(self):
+    def check(self):
         """check if the reg info need written"""
         if platform.system() != "Linux":
             return False
@@ -153,12 +156,12 @@ class _CustomInstaller:
         # attr
         attrs_name = []
         for _, item in enumerate(self.op_info.get("attr", [])):
-            attr_name = item.get("name")
+            attr_name = item.get(KEY_NAME)
             attrs_name.append(attr_name)
             key = "attr_" + attr_name
             op_info[key] = {}
             for k, v in item.items():
-                if k != "name":
+                if k != KEY_NAME:
                     op_info[key][k] = v
         if attrs_name:
             op_info["attr"] = {"list": ",".join(attrs_name)}
@@ -171,7 +174,7 @@ class _CustomInstaller:
             item = inputs[i] if i < input_num else outputs[i - input_num]
             key = "input" if i < input_num else "output"
             key += str(item.get("index"))
-            op_info[key] = {"name": item.get("name"),
+            op_info[key] = {KEY_NAME: item.get(KEY_NAME),
                             "paramType": item.get("paramType", "required"),
                             "shape": item.get("shape", "all")}
             dtype, formats = _get_dtype_format(i)
@@ -181,7 +184,8 @@ class _CustomInstaller:
                 op_info[key]["format"] = ",".join(formats)
         return op_info
 
-    def _gen_ai_cpu_reg_info(self, so_file):
+    @staticmethod
+    def _gen_ai_cpu_reg_info(so_file):
         """generate reg info"""
         op_info = {"opInfo": {"computeCost": "100",
                               "engine": "DNN_VM_AICPU",
@@ -198,7 +202,7 @@ class _CustomInstaller:
         repo = {}
         save_path = os.path.join(dst_dir, file_name)
         lock_file = os.path.join(dst_dir, "file.lock")
-        with open(lock_file, "w") as f:
+        with os.fdopen(os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
             fcntl.flock(f.fileno(), fcntl.LOCK_EX)
             if os.path.isfile(save_path):
                 with open(save_path, 'r') as fr:
@@ -211,7 +215,7 @@ class _CustomInstaller:
 
     def run(self):
         """save reg info to file"""
-        if not self._check():
+        if not self.check():
             return
         so_name = _get_reg_info_attr(self.op_info, "cust_aicpu")
         if so_name:
@@ -380,7 +384,6 @@ class RegOp:
         """
         if not isinstance(value, str):
             raise TypeError("%s value must be str" % str(value))
-        return True
 
     def _is_int(self, value):
         """
@@ -394,7 +397,6 @@ class RegOp:
         """
         if not isinstance(value, int):
             raise TypeError("%s value must be int" % str(value))
-        return True
 
     def _is_bool(self, value):
         """
@@ -408,7 +410,6 @@ class RegOp:
         """
         if not isinstance(value, bool):
             raise TypeError("%s value must be bool" % str(value))
-        return True
 
     @staticmethod
     def _is_list(value):
@@ -423,7 +424,6 @@ class RegOp:
         """
         if not isinstance(value, list):
             raise TypeError("%s value must be list" % str(value))
-        return True
 
     def _check_param(self, param_list, key_list, fn_list, kwargs):
         """
@@ -491,7 +491,9 @@ class RegOp:
             self._is_string(arg[1])
             if len(arg) == 3:
                 self._is_string(arg[2])
-            dtype_format.append(arg)
+                dtype_format.append(arg)
+            else:
+                dtype_format.append(arg)
         self.dtype_format_.append(tuple(dtype_format))
         return self
 
@@ -920,7 +922,8 @@ class TBERegOp(RegOp):
         Args:
             pattern (str): Value of op pattern, e.g. "broadcast", "reduce". Default: ``None`` .
         """
-        if pattern is not None and self._is_string(pattern):
+        if pattern is not None:
+            self._is_string(pattern)
             self.op_pattern_ = pattern
         return self
 

mindspore/ops/operations/__init__.py

@@ -118,7 +118,7 @@ from .nn_ops import (LSTM, SGD, Adam, AdamWeightDecay, FusedSparseAdam, FusedSpa
                      Dilation2D, DataFormatVecPermute, DeformableOffsets, Dense, FractionalAvgPool,
                      FractionalMaxPool, FractionalMaxPool3DWithFixedKsize, FractionalMaxPoolWithFixedKsize,
                      GridSampler2D, TripletMarginLoss, UpsampleNearest3D, UpsampleTrilinear3D, PadV3, ChannelShuffle,
-                     GLU, MaxUnpool3D, Pdist)
+                     GLU, MaxUnpool3D, Pdist, RmsNorm, PagedAttention, PagedAttentionMask, ReshapeAndCache)
 from .other_ops import (Assign, IOU, BoundingBoxDecode, BoundingBoxEncode,
                         ConfusionMatrix, UpdateState, Load, StopGradient,
                         CheckValid, Partial, Depend, Push, Pull, PyExecute, PyFunc, _DynamicLossScale,
@@ -691,7 +691,11 @@ __all__ = [
     "IndexPut",
     "MaskedScatter",
     "Ormqr",
-    "RandpermV2"
+    "RandpermV2",
+    "RmsNorm",
+    "PagedAttention",
+    "PagedAttentionMask",
+    "ReshapeAndCache"
 ]
 
 __custom__ = [

mindspore/ops/operations/_grad_ops.py

@@ -3845,7 +3845,7 @@ class FlashAttentionScoreGrad(Primitive):
     """
     @prim_attr_register
     def __init__(self, head_num, keep_prob=1.0, scale_value=1.0, pre_tokens=65536, next_tokens=65536, inner_precise=1,
-                 input_layout='BSH'):
+                 input_layout='BSH', sparse_mode=0):
         """Initialize FlashAttentionScoreGrad."""
         validator.check_value_type('head_num', head_num, [int], self.name)
         validator.check_value_type('keep_prob', keep_prob, [int, float], self.name)
@@ -3855,11 +3855,30 @@ class FlashAttentionScoreGrad(Primitive):
         validator.check_value_type('pre_tokens', pre_tokens, [int], self.name)
         validator.check_value_type('next_tokens', next_tokens, [int], self.name)
         validator.check_value_type('inner_precise', inner_precise, [int], self.name)
+        validator.check_value_type('sparse_mode', sparse_mode, [int], self.name)
         if inner_precise not in [0, 1]:
             raise ValueError(f"Attribute 'inner_precise' must be either 0 or 1, but got {inner_precise}")
         validator.check_value_type('input_layout', input_layout, [str], self.name)
-        if input_layout not in ["BSH"]:
-            raise ValueError(f"Attribute 'input_layout' must be either 'bsh' or 'sbh', but got {input_layout}")
-        self.init_prim_io_names(inputs=['query', 'key', 'value', 'attn_mask', 'attention_in', 'softmax_max',
-                                        'softmax_sum', 'dy', 'drop_mask', 'real_shift', "padding_mask", 'softmax_out'],
-                                outputs=['dq', 'dk', 'dv'])
+        if input_layout not in ["BSH", "BNSD"]:
+            raise ValueError(f"Attribute 'input_layout' must be either 'BSH' or 'BNSD', but got {input_layout}")
+        self.init_prim_io_names(inputs=['query', 'key', 'value', 'dy', 'pse_shift', 'drop_mask', "padding_mask",
+                                        'attn_mask', 'softmax_max', 'softmax_sum', 'softmax_out', 'attention_in',
+                                        'prefix'],
+                                outputs=['dq', 'dk', 'dv', 'dpse'])
+
+
+class RmsNormGrad(Primitive):
+    r"""
+    Calculates the gradient of RmsNorm operation.
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Supported Platforms:
+        ``Ascend``
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        """Initialize RmsNormGrad."""
+        self.init_prim_io_names(inputs=["dy", "x", "rstd", "gamma"],
+                                outputs=["dx", "dgamma"])