mindspore 2.3.0__cp39-cp39-win_amd64.whl → 2.4.1__cp39-cp39-win_amd64.whl

mindspore/ops/operations/custom_ops.py

@@ -28,6 +28,8 @@ import subprocess
 import numpy as np
 import mindspore as ms
 from mindspore._c_expression import Oplib, typing
+from mindspore._c_expression import pyboost_custom_ext
+from mindspore.common._stub_tensor import _convert_stub
 from mindspore import context
 from mindspore.common import Tensor
 from mindspore.common import dtype as mstype
@@ -156,6 +158,55 @@ def _compile_aot(file):
     return func_path
 
 
+class _CustomExt(ops.PrimitiveWithInfer):
+    """
+    `Custom` primitive is used for PyBoost.
+    """
+
+    def __init__(self, func, out_shape=None, out_dtype=None, bprop=None):
+        super().__init__("CustomExt")
+        self.func = func
+        self.out_shape = out_shape
+        self.out_dtype = out_dtype
+        self.bprop = bprop
+
+    def __infer__(self, *args):
+        if callable(self.out_shape):
+            infer_shape = self.out_shape(*(x["shape"] for x in args))
+        else:
+            infer_shape = self.out_shape
+
+        if callable(self.out_dtype):
+            infer_dtype = self.out_dtype(*(x["dtype"] for x in args))
+        else:
+            infer_dtype = self.out_dtype
+
+        infer_value = None
+        if infer_shape is None:
+            logger.warning("'out_shape' is None. Add a placeholder instead. "
+                           "A CPP version of infer shape function is required "
+                           "in this case.")
+            infer_shape = (1,)
+        # after all automatic infer information fulfillment, throw error if infer_shape/infer_dtype is still None
+        if not isinstance(infer_shape, (tuple, list)):
+            raise TypeError("'out_shape' must be one of [tuple, list, function], but got {}".format(type(infer_shape)))
+
+        if not isinstance(infer_dtype, (typing.Type, tuple, list)):
+            raise TypeError("'out_dtype' must be one of [mindspore.dtype, tuple, list, function], but got {}"
+                            .format(type(infer_dtype)))
+
+        out = {
+            "shape": infer_shape,
+            "dtype": infer_dtype,
+            "value": infer_value,
+        }
+        return out
+
+    def get_bprop(self):
+        """return back propagation function"""
+        return self.bprop
+
+
 class Custom(ops.PrimitiveWithInfer):
     r"""
     `Custom` primitive is used for user defined operators and is to enhance the expressive ability of built-in
@@ -164,7 +215,7 @@ class Custom(ops.PrimitiveWithInfer):
     function if needed. Then these `Custom` objects can be directly used in neural networks.
     Detailed description and introduction of user-defined operators, including correct writing of parameters,
     please refer to `Custom Operators Tutorial
-    <https://www.mindspore.cn/tutorials/experts/en/master/operation/op_custom.html>`_ .
+    <https://www.mindspore.cn/docs/en/master/model_train/custom_program/op_custom.html>`_ .
 
     .. warning::
         - This is an experimental API that is subject to change.
@@ -174,7 +225,7 @@ class Custom(ops.PrimitiveWithInfer):
 
         - "hybrid": supports ["GPU", "CPU"].
         - "akg": supports ["GPU", "CPU"].
-        - "aot": supports ["GPU", "CPU", "ASCEDN"].
+        - "aot": supports ["GPU", "CPU", "Ascend"].
         - "pyfunc": supports ["CPU"].
         - "julia": supports ["CPU"].
 
@@ -249,20 +300,18 @@ class Custom(ops.PrimitiveWithInfer):
                        (ex. Custom(func="./reorganize.so:CustomReorganize", out_shape=[1], out_dtype=mstype.float32,
                        "aot"))
 
-                 b) ASCEND platform
-                 Before using Custom operators on the ASCEND platform, users must first develop custom operators
-                 based on Ascend C and compile them. For operator development, you can refer to the tutorial on
-                 `Quick Start for End-to-End Operator Development
-                 <https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/operatordev/Ascendcopdevg/atlas_ascendc_10_0022.html>`_,
-                 and for compiling custom operators, you can use the `Offline Compilation of Ascend C Custom Operators
-                 <https://www.mindspore.cn/tutorials/experts/en/master/operation/op_custom_ascendc.html>` tool.
-                 When passing the operator's name into the func parameter, taking AddCustom as an example for the
-                 name given in the custom operator implementation, there are several ways to use it:
-
-                 - Usin TBE: func="AddCustom"
-                 - Using AclNN: func="aclnnAddCustom"
-                 - Inferring the shape of the operator through C++ derivation: func="infer_shape.cc:aclnnAddCustom",
-                   where infer_shape.cc is the shape derivation implemented in C++.
+                 b) Ascend platform.
+                 Before using Custom operators on the Ascend platform, users must first develop custom operators
+                 based on Ascend C and compile them. The complete development and usage process can refer to the
+                 tutorial `AOT-Type Custom Operators(Ascend) <https://www.mindspore.cn/docs/en/master/model_train/custom_program/operation/op_custom_ascendc.html>`_.
+                 By passing the name of the operator through the input parameter `func`, there are two usage methods
+                 based on the implementation of the infer shape function:
+
+                 - Python infer: If the operator's infer shape is implemented in Python, that is, the infer shape
+                   function is passed through the `out_shape` parameter, specify `func="CustomName"` .
+                 - C++ infer: If the operator's infer shape is implemented through C++, then pass the path of the
+                   infer shape implementation file in `func` and separate the operator name with `:`,
+                   for example: `func="add_custom_infer.cc:AddCustom"` .
 
               2. for "julia":
 
@@ -338,7 +387,7 @@ class Custom(ops.PrimitiveWithInfer):
             or the attributes of `func` differs in different targets.
 
     Supported Platforms:
-        ``GPU`` ``CPU`` ``ASCEND``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
         >>> import numpy as np
@@ -457,6 +506,12 @@ class Custom(ops.PrimitiveWithInfer):
 
         self.add_prim_attr("func_type", self.func_type)
         self._update_attr()
+        self.enable_pyboost = False
+        self.custom_pyboost = _CustomExt(self.func, self.out_shape, self.out_dtype, self.bprop)
+        if context.get_context("device_target") == "Ascend" and self.func_type == "aot":
+            self.enable_pyboost = True
+            for key, value in super().get_attr_dict().items():
+                self.custom_pyboost.add_prim_attr(key, value)
 
     def __infer__(self, *args):
         if callable(self.out_shape):
@@ -559,7 +614,7 @@ class Custom(ops.PrimitiveWithInfer):
             raise TypeError(
                 "{}, 'func' should be like 'file_name:func_name', but got {}".format(
                     self.log_prefix, self.func))
-        file_path = os.path.abspath(file_name_list[0])
+        file_path = os.path.realpath(file_name_list[0])
         if os.environ.get('MS_CUSTOM_AOT_WHITE_LIST') is None:
             if Custom.custom_aot_warning:
                 logger.info("{}, no white list is set and it might cause problems. "
@@ -567,7 +622,7 @@ class Custom(ops.PrimitiveWithInfer):
                             .format(self.log_prefix))
                 Custom.custom_aot_warning = False
         else:
-            legal_path = os.path.abspath(os.environ.get('MS_CUSTOM_AOT_WHITE_LIST'))
+            legal_path = os.path.realpath(os.environ.get('MS_CUSTOM_AOT_WHITE_LIST'))
             if legal_path not in file_path:
                 raise TypeError(
                     "{}, the legal path for the file is {}, but the file is {}".format(
@@ -1063,3 +1118,12 @@ class Custom(ops.PrimitiveWithInfer):
         infer_value = Tensor(fake_output) if enable_infer_value else None
 
         return infer_shape, infer_dtype, infer_value
+
+    def __call__(self, *args):
+        if self.enable_pyboost:
+            return _convert_stub(pyboost_custom_ext(self.custom_pyboost, [args]))
+        should_elim, output = self.check_elim(*args)
+        if should_elim:
+            return output
+        # pylint: disable=protected-access
+        return ops.primitive._run_op(self, self.name, args)

mindspore/ops/operations/debug_ops.py

@@ -15,17 +15,17 @@
 """debug_ops"""
 import os
 import stat
-from types import FunctionType, MethodType
 
 import numpy as np
 from mindspore import log as logger
-from mindspore._c_expression import security
+from mindspore._c_expression import security, HookType
 from mindspore._c_expression import Tensor as Tensor_
 from mindspore import _checkparam as validator
 from mindspore.common import dtype as mstype
 from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
 from mindspore.ops.primitive import prim_attr_register, Primitive, PrimitiveWithInfer
+from mindspore._checkparam import check_hook_fn
 
 
 SUMMARY_TENSOR_CACHE = []
@@ -64,6 +64,8 @@ class ScalarSummary(Primitive):
     which specify the directory of the summary file. The summary file can
     be loaded and shown by MindInsight, see `MindInsight documents <https://www.mindspore.cn/
     mindinsight/docs/en/master/index.html>`_ for details.
+    In Ascend platform with graph mode, can set environment variables `MS_DUMP_SLICE_SIZE` and `MS_DUMP_WAIT_TIME`
+    to solve operator execution failure when calling this operator intensively.
 
     Inputs:
         - **name** (str) - The name of the input variable, it must not be an empty string.
@@ -122,6 +124,8 @@ class ImageSummary(Primitive):
     SummaryRecord or SummaryCollector, which specify the directory of the summary file. The summary file can
     be loaded and shown by MindInsight, see `MindInsight documents <https://www.mindspore.cn/
     mindinsight/docs/en/master/index.html>`_ for details.
+    In Ascend platform with graph mode, can set environment variables `MS_DUMP_SLICE_SIZE` and `MS_DUMP_WAIT_TIME`
+    to solve operator execution failure when calling this operator intensively.
 
     Inputs:
         - **name** (str) - The name of the input variable, it must not be an empty string.
@@ -173,6 +177,8 @@ class TensorSummary(Primitive):
     or SummaryCollector, which specify the directory of the summary file. The summary file can
     be loaded and shown by MindInsight, see `MindInsight documents <https://www.mindspore.cn/
     mindinsight/docs/en/master/index.html>`_ for details.
+    In Ascend platform with graph mode, can set environment variables `MS_DUMP_SLICE_SIZE` and `MS_DUMP_WAIT_TIME`
+    to solve operator execution failure when calling this operator intensively.
 
     Inputs:
         - **name** (str) - The name of the input variable.
@@ -228,9 +234,6 @@ class TensorDump(Primitive):
     """
     Save the Tensor as an npy file in numpy format.
 
-    The file name will automatically have a prefix added based on the execution order. For example, if `file` is `a`,
-    the first saved file will be named `0_a.npy`, and the second one will be named `1_a.npy`, and so on.
-
     .. warning::
         - If a large amount of data is stored within a short period, it may lead to memory overflow on the device side.
           Consider slicing the data to reduce the data scale.
@@ -238,6 +241,34 @@ class TensorDump(Primitive):
           too quickly, data loss may occur. You need to actively control the destruction time of the main process,
           such as using sleep.
 
+    Args:
+        input_output (str, optional): Used to control Tensordump behavior.
+            Available value is one of ['in', 'out', 'all']. Default value is ``out``.
+
+            In case of OpA --> RedistributionOps --> OpB,
+            The dump data of OpA's output is not equal to OpB's input (Due to the redistribution operators).
+            So the parameter input_output is to handle this situation.
+
+            Assuming OpA's output is used as both Tensordump's input parameter and OpB's input parameter.
+            Different requirements of saving dump data can be achieved by configuring parameter input_output:
+
+            - If the input_output is 'out', the dump data contains only OpA's output slice.
+            - If the input_output is 'all', the dump data contains both OpA's output slice and OpB's input slice.
+            - If the input_output is 'in', the dump data contains only OpB's input slice.
+
+            For input_output is 'all' or 'in', the input slice npy file format is:
+            id_fileName_cNodeID_dumpMode_rankID.npy.
+
+            For input_output is 'out' or 'all' the output slice npy file format is:
+            id_fileName.npy.
+
+            - id: An auto increment ID.
+            - fileName: Value of the parameter file
+              (if parameter file_name is a user-specified path, the value of fileName is the last level of the path).
+            - cNodeID: The node ID of the Tensordump node in the step_parallel_end.ir file.
+            - dumpMode: Value of the parameter input_output.
+            - rankID: Logical device id.
+
     Inputs:
         - **file** (str) - The path of the file to be saved.
         - **input_x** (Tensor) - Input Tensor of any dimension.
@@ -280,7 +311,7 @@ class TensorDump(Primitive):
          [6. 7. 8. 9.]]
     """
     @prim_attr_register
-    def __init__(self):
+    def __init__(self, input_output='out'):
         """Initialize TensorDump."""
         if security.enable_security():
             raise ValueError('The TensorDump is not supported, please without `-s on` and recompile source.')
@@ -314,6 +345,8 @@ class HistogramSummary(Primitive):
     It must be used with SummaryRecord or SummaryCollector, which specify the directory of the summary file.
     The summary file can be loaded and shown by MindInsight, see `MindInsight documents <https://www.mindspore.cn/
     mindinsight/docs/en/master/index.html>`_ for details.
+    In Ascend platform with graph mode, can set environment variables `MS_DUMP_SLICE_SIZE` and `MS_DUMP_WAIT_TIME`
+    to solve operator execution failure when calling this operator intensively.
 
     Inputs:
         - **name** (str) - The name of the input variable.
@@ -499,16 +532,15 @@ class HookBackward(PrimitiveWithInfer):
     def __init__(self, hook_fn, cell_id=""):
         """Initialize HookBackward."""
         super(HookBackward, self).__init__(self.__class__.__name__)
-        if not isinstance(hook_fn, (FunctionType, MethodType)):
-            raise TypeError(f"For '{self.name}', the type of 'hook_fn' must be python function, "
-                            f"but got {type(hook_fn)}.")
+        if not check_hook_fn("HookBackward", hook_fn):
+            return
         if cell_id != "":
             logger.warning(f"The args 'cell_id' of HookBackward will be removed in a future version. If the value of "
                            f"'cell_id' is set, the hook function will not work.")
         self.add_prim_attr("cell_id", cell_id)
         self.init_attrs["cell_id"] = cell_id
         self.cell_id = cell_id
-        self.add_backward_hook_fn(hook_fn)
+        self.set_hook_fn(hook_fn, HookType.HookBackward)
 
     def infer_shape(self, *inputs_shape):
         if len(inputs_shape) == 1:

mindspore/ops/operations/manually_defined/_inner.py

@@ -23,6 +23,7 @@ from mindspore.common import dtype as mstype
 from mindspore import _checkparam as validator
 from mindspore.common._decorator import deprecated
 from mindspore.ops.primitive import prim_attr_register, Primitive
+from mindspore import log as logger
 
 
 class ScalarCast(Primitive):
@@ -59,3 +60,14 @@ class ScalarCast(Primitive):
         value = np.cast[np_dtype.lower()](input_x)
         value = value.item()
         return value
+
+
+class TensorReport(Primitive):
+    @prim_attr_register
+    def __init__(self):
+        """Initialize TensorReport"""
+        self.add_prim_attr("side_effect_io", True)
+        self.add_prim_attr("channel_name", "ms_tensor_report")
+
+    def __call__(self, file, input_x):
+        logger.warning("TensorReport doesn't support pynative mode.")

mindspore/ops/operations/manually_defined/ops_def.py

@@ -18,6 +18,7 @@ from __future__ import division
 
 import numbers
 import math
+import types
 import numpy as np
 from mindspore.ops import signature as sig
 from mindspore.ops.primitive import Primitive, prim_attr_register, prim_arg_register, PrimitiveWithInfer
@@ -937,6 +938,10 @@ class Tile(Primitive):
 
     Refer to :func:`mindspore.ops.tile` for more details.
 
+    Note:
+        On Ascend, the number of `dims` should not exceed 8, and currently does not support scenarios
+        where more than 4 dimensions are repeated simultaneously.
+
     Inputs:
         - **input** (Tensor) - The tensor whose elements need to be repeated. Set the shape of input tensor as
           :math:`(x_1, x_2, ..., x_S)` .
@@ -1025,6 +1030,10 @@ def tile(input, dims):
     output tensor has `input.shape[i] * dims[i]` elements, and the values of `input`
     are replicated `dims[i]` times along the i'th dimension.
 
+    Note:
+        On Ascend, the number of `dims` should not exceed 8, and currently does not support scenarios
+        where more than 4 dimensions are repeated simultaneously.
+
     Args:
         input (Tensor): The tensor whose elements need to be repeated. Set the shape of input tensor as
             :math:`(x_1, x_2, ..., x_S)` .
@@ -1127,16 +1136,16 @@ class Cast(Primitive):
         taken into account. As long as the real part is non-zero, it returns True; otherwise, it returns False.
 
     Inputs:
-        - **input_x** (Union[Tensor, Number]) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`.
+        - **input** (Union[Tensor, Number]) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`.
           The tensor to be cast.
-        - **type** (dtype.Number) - The valid data type of the output tensor. Only constant value is allowed.
+        - **dtype** (dtype.Number) - The valid data type of the output tensor. Only constant value is allowed.
 
     Outputs:
-        Tensor, the shape of tensor is the same as `input_x`, :math:`(x_1, x_2, ..., x_R)`.
+        Tensor, the shape of tensor is the same as `input`, :math:`(x_1, x_2, ..., x_R)`.
 
     Raises:
-        TypeError: If `input_x` is neither Tensor nor Number.
-        TypeError: If `type` is not a Number.
+        TypeError: If `input` is neither Tensor nor Number.
+        TypeError: If `dtype` is not a Number.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1146,10 +1155,10 @@ class Cast(Primitive):
         >>> import numpy as np
         >>> from mindspore import Tensor, ops
         >>> input_np = np.random.randn(2, 3, 4, 5).astype(np.float32)
-        >>> input_x = Tensor(input_np)
-        >>> type_dst = mindspore.int32
+        >>> input = Tensor(input_np)
+        >>> dtype = mindspore.int32
         >>> cast = ops.Cast()
-        >>> output = cast(input_x, type_dst)
+        >>> output = cast(input, dtype)
         >>> print(output.dtype)
         Int32
         >>> print(output.shape)
@@ -1162,17 +1171,15 @@ class Cast(Primitive):
         self.init_prim_io_names(inputs=['x', 'dst_type'], outputs=['output'])
 
     def check_elim(self, x, dtype):
-        if isinstance(x, (Tensor, numbers.Number, Parameter)):
-            if isinstance(x, Parameter):
-                data = x.data
-                if data.dtype == dtype:
-                    return (True, x)
-            if isinstance(x, Tensor) and x.dtype == dtype:
-                x = Tensor(x)
-                x.set_cast_dtype()
+        if isinstance(x, Parameter):
+            data = x.data
+            if data.dtype == dtype:
                 return (True, x)
-            if isinstance(x, numbers.Number):
-                return (True, Tensor(x, dtype=dtype))
+        if isinstance(x, Tensor) and x.dtype == dtype:
+            x.set_cast_dtype()
+            return (True, x)
+        if isinstance(x, numbers.Number):
+            return (True, Tensor(x, dtype=dtype))
         return (False, None)
 
     def __call__(self, input_x, dtype):
@@ -1187,7 +1194,7 @@ def to_sequence(val):
     to_sequence
     """
     if isinstance(val, (tuple, list)):
-        return val
+        return tuple(val)
     return (val,)
 
 
@@ -1891,7 +1898,7 @@ def flash_attention_score(query, key, value, head_num, real_shift=None, drop_mas
     H2 -- Hidden size of key and value, which equals to N2 * D.
 
     .. warning::
-        This is an experimental API that is subject to change or deletion. Only support on Atlas training series.
+        This is an experimental API that is subject to change or deletion. Only support on Atlas A2 training series.
 
     Args:
         query (Tensor[float16, bfloat16]): The query tensor. Input tensor of shape :math:`(B, S1, H1)`,
@@ -2014,3 +2021,249 @@ def flash_attention_score(query, key, value, head_num, real_shift=None, drop_mas
                                                    inner_precise, input_layout, sparse_mode)
     return rank_op(query, key, value, real_shift, drop_mask, padding_mask, attn_mask, prefix, actual_seq_qlen,
                    actual_seq_kvlen)[3]
+
+
+class WhileLoop(Primitive):
+    """
+    Provide a useful op for reducing compilation times of while loop.
+    The execution logic of the WhileLoop operator can be roughly represented by the following code:
+
+    .. code-block:: python
+
+        def WhileLoop(cond_func, loop_func, init_val):
+            while(cond_func(init_val)):
+                init_val = loop_func(init_val)
+            return init_val
+
+    The current WhileLoop operator has the following syntactic limitations:
+
+    - Using a side-effect function as `loop_func` is currently not support,
+      such as operations that modify parameters, global variables, etc.
+    - The return value of `loop_func` being of a different type or shape
+      from the `init_val` is currently not support.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Inputs:
+        - **cond_func** (Function) - The condition function.
+        - **loop_func** (Function) - The loop function, take one argument and
+          return value has the same type with input argument.
+        - **init_val** (Union[Tensor, number, str, bool, list, tuple, dict]) - The initial value.
+
+    Outputs:
+        Union[Tensor, number, str, bool, list, tuple, dict], the final result of the while loop,
+        has same type and shape with input `init_val` .
+
+    Raises:
+        TypeError: If `cond_func` is not a function.
+        TypeError: If `loop_func` is not a function.
+        ValueError: If `loop_func` cannot take `init_val` as input or has different
+                    output type or shape with `init_val` .
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import ops
+        >>> def loop_while_fun(init_val):
+        ...     val = init_val
+        ...     val = val + 1
+        ...     return val
+        ...
+        >>> init_state = 10
+        >>> while_loop = ops.WhileLoop()
+        >>> result = while_loop(lambda x : x < 100, loop_while_fun, init_state)
+        >>> print(result)
+        100
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        """Initialize WhileLoop."""
+
+    def __call__(self, cond_func, loop_func, init_val):
+        validator.check_value_type("cond_func", cond_func,
+                                   [types.FunctionType, types.MethodType], "WhileLoop")
+        validator.check_value_type("loop_func", loop_func,
+                                   [types.FunctionType, types.MethodType], "WhileLoop")
+        val = init_val
+        try:
+            while cond_func(val):
+                val = loop_func(val)
+        except Exception as e:
+            raise ValueError("Invalid loop_func, please check input arguments and \
+                             return value, error info: {}".format(e))
+        return val
+
+
+class Scan(Primitive):
+    """
+    Scan a function over an array while the processing of the current element
+    depends on the execution result of the previous element.
+    The execution logic of the Scan operator can be roughly represented by the following code:
+
+    .. code-block:: python
+
+        def Scan(loop_func, init, xs, length=None):
+            if xs is None:
+                xs = [None] * length
+            carry = init
+            ys = []
+            for x in xs:
+                carry, y = loop_func(carry, x)
+                ys.append(y)
+            return carry, ys
+
+    The current Scan operator has the following syntactic limitations:
+
+    - Using a side-effect function as `loop_func` is currently not support,
+      such as operations that modify parameters, global variables, etc.
+    - The first element of the return value of `loop_func` being of a different
+      type or shape from the `init_val` is currently not support.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Inputs:
+        - **loop_func** (Function) - The loop function.
+        - **init** (Union[Tensor, number, str, bool, list, tuple, dict]) - An initial loop carry value.
+        - **xs** (Union[tuple, list, None]) - The value over which to scan.
+        - **length** (Union[int, None], optional) - The size of xs. Default: ``None`` .
+        - **unroll** (bool, optional) - The flag for whether to perform loop unrolling in compile process.
+          Default: ``True`` .
+
+    Outputs:
+        Tuple(Union[Tensor, number, str, bool, list, tuple, dict], list). Output of scan loop,
+        a tuple with two elements, the first element has same type and shape with init argument,
+        and the second is a list containing the results of each loop.
+
+    Raises:
+        TypeError: If `loop_func` is not a function.
+        TypeError: If `xs` is not a tuple, a list or None.
+        TypeError: If `length` is not an int or None.
+        TypeError: If `unroll` is not a bool.
+        ValueError: If `loop_func` cannot take `init` and element of `xs` as inputs.
+        ValueError: If the return value of `loop_func` is not a tuple with two elements,
+                    or the first element of the tuple has different type or shape from `init` .
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import ops
+        >>> def cumsum(res, el):
+        ...     res = res + el
+        ...     return res, res
+        ...
+        >>> a = [1, 2, 3, 4]
+        >>> result_init = 0
+        >>> scan_op = ops.Scan()
+        >>> result = scan_op(cumsum, result_init, a)
+        >>> print(result == (10, [1, 3, 6, 10]))
+        True
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        """Initialize Scan."""
+
+    def __call__(self, loop_func, init, xs, length=None, unroll=True):
+        validator.check_value_type("loop_func", loop_func,
+                                   [types.FunctionType, types.MethodType], "Scan")
+        validator.check_value_type("xs", xs, [list, tuple, None], "Scan")
+        if xs is None:
+            validator.check_value_type("length", length, [int], "Scan")
+            xs = [None] * length
+        carry = init
+        length = len(xs)
+        if not length:
+            return init, []
+        try:
+            carry, y = loop_func(carry, xs[0])
+            ys = [y]
+            i = 1
+            while i < length:
+                carry, y = loop_func(carry, xs[i])
+                ys.append(y)
+                i = i + 1
+        except Exception as e:
+            raise ValueError("Invalid loop_func, please check input arguments and \
+                             return value, error info: {}".format(e))
+        return carry, ys
+
+
+class ForiLoop(Primitive):
+    """
+    Provide a useful op for loop from lower to upper.
+    The execution logic of the ForiLoop operator can be roughly represented by the following code:
+
+    .. code-block:: python
+
+        def ForiLoop(lower, upper, loop_func, init_val):
+            for i in range(lower, upper):
+                init_val = loop_func(i, init_val)
+            return init_val
+
+    The current ForiLoop operator has the following syntactic limitations:
+
+    - Using a side-effect function as `loop_func` is currently not support,
+      such as operations that modify parameters, global variables, etc.
+    - The return value of `loop_func` being of a different type or shape
+      from the `init_val` is currently not support.
+    - Negative numbers or custom increments is currently not support.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Inputs:
+        - **lower** (Union[int, Tensor]) - The start index of loop.
+        - **upper** (Union[int, Tensor]) - The end index of loop.
+        - **loop_func** (Function) - The loop function, takes two arguments.
+        - **init_val** (Union[Tensor, number, str, bool, list, tuple, dict]) - The init value.
+        - **unroll** (bool, optional) - The flag for whether unroll in compile process,
+          only valid when the number of loop iterations is determined. Default: ``True`` .
+
+    Outputs:
+        Union[Tensor, number, str, bool, list, tuple, dict], the final result of the loop,
+        has same type and shape with input `init_val` .
+
+    Raises:
+        TypeError: If `lower` is not an int or a Tensor.
+        TypeError: If `upper` is not an int or a Tensor.
+        TypeError: If `loop_func` is not a function.
+        ValueError: If `loop_func` cannot take index and `init_val` as arguments or if the type
+                    of output it produces is different from the type or shape of `init_val` .
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import ops
+        >>> def cumsum(index, res):
+        ...     return index + res
+        ...
+        >>> result_init = 0
+        >>> fori_loop = ops.ForiLoop()
+        >>> result = fori_loop(0, 4, cumsum, result_init)
+        >>> print(result)
+        6
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        """Initialize ForiLoop."""
+
+    def __call__(self, lower, upper, loop_func, init_val, unroll=True):
+        validator.check_value_type("lower", lower, [int, Tensor], "ForiLoop")
+        validator.check_value_type("upper", upper, [int, Tensor], "ForiLoop")
+        validator.check_value_type("loop_func", loop_func,
+                                   [types.FunctionType, types.MethodType], "ForiLoop")
+        val = init_val
+        try:
+            for i in range(lower, upper):
+                val = loop_func(i, val)
+        except Exception as e:
+            raise ValueError("Invalid loop_func, please check input arguments and \
+                             return value, error info: {}".format(e))
+        return val