mindspore 2.4.0__cp310-cp310-macosx_11_0_arm64.whl

mindspore/ops/function/clip_func.py

@@ -0,0 +1,384 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Defines clip operators with functional form."""
+from __future__ import absolute_import
+
+from mindspore.ops import composite as C
+from mindspore.ops import operations as P
+from mindspore.ops import functional as F
+from mindspore.nn.cell import Cell
+from mindspore.common.tensor import Tensor
+from mindspore.common import dtype as mstype
+from mindspore.ops.primitive import _primexpr
+from mindspore.ops._primitive_cache import _get_cache_prim
+from mindspore import _checkparam as Validator
+from mindspore.ops.auto_generate import clamp_tensor, clamp_scalar
+
+__all__ = [
+    'clip_by_value',
+    'clip_by_norm',
+    'clamp',
+    'clip',
+    'clip_by_global_norm',
+]
+
+apply_global_norm = C.MultitypeFuncGraph("apply_global_norm")
+cast_op = P.Cast()
+expand_dims = P.ExpandDims().add_prim_attr("grad_scale", True)
+get_square_sum = C.MultitypeFuncGraph("get_square_sum")
+greater_equal_ = P.GreaterEqual()
+hyper_map = C.HyperMap()
+max_op = P.Maximum()
+min_op = P.Minimum()
+partial_op = P.Partial()
+scalar2tensor_op = P.ScalarToTensor()
+
+
+def _old_norm(norm_type, x):
+    """Add norm function"""
+    out = F.pow((F.reduce_sum(F.pow(x, norm_type))), 1. / norm_type).astype(x.dtype)
+    return out
+
+
+def _cal_total_norm(x, norm_type):
+    if norm_type == float('inf'):
+        func = lambda data: data.abs().max()
+        total_norm = max(hyper_map(func, x))
+    else:
+        total_norm = _old_norm(norm_type, F.stack(hyper_map(partial_op(_old_norm, norm_type), x)))
+    return total_norm
+
+
+def clip_by_norm(x, max_norm, norm_type=2.0, error_if_nonfinite=False):
+    r"""
+    Clip norm of a set of input Tensors. This norm is the result of calculating the norm of all elements in the input
+    separately, connecting them into a vector, and then calculating the norm.
+
+    Note:
+        The interface is suitable for gradient clipping scenarios, and only supports input of type float.
+
+    Args:
+          x (Union(Tensor, list[Tensor], tuple[Tensor])): Input that wishes to be clipped.
+          max_norm (Union(float, int)): The upper limit of the norm for this group of network parameters.
+          norm_type (Union(float, int)): Norm type. Default: ``2.0``.
+          error_if_nonfinite (bool): If it is ``True``, an exception is thrown if the total norm from the input
+              is nan, inf or -inf. If it is ``False``, no exception will be thrown.Default: ``False`` .
+
+    Returns:
+        Tensors, a list or tuple of Tensors, representing clipped Tensors.
+
+    Raises:
+          RuntimeError: If the total norm from the `x` is nan, inf or -inf.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import Tensor, ops
+        >>> x = Tensor([[0.8748, 0.1425, 0.0076], [0.7721, 0.4084, 0.0552], [4.6376, 0.2914, 2.1120]])
+        >>> out = ops.clip_by_norm(x, max_norm=1)
+        >>> print(out)
+        [[0.16650201 0.02712224 0.00144652]
+         [0.14695495 0.07773139 0.0105063 ]
+         [0.8826814  0.0554626  0.40198016]]
+    """
+    is_tensor = False
+    if isinstance(x, Tensor):
+        x = [x]
+        is_tensor = True
+    total_norm = _cal_total_norm(x, norm_type)
+    if error_if_nonfinite and F.logical_or(total_norm.isnan(), total_norm.isinf()):
+        raise RuntimeError(f"For clip_by_norm, the total norm of order {norm_type} from input is non-finite.")
+    clip_coef = max_norm / (total_norm + 1e-6)
+    if clip_coef < 1:
+        ret = hyper_map(partial_op(F.mul, clip_coef), x)
+    else:
+        ret = x
+    if is_tensor:
+        return ret[0]
+    return ret
+
+
+def clip_by_value(x, clip_value_min=None, clip_value_max=None):
+    r"""
+    Clips tensor values to a specified min and max.
+
+    Limits the value of :math:`x` to a range, whose lower limit is `clip_value_min`
+    and upper limit is `clip_value_max` .
+
+    .. math::
+
+        out_i= \left\{
+        \begin{array}{align}
+            clip\_value\_max & \text{ if } x_i\ge  clip\_value\_max \\
+            x_i & \text{ if } clip\_value\_min \lt x_i \lt clip\_value\_max \\
+            clip\_value\_min & \text{ if } x_i \le clip\_value\_min \\
+        \end{array}\right.
+
+    Note:
+        - `clip_value_min` and `clip_value_max` cannot be None at the same time;
+        - When `clip_value_min` is None and `clip_value_max` is not None, the elements in Tensor
+          larger than `clip_value_max` will become `clip_value_max`;
+        - When `clip_value_min` is not None and `clip_value_max` is None, the elements in Tensor
+          smaller than `clip_value_min` will become `clip_value_min`;
+        - If `clip_value_min` is greater than `clip_value_max`, the value of all elements in Tensor
+          will be set to `clip_value_max`;
+        - The data type of `x`, `clip_value_min` and `clip_value_max` should support implicit type
+          conversion and cannot be bool type.
+
+    Args:
+          x (Union(Tensor, list[Tensor], tuple[Tensor])): Input data, which type is Tensor or a list or tuple of Tensor.
+            Tensors of arbitrary dimensions are supported.
+          clip_value_min (Union(Tensor, float, int)): The minimum value. Default: ``None`` .
+          clip_value_max (Union(Tensor, float, int)): The maximum value. Default: ``None`` .
+
+    Returns:
+          (Union(Tensor, tuple[Tensor], list[Tensor])), a clipped Tensor or a tuple or a list of clipped Tensor.
+          The data type and shape are the same as x.
+
+    Raises:
+          ValueError: If both `clip_value_min` and `clip_value_max` are None.
+          TypeError: If the type of `x` is not in Tensor or list[Tensor] or tuple[Tensor].
+          TypeError: If the type of `clip_value_min` is not in None, Tensor, float or int.
+          TypeError: If the type of `clip_value_max` is not in None, Tensor, float or int.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> # case 1: the data type of x is Tensor
+        >>> import mindspore
+        >>> from mindspore import Tensor, ops
+        >>> import numpy as np
+        >>> min_value = Tensor(5, mindspore.float32)
+        >>> max_value = Tensor(20, mindspore.float32)
+        >>> x = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> output = ops.clip_by_value(x, min_value, max_value)
+        >>> print(output)
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+        >>> # case 2: the data type of x is list[Tensor]
+        >>> min_value = 5
+        >>> max_value = 20
+        >>> x = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> y = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> output = ops.clip_by_value([x,y], min_value, max_value)
+        >>> for out in output:
+        ...     print(out)
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+    """
+
+    def _clip_by_value(clip_min, clip_max, x):
+        if not isinstance(x, Tensor):
+            raise TypeError("For 'clip_by_value', the type of argument 'x' must be "
+                            "Tensor or tuple[Tensor] or list[Tensor]")
+        result = x
+        if clip_min is not None:
+            result = max_op(result, cast_op(clip_min, x.dtype))
+        if clip_max is not None:
+            result = min_op(result, cast_op(clip_max, x.dtype))
+        return result
+
+    if clip_value_min is None and clip_value_max is None:
+        raise ValueError("For 'clip_by_value', at least one of "
+                         "'clip_value_min' or 'clip_value_max' must not be None")
+    if not isinstance(x, (Tensor, tuple, list)):
+        raise TypeError("For 'clip_by_value', the type of argument 'x' must be "
+                        "Tensor or tuple[Tensor] or list[Tensor]")
+    if not isinstance(clip_value_min, (type(None), Tensor, float, int)):
+        raise TypeError("For 'clip_by_value', the type of argument 'clip_value_min' must be "
+                        "not one of None, Tensor, float, int")
+    if not isinstance(clip_value_max, (type(None), Tensor, float, int)):
+        raise TypeError("For 'clip_by_value', the type of argument 'clip_value_max' must be "
+                        "not one of None, Tensor, float, int")
+    if isinstance(clip_value_min, (float, int)):
+        clip_value_min = scalar2tensor_op(clip_value_min)
+    if isinstance(clip_value_max, (float, int)):
+        clip_value_max = scalar2tensor_op(clip_value_max)
+
+    if isinstance(x, Tensor):
+        return _clip_by_value(clip_value_min, clip_value_max, x)
+    results = hyper_map(partial_op(_clip_by_value, clip_value_min, clip_value_max), x)
+    if isinstance(x, tuple):
+        results = tuple(results)
+    return results
+
+
+def clamp(input, min=None, max=None):
+    r"""
+    Clamps tensor values between the specified minimum value and maximum value.
+
+    Limits the value of :math:`input` to a range, whose lower limit is `min` and upper limit is `max` .
+
+    .. math::
+
+        out_i= \left\{
+        \begin{array}{align}
+            max & \text{ if } input_i\ge max \\
+            input_i & \text{ if } min \lt input_i \lt max \\
+            min & \text{ if } input_i \le min \\
+        \end{array}\right.
+
+    Note:
+        - `min` and `max` cannot be None at the same time;
+        - When `min` is None and `max` is not None, the elements in Tensor larger than `max` will become `max`;
+        - When `min` is not None and `max` is None, the elements in Tensor smaller than `min` will become `min`;
+        - If `min` is greater than `max`, the value of all elements in Tensor will be set to `max`;
+        - The data type of `input`, `min` and `max` should support implicit type conversion and cannot be bool type.
+
+    Args:
+          input (Tensor): Input data, which type is Tensor. Tensors of arbitrary dimensions are supported.
+          min (Union(Tensor, float, int), optional): The minimum value. Default: ``None`` .
+          max (Union(Tensor, float, int), optional): The maximum value. Default: ``None`` .
+
+    Returns:
+          Tensor, a clipped Tensor.
+          The data type and shape are the same as input.
+
+    Raises:
+          ValueError: If both `min` and `max` are None.
+          TypeError: If the type of `input` is not in Tensor.
+          TypeError: If the type of `min` is not in None, Tensor, float or int.
+          TypeError: If the type of `max` is not in None, Tensor, float or int.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> # case 1: the data type of input is Tensor
+        >>> import mindspore
+        >>> from mindspore import Tensor, ops
+        >>> import numpy as np
+        >>> min_value = Tensor(5, mindspore.float32)
+        >>> max_value = Tensor(20, mindspore.float32)
+        >>> input = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> output = ops.clamp(input, min_value, max_value)
+        >>> print(output)
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+    """
+    if isinstance(min, Tensor) or isinstance(max, Tensor):
+        return clamp_tensor(input, min, max)
+    return clamp_scalar(input, min, max)
+
+def clip(input, min=None, max=None):
+    r"""
+    Alias for :func:`mindspore.ops.clamp` .
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+    """
+    return clamp(input, min, max)
+
+
+@get_square_sum.register("Tensor")
+def _get_square_sum(x):
+    norm = _get_cache_prim(P.ReduceSum)(False)(F.square(x), ())
+    norm = expand_dims(F.cast(norm, mstype.float32), 0)
+    return norm
+
+
+@apply_global_norm.register("Tensor", "Tensor", "Tensor")
+def _apply_global_norm(clip_norm, global_norm, x):
+    x_dtype = F.dtype(x)
+    x = x * clip_norm / global_norm
+    x = F.cast(x, x_dtype)
+    return x
+
+
+class _ClipByGlobalNorm(Cell):
+    r"""
+    Clips tensor values by the ratio of the sum of their norms.
+
+    Args:
+        clip_norm (Union(float, int)): The clipping ratio. Default: 1.0
+        use_norm (Union(float, None)): The global norm. Default: ``None``
+
+    Inputs:
+        - **x** (Union(tuple[Tensor], list[Tensor])) - Input data to clip.
+
+    Outputs:
+        Tensor, a clipped Tensor.
+    """
+
+    def __init__(self, clip_norm=1.0, use_norm=None):
+        """Initialize _ClipByGlobalNorm."""
+        super(_ClipByGlobalNorm, self).__init__()
+        # Add interface. This parameter is not used at present
+        if use_norm is not None:
+            raise ValueError(f"For '{self.cls_name}', input 'use_norm' only supports None currently, "
+                             f"but got 'use_norm': {use_norm}")
+        Validator.check_number("clip_norm", clip_norm, 0.0, Validator.GT, self.cls_name)
+        self.clip_norm = Tensor([clip_norm], mstype.float32)
+        self.hyper_map = C.HyperMap()
+
+    def construct(self, x):
+        square_sum = self.hyper_map(get_square_sum, x)
+        global_norm = F.sqrt(F.addn(square_sum))
+        cond = greater_equal_(global_norm, self.clip_norm)
+        global_norm = F.select(cond, global_norm, self.clip_norm)
+        clip_x = self.hyper_map(F.partial(apply_global_norm, self.clip_norm, global_norm), x)
+        return clip_x
+
+
+@_primexpr
+def _check_value(clip_norm):
+    Validator.check_number("clip_norm", clip_norm, 0.0, Validator.GT, "clip_by_global_norm")
+
+
+def clip_by_global_norm(x, clip_norm=1.0, use_norm=None):
+    r"""
+    Clips tensor values by the ratio of the sum of their norms.
+
+    Note:
+        - Input `x` should be a tuple or list of tensors. Otherwise, it will raise an error.
+        - On the SEMI_AUTO_PARALLEL mode or AUTO_PARALLEL mode, if the input `x` is the gradient,
+          the gradient norm values on all devices will be automatically aggregated by allreduce inserted after
+          the local square sum of the gradients.
+
+    Args:
+        x (Union(tuple[Tensor], list[Tensor])): Input data to clip.
+        clip_norm (Union(float, int)): The clipping ratio, it should be greater than 0. Default: ``1.0`` .
+        use_norm (None): The global norm. Default: ``None`` . Currently only none is supported.
+
+    Returns:
+        tuple[Tensor], a clipped Tensor. It has the same data type as `x` and each Tensor in the output tuple is the
+        same as the original input shape.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import Tensor, ops
+        >>> import numpy as np
+        >>> x1 = np.array([[2., 3.], [1., 2.]]).astype(np.float32)
+        >>> x2 = np.array([[1., 4.], [3., 1.]]).astype(np.float32)
+        >>> input_x = (Tensor(x1), Tensor(x2))
+        >>> out = ops.clip_by_global_norm(input_x, 1.0)
+        >>> print(out)
+        (Tensor(shape=[2, 2], dtype=Float32, value=
+        [[ 2.98142403e-01,  4.47213590e-01],
+         [ 1.49071202e-01,  2.98142403e-01]]), Tensor(shape=[2, 2], dtype=Float32, value=
+        [[ 1.49071202e-01,  5.96284807e-01],
+         [ 4.47213590e-01,  1.49071202e-01]]))
+    """
+
+    _check_value(clip_norm)
+    clip_val = _ClipByGlobalNorm(clip_norm, use_norm)(x)
+    return clip_val

mindspore/ops/function/debug_func.py

@@ -0,0 +1,181 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Operators for debug function."""
+
+from mindspore.ops.operations.debug_ops import Print
+from mindspore.common.tensor import Tensor
+from mindspore.ops import operations as P
+from .._primitive_cache import _get_cache_prim
+
+
+def print_(*input_x):
+    """
+    Outputs the inputs to stdout. The outputs are printed to screen by default.
+    It can also be saved in a file by setting the parameter  `print_file_path` in `context`.
+    Once set, the output will be saved in the file specified by print_file_path.
+    :func:`mindspore.parse_print` can be employed to reload the data.
+    For more information, please refer to :func:`mindspore.set_context` and :func:`mindspore.parse_print`.
+    In Ascend platform with graph mode, can set environment variables `MS_DUMP_SLICE_SIZE` and `MS_DUMP_WAIT_TIME`
+    to solve operator execution failure when outputting big tensor or outputting tensor intensively.
+
+    Note:
+        In pynative mode, please use python print function.
+        In Ascend platform with graph mode, the bool, int and float would be converted into Tensor to print, and
+        str remains unchanged.
+        This function is used for debugging.
+
+    Args:
+        input_x (Union[Tensor, bool, int, float, str, tuple, list]): The inputs of print_.
+            Supports multiple inputs which are separated by ','.
+
+    Returns:
+        Invalid value, should be ignored.
+
+    Raises:
+        TypeError: If `input_x` is not one of the following: Tensor, bool, int, float, str, tuple or list.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import numpy as np
+        >>> from mindspore import Tensor, ops
+        >>> x = Tensor(np.ones([2, 1]).astype(np.int32))
+        >>> y = Tensor(np.ones([2, 2]).astype(np.int32))
+        >>> result = ops.print_('Print Tensor x and Tensor y:', x, y)
+        Print Tensor x and Tensor y:
+        Tensor(shape=[2, 1], dtype=Int32, value=
+        [[1],
+         [1]])
+        Tensor(shape=[2, 2], dtype=Int32, value=
+        [[1, 1],
+         [1, 1]])
+    """
+    print_op = _get_cache_prim(Print)()
+    return print_op(*input_x)
+
+
+
+def tensordump(file_name, tensor, mode='out'):
+    """
+    Save Tensor in numpy's npy format.
+
+    In Parallel situation, tensordump will dump slice of data at each rank.
+
+    In Ascend platform with graph mode,
+    Your code OpA --> OpB may compiled as OpA --> RedistributionOps --> OpB.
+
+    Note: The redistribution operator is introduced,
+    Due to inter-device communication and shard strategies in the static graph parallel scenario.
+
+    In case of OpA --> OpB, the dump data of OpA's output is equal to OpB's input.
+
+    But in case of OpA --> RedistributionOps --> OpB,
+    The dump data of OpA's output is not equal to OpB's input (Due to the redistribution operators).
+    So the parameter mode is to handle this situation.
+
+    Assuming OpA's output is used as both tensordump's input parameter and OpB's input parameter.
+    Different requirements of saving dump data can be achieved by configuring parameter mode:
+
+    - If the mode is 'out', the dump data contains only OpA's output slice.
+    - If the mode is 'all', the dump data contains both OpA's output slice and OpB's input slice.
+    - If the mode is 'in', the dump data contains only OpB's input slice.
+
+    For mode 'all' or 'in', the input slice npy file format is: id_fileName_cNodeID_dumpMode_rankID.npy.
+
+    For mode 'out' or 'all' the output slice npy file format is: id_filename.npy.
+
+    - id: An auto increment ID.
+    - fileName: Value of the parameter file_name
+      (if parameter file_name is a user-specified path, the value of fileName is the last level of the path).
+    - cNodeID: The cnode ID in ir graph of step_parallel_end.ir.
+    - dumpMode: Value of the parameter mode.
+    - rankID: Logical device id.
+
+    Note:
+        - The operator of tensordump doesn't support in control flow.
+        - If current parallel mode is STAND_ALONE, mode should only be 'out'.
+        - Parameter mode will be set to 'out' if user doesn't configure it.
+        - This function is used for debugging.
+
+    Args:
+        file_name (str): The path of the npy file saves.
+        tensor (Tensor): The tensor that user want to dump.
+        mode (str, optional): Used to control tensordump behavior, available value is one of ['in', 'out', 'all'].
+            Default value is ``out``.
+
+    Raises:
+        TypeError: If `file_name` is not str.
+        TypeError: If `tensor` is not Tensor.
+        TypeError: If `mode` is not str.
+        ValueError: If `mode` is not in one of ['in', 'out', 'all'].
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Using msrun command to run below example: msrun --worker_num=2 --local_worker_num=2 --master_port=11450
+            --log_dir=msrun_log --join=True --cluster_time_out=300 tensordump_example.py
+
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import nn, Tensor, ops, context
+        >>> from mindspore.ops import operations as P
+        >>> from mindspore.communication import init, get_rank
+        >>> init()
+        >>> rank_id = get_rank()
+        >>> dump_path = f'dumps/rank_{rank_id}/mul1_mul2.npy'
+        >>> class Net(nn.Cell):
+        ...     def __init__(self, strategy1, strategy2):
+        ...         super(Net, self).__init__()
+        ...         self.matmul1 = P.MatMul().shard(strategy1)
+        ...         self.matmul2 = P.MatMul().shard(strategy2)
+        ...
+        ...     def construct(self, x, y, b):
+        ...         out1 = self.matmul1(x, y)
+        ...         ops.tensordump(dump_path, out1, 'all')
+        ...         out2 = self.matmul2(out1, b)
+        ...         return out2
+        ...
+        >>> ms.set_context(mode=ms.GRAPH_MODE, save_graphs=2)
+        >>> context.set_auto_parallel_context(parallel_mode='semi_auto_parallel', full_batch=True)
+        >>> strategy1 = ((1, 2), (2, 1))
+        >>> strategy2 = ((1, 2), (2, 1))
+        >>> net = Net(strategy1, strategy2)
+        >>> x = Tensor(0.1 * np.random.randn(64, 64).astype(np.float32))
+        >>> y = Tensor(0.1 * np.random.randn(64, 64).astype(np.float32))
+        >>> b = Tensor(0.1 * np.random.randn(64, 64).astype(np.float32))
+        >>> out = net(x, y, b)
+        >>> print(f"out shape is: {out.shape}")
+        >>> matmul1_output_slice = np.load('0_mul1_mul2.npy')                       # load matmul1's output slice
+        >>> matmul2_input_slice = np.load('1_mul1_mul2_CNode_64_all_rank_0.npy')    # load matmul2's input slice
+    """
+    if not isinstance(file_name, str):
+        raise TypeError(f"Parameter file_name should only be build_in str type but got: {type(file_name)}")
+    if not isinstance(tensor, Tensor):
+        raise TypeError(f"Parameter tensor should only be Tensor type, but got: {type(tensor)}")
+    if not isinstance(mode, str):
+        raise TypeError(f"Parameter mode should only be build_in str type, but got: {type(mode)}")
+    mode_list = ['out', 'in', 'all']
+    if mode not in mode_list:
+        raise ValueError(f"Parameter mode should in {mode_list}, but got {mode}")
+    _tensordump = _get_cache_prim(P.TensorDump)(input_output=mode)
+    return _tensordump(file_name, tensor)
+
+__all__ = ['print_', 'tensordump']
+
+__all__.sort()

mindspore/ops/function/fft_func.py

@@ -0,0 +1,44 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Defines Fast Fourier Transform operators with functional form."""
+from mindspore.ops.auto_generate import fft, fft2, fftn, ifft, ifft2, ifftn, fftshift, ifftshift, \
+    rfft, irfft, rfft2, irfft2, rfftn, irfftn, hfft, ihfft, hfft2, ihfft2, hfftn, ihfftn, fftfreq, rfftfreq
+
+__all__ = [
+    'fftshift',
+    'ifftshift',
+    'fft',
+    'fft2',
+    'fftn',
+    'ifft',
+    'ifft2',
+    'ifftn',
+    'rfft',
+    'irfft',
+    'rfft2',
+    'irfft2',
+    'rfftn',
+    'irfftn',
+    'hfft',
+    'ihfft',
+    'hfft2',
+    'ihfft2',
+    'hfftn',
+    'ihfftn',
+    'fftfreq',
+    'rfftfreq'
+]
+
+__all__.sort()

mindspore/ops/function/grad/__init__.py

@@ -0,0 +1,34 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Defines gradient related operators with functional form."""
+
+from __future__ import absolute_import
+from .grad_func import (
+    grad,
+    value_and_grad,
+    jacfwd,
+    jacrev,
+    jet,
+    derivative,
+    jvp,
+    vjp,
+    linearize,
+    stop_gradient,
+    get_grad
+)
+
+__all__ = []
+__all__.extend(grad_func.__all__)