mindspore 2.0.0a0__cp39-cp39-win_amd64.whl → 2.0.0rc1__cp39-cp39-win_amd64.whl

mindspore/ops/function/clip_func.py

@@ -14,14 +14,23 @@
 # ============================================================================
 
 """Defines clip operators with functional form."""
+from __future__ import absolute_import
 
 from mindspore.ops import composite as C
 from mindspore.ops import operations as P
-from mindspore.ops._primitive_cache import _get_cache_prim
+from mindspore.ops import functional as F
+from mindspore.nn.cell import Cell
 from mindspore.common.tensor import Tensor
+from mindspore.common import dtype as mstype
+from mindspore.ops.primitive import constexpr
+from mindspore import _checkparam as Validator
+from mindspore.ops._primitive_cache import _get_cache_prim
 
 __all__ = [
     'clip_by_value',
+    'clamp',
+    'clip',
+    'clip_by_global_norm',
 ]
 
 hyper_map = C.HyperMap()
@@ -30,6 +39,9 @@ min_op = _get_cache_prim(P.Minimum)()
 cast_op = _get_cache_prim(P.Cast)()
 scalar2tensor_op = _get_cache_prim(P.ScalarToTensor)()
 partial_op = _get_cache_prim(P.Partial)()
+expand_dims = P.ExpandDims().add_prim_attr("grad_scale", True)
+get_square_sum = C.MultitypeFuncGraph("get_square_sum")
+apply_global_norm = C.MultitypeFuncGraph("apply_global_norm")
 
 
 def clip_by_value(x, clip_value_min=None, clip_value_max=None):
@@ -61,8 +73,7 @@ def clip_by_value(x, clip_value_min=None, clip_value_max=None):
 
     Args:
           x (Union(Tensor, list[Tensor], tuple[Tensor])): Input data, which type is Tensor or a list or tuple of Tensor.
-                                                         The shape of Tensor is :math:`(N,*)` where :math:`*` means,
-                                                         any number of additional dimensions.
+            Tensors of arbitrary dimensions are supported.
           clip_value_min (Union(Tensor, float, int)): The minimum value. Default: None.
           clip_value_max (Union(Tensor, float, int)): The maximum value. Default: None.
 
@@ -81,6 +92,7 @@ def clip_by_value(x, clip_value_min=None, clip_value_max=None):
 
     Examples:
         >>> # case 1: the data type of x is Tensor
+        >>> import mindspore
         >>> from mindspore import Tensor, ops
         >>> import numpy as np
         >>> min_value = Tensor(5, mindspore.float32)
@@ -96,15 +108,17 @@ def clip_by_value(x, clip_value_min=None, clip_value_max=None):
         >>> x = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
         >>> y = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
         >>> output = ops.clip_by_value([x,y], min_value, max_value)
-        >>> print(output)
-        [[[ 5. 20.  5.  7.]
-          [ 5. 11.  6. 20.]],
-         [[ 5. 20.  5.  7.]
-          [ 5. 11.  6. 20.]]]
+        >>> for out in output:
+        ...     print(out)
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
     """
     def _clip_by_value(clip_min, clip_max, x):
         if not isinstance(x, Tensor):
-            TypeError("Then type of 'x' must be Tensor")
+            raise TypeError("For 'clip_by_value', the type of argument 'x' must be "
+                            "Tensor or tuple[Tensor] or list[Tensor]")
         result = x
         if clip_min is not None:
             result = max_op(result, cast_op(clip_min, x.dtype))
@@ -113,13 +127,17 @@ def clip_by_value(x, clip_value_min=None, clip_value_max=None):
         return result
 
     if clip_value_min is None and clip_value_max is None:
-        ValueError("At least one of 'clip_value_min' or 'clip_value_max' must not be None")
+        raise ValueError("For 'clip_by_value', at least one of "
+                         "'clip_value_min' or 'clip_value_max' must not be None")
     if not isinstance(x, (Tensor, tuple, list)):
-        TypeError("The input of 'clip_by_value' must be tensor or tuple[Tensor] or list[Tensor]")
+        raise TypeError("For 'clip_by_value', the type of argument 'x' must be "
+                        "Tensor or tuple[Tensor] or list[Tensor]")
     if not isinstance(clip_value_min, (type(None), Tensor, float, int)):
-        TypeError("Then type of 'clip_value_min' must be not one of None, Tensor, float, int.")
+        raise TypeError("For 'clip_by_value', the type of argument 'clip_value_min' must be "
+                        "not one of None, Tensor, float, int")
     if not isinstance(clip_value_max, (type(None), Tensor, float, int)):
-        TypeError("Then type of 'clip_value_max' must be not one of None, Tensor, float, int.")
+        raise TypeError("For 'clip_by_value', the type of argument 'clip_value_max' must be "
+                        "not one of None, Tensor, float, int")
     if isinstance(clip_value_min, (float, int)):
         clip_value_min = scalar2tensor_op(clip_value_min)
     if isinstance(clip_value_max, (float, int)):
@@ -131,3 +149,181 @@ def clip_by_value(x, clip_value_min=None, clip_value_max=None):
     if isinstance(x, tuple):
         results = tuple(results)
     return results
+
+
+def clamp(input, min=None, max=None):
+    r"""
+    Clamps tensor values between the specified minimum value and maximum value.
+
+    Limits the value of :math:`input` to a range, whose lower limit is `min` and upper limit is `max` .
+
+    .. math::
+
+        out_i= \left\{
+        \begin{array}{align}
+            max & \text{ if } x_i\ge  max \\
+            x_i & \text{ if } min \lt x_i \lt max \\
+            min & \text{ if } x_i \le min \\
+        \end{array}\right.
+
+    Note:
+        - `min` and `max` cannot be None at the same time;
+        - When `min` is None and `max` is not None, the elements in Tensor larger than `max` will become `max`;
+        - When `min` is not None and `max` is None, the elements in Tensor smaller than `min` will become `min`;
+        - If `min` is greater than `max`, the value of all elements in Tensor will be set to `max`;
+        - The data type of `input`, `min` and `max` should support implicit type conversion and cannot be bool type.
+
+    Args:
+          input (Union(Tensor, list[Tensor], tuple[Tensor])): Input data, which type is Tensor or a list or tuple of
+            Tensor. Tensors of arbitrary dimensions are supported.
+          min (Union(Tensor, float, int), optional): The minimum value. Default: None.
+          max (Union(Tensor, float, int), optional): The maximum value. Default: None.
+
+    Returns:
+          Union(Tensor, tuple[Tensor], list[Tensor]), a clipped Tensor or a tuple or a list of clipped Tensor.
+          The data type and shape are the same as input.
+
+    Raises:
+          ValueError: If both `min` and `max` are None.
+          TypeError: If the type of `input` is not in Tensor or list[Tensor] or tuple[Tensor].
+          TypeError: If the type of `min` is not in None, Tensor, float or int.
+          TypeError: If the type of `max` is not in None, Tensor, float or int.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> # case 1: the data type of x is Tensor
+        >>> import mindspore
+        >>> from mindspore import Tensor, ops
+        >>> import numpy as np
+        >>> min_value = Tensor(5, mindspore.float32)
+        >>> max_value = Tensor(20, mindspore.float32)
+        >>> x = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> output = ops.clamp(x, min_value, max_value)
+        >>> print(output)
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+        >>> # case 2: the data type of x is list[Tensor]
+        >>> min_value = 5
+        >>> max_value = 20
+        >>> x = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> y = Tensor(np.array([[1., 25., 5., 7.], [4., 11., 6., 21.]]), mindspore.float32)
+        >>> output = ops.clamp([x,y], min_value, max_value)
+        >>> for out in output:
+        ...     print(out)
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+        [[ 5. 20.  5.  7.]
+         [ 5. 11.  6. 20.]]
+    """
+    return clip_by_value(input, min, max)
+
+
+def clip(x, min=None, max=None):
+    r"""
+    Alias for :func:`mindspore.ops.clamp` .
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+    """
+    return clamp(x, min, max)
+
+
+@get_square_sum.register("Tensor")
+def _get_square_sum(x):
+    norm = P.ReduceSum(False)(F.square(x), ())
+    norm = expand_dims(F.cast(norm, mstype.float32), 0)
+    return norm
+
+
+@apply_global_norm.register("Tensor", "Tensor", "Tensor")
+def _apply_global_norm(clip_norm, global_norm, x):
+    x_dtype = F.dtype(x)
+    x = x * clip_norm / global_norm
+    x = F.cast(x, x_dtype)
+    return x
+
+
+class _ClipByGlobalNorm(Cell):
+    r"""
+    Clips tensor values by the ratio of the sum of their norms.
+
+    Args:
+        clip_norm (Union(float, int)): The clipping ratio. Default: 1.0
+        use_norm (Union(float, None)): The global norm. Default: None
+
+    Inputs:
+        - **x** (Union(tuple[Tensor], list[Tensor])) - Input data to clip.
+
+    Outputs:
+        Tensor, a clipped Tensor.
+    """
+
+    def __init__(self, clip_norm=1.0, use_norm=None):
+        """Initialize _ClipByGlobalNorm."""
+        super(_ClipByGlobalNorm, self).__init__()
+        # Add interface. This parameter is not used at present
+        if use_norm is not None:
+            raise ValueError(f"For '{self.cls_name}', input 'use_norm' only supports None currently, "
+                             f"but got 'use_norm': {use_norm}")
+        Validator.check_number("clip_norm", clip_norm, 0.0, Validator.GT, self.cls_name)
+        self.clip_norm = Tensor([clip_norm], mstype.float32)
+        self.hyper_map = C.HyperMap()
+        self.greater_equal = P.GreaterEqual()
+
+    def construct(self, x):
+        square_sum = self.hyper_map(get_square_sum, x)
+        global_norm = F.sqrt(F.addn(square_sum))
+        cond = self.greater_equal(global_norm, self.clip_norm)
+        global_norm = F.select(cond, global_norm, self.clip_norm)
+        clip_x = self.hyper_map(F.partial(apply_global_norm, self.clip_norm, global_norm), x)
+        return clip_x
+
+
+@constexpr
+def _check_value(clip_norm):
+    Validator.check_number("clip_norm", clip_norm, 0.0, Validator.GT, "clip_by_global_norm")
+    return clip_norm
+
+
+def clip_by_global_norm(x, clip_norm=1.0, use_norm=None):
+    r"""
+    Clips tensor values by the ratio of the sum of their norms.
+
+    Note:
+        - Input `x` should be a tuple or list of tensors. Otherwise, it will raise an error.
+        - On the SEMI_AUTO_PARALLEL mode or AUTO_PARALLEL mode, if the input `x` is the gradient,
+          the gradient norm values on all devices will be automatically aggregated by allreduce inserted after
+          the local square sum of the gradients.
+
+    Args:
+        x (Union(tuple[Tensor], list[Tensor])): Input data to clip.
+        clip_norm (Union(float, int)): The clipping ratio, it should be greater than 0. Default: 1.0
+        use_norm (None): The global norm. Default: None. Currently only none is supported.
+
+    Returns:
+        tuple[Tensor], a clipped Tensor. It has the same data type as `x` and each Tensor in the output tuple is the
+        same as the original input shape.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> from mindspore import Tensor, ops
+        >>> import numpy as np
+        >>> x1 = np.array([[2., 3.], [1., 2.]]).astype(np.float32)
+        >>> x2 = np.array([[1., 4.], [3., 1.]]).astype(np.float32)
+        >>> input_x = (Tensor(x1), Tensor(x2))
+        >>> out = ops.clip_by_global_norm(input_x, 1.0)
+        >>> print(out)
+        (Tensor(shape=[2, 2], dtype=Float32, value=
+        [[ 2.98142403e-01,  4.47213590e-01],
+         [ 1.49071202e-01,  2.98142403e-01]]), Tensor(shape=[2, 2], dtype=Float32, value=
+        [[ 1.49071202e-01,  5.96284807e-01],
+         [ 4.47213590e-01,  1.49071202e-01]]))
+    """
+
+    clip_norm = _check_value(clip_norm)
+    clip_val = _ClipByGlobalNorm(clip_norm, use_norm)(x)
+    return clip_val

mindspore/ops/function/debug_func.py

@@ -34,7 +34,7 @@ def print_(*input_x):
         This function is used for debugging. When too much data is printed at the same time,
         in order not to affect the main process, the framework may discard some data. If you need to record the
         data completely, you are recommended to use the `Summary` function, and can check
-        `Summary <https://www.mindspore.cn/mindinsight/docs/en/r2.0.0-alpha/summary_record.html?highlight=summary#>`_.
+        `Summary <https://www.mindspore.cn/mindinsight/docs/en/r2.0/summary_record.html>`_.
 
     Args:
         input_x (Union[Tensor, bool, int, float, str, tuple, list]): The inputs of print_.
@@ -44,7 +44,7 @@ def print_(*input_x):
         Invalid value, should be ignored.
 
     Raises:
-        TypeError: If `input_x` is not one of the following: Tensor, bool, int, float, str.
+        TypeError: If `input_x` is not one of the following: Tensor, bool, int, float, str, tuple or list.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``

mindspore/ops/function/grad/__init__.py

@@ -26,7 +26,8 @@ from .grad_func import (
     jvp,
     vjp,
     linearize,
-    stop_gradient
+    stop_gradient,
+    get_grad
 )
 
 __all__ = []