mindspore 2.1.0__cp38-cp38-win_amd64.whl → 2.2.11__cp38-cp38-win_amd64.whl

mindspore/ops/composite/multitype_ops/_constexpr_utils.py

@@ -398,10 +398,16 @@ def slice2indices(input_slice, shape):
         return False
     ndim = len(shape)
     mesh = list()
-    grids = [P.Range()(P.Fill()(mstype.int64, (), start), P.Fill()(
-        mstype.int64, (), stop), P.Fill()(mstype.int64, (), step))]
-    grids += [P.Range()(Tensor(0, mstype.int64), P.Fill()(mstype.int64, (), dim_size),
-                        Tensor(1, mstype.int64)) for dim_size in shape[1:]]
+    range_op = P.Range()
+    cast_op = P.Cast()
+    grids = [
+        range_op(cast_op(start, mstype.int64), cast_op(stop, mstype.int64),
+                 cast_op(step, mstype.int64))
+    ]
+    grids += [
+        range_op(Tensor(0, mstype.int64), cast_op(dim_size, mstype.int64),
+                 Tensor(1, mstype.int64)) for dim_size in shape[1:]
+    ]
     for j, grid in enumerate(grids):
         mesh.append(P.Reshape()(grid, tuple(
             [grid.size if j == t else 1 for t in range(ndim)])))
@@ -543,7 +549,7 @@ def convert_scalar_to_tensor(data_shape, data_dtype, indices_shape, value, op_ty
         updates_shape = indices_shape + data_shape[1:]
     else:
         updates_shape = indices_shape[:-1] + data_shape[indices_shape[-1]:]
-    return P.Fill()(data_dtype, updates_shape, value)
+    return P.FillV2()(updates_shape, P.Cast()(value, data_dtype))
 
 
 def generate_updates_shape(data_shape, index_shape, op_type, is_dynamic):
@@ -844,7 +850,7 @@ def sequence_to_index(sequence, dim_size):
         return False
     if all(isinstance(i, bool) for i in sequence):
         if dim_size is None:
-            raise IndexError("Not supported to take the subscript of dynamic shape tensor using Boolean type")
+            return Tensor(sequence)
         seq_size = len(sequence)
         if seq_size != dim_size:
             raise IndexError(f'dimension is {dim_size} but corresponding boolean dimension is {seq_size}')
@@ -865,11 +871,11 @@ def int_to_index(i, shape):
     _check(i, dim_size)
     i = (i + dim_size) % dim_size
     if len(shape) == 1:
-        return P.Fill()(mstype.int64, (1, 1), i)
+        return P.FillV2()((1, 1), P.Cast()(i, mstype.int64))
     mesh = list()
     ndim = len(shape) - 1
     for j, size in enumerate(shape[1:]):
-        grid = P.Range()(Tensor(0, mstype.int64), P.Fill()(mstype.int64, (), size), Tensor(1, mstype.int64))
+        grid = P.Range()(Tensor(0, mstype.int64), P.Cast()(size, mstype.int64), Tensor(1, mstype.int64))
         mesh.append(P.Reshape()(grid, tuple([size if j == t else 1 for t in range(ndim)])))
     shapes = map(P.Shape(), mesh)
     out_shape = infer_out_shape(*shapes)
@@ -877,7 +883,8 @@ def int_to_index(i, shape):
     for arr in mesh:
         mesh_arrays.append(P.BroadcastTo(out_shape)(arr))
     index = P.Stack(-1)(mesh_arrays)
-    return P.Concat(-1)((P.Fill()(mstype.int64, P.Shape()(index)[:-1] + (1,), i), index))
+    return P.Concat(-1)((P.FillV2()(P.Shape()(index)[:-1] + (1,),
+                                    P.Cast()(i, mstype.int64)), index))
 
 
 @constexpr

mindspore/ops/composite/multitype_ops/add_impl.py

@@ -60,6 +60,26 @@ _tuple_add = _TupleAdd('tuple_add')
 """`_tuple_add` is an metafuncgraph object which will concatenate two tuples to form a tuple."""
 
 
+class _DictUpdate(base.DictUpdate_):
+    """
+    A metafuncgraph class that append another dict to the end of the dict.
+
+    Args:
+        name (str): The name of the metafuncgraph object.
+    """
+
+    def __init__(self, name):
+        """Initialize _DictUpdate."""
+        base.DictUpdate_.__init__(self, name)
+
+    def __call__(self, *args):
+        pass
+
+
+_dict_update = _DictUpdate('dict_update')
+"""`_dict_update` is an metafuncgraph object which will concatenate two dict to form a dict."""
+
+
 class _ListAdd(base.ListAdd_):
     """
     Adding two lists.
@@ -315,7 +335,8 @@ def _add_csrtensor(x, y):
     Returns:
         CSRTensor, consists of elements of x and elements of y.
     """
-    check_equal(x.shape, y.shape, "input1 (shape={}) and input2(shape={}) should be the same shape.")
+    check_equal(x.shape, y.shape,
+                "input1 (shape={}) and input2(shape={}) should be the same shape.")
     return F.csr_add(x, y, make_tensor(1, x.values.dtype), make_tensor(1, x.values.dtype))
 
 
@@ -331,7 +352,8 @@ def _add_cootensor(x, y):
     Returns:
         COOTensor, consists of elements of x and elements of y.
     """
-    check_equal(x.shape, y.shape, "input1 (shape={}) and input2(shape={}) should be the same shape.")
+    check_equal(x.shape, y.shape,
+                "input1 (shape={}) and input2(shape={}) should be the same shape.")
     return F.coo_add(x, y, make_tensor(0, x.values.dtype))
 
 
@@ -347,7 +369,8 @@ def _add_cootensor_tensor(x, y):
     Returns:
         Tensor, consists of elements of x and elements of y.
     """
-    check_equal(x.shape, y.shape, "input1 (shape={}) and input2(shape={}) should be the same shape.")
+    check_equal(x.shape, y.shape,
+                "input1 (shape={}) and input2(shape={}) should be the same shape.")
     return F.tensor_scatter_add(y, x.indices, x.values)
 
 
@@ -363,7 +386,8 @@ def _add_tensor_cootensor(x, y):
     Returns:
         Tensor, consists of elements of x and elements of y.
     """
-    check_equal(x.shape, y.shape, "input1 (shape={}) and input2(shape={}) should be the same shape.")
+    check_equal(x.shape, y.shape,
+                "input1 (shape={}) and input2(shape={}) should be the same shape.")
     return F.tensor_scatter_add(x, y.indices, y.values)
 
 
@@ -558,4 +582,19 @@ def _map_tensor_add_backward(x, y):
     return x
 
 
+@add.register("Dictionary", "Dictionary")
+def _dict_add_dict(x, y):
+    """
+        dictionary is added to dictionary.
+
+        Args:
+            x (dictionary): x
+            y (dictionary): y.
+
+        Returns:
+            dictionary, has the same dtype as x.
+    """
+    return _dict_update(x, y)
+
+
 hyper_add = base.HyperMap(_add_backward)

mindspore/ops/composite/multitype_ops/getitem_impl.py

@@ -106,7 +106,26 @@ _tuple_get_item_tensor = _TupleGetItemTensor('tuple_get_item_tensor')
 """_tuple_get_item_tensor is an metafuncgraph object which will select indexed item."""
 
 
-@getitem.register("Tuple", "Number")
+@getitem.register("Tuple", "Bool")
+def _tuple_getitem_by_bool(data, number_index):
+    """
+    Getting item of tuple by bool index.
+
+    Inputs:
+        data (tuple): A tuple to be sliced.
+        number_index (Number): Index in scalar.
+
+    Outputs:
+        Type, is the same as the element type of data.
+    """
+    if number_index:
+        return F.tuple_getitem(data, 1)
+    return F.tuple_getitem(data, 0)
+
+
+@getitem.register("Tuple", "Int")
+@getitem.register("Tuple", "UInt")
+@getitem.register("Tuple", "Float")
 def _tuple_getitem_by_number(data, number_index):
     """
     Getting item of tuple by number index.
@@ -142,7 +161,7 @@ def _tuple_getitem_by_slice(data, slice_index):
         if start is None:
             start = 0 if step >= 1 else -1
         if stop is None:
-            stop = (2**31-1) if step >= 1 else -(2**31-1)
+            stop = (2**31 - 1) if step >= 1 else -(2**31 - 1)
         return sequence_slice(data, start, stop, step)
     return _tuple_slice(data, slice_index)
 
@@ -162,7 +181,26 @@ def _tuple_getitem_by_tensor(data, tensor_index):
     return _tuple_get_item_tensor(data, tensor_index)
 
 
-@getitem.register("List", "Number")
+@getitem.register("List", "Bool")
+def _list_getitem_by_bool(data, number_index):
+    """
+    Getting item of list by bool index.
+
+    Inputs:
+        data (tuple): A list to be sliced.
+        number_index (Bool): Index in scalar.
+
+    Outputs:
+        Type, is the same as the element type of data.
+    """
+    if number_index:
+        return F.list_getitem(data, 1)
+    return F.list_getitem(data, 0)
+
+
+@getitem.register("List", "Int")
+@getitem.register("List", "UInt")
+@getitem.register("List", "Float")
 def _list_getitem_by_number(data, number_index):
     """
     Getting item of list by number index.
@@ -198,7 +236,7 @@ def _list_getitem_by_slice(data, slice_index):
         if start is None:
             start = 0 if step >= 1 else -1
         if stop is None:
-            stop = (2**31-1) if step >= 1 else -(2**31-1)
+            stop = (2**31 - 1) if step >= 1 else -(2**31 - 1)
         return sequence_slice(data, start, stop, step)
     return _list_slice(data, slice_index)
 

mindspore/ops/composite/multitype_ops/ones_like_impl.py

@@ -78,6 +78,12 @@ def _ones_like_func(x):
     return F.environ_create()
 
 
+@ones_like_leaf.register("None")
+def _ones_like_none(x):
+    """Returns none"""
+    return None
+
+
 ones_like = base.HyperMap(ones_like_leaf)
 """
 `ones_like` is a function which can generate a graph of `ones_like` operation according to input tensor dtype.

mindspore/ops/composite/multitype_ops/setitem_impl.py

@@ -22,6 +22,7 @@ from mindspore.ops.operations._inner_ops import SliceGetItem
 from mindspore.ops.operations import _map_tensor_ops
 from mindspore.ops.composite import base
 from mindspore.common import Tensor
+from mindspore.ops.composite.base import _dict_setitem
 from ...operations._sequence_ops import SequenceSliceSetItem
 
 DOC_URL = "https://mindspore.cn/search/en?inputValue=Index%20value%20assignment"
@@ -252,7 +253,7 @@ def _dict_setitem_with_tensor(data, key, value):
     Outputs:
         dict, type is as same as the element type of data.
     """
-    return F.dict_setitem(data, key, value)
+    return _dict_setitem(data, key, value)
 
 
 @setitem.register("Tensor", "Tensor", "Tensor")

mindspore/ops/composite/multitype_ops/zeros_like_impl.py

@@ -41,6 +41,15 @@ def _zeros_like_bool(x):
     """Returns False if x is a bool."""
     return False
 
+@zeros_like_leaf.register("String")
+def _zeros_like_string(x):
+    """Returns False if x is a string."""
+    return 0
+
+@zeros_like_leaf.register("Tuple")
+def _zeros_like_tuple(x):
+    """Returns False if x is a tuple."""
+    return 0
 
 @zeros_like_leaf.register("Function")
 def _zeros_like_func(x):

mindspore/ops/deprecated.py

@@ -0,0 +1,304 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Defines deprecated operators."""
+import itertools
+import numpy as np
+from mindspore.common._decorator import deprecated
+from mindspore import context
+from mindspore import _checkparam as validator
+from mindspore.ops import signature as sig
+from mindspore.ops.primitive import Primitive, prim_attr_register
+from mindspore.ops.operations.math_ops import _MathBinaryOp
+from mindspore.ops.operations.nn_ops import _check_positive_int_or_tuple
+
+
+class BNTrainingReduce(Primitive):
+    """
+    Please use BatchNorm instead.
+    """
+    @deprecated("1.5", "ops.BatchNorm", False)
+    @prim_attr_register
+    def __init__(self, data_format="NCHW"):
+        """Initialize BNTrainingReduce."""
+        super().__init__(name="BNTrainingReduce")
+        self.init_prim_io_names(inputs=['x'], outputs=['sum', 'square_sum'])
+        self.format = validator.check_string(data_format, ['NCHW', 'NHWC'], 'format', self.name)
+        if context.get_context("device_target") != "GPU" and self.format == "NHWC":
+            raise ValueError(f"For '{self.name}', the 'NHWC' format is only supported in GPU target, "
+                             f"but got the 'data_format' is {self.format} and "
+                             f"the platform is {context.get_context('device_target')}.")
+        self.add_prim_attr('data_format', self.format)
+
+
+class BNTrainingUpdate(Primitive):
+    """
+    Please use BatchNorm instead.
+    """
+    @deprecated("1.5", "ops.BatchNorm", False)
+    @prim_attr_register
+    def __init__(self, isRef=True, epsilon=1e-5, factor=0.1, data_format="NCHW"):
+        """Initialize BNTrainingUpdate."""
+        super().__init__(name="BNTrainingUpdate")
+        self.init_prim_io_names(inputs=['x', 'sum', 'square_sum', 'scale', 'b', 'mean', 'variance'],
+                                outputs=['y', 'running_mean', 'running_variance', 'save_mean', 'save_inv_variance'])
+        validator.check_value_type("isRef", isRef, [bool], self.name)
+        validator.check_value_type("epsilon", epsilon, [float], self.name)
+        validator.check_value_type("factor", factor, [float], self.name)
+        self.epsilon = validator.check_float_range(epsilon, 0, 1, validator.INC_RIGHT, 'epsilon', 'BNTrainingUpdate')
+        self.factor = validator.check_float_range(factor, 0, 1, validator.INC_BOTH, 'factor', 'BNTrainingUpdate')
+        self.format = validator.check_string(data_format, ['NCHW', 'NHWC'], 'format', self.name)
+        if context.get_context("device_target") != "GPU" and self.format == "NHWC":
+            raise ValueError(f"For '{self.name}', the 'NHWC' format is only supported in GPU target, "
+                             f"but got the 'data_format' is {self.format} and "
+                             f"the platform is {context.get_context('device_target')}.")
+        self.add_prim_attr('data_format', self.format)
+
+
+class MaxPoolWithArgmax(Primitive):
+    """
+    Please use MaxPoolWithArgmaxV2 instead.
+    """
+    @deprecated("2.0", "ops.MaxPoolWithArgmaxV2", False)
+    @prim_attr_register
+    def __init__(self, kernel_size=1, strides=1, pad_mode="valid", data_format="NCHW"):
+        """Initialize MaxPoolWithArgmax."""
+        super().__init__(name="MaxPoolWithArgmax")
+        self.init_prim_io_names(inputs=['x'], outputs=['output', 'mask'])
+        validator.check_value_type('kernel_size', kernel_size, [int, tuple], self.name)
+        validator.check_value_type('strides', strides, [int, tuple], self.name)
+        validator.check_value_type('pad_mode', pad_mode, [str], self.name)
+        self.pad_mode = validator.check_string(pad_mode.upper(), ['VALID', 'SAME'], 'pad_mode', self.name)
+        self.add_prim_attr("pad_mode", self.pad_mode)
+        self.format = validator.check_string(data_format, ['NCHW', 'NHWC'], 'format', self.name)
+        if context.get_context("device_target") != "GPU" and self.format == "NHWC":
+            raise ValueError(f"For '{self.name}', the 'NHWC' format is only supported in GPU target, "
+                             f"but got the 'data_format' is {self.format} and "
+                             f"the platform is {context.get_context('device_target')}.")
+        self.kernel_size = _check_positive_int_or_tuple(
+            "kernel_size", kernel_size, self.name, allow_four=False, ret_four=True)
+        self.kernel_size = (1, self.kernel_size[-2], self.kernel_size[-1], 1)
+        self.add_prim_attr("kernel_size", self.kernel_size)
+
+        self.strides = _check_positive_int_or_tuple("strides", strides, self.name, allow_four=False, ret_four=True)
+        self.strides = (1, self.strides[-2], self.strides[-1], 1)
+        self.add_prim_attr("strides", self.strides)
+
+
+class DropoutGenMask(Primitive):
+    """
+    Please use Dropout instead.
+    """
+    @deprecated("1.5", "ops.Dropout", False)
+    @prim_attr_register
+    def __init__(self, Seed0=0, Seed1=0):
+        """Initialize DropoutGenMask."""
+        super().__init__(name="DropoutGenMask")
+        self.init_prim_io_names(inputs=['shape', 'keep_prob'], outputs=['output'])
+        validator.check_value_type("Seed0", Seed0, [int], self.name)
+        validator.check_value_type("Seed1", Seed1, [int], self.name)
+        self.add_prim_attr("side_effect_hidden", True)
+
+
+class DropoutDoMask(Primitive):
+    """
+    Please use Dropout instead.
+    """
+    @deprecated("1.5", "ops.Dropout", False)
+    @prim_attr_register
+    def __init__(self):
+        super().__init__(name="DropoutDoMask")
+
+
+class Gelu(Primitive):
+    """
+    Please use GeLU instead.
+    """
+    @deprecated("1.1", "GeLU", True)
+    @prim_attr_register
+    def __init__(self):
+        """Initialize Gelu"""
+        super().__init__(name="Gelu")
+        self.init_prim_io_names(inputs=['x'], outputs=['output'])
+
+
+class FastGelu(Primitive):
+    """
+    Please use FastGeLU instead.
+    """
+    @deprecated("1.1", "FastGeLU", True)
+    @prim_attr_register
+    def __init__(self):
+        """Initialize FastGelu."""
+        super().__init__(name="FastGelu")
+        self.init_prim_io_names(inputs=['x'], outputs=['output'])
+
+
+class TensorAdd(_MathBinaryOp):
+    """
+    Please use Add instead.
+    """
+    @deprecated("1.1", "Add", True)
+    @prim_attr_register
+    def __init__(self):
+        """Initialize TensorAdd."""
+        _MathBinaryOp.__init__(self)
+
+
+class InplaceUpdate(Primitive):
+    """
+    Please use InplaceUpdateV2 instead.
+    """
+    @deprecated("2.0", "ops.InplaceUpdateV2", False)
+    @prim_attr_register
+    def __init__(self, indices):
+        """Initialize InplaceUpdate"""
+        self.init_prim_io_names(inputs=['x', 'v'], outputs=['y'])
+        self.indices = indices
+        validator.check_value_type("indices", indices, [int, tuple], self.name)
+        if isinstance(indices, int):
+            self.indices = (indices,)
+        for item in self.indices:
+            validator.check_value_type("item of indices", item, [int], self.name)
+
+
+class DynamicShape(Primitive):
+    """
+    Please use TensorShape instead.
+    """
+    @deprecated("1.7", "TensorShape", True)
+    @prim_attr_register
+    def __init__(self, dtype=9):
+        """init Shape"""
+        super().__init__(name="DynamicShape")
+        self.init_prim_io_names(inputs=['tensor'], outputs=['output'])
+        self.add_prim_attr('is_dynamic_shape', True)
+
+
+class GatherV2(Primitive):
+    """
+    Please use Gather instead.
+    """
+    @deprecated("1.1", "Gather", True)
+    @prim_attr_register
+    def __init__(self):
+        """Initialize GatherV2"""
+        super().__init__(name="GatherV2")
+        self.add_prim_attr("batch_dims", 0)
+        self.init_prim_io_names(inputs=['params', 'indices', 'axis'], outputs=['output'])
+
+
+class ScalarToArray(Primitive):
+    """
+    Please use scalar_to_tensor instead.
+    """
+    @deprecated("2.0", "ops.scalar_to_tensor", False)
+    @prim_attr_register
+    def __init__(self):
+        super().__init__(name="ScalarToArray")
+
+
+class Pack(Primitive):
+    """
+    Please use Stack instead.
+    """
+    @deprecated("1.1", "Stack", True)
+    @prim_attr_register
+    def __init__(self, axis=0):
+        """Initialize Pack"""
+        super().__init__(name="Pack")
+        validator.check_value_type("axis", axis, [int], self.name)
+        self.axis = axis
+
+
+class Unpack(Primitive):
+    """
+    Please use Unstack instead.
+    """
+    @deprecated("1.1", "Unstack", True)
+    @prim_attr_register
+    def __init__(self, axis=0):
+        """Initialize Unpack"""
+        super().__init__(name="Unpack")
+        validator.check_value_type("axis", axis, [int], self.name)
+        self.axis = axis
+
+
+class ScatterNonAliasingAdd(Primitive):
+    """
+    Please use TensorScatterAdd instead.
+    """
+    __mindspore_signature__ = (
+        sig.make_sig('input_x', sig.sig_rw.RW_WRITE, dtype=sig.sig_dtype.T),
+        sig.make_sig('indices', dtype=sig.sig_dtype.T1),
+        sig.make_sig('updates', dtype=sig.sig_dtype.T)
+    )
+    @deprecated("2.1", "ops.ScatterNonAliasingAdd", False)
+    @prim_attr_register
+    def __init__(self):
+        """Initialize ScatterNonAliasingAdd"""
+        super().__init__(name="ScatterNonAliasingAdd")
+        self.init_prim_io_names(inputs=['input_x', 'indices', 'updates'], outputs=['y'])
+        self.add_prim_attr('side_effect_mem', True)
+
+
+class BatchToSpaceND(Primitive):
+    """
+    Please use batch_to_space_nd instead.
+    """
+    @deprecated("2.0", "ops.batch_to_space_nd", False)
+    @prim_attr_register
+    def __init__(self, block_shape, crops):
+        """Initialize BatchToSpaceND"""
+        super().__init__(name="BatchToSpaceND")
+        if isinstance(block_shape, int):
+            block_shape = (block_shape,) * np.array(crops).shape[0]
+        self.add_prim_attr("block_shape", block_shape)
+        validator.check_value_type('block_shape type', block_shape, [list, tuple], self.name)
+        validator.check('block_shape shape', len(np.array(block_shape).shape), '', 1, validator.EQ, self.name)
+        block_rank = len(block_shape)
+        if context.get_context("device_target") == "Ascend":
+            validator.check('block_shape length', block_rank, '', 2, validator.EQ, self.name)
+        for elem in block_shape:
+            validator.check('block_shape element', elem, '', 1, validator.GE, self.name)
+            validator.check_value_type('block_shape element', elem, [int], self.name)
+        self.block_shape = block_shape
+
+        validator.check_value_type('crops type', crops, [list, tuple], self.name)
+        validator.check('crops length', len(crops), '', 1, validator.GE, self.name)
+        validator.check('crops shape', np.array(crops).shape, '', (block_rank, 2), validator.EQ, self.name)
+        for elem in itertools.chain(*crops):
+            validator.check_non_negative_int(elem, 'crops element', self.name)
+            validator.check_value_type('crops element', elem, [int], self.name)
+        self.crops = crops
+
+
+class identity(Primitive):
+    """
+    Please use side_effect_propagate instead.
+    """
+
+    # Side effect will propagated from the first argument to return value.
+    side_effect_propagate = 1
+
+    @prim_attr_register
+    def __init__(self):
+        """Initialize identity."""
+        super().__init__(name="identity")
+        self.add_prim_attr('side_effect_propagate', 1)
+
+    @deprecated('2.0', 'nn.Identity', False)
+    def __call__(self, x):
+        return x

mindspore/ops/function/__init__.py

@@ -34,7 +34,6 @@ from .array_func import (
     matrix_band_part,
     padding,
     fill,
-    fill_,
     full,
     full_like,
     chunk,
@@ -243,6 +242,7 @@ from .math_func import (
     exp,
     tensor_expm1,
     expm1,
+    eq,
     equal,
     not_equal,
     ne,
@@ -445,6 +445,7 @@ from .math_func import (
     vecdot,
     dot,
     batch_dot,
+    eps,
 )
 from .nn_func import (
     adaptive_avg_pool1d,
@@ -502,6 +503,7 @@ from .nn_func import (
     softmax,
     softmin,
     softshrink,
+    softplus,
     pdist,
     pad,
     prelu,
@@ -709,6 +711,7 @@ from .sparse_unary_func import (
 )
 from .clip_func import (
     clip_by_value,
+    clip_by_norm,
     clamp,
     clip,
     clip_by_global_norm,