mindspore 2.1.0__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/ops/operations/_inner_ops.py

@@ -23,16 +23,17 @@ from mindspore.common._stub_tensor import StubTensor
 from mindspore.ops import composite as C
 from mindspore.ops.operations.array_ops import Cast
 from mindspore.ops.operations._scalar_ops import bit_or, bit_and
+from mindspore.ops.operations.comm_ops import ReduceOp
 from mindspore.ops import signature as sig
 from mindspore.ops.operations.math_ops import _infer_shape_reduce
-from mindspore.ops.primitive import PrimitiveWithCheck, PrimitiveWithInfer, prim_attr_register, Primitive, _run_op
-from mindspore import context
+from mindspore.ops.primitive import PrimitiveWithCheck, PrimitiveWithInfer, prim_attr_register, Primitive,\
+    _run_op, _check_contains_variable
 from mindspore._c_expression import Tensor as Tensor_
 from mindspore._c_expression import typing
 from mindspore import _checkparam as validator
 from mindspore.common import dtype as mstype
 from mindspore.common.parameter import Parameter
-from mindspore.communication.management import GlobalComm
+from mindspore.communication.management import GlobalComm, get_rank
 from mindspore.common.api import _pynative_executor
 from mindspore.common._register_for_adapter import ms_adapter_registry
 from mindspore import ops
@@ -74,11 +75,11 @@ class ExtractImagePatches(Primitive):
             - valid: Means that the taken patch area must be completely covered in the original image.
 
     Inputs:
-        - **input_x** (Tensor) - A 4-D tensor whose shape is :math:`(in_batch, in_depth, in_row, in_col)`.
+        - **input_x** (Tensor) - A 4-D tensor whose shape is :math:`(in\_batch, in\_depth, in\_row, in\_col)`.
 
     Outputs:
         Tensor, a 4-D tensor whose data type is same as 'input_x', and the shape
-        is :math:`(out_batch, out_depth, out_row, out_col)`,where the out_batch is the same as the in_batch
+        is :math:`(out\_batch, out\_depth, out\_row, out\_col)`,where the out_batch is the same as the in_batch
         and
 
         .. math::
@@ -121,7 +122,6 @@ class ExtractImagePatches(Primitive):
         validator.check_value_type('padding', padding, [str], self.name)
         self.padding = validator.check_string(padding.upper(), ['VALID', 'SAME'], 'padding', self.name)
         self.add_prim_attr("padding", self.padding)
-        self.is_ge = context.get_context("enable_ge")
 
 
 class Quant(PrimitiveWithInfer):
@@ -502,6 +502,109 @@ class Receive(PrimitiveWithInfer):
         return self.get_attr_dict()['dtype']
 
 
+class Reduce(PrimitiveWithInfer):
+    """
+    Reduces tensor across the processes in the specified communication group.
+
+    Note:
+        Only process with destination rank receives the reduced output.
+        Other processes only get a tensor with shape [1], which has no mathematical meaning.
+
+    Args:
+        dest_rank (int): Specifies the rank of the process that receives the reduced output.
+        op (str, optional): Specifies an operation used for element-wise reductions, like sum, prod, max, and min.
+                On the CPU, only 'sum' is supported. Default: ``ReduceOp.SUM`` .
+        group (str, optional): The communication group to work on.
+            Default: "hccl_world_group" on Ascend, "nccl_world_group" on GPU.
+
+    Inputs:
+        - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`.
+
+    Examples:
+        >>> import mindspore.ops as ops
+        >>> import mindspore.nn as nn
+        >>> from mindspore.communication import init
+        >>> from mindspore import Tensor
+        >>> import numpy as np
+        >>> # Launch 4 processes.
+        >>> init()
+        >>> class ReduceNet(nn.Cell):
+        >>>     def __init__(self):
+        >>>         super(Net, self).__init__()
+        >>>         self.reduce = ops.Reduce(dest_rank=1)
+        >>>
+        >>>     def construct(self, x):
+        >>>         out = self.reduce(x)
+        >>>         return out
+        >>> input = Tensor(np.ones([2, 8]).astype(np.float32))
+        >>> net = ReduceNet()
+        >>> output = net(input)
+        >>> print(output)
+        Process with rank 1: [[4. 4. 4. 4. 4. 4. 4. 4.]
+                             [4. 4. 4. 4. 4. 4. 4. 4.]],
+        Other proesses: [0.].
+    """
+
+    @prim_attr_register
+    def __init__(self, dest_rank, op=ReduceOp.SUM, group=GlobalComm.WORLD_COMM_GROUP):
+        self.dest_rank = dest_rank
+        self.op = op
+        self.group = group
+
+    def infer_shape(self, x_shape):
+        # The process with dest_rank returns the reduced output.
+        # Other processes only gets a tensor with shape [1], which has no mathematical meaning.
+        if self.dest_rank == get_rank():
+            return x_shape
+        return [1]
+
+    def infer_dtype(self, x_dtype):
+        return x_dtype
+
+
+class Barrier(PrimitiveWithInfer):
+    """
+    Synchronizes all processes in the specified group.
+
+    Note:
+        After calling this collective operator,
+        this process will be blocked until all other processes in the group call this operator.
+
+    Args:
+        group (str, optional): The communication group to work on.
+            Default: "hccl_world_group" on Ascend, "nccl_world_group" on GPU.
+
+    Examples:
+        >>> import mindspore.ops as ops
+        >>> import mindspore.nn as nn
+        >>> from mindspore.communication import init
+        >>> from mindspore import Tensor
+        >>> import numpy as np
+        >>> # Launch 4 processes.
+        >>> init()
+        >>> class BarrierNet(nn.Cell):
+        >>>     def __init__(self):
+        >>>         super(Net, self).__init__()
+        >>>         self.barrier = ops.Barrier()
+        >>>
+        >>>     def construct(self):
+        >>>         self.barrier()
+        >>> net = BarrierNet()
+        >>> net()
+    """
+
+    @prim_attr_register
+    def __init__(self, group=GlobalComm.WORLD_COMM_GROUP):
+        self.group = group
+        self.add_prim_attr("side_effect_mem", True)
+
+    def infer_shape(self):
+        return [1]
+
+    def infer_dtype(self):
+        return mstype.float32
+
+
 class MatrixSetDiag(PrimitiveWithInfer):
     r"""
     Modifies the batched diagonal part of a batched tensor.
@@ -1843,16 +1946,32 @@ class Format(PrimitiveWithInfer):
     def __init__(self):
         self.init_prim_io_names(inputs=['string', 'args'], outputs=['string'])
 
+
     def __infer__(self, str_, *var):
-        str_value = str_["value"]
+        def check_variable(str_, var):
+            if _check_contains_variable(str_['dtype'], str_['value']):
+                return True
+
+            for item in var:
+                if _check_contains_variable(item['dtype'], item['value']):
+                    return True
+            return False
+
+
+        if check_variable(str_, var):
+            return {'dtype': mstype.string, 'shape': [], 'value': None}
+
+
+        str_value = str_['value']
+        kwargs = dict()
         var_value = list()
-        if str_value is None and str_["dtype"] is not None:
-            raise ValueError("str.format not support to input a variable.")
+
         for item in var:
-            if item["value"] is None and item["dtype"] is not None:
-                raise ValueError("str.format not support to input a variable.")
+            if isinstance(item["dtype"], typing.Keyword):
+                kwargs.update(item["value"])
             var_value.append(item["value"])
-        value = str_value.format(*var_value)
+
+        value = str_value.format(*var_value, **kwargs)
         return {'dtype': mstype.string, 'shape': [], 'value': value}
 
 
@@ -2588,3 +2707,85 @@ class IsConstant(Primitive):
 
     def __call__(self, x):
         return True
+
+
+class SelectView(Primitive):
+    r"""
+        Select tensor of view
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        self.init_prim_io_names(inputs=['input_tensor', 'input_indices', 'axis'], outputs=['output'])
+
+
+class CopyWithSlice(Primitive):
+    r"""
+        Copy data to discontinuous tensor
+    """
+    @prim_attr_register
+    def __init__(self):
+        self.add_prim_attr('side_effect_mem', True)
+        self.init_prim_io_names(inputs=['x', 'y'], outputs=['x'])
+
+
+class MoeFFN(Primitive):
+    r"""
+    The MoeFFN computation is similar to Feed-Forward Network, it contains matmul + gelu + matmul.
+
+    Args:
+        activation (string): The activation type, set to 'fastgelu' or 'gelu'.
+        Only support 'fastgelu' for now. Default: "fastgelu".
+
+    Inputs:
+        - **x** (Tensor) - The input tensor with data type of int8, float16.
+          Input tensor of shape :math:`(batch\_size * seq\_length, hidden\_size)`.
+        - **expert_tokens** (Tensor]) - The expert tokens tensor with data type of int64.
+          Expert tokens tensor of shape :math:`(16,)`. For example, `(2, 1, 0, .., 9)`
+          indicate that the 0th expert deals with 2 tokens, the 1th expert deals with 1 tokens,
+          the 2th expert do noting and so on.
+        - **weight1** (Tensor) - The weight1 tensor with data type of float16.
+          Weight1 tensor of shape :math:`(expert\_num, hidden\_size, ffn\_hidden\_size)`.
+        - **bias1** (Tensor) - The bias1 tensor with data type of float16.
+          Bias1 tensor of shape :math:`(expert\_num, ffn\_hidden\_size)`.
+        - **weight2** (Tensor) - The weight2 tensor with data type of float16.
+          Weight2 tensor of shape :math:`(expert\_num, ffn\_hidden\_size, hidden\_size)`.
+        - **bias2** (Tensor) - The bias2 tensor with data type of float16.
+          Bias2 tensor of shape :math:`(expert\_num, hidden\_size)`.
+        - **scale** (Tensor) - The scale tensor with data type of float16. Not enable now.
+        - **offset** (Tensor) - The offset tensor with data type of float16. Not enable now.
+        - **deq_scale1** (Tensor) - The deq_scale1 tensor with data type of float16. Not enable now.
+        - **deq_scale2** (Tensor) - The deq_scale2 tensor with data type of float16. Not enable now.
+
+    Outputs:
+        Tensor of shape :math:`(batch\_size * seq\_length, hidden\_size)`. With data type of float16.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.ops.operations import _inner_ops
+        >>> b = 4
+        >>> s = 128
+        >>> h = 1024
+        >>> h_f = 4 * h
+        >>> e = 16
+        >>> x = Tensor(np.random.randn(b * s, h).astype(np.float16))
+        >>> expert_tokens = Tensor(np.random.randn(e).astype(np.int64))
+        >>> w1 = Tensor(np.random.randn(e, h, h_f).astype(np.float16))
+        >>> bias1 = Tensor(np.random.randn(e, h_f).astype(np.float16))
+        >>> w2 = Tensor(np.random.randn(e, h_f, h).astype(np.float16))
+        >>> bias2 = Tensor(np.random.randn(e, h).astype(np.float16))
+        >>> moe_ffn = _inner_ops.MoeFFN("fastgelu")
+        >>> output = moe_ffn(x, w1, bias1, w2, bias2)
+        >>> print(output)
+    """
+
+    @prim_attr_register
+    def __init__(self, activation):
+        """Initialize MoeFFN."""
+        self.init_prim_io_names(inputs=["x", "expert_tokens", "weight1", "bias1",
+                                        "weight2", "bias2", "scale", "offset", "deq_scale1"
+                                        "deq_scale2"],
+                                outputs=["y"])
+        self.activation = activation

mindspore/ops/operations/_quant_ops.py

@@ -269,7 +269,7 @@ class FakeLearnedScaleQuantPerLayer(PrimitiveWithInfer):
         - **quant_max** (Tensor) : Value of the quantization range.
 
     Outputs:
-        - Tensor: Simulates quantize tensor of `input_x`，with the same type and shape as the `input_x`.
+        - Tensor: Simulates quantize tensor of `input_x`, with the same type and shape as the `input_x`.
 
     Examples:
         >>> input_tensor = Tensor(np.random.rand(3, 16, 5, 5), mstype.float32)
@@ -419,7 +419,7 @@ class FakeLearnedScaleQuantPerChannel(PrimitiveWithInfer):
         - **quant_max** (Tensor) : Value of the quantization range.
 
     Outputs:
-        - Tensor: Simulates quantize tensor of `input_x`，with the same type and shape as the `input_x`.
+        - Tensor: Simulates quantize tensor of `input_x`, with the same type and shape as the `input_x`.
 
     Examples:
         >>> input_tensor = Tensor(np.random.rand(3, 16, 5, 5), mstype.float32)
@@ -975,7 +975,7 @@ class FakeQuantPerChannel(PrimitiveWithInfer):
         >>> result = fake_quant(input_x, _min, _max)
     """
     support_quant_bit = [4, 7, 8]
-    ascend_support_x_rank = [2, 4]
+    ascend_support_x_rank = [2, 3, 4]
 
     @prim_attr_register
     def __init__(self,
@@ -1008,11 +1008,7 @@ class FakeQuantPerChannel(PrimitiveWithInfer):
         self.ema_decay = validator.check_float_range(ema_decay, 0, 1, validator.INC_BOTH, 'ema_decay', self.name)
         self.num_bits = validator.check_positive_int(num_bits, 'num_bits', self.name)
         self.quant_delay = validator.check_non_negative_int(quant_delay, 'quant_delay', self.name)
-        if self.is_ascend:
-            self.channel_axis = validator.check_int_range(channel_axis, 0, 1, validator.INC_BOTH,
-                                                          'channel_axis', self.name)
-        else:
-            self.channel_axis = validator.check_non_negative_int(channel_axis, 'channel_axis', self.name)
+        self.channel_axis = validator.check_non_negative_int(channel_axis, 'channel_axis', self.name)
         self.init_prim_io_names(inputs=['x', 'min', 'max'], outputs=['out'])
 
     def infer_shape(self, x_shape, min_shape, max_shape):

mindspore/ops/operations/_sequence_ops.py

@@ -196,6 +196,9 @@ class SequenceSlice(Primitive):
         """Initialize SequenceSlice"""
         self.init_prim_io_names(inputs=['seq', 'start', 'stop', 'step'], outputs=['output_data'])
 
+    def __call__(self, sequence, start, stop, step):
+        return sequence[start:stop:step]
+
 
 class InSequence(Primitive):
     r"""
@@ -223,6 +226,9 @@ class InSequence(Primitive):
         """Initialize ListAppend"""
         self.init_prim_io_names(inputs=['element', 'input_data'], outputs=['output_data'])
 
+    def __call__(self, target, sequence):
+        return target in sequence
+
 
 class SequenceSliceSetItem(Primitive):
     r"""
@@ -291,6 +297,9 @@ class SequenceAdd(Primitive):
         """Initialize SequenceAdd"""
         self.init_prim_io_names(inputs=['input_1', 'input_2'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x + y
+
 
 class SequenceAddOffset(Primitive):
     r"""
@@ -536,6 +545,9 @@ class SequenceCount(Primitive):
         """Initialize SequenceCount"""
         self.init_prim_io_names(inputs=['sequence', 'target'], outputs=['output_data'])
 
+    def __call__(self, sequence, target):
+        return sequence.count(target)
+
 
 class SequenceIndex(Primitive):
     r"""
@@ -566,6 +578,9 @@ class SequenceIndex(Primitive):
         """Initialize SequenceIndex"""
         self.init_prim_io_names(inputs=['sequence', 'target', 'start', 'end'], outputs=['output_data'])
 
+    def __call__(self, sequence, target, start=None, end=None):
+        return sequence.index(target, start, end)
+
 
 class SequenceMul(Primitive):
     r"""
@@ -594,6 +609,9 @@ class SequenceMul(Primitive):
         """Initialize SequenceMul"""
         self.init_prim_io_names(inputs=['sequence', 'scalar'], outputs=['output_data'])
 
+    def __call__(self, sequence, scalar):
+        return sequence * scalar
+
 
 class SequenceZerosLike(Primitive):
     r"""
@@ -946,6 +964,9 @@ class tuple_greater_than(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x > y
+
 
 class tuple_greater_equal(Primitive):
     r"""
@@ -974,6 +995,9 @@ class tuple_greater_equal(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x >= y
+
 
 class list_greater_than(Primitive):
     r"""
@@ -1002,6 +1026,9 @@ class list_greater_than(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x > y
+
 
 class list_greater_equal(Primitive):
     r"""
@@ -1030,6 +1057,9 @@ class list_greater_equal(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x >= y
+
 
 class tuple_lt(Primitive):
     r"""
@@ -1058,6 +1088,9 @@ class tuple_lt(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x < y
+
 
 class list_lt(Primitive):
     r"""
@@ -1086,6 +1119,9 @@ class list_lt(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x < y
+
 
 class tuple_le(Primitive):
     r"""
@@ -1114,6 +1150,9 @@ class tuple_le(Primitive):
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
 
+    def __call__(self, x, y):
+        return x <= y
+
 
 class list_le(Primitive):
     r"""
@@ -1141,3 +1180,6 @@ class list_le(Primitive):
         """Initialize list_le"""
         self.init_prim_io_names(
             inputs=['input_0', 'input_1'], outputs=['output_data'])
+
+    def __call__(self, x, y):
+        return x <= y