mindspore 2.1.0__cp37-cp37m-manylinux1_x86_64.whl → 2.2.10__cp37-cp37m-manylinux1_x86_64.whl

mindspore/ops/operations/comm_ops.py

@@ -52,12 +52,15 @@ class ReduceOp:
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with multiple devices.
 
@@ -91,7 +94,7 @@ class ReduceOp:
 
 def check_collective_target_dtype(data_name, data_dtype, prim_name):
     """Check if data type is valid."""
-    default_target_dtypes = (mstype.int8, mstype.int32, mstype.float16, mstype.float32)
+    default_target_dtypes = (mstype.int8, mstype.uint8, mstype.int32, mstype.float16, mstype.bfloat16, mstype.float32)
     gpu_target_dtypes = (mstype.bool_, mstype.int8, mstype.int32, mstype.int64, mstype.uint32, mstype.uint64,
                          mstype.float16, mstype.float32, mstype.float64)
 
@@ -140,14 +143,17 @@ class AllReduce(Primitive):
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
 
-            This example should be run with multiple devices.
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
+
+            This example should be run with 2 devices.
 
         >>> import numpy as np
         >>> from mindspore.communication import init
@@ -174,7 +180,7 @@ class AllReduce(Primitive):
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - AllReduce
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#AllReduce>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#allreduce>`_
 
     """
 
@@ -226,12 +232,15 @@ class AllGather(PrimitiveWithInfer):
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with 2 devices.
 
@@ -263,7 +272,7 @@ class AllGather(PrimitiveWithInfer):
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - AllGather
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#AllGather>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#allgather>`_
 
     """
 
@@ -448,12 +457,15 @@ class ReduceScatter(Primitive):
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with 2 devices.
 
@@ -486,7 +498,7 @@ class ReduceScatter(Primitive):
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - ReduceScatter
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#ReduceScatter>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#reducescatter>`_
 
     """
 
@@ -587,12 +599,15 @@ class Broadcast(PrimitiveWithInfer):
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with multiple devices.
 
@@ -623,7 +638,7 @@ class Broadcast(PrimitiveWithInfer):
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - Broadcast
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#Broadcast>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#broadcast>`_
 
     """
 
@@ -643,7 +658,7 @@ class Broadcast(PrimitiveWithInfer):
         if not isinstance(x_dtype, tuple):
             raise TypeError(f"For '{self.name}', the 'input_x' must be a tuple, but got {type(x_dtype).__name__}!")
         for _ele in x_dtype:
-            check_collective_target_dtype('input_x', _ele, self.name)
+            check_collective_target_dtype('tuple input_x', _ele, self.name)
         return x_dtype
 
 
@@ -713,11 +728,11 @@ class NeighborExchange(Primitive):
         The user needs to preset
         communication environment variables before running the following example, please check the details on the
         official website of `MindSpore \
-        <https://www.mindspore.cn/docs/en/r2.1/api_python/mindspore.ops.html#communication-operator>`_.
+        <https://www.mindspore.cn/docs/en/r2.2/api_python/mindspore.ops.primitive.html#communication-operator>`_.
 
         This operator requires a full-mesh network topology, each device has the same vlan id, and the ip & mask are
         in the same subnet, please check the `details \
-        <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#notes>`_.
+        <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#notes>`_.
 
     Args:
         send_rank_ids (list(int)): Ranks which the data is sent to.
@@ -756,7 +771,7 @@ class NeighborExchange(Primitive):
         ...     def construct(self, x):
         ...         out = self.neighborexchange((x,))
         ...
-        >>> ms.set_context(mode=ms.GRAPH_MODE, device_target='Ascend')
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> init()
         >>> net = Net()
         >>> input_x = Tensor(np.ones([3, 3]), dtype = ms.float32)
@@ -766,7 +781,7 @@ class NeighborExchange(Primitive):
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - NeighborExchange
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#NeighborExchange>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#neighborexchange>`_
 
     """
 
@@ -799,7 +814,7 @@ class AlltoAll(PrimitiveWithInfer):
     Note:
         This operator requires a full-mesh network topology, each device has the same vlan id, and the ip & mask are
         in the same subnet, please check the `details \
-        <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#notes>`_.
+        <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#notes>`_.
 
     Args:
         split_count (int): On each process, divide blocks into split_count number.
@@ -829,12 +844,15 @@ class AlltoAll(PrimitiveWithInfer):
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with 8 devices.
 
@@ -854,7 +872,7 @@ class AlltoAll(PrimitiveWithInfer):
         ...         out = self.alltoall(x)
         ...         return out
         ...
-        >>> ms.set_context(mode=ms.GRAPH_MODE, device_target='Ascend')
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> init()
         >>> net = Net()
         >>> rank_id = int(os.getenv("RANK_ID"))
@@ -865,7 +883,7 @@ class AlltoAll(PrimitiveWithInfer):
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - AlltoAll
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#AlltoAll>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#alltoall>`_
 
     """
 
@@ -913,7 +931,7 @@ class NeighborExchangeV2(Primitive):
     Note:
         This operator requires a full-mesh network topology, each device has the same vlan id, and the ip & mask are
         in the same subnet, please check the `details \
-        <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#notes>`_.
+        <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#notes>`_.
 
     Args:
         send_rank_ids (list(int)): Ranks which the data is sent to. 8 rank_ids represents 8 directions, if one
@@ -950,46 +968,67 @@ class NeighborExchangeV2(Primitive):
             Before running the following examples, you need to configure the communication environment variables.
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
-            Please see the `Ascend tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_ascend.html#preparations>`_
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
             for more details.
 
-            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/parallel/train_gpu.html#preparation>`_ .
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+
+            For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with 2 devices.
 
         >>> import os
         >>> import mindspore as ms
-        >>> from mindspore import Tensor
         >>> from mindspore.communication import init
         >>> import mindspore.nn as nn
         >>> import mindspore.ops as ops
         >>> import numpy as np
-        >>> class Net(nn.Cell):
+        >>>
+        >>> class Net0(nn.Cell):
         ...     def __init__(self):
-        ...         super(Net, self).__init__()
-        ...         self.neighborexchangev2 = ops.NeighborExchangeV2(send_rank_ids=[-1, -1, -1, -1, 1, -1, -1, -1],
-        ...                                                          send_lens=[0, 1, 0, 0],
-        ...                                                          recv_rank_ids=[-1, -1, -1, -1, 1, -1, -1, -1],
-        ...                                                          recv_lens=[0, 1, 0, 0],
-        ...                                                          data_format="NCHW")
+        ...         super(Net0, self).__init__()
+        ...         self.neighbor_exchangev2 = ops.NeighborExchangeV2(send_rank_ids=[-1, -1, -1, -1, 1, -1, -1, -1],
+        ...                                                           send_lens=[0, 1, 0, 0],
+        ...                                                           recv_rank_ids=[-1, -1, -1, -1, 1, -1, -1, -1],
+        ...                                                           recv_lens=[0, 1, 0, 0], data_format="NCHW")
         ...
         ...     def construct(self, x):
-        ...         out = self.neighborexchangev2(x)
+        ...         out = self.neighbor_exchangev2(x)
         ...         return out
+        >>>
+        ... class Net1(nn.Cell):
+        ...     def __init__(self):
+        ...         super(Net1, self).__init__()
+        ...         self.neighbor_exchangev2 = ops.NeighborExchangeV2(send_rank_ids=[0, -1, -1, -1, -1, -1, -1, -1],
+        ...                                                           send_lens=[1, 0, 0, 0],
+        ...                                                           recv_rank_ids=[0, -1, -1, -1, -1, -1, -1, -1],
+        ...                                                           recv_lens=[1, 0, 0, 0], data_format="NCHW")
         ...
-        >>> ms.set_context(mode=ms.GRAPH_MODE, device_target='Ascend')
+        ...     def construct(self, x):
+        ...         out = self.neighbor_exchangev2(x)
+        ...         return out
+        >>>
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> init()
-        >>> input_x = Tensor(np.ones([1, 1, 2, 2]), dtype = ms.float32)
-        >>> net = Net()
-        >>> output = net(input_x)
-        >>> print(output)
+        >>> rank_id = int(os.getenv("RANK_ID"))
+        >>> if (rank_id % 2 == 0):
+        >>>     input_x = ms.Tensor(np.ones([1, 1, 2, 2]), dtype = ms.float32)
+        >>>     net = Net0()
+        >>>     output = net(input_x)
+        >>>     print(output)
+        >>> else:
+        >>>     input_x = ms.Tensor(np.ones([1, 1, 2, 2]) * 2, dtype = ms.float32)
+        >>>     net = Net1()
+        >>>     output = net(input_x)
+        >>>     print(output)
         [[[[1. 1.], [1. 1.], [2. 2.]]]]
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - NeighborExchangeV2
-          <https://www.mindspore.cn/docs/en/r2.1/api_python/samples/ops/communicate_ops.html#NeighborExchangeV2>`_
+          <https://www.mindspore.cn/docs/en/r2.2/api_python/samples/ops/communicate_ops.html#neighborexchangev2>`_
 
     """
 
@@ -1004,6 +1043,15 @@ class NeighborExchangeV2(Primitive):
         self.format = data_format
         self.add_prim_attr('group', _get_group(group))
         self.add_prim_attr('no_eliminate', True)
+        self.rank_size = get_group_size(_get_group(group))
+        for rank_id in send_rank_ids:
+            if rank_id != -1:
+                validator.check_number_range(rank_id, 0, self.rank_size, validator.INC_LEFT, int,
+                                             "rank_id in send_rank_ids")
+        for rank_id in recv_rank_ids:
+            if rank_id != -1:
+                validator.check_number_range(rank_id, 0, self.rank_size, validator.INC_LEFT, int,
+                                             "rank_id in recv_rank_ids")
 
     def __call__(self, tensor):
         raise NotImplementedError
@@ -1262,4 +1310,4 @@ class _GetTensorSlice(PrimitiveWithInfer):
         from mindspore.parallel._tensor import _load_tensor
         validator.check_value_type("dev_mat", dev_mat, [tuple], self.name)
         validator.check_value_type("tensor_map", tensor_map, [tuple], self.name)
-        return Tensor(_load_tensor(x, dev_mat, tensor_map))
+        return Tensor(_load_tensor(x, dev_mat, tensor_map), x.dtype)