mindspore 2.5.0__cp310-cp310-win_amd64.whl → 2.6.0__cp310-cp310-win_amd64.whl

mindspore/communication/__init__.py

@@ -20,18 +20,20 @@ Note that the APIs in the following list need to preset communication environmen
 For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
 without any third-party or configuration file dependencies.
 Please see the `msrun start up
-<https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+<https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
 for more details.
 """
 
 from mindspore.communication.management import GlobalComm, init, release, get_rank, \
     get_group_size, get_world_rank_from_group_rank, \
-    get_group_rank_from_world_rank, create_group, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, \
-    MCCL_WORLD_COMM_GROUP, get_local_rank, get_local_rank_size, destroy_group, get_process_group_ranks
+    get_group_rank_from_world_rank, create_group, get_comm_name, \
+    HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, MCCL_WORLD_COMM_GROUP, get_local_rank, \
+    get_local_rank_size, destroy_group, get_process_group_ranks
 
 
 __all__ = [
     "GlobalComm", "init", "release", "get_rank", "get_group_size", "get_world_rank_from_group_rank",
-    "get_group_rank_from_world_rank", "create_group", "HCCL_WORLD_COMM_GROUP", "NCCL_WORLD_COMM_GROUP",
-    "MCCL_WORLD_COMM_GROUP", "get_local_rank", "get_local_rank_size", "destroy_group", "get_process_group_ranks"
+    "get_group_rank_from_world_rank", "create_group", "get_comm_name",
+    "HCCL_WORLD_COMM_GROUP", "NCCL_WORLD_COMM_GROUP", "MCCL_WORLD_COMM_GROUP", "get_local_rank",
+    "get_local_rank_size", "destroy_group", "get_process_group_ranks"
 ]

mindspore/communication/_comm_helper.py

@@ -178,8 +178,17 @@ def check_parameter_available(func):
         Wrapper. If not available, raise Error.
     """
     def wrapper(*args, **kargs):
-        if not GlobalComm.INITED:
-            raise RuntimeError("Distributed Communication has not been inited")
+        # This function list indicates these functions will return 0 or 1 value in standalone mode or
+        # not calling 'init' method.
+        standalone_bypass_check_func_list = [
+            "_get_rank_helper",
+            "_get_local_rank_helper",
+            "_get_size_helper",
+            "_get_local_size_helper"
+        ]
+        if not GlobalComm.INITED and func.__name__ not in standalone_bypass_check_func_list:
+            raise RuntimeError(f"Distributed Communication has not been inited."
+                               f"You can't invoke this interface yet. Please call `init()` method first.")
         group = None
         if "group" in kargs.keys():
             group = kargs.get("group")
@@ -264,6 +273,11 @@ def _get_rank_helper(group):
     if _check_bypass_rank_id_and_size():
         rank_id = 0
         return rank_id
+    if not GlobalComm.INITED:
+        # If 'RANK_ID' is not set, return 0 as default value.
+        logger.info(f"You are invoking this interface without calling `init` method."
+                    "Return 'RANK_ID' env value instead. If 'RANK_ID' is not set, return 0 as default value.")
+        return int(os.getenv("RANK_ID", "0"))
     if _hccl_test():
         return hccl.get_rank_id(group)
     rank_id = CollectiveManager.get_instance().get_rank_id(group)
@@ -288,6 +302,11 @@ def _get_local_rank_helper(group):
     if _check_bypass_rank_id_and_size():
         local_rank_id = 0
         return local_rank_id
+    if not GlobalComm.INITED:
+        # If 'LOCAL_RANK' env is not set, return 0 as default value.
+        logger.info(f"You are invoking this interface without calling `init` method."
+                    "Return 'LOCAL_RANK' env value instead. If 'LOCAL_RANK' is not set, return 0 as default value.")
+        return int(os.getenv("LOCAL_RANK", "0"))
     if _hccl_test():
         return hccl.get_local_rank_id(group)
     rank_id = CollectiveManager.get_instance().get_local_rank_id(group)
@@ -312,6 +331,11 @@ def _get_size_helper(group):
     if _check_bypass_rank_id_and_size():
         size = 1
         return size
+    if not GlobalComm.INITED:
+        # If 'LOCAL_RANK' env is not set, return 0 as default value.
+        logger.info(f"You are invoking this interface without calling `init` method."
+                    "Return 'RANK_SIZE' env value instead. If 'RANK_SIZE' is not set, return 1 as default value.")
+        return int(os.getenv("RANK_SIZE", "1"))
     if _hccl_test():
         return hccl.get_rank_size(group)
     size = CollectiveManager.get_instance().get_group_size(group)
@@ -333,6 +357,15 @@ def _get_local_size_helper(group):
     Returns:
         Integer. The local rank size where the calling process is being within specified group.
     """
+    if _check_bypass_rank_id_and_size():
+        size = 1
+        return size
+    if not GlobalComm.INITED:
+        # If 'LOCAL_RANK_SIZE' env is not set, return 0 as default value.
+        logger.info(f"You are invoking this interface without calling `init` method."
+                    "Return 'LOCAL_RANK_SIZE' env value instead. If 'LOCAL_RANK_SIZE' is not set,"
+                    "return 1 as default value.")
+        return int(os.getenv("LOCAL_RANK_SIZE", "1"))
     size = CollectiveManager.get_instance().get_local_group_size(group)
     return size
 
@@ -501,6 +534,18 @@ def _destroy_group_helper(group):
         CollectiveManager.get_instance().destroy_group(group)
 
 
+@check_parameter_available
+def _get_comm_name_helper(group):
+    """
+    The Helper to get inner_comm_name.
+
+    Args:
+        group (str): The user communication group.
+
+    """
+    return CollectiveManager.get_instance().get_comm_name(group)
+
+
 def _get_group_map():
     """Get the group map"""
     return CollectiveManager.get_instance().get_group_map()

mindspore/communication/comm_func.py

@@ -20,7 +20,7 @@ from mindspore.communication import GlobalComm, get_group_rank_from_world_rank,
 from mindspore.communication.management import _get_group
 from mindspore.communication._comm_helper import _get_group_rank_from_world_rank_from_cache_helper
 from mindspore.common.tensor import Tensor
-from mindspore._c_expression import Tensor as Tensor_
+from mindspore._c_expression import TensorPy as Tensor_
 from mindspore.ops import ReduceOp, cat
 from mindspore.ops._primitive_cache import _get_cache_prim
 from mindspore.ops.primitive import _primexpr
@@ -30,7 +30,7 @@ from mindspore.ops.auto_generate.gen_ops_prim import (inner_comm_all_reduce_op,
 from mindspore._c_expression import CommHandle as CommHandle_
 from mindspore._c_expression.typing import Type
 from mindspore import jit_class
-from mindspore.common.api import _pynative_executor
+import mindspore as ms
 
 __all__ = [
     'all_reduce',
@@ -63,6 +63,12 @@ class CommHandle(CommHandle_):
     handles will be created using Python.
     """
 
+    def __init__(self, handle=None, exec_sync=False):
+        super(CommHandle, self).__init__()
+        self.handle = handle
+        self.exec_sync = exec_sync
+
+
     def wait(self):
         r"""
         The wait for asynchronous handles will not take effect for handles created on the Python side.
@@ -80,6 +86,10 @@ class CommHandle(CommHandle_):
         [[2. 2. 2. 2. 2. 2. 2. 2.]
          [2. 2. 2. 2. 2. 2. 2. 2.]]
         """
+        if self.handle:
+            self.handle.wait()
+        if self.exec_sync:
+            ms.runtime.synchronize()
 
 
 default_handle = CommHandle()
@@ -220,7 +230,7 @@ def all_reduce(tensor, op=ReduceOp.SUM, group=GlobalComm.WORLD_COMM_GROUP, async
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -285,7 +295,7 @@ def all_gather_into_tensor(tensor, group=GlobalComm.WORLD_COMM_GROUP, async_op=F
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -355,7 +365,7 @@ def reduce_scatter_tensor(tensor, op=ReduceOp.SUM, group=GlobalComm.WORLD_COMM_G
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -424,7 +434,7 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=GlobalComm.WORLD_COMM_GROUP):
             without any third-party or configuration file dependencies.
 
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 4 devices.
@@ -561,7 +571,7 @@ def batch_isend_irecv(p2p_op_list):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -679,7 +689,7 @@ def scatter_tensor(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -744,7 +754,7 @@ def gather_into_tensor(tensor, dst=0, group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -805,7 +815,7 @@ def broadcast(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -819,6 +829,7 @@ def broadcast(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP):
         >>> comm.init()
         >>> data = ms.Tensor(np.arange(8).reshape([2, 4]).astype(np.float32))
         >>> out = comm.comm_func.broadcast(tensor=data, src=0)
+        >>> print(out)
         [[0. 1. 2. 3.]
          [4. 5. 6. 7.]]
 
@@ -858,7 +869,7 @@ def barrier(group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -869,6 +880,8 @@ def barrier(group=GlobalComm.WORLD_COMM_GROUP):
         >>> # Launch 2 processes.
         >>> comm.init()
         >>> comm.comm_func.barrier()
+        >>> print("barrier finish!")
+        barrier finish!
 
     Tutorial Examples:
         - `Distributed Set Communication Primitives - Barrier
@@ -888,9 +901,9 @@ def _deal_comm_outputs(output, async_op, exec_sync=False):
         if not async_op:
             output[1].wait()
             if exec_sync:
-                _pynative_executor.sync()
+                ms.runtime.synchronize()
             return (output[0], None)
-        return output
+        return (output[0], CommHandle(output[1], exec_sync))
 
     if not async_op:
         return (output, None)
@@ -926,7 +939,7 @@ def send(tensor, dst=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -946,12 +959,13 @@ def send(tensor, dst=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
         >>>
         >>>
         >>> if rank < size / 2:
-        >>>     _x = ms.Tensor(x)
-        >>>     send(_x, rank + size // 2)
-        >>> else:
-        >>>     _x2 = ms.Tensor(x2)
-        >>>     output = recv(_x2, rank - size // 2)
-        >>>     print(output)
+        ...     _x = ms.Tensor(x)
+        ...     send(_x, rank + size // 2)
+        ... else:
+        ...     _x2 = ms.Tensor(x2)
+        ...     output = recv(_x2, rank - size // 2)
+        ...     print(output)
+        rank1:
         [[0.01  0.01]
          [0.01  0.01]]
     """
@@ -1000,7 +1014,7 @@ def recv(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -1020,12 +1034,13 @@ def recv(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
         >>>
         >>>
         >>> if rank < size / 2:
-        >>>     _x = ms.Tensor(x)
-        >>>     send(_x, rank + size // 2)
-        >>> else:
-        >>>     _x2 = ms.Tensor(x2)
-        >>>     output = recv(_x2, rank - size // 2)
-        >>>     print(output)
+        ...     _x = ms.Tensor(x)
+        ...     send(_x, rank + size // 2)
+        ... else:
+        ...     _x2 = ms.Tensor(x2)
+        ...     output = recv(_x2, rank - size // 2)
+        ...     print(output)
+        rank1:
         [[0.01  0.01]
          [0.01  0.01]]
     """
@@ -1075,7 +1090,7 @@ def isend(tensor, dst=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -1095,13 +1110,14 @@ def isend(tensor, dst=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
         >>>
         >>>
         >>> if rank < size / 2:
-        >>>     _x = ms.Tensor(x)
-        >>>     isend(_x, rank + size // 2)
-        >>> else:
-        >>>     _x2 = ms.Tensor(x2)
-        >>>     output, handle = irecv(_x2, rank - size // 2)
-        >>>     handle.wait()
-        >>>     print(output)
+        ...     _x = ms.Tensor(x)
+        ...     isend(_x, rank + size // 2)
+        ... else:
+        ...     _x2 = ms.Tensor(x2)
+        ...     output, handle = irecv(_x2, rank - size // 2)
+        ...     handle.wait()
+        ...     print(output)
+        rank1:
         [[0.01  0.01]
          [0.01  0.01]]
     """
@@ -1153,7 +1169,7 @@ def irecv(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -1173,13 +1189,14 @@ def irecv(tensor, src=0, group=GlobalComm.WORLD_COMM_GROUP, tag=0):
         >>>
         >>>
         >>> if rank < size / 2:
-        >>>     _x = ms.Tensor(x)
-        >>>     isend(_x, rank + size // 2)
-        >>> else:
-        >>>     _x2 = ms.Tensor(x2)
-        >>>     output, handle = irecv(_x2, rank - size // 2)
-        >>>     handle.wait()
-        >>>     print(output)
+        ...     _x = ms.Tensor(x)
+        ...     isend(_x, rank + size // 2)
+        ... else:
+        ...     _x2 = ms.Tensor(x2)
+        ...     output, handle = irecv(_x2, rank - size // 2)
+        ...     handle.wait()
+        ...     print(output)
+        rank1:
         [[0.01  0.01]
          [0.01  0.01]]
     """
@@ -1229,7 +1246,7 @@ def all_to_all_with_output_shape(output_shape_list, input_tensor_list, group=Non
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.
@@ -1241,11 +1258,11 @@ def all_to_all_with_output_shape(output_shape_list, input_tensor_list, group=Non
         >>> comm.init()
         >>> this_rank = comm.get_rank()
         >>> if this_rank == 0:
-        >>>     send_tensor_list = [ms.Tensor(1.), ms.Tensor([[2, 3], [4, 5.]])]
-        >>>     recv_tensor_list = [(), (2,)]
+        ...     send_tensor_list = [ms.Tensor(1.), ms.Tensor([[2, 3], [4, 5.]])]
+        ...     recv_tensor_list = [(), (2,)]
         >>> if this_rank == 1:
-        >>>     send_tensor_list = [ms.Tensor([2, 2.]), ms.Tensor([4, 5, 6, 7.])]
-        >>>     recv_tensor_list = [(2, 2), (4,)]
+        ...     send_tensor_list = [ms.Tensor([2, 2.]), ms.Tensor([4, 5, 6, 7.])]
+        ...     recv_tensor_list = [(2, 2), (4,)]
         >>> output, _ = comm.comm_func.all_to_all_with_output_shape(recv_tensor_list, send_tensor_list)
         >>> print(output)
         rank 0:
@@ -1280,7 +1297,6 @@ def all_to_all_with_output_shape(output_shape_list, input_tensor_list, group=Non
             recv_shape_list.append(_shape)
 
     send_flatten_tensor = cat(send_flatten_tensor)
-    send_flatten_tensor = _contiguous(send_flatten_tensor)
     group = GlobalComm.WORLD_COMM_GROUP if group is None else _get_group(group)
     global _GROPU_SIZE_CACHE
     if group not in _GROPU_SIZE_CACHE:
@@ -1345,7 +1361,8 @@ _ALL_TO_ALL_CACHE = {}
 def all_to_all_single_with_output_shape(output_shape, tensor, output_split_sizes=None,
                                         input_split_sizes=None, group=None, async_op=False):
     """
-    scatter and gather input with split size to/from all rank, and return result in a single tensor.
+    Based on the slice size of the user input, the input `tensor` is sliced and sent to other devices
+    and receives the sliced chunks from the other devices, which are then merged into an output Tensor.
 
     Note:
         'output_shape' and 'tensor' shape should be match across ranks.
@@ -1365,8 +1382,8 @@ def all_to_all_single_with_output_shape(output_shape, tensor, output_split_sizes
 
     Returns:
         Tuple(Tensor, CommHandle), the output tensor is gathered concatenated from remote ranks.
-        If the numel of tensor gathered from remote is zero, it will return a Tensor will value 0,
-        which has no actual meanning. CommHandle is an async work handle, if `async_op` is set to True.
+        If the numel of tensor gathered from remote is zero, it will return a Tensor with shape `()`,
+        and value has no actual meanning. CommHandle is an async work handle, if `async_op` is set to True.
         CommHandle will be None, when `async_op` is False.
 
     Raises:
@@ -1383,7 +1400,7 @@ def all_to_all_single_with_output_shape(output_shape, tensor, output_split_sizes
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/docs/en/master/model_train/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 2 devices.