mindspore 1.10.0__cp37-cp37m-win_amd64.whl → 2.0.0rc1__cp37-cp37m-win_amd64.whl

mindspore/communication/management.py

@@ -20,8 +20,9 @@ from mindspore.communication._comm_helper import Backend, _get_rank_helper, _get
     _get_world_rank_from_group_rank_helper, _get_group_rank_from_world_rank_helper, \
     _create_group_helper, _destroy_group_helper, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, \
     MCCL_WORLD_COMM_GROUP, _get_local_rank_helper, _get_local_size_helper, GlobalComm, \
-    _not_require_collective_comm_lib, _check_mpi_envs, _use_old_ps
-from mindspore._c_expression import init_hccl, finalize_hccl, init_gpu_collective, init_cluster
+    _check_mpi_envs, _set_elegant_exit_handle
+from mindspore._c_expression import init_hccl, finalize_hccl, init_cluster, MSContext, ms_ctx_param
+
 
 __all__ = ["init", "release", "get_rank", "get_local_rank", "get_group_size",
            "get_local_rank_size", "get_world_rank_from_group_rank",
@@ -40,6 +41,7 @@ def _set_rank_from_mpi():
         os.environ["RANK_ID"] = ompi_rank_id
     if ompi_device_id:
         os.environ["DEVICE_ID"] = ompi_device_id
+        MSContext.get_instance().set_param(ms_ctx_param.device_id, int(ompi_device_id))
     if ompi_rank_size:
         os.environ["RANK_SIZE"] = ompi_rank_size
 
@@ -54,6 +56,11 @@ def _get_group(group):
     return group
 
 
+def _host_distribute():
+    """Check whether host distribute needed."""
+    return os.getenv("MS_ROLE") or _check_mpi_envs()
+
+
 def _check_parallel_envs():
     """
     Check whether parallel environment variables have been exported or not.
@@ -82,19 +89,17 @@ def _check_parallel_envs():
 
 def init(backend_name=None):
     """
-    Initialize distributed backend required by communication services, e.g. HCCL/NCCL. It is usually used in
+    Initialize distributed backends required by communication services, e.g. HCCL/NCCL. It is usually used in
     distributed parallel scenarios and set before using communication services.
 
     Note:
         - The full name of HCCL is Huawei Collective Communication Library.
         - The full name of NCCL is NVIDIA Collective Communication Library.
         - The full name of MCCL is MindSpore Collective Communication Library.
-        - The user needs to preset communication environment variables
-        before running the following example, please see the docstring of the mindspore.communication.
 
     Args:
         backend_name (str): Backend, using HCCL/NCCL/MCCL. HCCL should be used for Ascend hardware platforms and
-                            NCCL for GPU hardware platform. If not set, inference is automatically made based on the
+                            NCCL for GPU hardware platforms. If not set, inference is automatically made based on the
                             hardware platform type (device_target). Default: None.
 
     Raises:
@@ -104,15 +109,24 @@ def init(backend_name=None):
                       have not been exported when backend is HCCL.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> from mindspore.communication import init
         >>> init()
     """
-    if _not_require_collective_comm_lib():
-        return
-    mpi_init = _check_mpi_envs()
+    host_init = _host_distribute()
     device_target = context.get_context("device_target")
 
     if backend_name is None:
@@ -130,37 +144,34 @@ def init(backend_name=None):
                         "but got the type : {}".format(type(backend_name)))
 
     if backend_name == "hccl":
-        if not _use_old_ps() and _is_ps_mode():
+        if _is_ps_mode():
             # Use MindSpore cluster to build network for Parameter Server traning.
             init_cluster()
             if _is_role_sched() or _is_role_pserver():
                 raise RuntimeError("Parameter server and scheduler should use 'CPU' as backend instead of 'Ascend'")
             if _get_ps_context("worker_num") == 1:
                 GlobalComm.INITED = True
+                _set_elegant_exit_handle()
                 return
         if device_target != "Ascend":
             raise RuntimeError("For 'init', the argument  'backend_name' should be 'Ascend' to init hccl, "
                                "but got {}".format(device_target))
-        if not mpi_init:
+        if not host_init:
             _check_parallel_envs()
-            GlobalComm.BACKEND = Backend("hccl")
-        else:
-            GlobalComm.BACKEND = Backend("hccl_mpi")
+        GlobalComm.BACKEND = Backend("hccl")
         init_hccl()
         GlobalComm.WORLD_COMM_GROUP = HCCL_WORLD_COMM_GROUP
     elif backend_name == "nccl":
-        init_gpu_collective()
-        GlobalComm.BACKEND = Backend("nccl")
+        init_cluster()
         GlobalComm.WORLD_COMM_GROUP = NCCL_WORLD_COMM_GROUP
     elif backend_name == "mccl":
-        GlobalComm.BACKEND = Backend("mccl")
-        # Init cluster.
         init_cluster()
         GlobalComm.WORLD_COMM_GROUP = MCCL_WORLD_COMM_GROUP
     else:
         raise RuntimeError("For 'init', the argument 'backend_name' must be nccl while 'device_target' is GPU, "
                            "but got the 'backend_name' : hccl.")
     GlobalComm.INITED = True
+    _set_elegant_exit_handle()
 
 
 def release():
@@ -168,8 +179,7 @@ def release():
     Release distributed resource. e.g. HCCL/NCCL.
 
     Note:
-        This method should be used after init(). The user needs to preset communication environment variables
-        before running the following example, please see the docstring of the mindspore.communication.
+        This method should be used after init().
 
     Raises:
         RuntimeError: If failed to release distributed resource.
@@ -178,6 +188,17 @@ def release():
         ``Ascend`` ``GPU``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> from mindspore.communication import init, release
         >>> init()
         >>> release()
@@ -190,8 +211,7 @@ def get_rank(group=GlobalComm.WORLD_COMM_GROUP):
     Get the rank ID for the current device in the specified collective communication group.
 
     Note:
-        This method should be used after init(). The user needs to preset communication environment variables
-        before running the following example, please see the docstring of the mindspore.communication.
+        This method should be used after init().
 
     Args:
         group (str): The communication group to work on. Normally, the group should be created by create_group,
@@ -206,9 +226,20 @@ def get_rank(group=GlobalComm.WORLD_COMM_GROUP):
         RuntimeError: If HCCL/NCCL is not available.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> from mindspore.communication import init, get_rank
         >>> init()
         >>> rank_id = get_rank()
@@ -218,7 +249,7 @@ def get_rank(group=GlobalComm.WORLD_COMM_GROUP):
     if not isinstance(group, str):
         raise TypeError("For 'get_rank', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    return _get_rank_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
+    return _get_rank_helper(group=_get_group(group))
 
 
 def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP):
@@ -227,8 +258,7 @@ def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP):
 
     Note:
         GPU version of MindSpore doesn't support this method.
-        This method should be used after init(). The user needs to preset communication environment variables
-        before running the following example, please see the docstring of the mindspore.communication.
+        This method should be used after init().
 
     Args:
         group (str): The communication group to work on. Normally, the group should be created by create_group,
@@ -246,6 +276,17 @@ def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP):
         ``Ascend``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> import mindspore as ms
         >>> from mindspore.communication.management import init, get_rank, get_local_rank
         >>> ms.set_context(device_target="Ascend")
@@ -259,7 +300,7 @@ def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP):
     if not isinstance(group, str):
         raise TypeError("For 'get_local_rank', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    return _get_local_rank_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
+    return _get_local_rank_helper(group=_get_group(group))
 
 
 def get_group_size(group=GlobalComm.WORLD_COMM_GROUP):
@@ -267,8 +308,7 @@ def get_group_size(group=GlobalComm.WORLD_COMM_GROUP):
     Get the rank size of the specified collective communication group.
 
     Note:
-        This method should be used after init(). The user needs to preset communication environment variables before
-        running the following example, please see the docstring of the mindspore.communication.
+        This method should be used after init().
 
     Args:
         group (str): The communication group to work on. Normally, the group should be created by create_group,
@@ -283,9 +323,20 @@ def get_group_size(group=GlobalComm.WORLD_COMM_GROUP):
         RuntimeError: If HCCL/NCCL is not available.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``
+        ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> import mindspore as ms
         >>> from mindspore.communication.management import init, get_group_size
         >>> ms.set_auto_parallel_context(device_num=8)
@@ -297,7 +348,7 @@ def get_group_size(group=GlobalComm.WORLD_COMM_GROUP):
     if not isinstance(group, str):
         raise TypeError("For 'get_group_size', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    return _get_size_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
+    return _get_size_helper(group=_get_group(group))
 
 
 def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP):
@@ -306,8 +357,7 @@ def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP):
 
     Note:
         GPU version of MindSpore doesn't support this method.
-        This method should be used after init(). The user needs to preset communication environment variables before
-        running the following example, please see the docstring of the mindspore.communication.
+        This method should be used after init().
 
     Args:
         group (str): The communication group to work on. The group is created by create_group
@@ -325,6 +375,17 @@ def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP):
         ``Ascend``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> import mindspore as ms
         >>> from mindspore.communication.management import init, get_local_rank_size
         >>> ms.set_context(device_target="Ascend")
@@ -337,7 +398,7 @@ def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP):
     if not isinstance(group, str):
         raise TypeError("For 'get_local_rank_size', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    return _get_local_size_helper(group=_get_group(group), backend=GlobalComm.BACKEND)
+    return _get_local_size_helper(group=_get_group(group))
 
 
 def get_world_rank_from_group_rank(group, group_rank_id):
@@ -348,8 +409,7 @@ def get_world_rank_from_group_rank(group, group_rank_id):
     Note:
         GPU version of MindSpore doesn't support this method.
         The parameter group should not be "hccl_world_group".
-        This method should be used after init(). The user needs to preset communication environment variables
-        before running the following example, please see the docstring of the mindspore.communication.
+        This method should be used after init().
 
     Args:
         group (str): The communication group to work on. The group is created by create_group.
@@ -367,6 +427,17 @@ def get_world_rank_from_group_rank(group, group_rank_id):
         ``Ascend``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> from mindspore import set_context
         >>> from mindspore.communication.management import init, create_group, get_world_rank_from_group_rank
         >>> set_context(device_target="Ascend")
@@ -381,7 +452,7 @@ def get_world_rank_from_group_rank(group, group_rank_id):
     if not isinstance(group, str):
         raise TypeError("For 'get_world_rank_from_group_rank', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    return _get_world_rank_from_group_rank_helper(group=group, group_rank_id=group_rank_id, backend=GlobalComm.BACKEND)
+    return _get_world_rank_from_group_rank_helper(group=group, group_rank_id=group_rank_id)
 
 
 def get_group_rank_from_world_rank(world_rank_id, group):
@@ -393,8 +464,6 @@ def get_group_rank_from_world_rank(world_rank_id, group):
         GPU version of MindSpore doesn't support this method.
         The parameter group should not be "hccl_world_group".
         This method should be used after init().
-        The user needs to preset communication environment variables before running the following example, please see
-        the docstring of the mindspore.managerment.
 
     Args:
         world_rank_id (int): A rank ID in the world communication group.
@@ -412,6 +481,17 @@ def get_group_rank_from_world_rank(world_rank_id, group):
         ``Ascend``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> from mindspore import set_context
         >>> from mindspore.communication.management import init, create_group, get_group_rank_from_world_rank
         >>> set_context(device_target="Ascend")
@@ -426,7 +506,7 @@ def get_group_rank_from_world_rank(world_rank_id, group):
     if not isinstance(group, str):
         raise TypeError("For 'get_group_rank_from_world_rank', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    return _get_group_rank_from_world_rank_helper(world_rank_id=world_rank_id, group=group, backend=GlobalComm.BACKEND)
+    return _get_group_rank_from_world_rank_helper(world_rank_id=world_rank_id, group=group)
 
 
 def create_group(group, rank_ids):
@@ -437,9 +517,7 @@ def create_group(group, rank_ids):
         GPU version of MindSpore doesn't support this method.
         The size of rank_ids should be larger than 1, rank_ids should not have duplicate data.
         This method should be used after init().
-        Only support global single communication group in PyNative mode.
-        The user needs to preset communication environment variables before running the following example, please see
-        the docstring of the mindspore.managerment.
+        Only support global single communication group in PyNative mode if you do not start with mpirun.
 
     Args:
         group (str): The name of the communication group to be created.
@@ -454,8 +532,19 @@ def create_group(group, rank_ids):
         ``Ascend``
 
     Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `Ascend tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+            <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
+
         >>> from mindspore import set_context
-        >>> from mindspore.ops import operations as ops
+        >>> import mindspore.ops as ops
         >>> from mindspore.communication.management import init, create_group
         >>> set_context(device_target="Ascend")
         >>> init()
@@ -467,7 +556,7 @@ def create_group(group, rank_ids):
     if not isinstance(group, str):
         raise TypeError("For 'create_group', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    _create_group_helper(group, rank_ids, backend=GlobalComm.BACKEND)
+    _create_group_helper(group, rank_ids)
 
 
 def destroy_group(group):
@@ -493,4 +582,4 @@ def destroy_group(group):
     if not isinstance(group, str):
         raise TypeError("For 'destroy_group', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    _destroy_group_helper(group, backend=GlobalComm.BACKEND)
+    _destroy_group_helper(group)