mindspore 1.10.0__cp37-cp37m-win_amd64.whl → 2.0.0rc1__cp37-cp37m-win_amd64.whl

mindspore/communication/__init__.py

@@ -13,12 +13,17 @@
 # limitations under the License.
 # ============================================================================
 """
-Collective communication interface. Note the API in the file needs to preset communication environment variables. For
-the Ascend cards, users need to prepare the rank table, set rank_id and device_id. Please see the `Ascend tutorial \
-<https://www.mindspore.cn/tutorials/experts/zh-CN/r1.10/parallel/train_ascend.html>`_ for more details.
-For the GPU device, users need to prepare the host file and mpi, please see the `GPU tutorial \
-<https://www.mindspore.cn/tutorials/experts/zh-CN/r1.10/parallel/train_gpu.html>`_
+Collective communication interface.
+
+Note that the APIs in the following list need to preset communication environment variables.
+
+For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+Please see the `Ascend tutorial
+<https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_ascend.html#preparations>`_
 for more details.
+
+For the GPU devices, users need to prepare the host file and mpi, please see the `GPU tutorial
+<https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/train_gpu.html#preparation>`_ .
 """
 
 from mindspore.communication.management import GlobalComm, init, release, get_rank, \

mindspore/communication/_comm_helper.py

@@ -13,47 +13,44 @@
 # limitations under the License.
 # ============================================================================
 """comm_helper"""
-from __future__ import absolute_import
-from __future__ import division
 
 import os
+import ctypes
+
+import sys
+from sys import excepthook
+
 from mindspore import context
-from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _is_ps_mode, _get_ps_context
+from mindspore.parallel._ps_context import _is_role_worker, _is_role_pserver, _is_role_sched, _is_ps_mode,\
+                                           _get_ps_context
 from mindspore import log as logger
-from mindspore.communication._hccl_management import load_lib as hccl_load_lib
-from mindspore._c_expression import get_rank_id, get_rank_size, CollectiveManager
+from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception, MSContext
 
-_HCCL_AVAILABLE = False
-_HCCL_TEST_AVAILABLE = False
-_NCCL_AVAILABLE = False
-_MPI_AVAILABLE = False
-try:
-    import mindspore._ms_mpi as mpi
-    _NCCL_AVAILABLE = True
-except ImportError:
-    _NCCL_AVAILABLE = False
+HCCL_LIB = 'libhccl_plugin.so'
+
+
+def hccl_load_lib():
+    """load hccl lib"""
+    try:
+        base_dir = os.path.dirname(os.path.realpath(__file__))
+        lib_path = os.path.join(base_dir, "../lib/plugin/ascend", HCCL_LIB)
+        ctypes.CDLL(lib_path)
+    except Exception:
+        raise RuntimeError('Get hccl lib error.')
 
+_HCCL_TEST_AVAILABLE = False
 
 try:
-    hccl_load_lib()
-    _HCCL_AVAILABLE = True
+    if MSContext.get_instance().is_ascend_plugin_loaded():
+        hccl_load_lib()
 except RuntimeError:
-    _HCCL_AVAILABLE = False
+    _HCCL_TEST_AVAILABLE = True
 
-if _HCCL_AVAILABLE:
-    from mindspore.communication import _hccl_management as hccl
-    try:
-        import mindspore._ascend_mpi as mpi
-        _MPI_AVAILABLE = True
-    except ImportError:
-        _MPI_AVAILABLE = False
-else:
+if _HCCL_TEST_AVAILABLE:
     try:
         import hccl_test.manage.api as hccl
-        _HCCL_AVAILABLE = True
-        _HCCL_TEST_AVAILABLE = True
     except ImportError:
-        _HCCL_AVAILABLE = False
+        _HCCL_TEST_AVAILABLE = False
 
 
 HCCL_WORLD_COMM_GROUP = "hccl_world_group"
@@ -83,7 +80,6 @@ class Backend:
     UNDEFINED = "undefined"
     HCCL = "hccl"
     NCCL = "nccl"
-    HCCL_MPI = "hccl_mpi"
     MCCL = "mccl"
 
     def __new__(cls, name):
@@ -121,34 +117,8 @@ class _ExistingGroup:
     ITEMS = {}
 
 
-def is_hccl_available():
-    """
-    Check HCCL api is available.
-
-    Returns:
-        Boolean. Return whether HCCL is available or not.
-    """
-    return _HCCL_AVAILABLE
-
-
-def is_mpi_available():
-    """
-    Check HCCL & MPI api is available.
-
-    Returns:
-        Boolean. Return whether HCCL & MPI is available or not.
-    """
-    return _MPI_AVAILABLE
-
-
-def is_nccl_available():
-    """
-    Check NCCL api is available.
-
-    Returns:
-        Boolean. Return whether NCCL is available or not.
-    """
-    return _NCCL_AVAILABLE
+def _hccl_test():
+    return _HCCL_TEST_AVAILABLE and GlobalComm.BACKEND == Backend.HCCL
 
 
 def _check_mpi_envs():
@@ -164,25 +134,6 @@ def _check_mpi_envs():
     return False
 
 
-def _use_old_ps():
-    """
-    Whether use old framework to launch Parameter Server training.
-    """
-    return os.getenv("USE_OLD_PS") == "True"
-
-
-def _not_require_collective_comm_lib():
-    '''
-    Whether collective communication library is required in this training mode.
-    For example, scheduler and server do not require actual collective communication in parameter server mode.
-    '''
-    # Environment variable USE_OLD_PS is set by user and used to run
-    # parameter server training with old framework.
-    if _use_old_ps() and (_is_role_sched() or _is_role_pserver()):
-        return True
-    return False
-
-
 def _check_bypass_rank_id_and_size():
     '''
     Whether bypass calling c++ API to get rank id and size, instead, use fake rank id 0 and rank size 1.
@@ -190,14 +141,17 @@ def _check_bypass_rank_id_and_size():
     '''
     if _is_role_sched():
         return True
-    if _use_old_ps() and _is_role_pserver():
-        return True
     device_target = context.get_context("device_target")
-    if not _use_old_ps() and _is_ps_mode() and _get_ps_context("worker_num") == 1 and device_target == "Ascend":
+    if _is_ps_mode() and _get_ps_context("worker_num") == 1 and device_target == "Ascend":
         return True
     return False
 
 
+def _set_elegant_exit_handle():
+    if _is_role_worker() or _is_role_pserver() or _is_role_sched():
+        sys.excepthook = lambda *args: (set_cluster_exit_with_exception(), excepthook(*args))
+
+
 def check_parameter_available(func):
     """
     Check parameter is available. If not available, raise Error.
@@ -212,8 +166,6 @@ def check_parameter_available(func):
         Wrapper. If not available, raise Error.
     """
     def wrapper(*args, **kargs):
-        if _not_require_collective_comm_lib():
-            return func(*args, **kargs)
         if not GlobalComm.INITED:
             raise RuntimeError("Distributed Communication has not been inited")
         group = None
@@ -222,27 +174,14 @@ def check_parameter_available(func):
             if group is not None and not isinstance(group, str):
                 raise TypeError("The parameter 'group' should be str or None, "
                                 "but got the type : {}".format(type(group)))
-
-        if "backend" in kargs.keys():
-            backend = kargs.get("backend")
-            if backend is Backend.HCCL and not is_hccl_available():
-                raise RuntimeError("Distributed Communication doesn't have HCCL built in")
-            if backend is Backend.HCCL_MPI and not is_mpi_available():
-                raise RuntimeError("Distributed Communication doesn't have MPI built in")
-            if backend is Backend.NCCL and not is_nccl_available():
-                raise RuntimeError("Distributed Communication doesn't have NCCL built in")
-
         if group is None:
-            if backend is Backend.HCCL or Backend.HCCL_MPI:
-                group = HCCL_WORLD_COMM_GROUP
-            elif backend is Backend.NCCL:
-                group = NCCL_WORLD_COMM_GROUP
+            group = GlobalComm.WORLD_COMM_GROUP
         return func(*args, **kargs)
     return wrapper
 
 
 @check_parameter_available
-def _get_rank_helper(group, backend):
+def _get_rank_helper(group):
     """
     The Helper to do get_rank_id.
 
@@ -259,26 +198,14 @@ def _get_rank_helper(group, backend):
     if _check_bypass_rank_id_and_size():
         rank_id = 0
         return rank_id
-    if backend == Backend.HCCL_MPI:
-        rank_id = mpi.get_rank_id(group)
-    elif backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            rank_id = hccl.get_rank_id()
-        else:
-            rank_id = hccl.get_rank_id(group)
-    elif backend == Backend.NCCL:
-        rank_id = get_rank_id(group)
-    elif backend == Backend.MCCL:
-        # Call cluster getting rank function.
-        rank_id = CollectiveManager.get_instance().get_rank_id(group)
-    else:
-        raise ValueError("For '_get_rank_helper', the argument 'backend' {} is not supported, "
-                         "please use hccl_mpi, hccl or nccl.".format(backend))
+    if _hccl_test():
+        return hccl.get_rank_id(group)
+    rank_id = CollectiveManager.get_instance().get_rank_id(group)
     return rank_id
 
 
 @check_parameter_available
-def _get_local_rank_helper(group, backend):
+def _get_local_rank_helper(group):
     """
     The Helper to do get_local_rank_id.
 
@@ -292,24 +219,12 @@ def _get_local_rank_helper(group, backend):
     Returns:
         Integer. The local rank id of the calling process.
     """
-    rank_id = None
-    if backend == Backend.HCCL_MPI:
-        rank_id = mpi.get_rank_id(group)
-    elif backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            rank_id = hccl.get_local_rank_id()
-        else:
-            rank_id = hccl.get_local_rank_id(group)
-    elif backend == Backend.NCCL:
-        raise RuntimeError("Nccl doesn't support get_local_rank_id now.")
-    else:
-        raise ValueError("For '_get_local_rank_helper', the argument 'backend' {} is not supported, "
-                         "please use hccl_mpi or hccl.".format(backend))
+    rank_id = CollectiveManager.get_instance().get_local_rank_id(group)
     return rank_id
 
 
 @check_parameter_available
-def _get_size_helper(group, backend):
+def _get_size_helper(group):
     """
     The Helper to do get_rank_size.
 
@@ -323,30 +238,17 @@ def _get_size_helper(group, backend):
     Returns:
         Integer. The rank size of specified group.
     """
-    size = None
     if _check_bypass_rank_id_and_size():
         size = 1
         return size
-    if backend == Backend.HCCL_MPI:
-        size = mpi.get_rank_size(group)
-    elif backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            size = hccl.get_rank_size()
-        else:
-            size = hccl.get_rank_size(group)
-    elif backend == Backend.NCCL:
-        size = get_rank_size(group)
-    elif backend == Backend.MCCL:
-        # Call cluster getting group size function.
-        size = CollectiveManager.get_instance().get_group_size(group)
-    else:
-        raise ValueError("For '_get_size_helper', the argument 'backend' {} is not supported, "
-                         "please use hccl or nccl.".format(backend))
+    if _hccl_test():
+        return hccl.get_rank_size(group)
+    size = CollectiveManager.get_instance().get_group_size(group)
     return size
 
 
 @check_parameter_available
-def _get_local_size_helper(group, backend):
+def _get_local_size_helper(group):
     """
     The Helper to do get_local_rank_size.
 
@@ -360,24 +262,12 @@ def _get_local_size_helper(group, backend):
     Returns:
         Integer. The local rank size where the calling process is being within specified group.
     """
-    size = None
-    if backend == Backend.HCCL_MPI:
-        size = mpi.get_local_rank_size(group)
-    elif backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            size = hccl.get_local_rank_size()
-        else:
-            size = hccl.get_local_rank_size(group)
-    elif backend == Backend.NCCL:
-        raise RuntimeError("Nccl doesn't support get_local_rank_size now.")
-    else:
-        raise ValueError("For '_get_local_size_helper', the argument 'backend' {} is not supported, "
-                         "please use hccl.".format(backend))
+    size = CollectiveManager.get_instance().get_local_group_size(group)
     return size
 
 
 @check_parameter_available
-def _get_world_rank_from_group_rank_helper(group, group_rank_id, backend):
+def _get_world_rank_from_group_rank_helper(group, group_rank_id):
     """
     The Helper to do get_world_rank_from_group_rank.
 
@@ -393,29 +283,17 @@ def _get_world_rank_from_group_rank_helper(group, group_rank_id, backend):
     Returns:
         Integer. A rank id in world communication group.
     """
-    world_rank_id = None
     if not isinstance(group_rank_id, int):
         raise TypeError("For 'get_world_rank_from_group_rank', the argument 'group_rank_id' must be"
                         " type of int, but got 'group_rank_id' type : {}.".format(type(group_rank_id)))
-    if backend == Backend.HCCL_MPI:
-        if group == HCCL_WORLD_COMM_GROUP:
-            raise ValueError("For 'get_world_rank_from_group_rank', the argument 'group' "
-                             "should not be 'HCCL_WORLD_COMM_GROUP'.")
-        world_rank_id = mpi.get_world_rank_from_group_rank(group, group_rank_id)
-    elif backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            raise ValueError("For 'get_world_rank_from_group_rank' on GPU, the argument 'group' "
-                             "should be 'NCCL_WORLD_COMM_GROUP', but got 'HCCL_WORLD_COMM_GROUP'.")
-        world_rank_id = hccl.get_world_rank_from_group_rank(group, group_rank_id)
-    elif backend == Backend.NCCL:
-        raise RuntimeError("Nccl doesn't support get_world_rank_from_group_rank now.")
-    else:
-        raise ValueError("The argument 'backend' {} is not supported, please use hccl.".format(backend))
+    if _hccl_test():
+        return hccl.get_world_rank_from_group_rank(group, group_rank_id)
+    world_rank_id = CollectiveManager.get_instance().get_world_rank_from_group_rank(group, group_rank_id)
     return world_rank_id
 
 
 @check_parameter_available
-def _get_group_rank_from_world_rank_helper(world_rank_id, group, backend):
+def _get_group_rank_from_world_rank_helper(world_rank_id, group):
     """
     The Helper to do get_group_rank_from_world_rank.
 
@@ -435,25 +313,14 @@ def _get_group_rank_from_world_rank_helper(world_rank_id, group, backend):
     if not isinstance(world_rank_id, int):
         raise TypeError("For 'get_group_rank_from_world_rank', the argument 'world_rank_id' must be type of int, "
                         "but got 'world_rank_id' type : {}.".format(type(world_rank_id)))
-    if backend == Backend.HCCL_MPI:
-        if group == HCCL_WORLD_COMM_GROUP:
-            raise ValueError("For 'get_group_rank_from_world_rank', the argument 'group' "
-                             "should not be 'HCCL_WORLD_COMM_GROUP'.")
-        group_rank_id = mpi.get_group_rank_from_world_rank(world_rank_id, group)
-    elif backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            raise ValueError("For 'get_group_rank_from_world_rank' on GPU, the argument 'group' "
-                             "should be 'NCCL_WORLD_COMM_GROUP', but got 'HCCL_WORLD_COMM_GROUP'.")
-        group_rank_id = hccl.get_group_rank_from_world_rank(world_rank_id, group)
-    elif backend == Backend.NCCL:
-        raise RuntimeError("Nccl doesn't support get_group_rank_from_world_rank now.")
-    else:
-        raise ValueError("The argument 'backend' {} is not supported, please use hccl.".format(backend))
+    if _hccl_test():
+        return hccl.get_group_rank_from_world_rank(world_rank_id, group)
+    group_rank_id = CollectiveManager.get_instance().get_group_rank_from_world_rank(world_rank_id, group)
     return group_rank_id
 
 
 @check_parameter_available
-def _create_group_helper(group, rank_ids, backend):
+def _create_group_helper(group, rank_ids):
     """
     The Helper to do create_group.
 
@@ -467,35 +334,35 @@ def _create_group_helper(group, rank_ids, backend):
         ValueError: If rank_ids size is not larger than 1 or rank_ids has duplicate data or backend is invalid.
     """
     if group in _ExistingGroup.ITEMS.keys():
-        if rank_ids != _ExistingGroup.ITEMS[group]:
+        if rank_ids != _ExistingGroup.ITEMS.get(group):
             raise ValueError("The group {} has been created, the rank_list is {}, "
                              "but current rank_list for the group is {}".
                              format(group, _ExistingGroup.ITEMS[group], rank_ids))
         logger.warning("%r group has existed.", group)
         return
-    if backend == Backend.HCCL:
-        if not isinstance(rank_ids, list):
-            raise TypeError("For 'create_group', the argument 'rank_ids' must be type of list, "
-                            "but got 'rank_ids' type : {}.".format(type(rank_ids)))
-        rank_size = len(rank_ids)
-        if rank_size < 1:
-            raise ValueError("For 'create_group', the argument 'rank_ids' size should be greater than 1, "
-                             "but got 'rank_ids' size : {}.".format(len(rank_ids)))
-        if len(rank_ids) - len(list(set(rank_ids))) > 0:
-            raise ValueError("List rank_ids in Group {} has duplicate data!".format(group))
+    if not isinstance(rank_ids, list):
+        raise TypeError("For 'create_group', the argument 'rank_ids' must be type of list, "
+                        "but got 'rank_ids' type : {}.".format(type(rank_ids)))
+    rank_size = len(rank_ids)
+    if rank_size < 1:
+        raise ValueError("For 'create_group', the argument 'rank_ids' size should be greater than 1, "
+                         "but got 'rank_ids' size : {}.".format(len(rank_ids)))
+    if len(rank_ids) - len(list(set(rank_ids))) > 0:
+        raise ValueError("List rank_ids in Group {} has duplicate data!".format(group))
+    if _hccl_test():
         hccl.create_group(group, rank_size, rank_ids)
-    elif backend == Backend.HCCL_MPI:
-        mpi.create_group(group, rank_ids)
-    elif backend == Backend.NCCL:
-        raise RuntimeError("Nccl doesn't support create_group now.")
     else:
-        raise ValueError("The context configuration parameter 'backend' {} is not supported, "
-                         "please use hccl.".format(backend))
+        result = CollectiveManager.get_instance().create_group(group, rank_ids)
+        if not result:
+            raise RuntimeError("Failed to create communication group for {} with rank ids {}. "
+                               "If NCCL is used, 'export NCCL_DEBUG=INFO' "
+                               "is suggested before launching jobs.".format(group, rank_ids))
+
     _ExistingGroup.ITEMS[group] = rank_ids
 
 
 @check_parameter_available
-def _destroy_group_helper(group, backend):
+def _destroy_group_helper(group):
     """
     The Helper to do destroy_group.
 
@@ -506,12 +373,9 @@ def _destroy_group_helper(group, backend):
     Raises:
         ValueError: If group is "hccl_world_group" or backend is invalid.
     """
-    if backend == Backend.HCCL:
-        if group == HCCL_WORLD_COMM_GROUP:
-            raise ValueError("The hccl_world_group does not support destruction.")
-        hccl.destroy_group(group)
-    elif backend == Backend.NCCL:
-        raise RuntimeError("Nccl doesn't support destroy_group now.")
+    if group == GlobalComm.WORLD_COMM_GROUP:
+        raise ValueError("The world_group does not support destruction.")
+    if _hccl_test():
+        hccl.create_group(group)
     else:
-        raise ValueError("The context configuration parameter 'backend' {} is not supported, "
-                         "please use hccl.".format(backend))
+        CollectiveManager.get_instance().destroy_group(group)

mindspore/communication/_hccl_management.py

@@ -19,6 +19,7 @@ from __future__ import division
 
 import ctypes
 import os
+
 from mindspore import context
 from mindspore._c_expression import get_hccl_rank_id, get_hccl_rank_size
 
@@ -81,7 +82,7 @@ def load_lib():
     """load hccl lib"""
     try:
         base_dir = os.path.dirname(os.path.realpath(__file__))
-        lib_path = os.path.join(base_dir, "../lib", HCCL_LIB)
+        lib_path = os.path.join(base_dir, "../lib/plugin/ascend", HCCL_LIB)
         hccl_lib = ctypes.CDLL(lib_path)
     except Exception:
         raise RuntimeError('Get hccl lib error.')