mindspore 2.7.0__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/common/tensor.py

@@ -17,7 +17,6 @@
 __all__ = ['Tensor']
 
 import abc
-import math
 import numbers
 import numpy as np
 
@@ -29,7 +28,6 @@ from mindspore import log as logger
 from mindspore.common import dtype as mstype
 from mindspore.common.hook_handle import _TensorHookHandle
 
-from mindspore.common._utils import get_slice_num
 from mindspore.common._register_for_tensor import tensor_operator_registry
 from mindspore._c_expression import TensorPy as TensorPy_
 from mindspore._c_expression import _rmod_instance
@@ -296,6 +294,7 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         ...
         >>> # initialize a tensor with init
         >>> t4 = Tensor(shape = (1, 3), dtype=ms.float32, init=One())
+        >>> t4.init_data()
         >>> print(t4)
         [[1. 1. 1.]]
         >>> print(type(t4))
@@ -326,7 +325,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
 
     def __repr__(self):
         if self.init_finished:
-            TensorPy_.data_sync(self, True)
             return TensorPy_.__repr__(self)
         return ''
 
@@ -453,6 +451,8 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
     def __str__(self):
         if self.dtype == mstype.type_none:
             return "Unknown Tensor type!"
+        if not self._data_ptr():
+            return TensorPy_.__str__(self)
         return str(self.asnumpy())
 
     def __getstate__(self):
@@ -983,37 +983,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return self.asnumpy()
 
-    def is_persistent_data(self):
-        """
-        Check if size of tensor is huge, and need save data to persistent storage.
-        If size of tensor is bigger then MS_EMBEDDING_REMOTE_CACHE_MEMORY_SIZE, it will
-        use persistent storage to save tensor data. And will spilt data to some slice.
-
-        Returns:
-            True or False
-        """
-        return TensorPy_.is_persistent_data(self)
-
-    def asnumpy_of_slice_persistent_data(self, param_key, slice_index):
-        """
-        Convert a slice of tensor data to numpy array. A slice is part of tensor data.
-        Returns as a NumPy ndarray. This slice tensor data and the returned ndarray
-        share the same underlying storage. Changes to self tensor will be reflected in the ndarray.
-
-        Returns:
-            A numpy ndarray which shares the same underlying storage with the slice of tensor data.
-        """
-        return TensorPy_.asnumpy_of_slice_persistent_data(self, param_key, slice_index)
-
-    def slice_num_of_persistent_data(self):
-        """
-        Get slice num of a tensor which use persistent storage.
-
-        Returns:
-            Num of slice.
-        """
-        return self.slice_num_of_persistent_data_
-
     def slice_scatter(self, src, axis=0, start=None, end=None, step=1):
         """
         For details, please refer to :func:`mindspore.ops.slice_scatter`.
@@ -1032,15 +1001,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('geqrf')(self)
 
-    def slice_shape_of_persistent_data(self):
-        """
-        Get slice shape of tensor after cut to slice size.
-
-        Returns:
-            The slice shape of tensor.
-        """
-        return self.slice_shape_of_persistent_data_
-
     def value(self):
         """
         Get the value of the tensor or the parameter.
@@ -1241,35 +1201,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('angle')(self)
 
-    def view(self, *shape):
-        """
-        Reshape the tensor according to the input shape. It's the same as :func:`mindspore.Tensor.reshape`,
-        implemented by the underlying reshape operator.
-
-        Args:
-            shape (Union[tuple(int), int]): Dimension of the output tensor.
-
-        Returns:
-            Tensor, which dimension is the input shape's value.
-
-        Examples:
-            >>> from mindspore import Tensor
-            >>> import numpy as np
-            >>> a = Tensor(np.array([[1, 2, 3], [2, 3, 4]], dtype=np.float32))
-            >>> output = a.view((3, 2))
-            >>> print(output)
-            [[1. 2.]
-            [3. 2.]
-            [3. 4.]]
-        """
-        if not shape:
-            raise ValueError("The shape variable should not be empty")
-        if isinstance(shape[0], tuple):
-            if len(shape) != 1:
-                raise ValueError(f"Only one tuple is needed, but got {shape}")
-            shape = shape[0]
-        return tensor_operator_registry.get('reshape')(self, shape)
-
     def bitwise_left_shift(self, other):
         """
         For details, please refer to :func:`mindspore.ops.bitwise_left_shift`.
@@ -1302,18 +1233,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('ger')(self, vec2)
 
-    def broadcast_to(self, shape):
-        """
-        For details, please refer to :func:`mindspore.ops.broadcast_to`.
-        """
-        return tensor_operator_registry.get('broadcast_to')(self, shape)
-
-    def real(self):
-        r"""
-        For details, please refer to :func:`mindspore.ops.real`.
-        """
-        return tensor_operator_registry.get('real')(self)
-
     def tanh_(self):
         r"""
         Computes hyperbolic tangent of self inplace element-wise. The Tanh function is defined as:
@@ -1538,8 +1457,7 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
             >>> print(output.shape)
             (24,)
         """
-        reshape_op = tensor_operator_registry.get('reshape')
-        return reshape_op(self, (-1,))
+        return self.reshape((-1,))
 
     def rot90(self, k, dims):
         r"""
@@ -1577,15 +1495,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return self._size
 
-    def permute(self, *axis):
-        """
-        Tensor.permute supports unpacking the `axis` argument automatically when it is passed as an indefinite number of
-        positional arguments, which has a slight difference from the input parameter of :func:`mindspore.ops.permute`.
-        For details, please refer to :func:`mindspore.ops.permute`.
-        """
-        perm = validator.check_transpose_axis(axis, self.ndim)
-        return tensor_operator_registry.get('permute')(self, perm)
-
     def positive(self):
         """
         For details, please refer to :func:`mindspore.ops.positive`.
@@ -1622,12 +1531,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('swapdims')(self, dim0, dim1)
 
-    def squeeze(self, axis=None):
-        """
-        For details, please refer to :func:`mindspore.ops.squeeze`.
-        """
-        return tensor_operator_registry.get('squeeze')(self, axis)
-
     def slogdet(self):
         """
         For details, please refer to :func:`mindspore.ops.slogdet`.
@@ -2085,15 +1988,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
 
         if shape is None:
             shape = self.shape
-        # At embedding cache scenes, we need limit the size of memory for tensor.
-        # And save out of range data to persistent storage to support TB-Level size of tensor.
-        data_shape = list(shape)
-        slice_num_of_persistent_data = get_slice_num(self.dtype, shape)
-        if slice_num_of_persistent_data > 1:
-            slice_first_dim = math.ceil(shape[0] / slice_num_of_persistent_data)
-            data_shape[0] = slice_first_dim
-            self.slice_shape_of_persistent_data_ = data_shape
-            self.slice_num_of_persistent_data_ = slice_num_of_persistent_data
 
         from mindspore.common.initializer import Zero as ZeroInitializer
 
@@ -2101,9 +1995,9 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         try:
             dtype_ = mstype.int8 if is_qint4x2 else self.dtype
             if isinstance(self.init, ZeroInitializer):
-                data = np.zeros(data_shape, dtype=mstype._dtype_to_nptype(dtype_))  # pylint:disable=protected-access
+                data = np.zeros(shape, dtype=mstype._dtype_to_nptype(dtype_))  # pylint:disable=protected-access
             else:
-                data = np.ndarray(data_shape, dtype=mstype._dtype_to_nptype(dtype_))  # pylint:disable=protected-access
+                data = np.ndarray(shape, dtype=mstype._dtype_to_nptype(dtype_))  # pylint:disable=protected-access
         except ValueError as e:
             msg = "Error shape={}".format(shape)
             logger.critical(msg)
@@ -2139,16 +2033,12 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
                     self.init.seed, _ = self.seed
 
         with seed_context(self.init):
-            if (not isinstance(self.init, ZeroInitializer) and slice_num_of_persistent_data == 1) \
+            if (not isinstance(self.init, ZeroInitializer)) \
                     and not is_reboot_node():
                 self.init(data)
         self.init = None
 
-        # At embedding cache scenes. When size of tensor is out of range, we store data to persistent storage
-        if slice_num_of_persistent_data > 1:
-            self.assign_value(TensorPy_.persistent_data_from_numpy(data, slice_num_of_persistent_data))
-        else:
-            self.assign_value(TensorPy_.from_numpy(data))
+        self.assign_value(TensorPy_.from_numpy(data))
 
         if is_qint4x2:
             self.set_dtype(mstype.qint4x2)
@@ -2956,41 +2846,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('bmm')(self, mat2)
 
-    def to(self, dtype):
-        r"""
-        Performs tensor dtype conversion.
-
-        Note:
-            - If the `self` Tensor already has the correct `mindspore.dtype`, then self is returned.
-              Otherwise, the returned tensor is a copy of `self` with the desired mindspore.dtype.
-            - When converting complex numbers to boolean type, the imaginary part of the complex number is not
-              taken into account. As long as the real part is non-zero, it returns True; otherwise, it returns False.
-
-        Args:
-            dtype (dtype.Number, bool): The valid data type of the output tensor. Only constant value is allowed.
-                Only Support type bool in PyNative mode.
-
-        Returns:
-            Tensor, converted to the specified `dtype`.
-
-        Raises:
-            TypeError: If `dtype` is not a Number.
-
-        Supported Platforms:
-            ``Ascend`` ``GPU`` ``CPU``
-
-        Examples:
-            >>> import numpy as np
-            >>> import mindspore
-            >>> from mindspore import Tensor
-            >>> input_np = np.random.randn(2, 3, 4, 5).astype(np.float32)
-            >>> input_x = Tensor(input_np)
-            >>> dtype = mindspore.int32
-            >>> output = input_x.to(dtype)
-            >>> print(output.dtype)
-            Int32
-        """
-        return self if self.dtype == dtype else self._to(dtype)
 
     def type(self, dtype=None):
         r"""
@@ -3024,9 +2879,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         r"""
         Returns self tensor cast to the type of the with the input other tensor.
 
-        .. warning::
-            This is an experimental API that is subject to change or deletion.
-
         Note:
             When converting complex numbers to boolean type, the imaginary part of the complex number is not
             taken into account. As long as the real part is non-zero, it returns True; otherwise, it returns False.
@@ -3329,14 +3181,12 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('unfold')(self, kernel_size, dilation, padding, stride)
 
-    def expand(self, size):
+    def expand(self, *size):
         r"""
         For details, please refer to :func:`mindspore.ops.broadcast_to`.
         The parameter `size` of the current interface is the same as the parameter `shape` of the reference interface.
         """
-        if isinstance(size, Tensor):
-            size = tensor_operator_registry.get('tensortotuple')()(size)
-        return tensor_operator_registry.get('expand')(self, size)
+        return self.broadcast_to(*size)
 
     def cumprod(self, dim, dtype=None):
         r"""
@@ -3537,9 +3387,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         r"""
         Return a tensor filled with zeros.
 
-        .. warning::
-            This is an experimental API that is subject to change or deletion.
-
         Returns:
             Return a tensor. Fill self tensor with zeros.
 
@@ -3563,6 +3410,12 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('sign')(self)
 
+    def sign_(self):
+        """
+        In-place version of :func:`mindspore.mint.sign`.
+        """
+        return tensor_operator_registry.get('sign_')(self)
+
     def signbit(self):
         """
         For details, please refer to :func:`mindspore.ops.signbit`.
@@ -3575,12 +3428,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('sgn')(self)
 
-    def imag(self):
-        r"""
-        For details, please refer to :func:`mindspore.ops.imag`.
-        """
-        return tensor_operator_registry.get('imag')(self)
-
     def quantile(self, q, axis=None, keepdims=False):
         r"""
         This interface is deprecated from version 2.4 and will be removed in a future version.

mindspore/communication/_comm_helper.py

@@ -25,7 +25,8 @@ from mindspore import context
 from mindspore.parallel._ps_context import _is_role_sched, _is_ps_mode,\
                                            _get_ps_context
 from mindspore import log as logger
-from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception, MSContext, GroupOptions
+from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception, MSContext, GroupOptions, \
+    ParallelCommManager
 from mindspore.common._utils import load_lib
 
 HCCL_LIB = 'libhccl_plugin.so'
@@ -523,6 +524,9 @@ def _create_group_helper(group, rank_ids, options=None):
             raise RuntimeError("Failed to create communication group for {} with rank ids {}. "
                                "If NCCL is used, 'export NCCL_DEBUG=INFO' "
                                "is suggested before launching jobs.".format(group, rank_ids))
+        group_info = ParallelCommManager.get_instance().hccl_groups(rank_ids)
+        if group_info is None or not group_info[1]:
+            ParallelCommManager.get_instance().set_hccl_groups(rank_ids, group, True)
 
     _ExistingGroup.ITEMS[group] = rank_ids
     sorted_ranks = sorted(rank_ids)
@@ -547,7 +551,13 @@ def _destroy_group_helper(group):
     if _hccl_test():
         hccl.create_group(group)
     else:
+        group_ranks_map = CollectiveManager.get_instance().get_group_map()
+        ranks = group_ranks_map.get(group)
+        if ranks is not None:
+            ParallelCommManager.get_instance().set_hccl_groups(ranks, group, False)
         CollectiveManager.get_instance().destroy_group(group)
+        del _ExistingGroup.ITEMS[group]
+        del _ExistingGroup.GROUP_RANKS[group]
 
 
 @check_parameter_available

mindspore/communication/comm_func.py

@@ -18,7 +18,7 @@ Defines communication operators with functional form.
 """
 from mindspore.communication import GlobalComm, get_group_rank_from_world_rank, get_group_size
 from mindspore.communication.management import _get_group
-from mindspore.communication._comm_helper import _get_group_rank_from_world_rank_from_cache_helper
+from mindspore.communication._comm_helper import _get_group_rank_from_world_rank_from_cache_helper, _get_rank_helper
 from mindspore.common.tensor import Tensor
 from mindspore._c_expression import TensorPy as Tensor_
 from mindspore.ops import ReduceOp, cat
@@ -26,7 +26,8 @@ from mindspore.ops._primitive_cache import _get_cache_prim
 from mindspore.ops.primitive import _primexpr
 from mindspore.ops.auto_generate.gen_ops_prim import (inner_comm_all_reduce_op, inner_comm_all_gather_op,
                                                       inner_comm_all_to_all_v_op, inner_comm_irecv_op,
-                                                      inner_comm_isend_op, inner_comm_reduce_scatter_op)
+                                                      inner_comm_isend_op, inner_comm_reduce_scatter_op,
+                                                      dist_comm_all_to_all_v_c_op)
 from mindspore._c_expression import CommHandle as CommHandle_
 from mindspore._c_expression.typing import Type
 from mindspore import jit_class
@@ -49,11 +50,13 @@ __all__ = [
     'recv',
     'P2POp',
     'batch_isend_irecv',
+    'all_to_all_v_c'
 ]
 
 import mindspore.ops.operations as P
 
 _GROPU_SIZE_CACHE = {}
+_GROPU_RANK_CACHE = {}
 
 @jit_class
 class CommHandle(CommHandle_):
@@ -733,7 +736,7 @@ def gather_into_tensor(tensor, dst=0, group=GlobalComm.WORLD_COMM_GROUP):
     Args:
         tensor (Tensor): The tensor to be gathered. The shape of tensor is :math:`(x_1, x_2, ..., x_R)`.
         dst(int, optional): Specifies the rank(global rank) of the process that receive the tensor.
-            And only process `dst` will receive the gathered tensor. Default: 0.
+            And only process `dst` will receive the gathered tensor. Default: ``0``.
         group (str, optional): The communication group to work on. Default: ``GlobalComm.WORLD_COMM_GROUP``.
 
     Returns:
@@ -890,7 +893,8 @@ def barrier(group=GlobalComm.WORLD_COMM_GROUP):
     if not isinstance(group, str):
         raise TypeError(f"group must be type of string, but got {type(group)}")
     _op = _get_cache_prim(P.Barrier)(group)
-    return _op()
+    _op()
+    ms.runtime.synchronize()
 
 
 def _deal_comm_outputs(output, async_op, exec_sync=False):
@@ -1452,3 +1456,133 @@ def all_to_all_single_with_output_shape(output_shape, tensor, output_split_sizes
         result = result.reshape((-1,) + recv_shape_without_first_dim)
 
     return result, handle
+
+
+def _get_all_to_all_v_c_numel_list(output, input, send_count_matrix_size):
+    """get numel list for all_to_all_v_c."""
+    send_size_without_first_dim = _get_size(input.shape[1:])
+    recv_size_without_first_dim = _get_size(output.shape[1:])
+    if send_size_without_first_dim != recv_size_without_first_dim:
+        raise ValueError("The input and output dimensions except 0 must be of equal size, "
+                         f"but got {send_size_without_first_dim} and {recv_size_without_first_dim}.")
+    send_count_matrix = [size * send_size_without_first_dim for size in send_count_matrix_size]
+    return send_count_matrix
+
+
+def get_cache_group_size(group=GlobalComm.WORLD_COMM_GROUP):
+    """get cache group size."""
+    global _GROPU_SIZE_CACHE
+    if group not in _GROPU_SIZE_CACHE:
+        _GROPU_SIZE_CACHE[group] = get_group_size(group)
+    group_size = _GROPU_SIZE_CACHE[group]
+    return group_size
+
+
+def get_cache_group_rank(group=GlobalComm.WORLD_COMM_GROUP):
+    """get cache rank id."""
+    global _GROPU_RANK_CACHE
+    if group not in _GROPU_RANK_CACHE:
+        _GROPU_RANK_CACHE[group] = _get_rank_helper(group)
+    group_rank = _GROPU_RANK_CACHE[group]
+    return group_rank
+
+
+def all_to_all_v_c(output, input, send_count_matrix, group=None, async_op=False):
+    r"""
+    Based on the user-specified split size, the input tensor is divided and sent to other devices, where split chunks
+    are received and then merged into a single output tensor.
+
+    Note:
+        Only support PyNative mode, Graph mode is not currently supported.
+
+    Args:
+        output (Tensor): the output tensor is gathered concatenated from remote ranks.
+        input (Tensor): tensor to be scattered to remote rank.
+        send_count_matrix (list[int]): The sending and receiving parameters of all ranks,
+            :math:`\text{send_count_matrix}[i*\text{rank_size}+j]` represents the amount of data sent by
+            rank i to rank j, and the basic unit is first dimension sizes. Among them, `rank_size`
+            indicates the size of the communication group.
+        group (str, optional): The communication group to work on. If ``None``, which means ``"hccl_world_group"`` in
+            Ascend. Default: ``None``.
+        async_op (bool, optional): Whether this operator should be an async operator. Default: ``False`` .
+
+    Returns:
+        CommHandle. CommHandle is an async work handle, if `async_op` is set to True.
+        CommHandle will be None, when `async_op` is False.
+
+    Raises:
+        TypeError: If `input` or `output` is not tensor. `group` is not a str, or async_op is not bool.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start up
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
+            for more details.
+
+            This example should be run with 2 devices.
+
+        >>> import numpy as np
+        >>> import mindspore
+        >>> from mindspore.mint.distributed import init_process_group, get_rank
+        >>> from mindspore.communication.comm_func import all_to_all_v_c
+        >>> from mindspore import Tensor
+        >>> from mindspore.ops import zeros
+        >>>
+        >>> init_process_group()
+        >>> this_rank = get_rank()
+        >>> if this_rank == 0:
+        ...     output = Tensor(np.zeros([3]).astype(np.float32))
+        ...     tensor = Tensor([0, 1, 2.]) * this_rank
+        ...     result = all_to_all_v_c(output, tensor, [0, 3, 3, 0])
+        ...     print(output)
+        >>> if this_rank == 1:
+        ...     output = Tensor(np.zeros([3]).astype(np.float32))
+        ...     tensor = Tensor([0, 1, 2.]) * this_rank
+        ...     result = all_to_all_v_c(output, tensor, [0, 3, 3, 0])
+        ...     print(output)
+        rank 0:
+        [0. 1. 2]
+        rank 1:
+        [0. 0. 0]
+    """
+
+    _check_all_tensors([input])
+    _check_all_tensors([output])
+    if group is None:
+        group = GlobalComm.WORLD_COMM_GROUP
+    if not isinstance(group, str):
+        raise TypeError(
+            "The argument 'group' must be type of string, "
+            "but got 'group' type : {}.".format(type(group))
+        )
+    if not isinstance(async_op, bool):
+        raise TypeError(
+            f"The argument 'async_op' must be a bool, but got {type(async_op)}."
+        )
+    if not isinstance(send_count_matrix, list):
+        raise TypeError("send_count_matrix must be list, but got {}".format(type(send_count_matrix)))
+    if not all(isinstance(x, int) for x in send_count_matrix):
+        raise TypeError("send_count_matrix elements must be of type int")
+    rank_size = get_cache_group_size(group)
+    if rank_size * rank_size != len(send_count_matrix):
+        raise TypeError(f"send_count_matrix must be square matrix, but got {len(send_count_matrix)}.")
+    _send_count_matrix = _get_all_to_all_v_c_numel_list(output, input, send_count_matrix)
+    _input = input.reshape(-1)
+    rank_id = get_cache_group_rank(group)
+    result = dist_comm_all_to_all_v_c_op(
+        output,
+        _input,
+        group,
+        _send_count_matrix,
+        rank_size,
+        rank_id,
+    )
+    _, handle = _deal_comm_outputs(result, async_op)
+    return handle

mindspore/communication/management.py

@@ -22,7 +22,8 @@ from mindspore.communication._comm_helper import Backend, _get_rank_helper, _get
     _create_group_helper, _destroy_group_helper, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, \
     MCCL_WORLD_COMM_GROUP, DEVICE_TO_BACKEND, _get_local_rank_helper, _get_local_size_helper, GlobalComm, \
     _check_mpi_envs, _set_elegant_exit_handle, _get_group_ranks, _get_comm_name_helper, _comm_switch_nic_helper
-from mindspore._c_expression import init_hccl, finalize_hccl, init_cluster, MSContext, ms_ctx_param
+from mindspore._c_expression import init_hccl, finalize_hccl, init_cluster, MSContext, ms_ctx_param, \
+    _init_hccl_with_store, _init_cluster_with_store
 from mindspore.hal.device import is_initialized
 
 __all__ = ["init", "release", "get_rank", "get_local_rank", "get_group_size",
@@ -221,6 +222,83 @@ def init(backend_name=None):
     _set_envs()
 
 
+def _init_without_sched(backend_name=None, init_method=None, timeout=None, world_size=-1, rank=-1, store=None):
+    """
+    Initialize the distributed backends required by the communication services through an existing TcpStore or
+    by creating a new TcpStore. This approach does not rely on an additional Scheduler process.
+
+    Args:
+        backend_name (str, optional): Backend, using ``"hccl"`` / ``"nccl"`` / ``"mccl"``.
+            ``"hccl"`` should be used for Ascend hardware platforms,
+            ``"nccl"`` for GPU hardware platforms and ``"mccl"`` for CPU hardware platforms.
+            If not set, inference is automatically made based on the hardware platform type (device_target).
+            Default: ``None`` .
+        init_method (str, optional): URL specifying how to init collective communication group. Default is ``None``.
+        timeout (timedelta, optional): Timeout for API executed. Default is ``None``. Currently, this parameter is
+            only supported for host-side cluster network configuration using `init_method` or `store`.
+        world_size (int, optional): Number of the processes participating in the job. Default is ``-1``.
+        rank (int, optional): Rank of the current process. Default is ``-1``.
+        store (Store, optional): An object that stores key/value data, facilitating the exchange of inter-process
+            communication addresses and connection information. Default is ``None``. Currently, only the
+            ``TCPStore`` type is supported.
+
+    Raises:
+        TypeError: If `backend_name` is not a string.
+        RuntimeError: If device target is invalid, or backend is invalid, or distributed initialization fails,
+                      or the environment variables RANK_ID/MINDSPORE_HCCL_CONFIG_PATH
+                      have not been exported when backend is HCCL.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+    """
+    device_target = context.get_context("device_target")
+
+    if backend_name is None:
+        if device_target == "Ascend":
+            backend_name = "hccl"
+        elif device_target == "GPU":
+            backend_name = "nccl"
+        elif device_target == "CPU":
+            backend_name = "mccl"
+        else:
+            raise RuntimeError("For 'set_context', the argument 'device_target' {} is not supported in "
+                               "parallel initialization, please use Ascend, GPU or CPU.".format(device_target))
+    if not isinstance(backend_name, str):
+        raise TypeError("For 'init', the argument 'backend_name' must be a string, "
+                        "but got the type : {}".format(type(backend_name)))
+
+    _set_elegant_exit_handle()
+    if backend_name == "hccl":
+        if device_target != "Ascend":
+            raise RuntimeError("For 'init', the argument 'backend_name' should be '{}' to init '{}', "
+                               "but got 'hccl'.".format(DEVICE_TO_BACKEND[device_target], device_target))
+        if is_initialized(device_target):
+            logger.warning(f"For 'init' in Ascend backend, the backend is already initialized, please set it before "
+                           "the definition of any Tensor and Parameter, and the instantiation and execution of any "
+                           "operation and net, otherwise the 'init' may not take effect.")
+        GlobalComm.BACKEND = Backend("hccl")
+        _check_hccl()
+        _init_hccl_with_store(init_method, timeout, world_size, rank, store)
+        GlobalComm.WORLD_COMM_GROUP = HCCL_WORLD_COMM_GROUP
+    elif backend_name == "nccl":
+        if device_target != "GPU":
+            raise RuntimeError("For 'init', the argument 'backend_name' should be '{}' to init '{}', "
+                               "but got 'nccl'.".format(DEVICE_TO_BACKEND[device_target], device_target))
+        _init_cluster_with_store(init_method, timeout, world_size, rank, store)
+        GlobalComm.BACKEND = Backend("nccl")
+        GlobalComm.WORLD_COMM_GROUP = NCCL_WORLD_COMM_GROUP
+    elif backend_name == "mccl":
+        _init_cluster_with_store(init_method, timeout, world_size, rank, store)
+        GlobalComm.BACKEND = Backend("mccl")
+        GlobalComm.WORLD_COMM_GROUP = MCCL_WORLD_COMM_GROUP
+    else:
+        raise RuntimeError("For 'init', the argument 'backend_name' must be one of 'hccl', 'nccl' and 'mccl', "
+                           "but got 'backend_name' : {}".format(backend_name))
+
+    GlobalComm.INITED = True
+    _set_envs()
+
+
 def release():
     """
     Release distributed resource. e.g. HCCL/NCCL/MCCL.
@@ -571,6 +649,12 @@ def create_group(group, rank_ids, options=None):
                     hccl_config(dict)
                 }
 
+            `hccl_config` currently only supports "hccl_buffer_size" or "hccl_comm".
+
+            - hccl_buffer_size (uint32): specifies the size of the HCCL communication buffer.
+            - hccl_comm (int64): specifies an existing HcclComm pointer. If "hccl_comm" is set,
+              "hccl_buffer_size" will be ignored.
+
     Raises:
         TypeError: If group is not a string or `rank_ids` is not a list.
         ValueError: If `rank_ids` size is not larger than 1, or `rank_ids` has duplicate data, or backend is invalid.