mindspore 2.6.0rc1__cp39-cp39-win_amd64.whl → 2.7.0rc1__cp39-cp39-win_amd64.whl

mindspore/common/tensor.py

@@ -32,11 +32,12 @@ from mindspore.common.hook_handle import _TensorHookHandle
 from mindspore.common._utils import get_slice_num
 from mindspore.common._register_for_tensor import tensor_operator_registry
 from mindspore._c_expression import TensorPy as TensorPy_
+from mindspore._c_expression import _rmod_instance
 from mindspore import _checkparam as validator
 from mindspore._checkparam import is_stub_tensor, check_hook_fn
 from mindspore._check_jit_forbidden_api import jit_forbidden_register
 from mindspore.common.symbol import Symbol
-
+from mindspore._c_expression import is_reboot_node
 
 np_types = (np.int8, np.int16, np.int32, np.int64,
             np.uint8, np.uint16, np.uint32, np.uint64, np.float16,
@@ -93,13 +94,6 @@ def _set_symbolic_shape(shape):
     return shape, symbolic_shape
 
 
-def _convert_stub_tensor(input_data):
-    """Convert input to stub tensor"""
-    if not is_stub_tensor(input_data):
-        return input_data
-    return input_data.stub_sync()
-
-
 def _convert_numpy_array(input_data):
     """Convert inpyt to numpy array"""
     if not isinstance(input_data, np_types):
@@ -144,8 +138,6 @@ def _init(input_data=None, dtype=None, shape=None, init=None, const_arg=False, d
         _cast = tensor_operator_registry.get("cast")
         input_data = _cast(input_data, dtype)
 
-    input_data = _convert_stub_tensor(input_data)
-
     if input_data is None and shape is None and init is None and dtype is not None:
         validator.check_type_name('dtype', dtype, mstype.number_type + (mstype.bool_, mstype.string), "Tensor")
         logger.warning(f"For 'Tensor', if 'dtype' is not None, 'input_data', 'shape' or 'init' must not be None.")
@@ -325,7 +317,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
                 return True
         return NotImplemented
 
-
     def __deepcopy__(self, memodict):
         new_obj = Tensor(self)
         new_obj.init = self.init
@@ -379,9 +370,8 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         try:
             data = self._item()
             return int(data)
-        except ValueError:
-            raise ValueError("Only one element tensors can be converted to Python scalars")
-
+        except ValueError as e:
+            raise ValueError("Only one element tensors can be converted to Python scalars") from e
 
     def __float__(self):
         try:
@@ -396,8 +386,8 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
             if not isinstance(data, (int, bool)):
                 raise ValueError
             return int(data)
-        except ValueError:
-            raise ValueError("Only integer tensors of a single element can be converted to an index.")
+        except ValueError as e:
+            raise ValueError("Only integer tensors of a single element can be converted to an index.") from e
 
     def __pos__(self):
         return self
@@ -426,11 +416,8 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
     def __rtruediv__(self, other):
         return tensor_operator_registry.get('__truediv__')(other, self)
 
-    def __mod__(self, other):
-        return tensor_operator_registry.get('__mod__')(self, other)
-
     def __rmod__(self, other):
-        return tensor_operator_registry.get('__mod__')(other, self)
+        return _rmod_instance(other, self)
 
     def __imod__(self, other):
         return self.__mod__(other)
@@ -2110,11 +2097,13 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
 
         from mindspore.common.initializer import Zero as ZeroInitializer
 
+        is_qint4x2 = self.dtype == mstype.qint4x2
         try:
+            dtype_ = mstype.int8 if is_qint4x2 else self.dtype
             if isinstance(self.init, ZeroInitializer):
-                data = np.zeros(data_shape, dtype=mstype.dtype_to_nptype(self.dtype))
+                data = np.zeros(data_shape, dtype=mstype.dtype_to_nptype(dtype_))
             else:
-                data = np.ndarray(data_shape, dtype=mstype.dtype_to_nptype(self.dtype))
+                data = np.ndarray(data_shape, dtype=mstype.dtype_to_nptype(dtype_))
         except ValueError as e:
             msg = "Error shape={}".format(shape)
             logger.critical(msg)
@@ -2127,7 +2116,7 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
                 self.init = init
                 global_seed = get_seed()
                 self._np_seed = np.random.get_state()[1][0]
-                self.need_set_seed = (slice_index is not None)
+                self.need_set_seed = slice_index is not None
                 self._global_seed = global_seed
                 self._seed_offset = 1
                 if self.need_set_seed:
@@ -2150,7 +2139,8 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
                     self.init.seed, _ = self.seed
 
         with seed_context(self.init):
-            if not isinstance(self.init, ZeroInitializer) and slice_num_of_persistent_data == 1:
+            if (not isinstance(self.init, ZeroInitializer) and slice_num_of_persistent_data == 1) \
+                    and not is_reboot_node():
                 self.init(data)
         self.init = None
 
@@ -2159,6 +2149,10 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
             self.assign_value(TensorPy_.persistent_data_from_numpy(data, slice_num_of_persistent_data))
         else:
             self.assign_value(TensorPy_.from_numpy(data))
+
+        if is_qint4x2:
+            self.set_dtype(mstype.qint4x2)
+
         return self
 
     def resize(self, *new_shape):
@@ -2495,6 +2489,39 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('uniform_')(self, from_=from_, to=to, generator=generator)
 
+    def exponential_(self, lambd=1, *, generator=None):
+        r"""
+        Fills `self` tensor with elements drawn from the exponential distribution:
+
+        .. math::
+            f(x) = \lambda \exp(-\lambda x)
+
+        .. warning::
+            - It is only supported on Atlas A2 Training Series Products.
+            - This is an experimental API that is subject to change or deletion.
+
+        Args:
+            lambd (float, optional): Parameters of exponential distribution. Default: ``1``.
+
+        Keyword Args:
+            generator (Generator, optional): a pseudorandom number generator.
+                Default: ``None`` .
+
+        Returns:
+            Tensor, with same shape and same data type with input.
+
+        Supported Platforms:
+            ``Ascend``
+
+        Examples:
+            >>> import mindspore
+            >>> x = mindspore.Tensor([1, 2, 3.0])
+            >>> out = x.exponential_(2)
+            >>> print(out.shape)
+            (3,)
+        """
+        return tensor_operator_registry.get('exponential_')(self, lambd=lambd, generator=generator)
+
     def sum_to_size(self, *size):
         r"""
         Sum self Tensor to the `size`. `size` must be expandable to the Tensor size.
@@ -2526,17 +2553,19 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         shape_x = x.shape
         if len(size) > x.ndim:
             raise ValueError(f"For sum_to_size, size {size} is not expandable to the tensor size {shape_x}.")
+        pre_len = 0
+        pre_axis = []
         if len(size) < x.ndim:
-            pre_axis = tuple([axis for axis in range(x.ndim - len(size))])
-            x = x.sum(pre_axis)
-        axes = []
+            pre_len = x.ndim - len(size)
+            pre_axis = [axis for axis in range(pre_len)]
+        axes = pre_axis
         for i, element in enumerate(size):
-            if element != x.shape[i] and element == 1:
-                axes.append(i)
-            elif element != x.shape[i]:
+            if element != x.shape[i + pre_len] and element == 1:
+                axes.append(i + pre_len)
+            elif element != x.shape[i + pre_len]:
                 raise ValueError(f"For sum_to_size, size {size} is not expandable to the tensor size {shape_x}.")
         if axes:
-            return x.sum(tuple(axes), keepdims=True)
+            return x.sum(tuple(axes), keepdims=True).reshape(size)
         return x
 
     def nanmean(self, axis=None, keepdims=False, *, dtype=None):
@@ -2619,7 +2648,7 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         Note:
            The rank of `self`.
 
-           - Ascend: its rank can be equal to 0 except O2 mode.
+           - Ascend: its rank can be equal to 0 except GE backend.
            - CPU/GPU: its rank should be greater than or eaqual to 1.
 
         Keyword Args:
@@ -2638,7 +2667,7 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         Raises:
             TypeError: If `self` is not Tensor.
             TypeError: If `as_tuple` is not bool.
-            RuntimeError: On GPU or CPU or Ascend O2 mode, if dim of `input` equals to 0.
+            RuntimeError: On GPU or CPU or Ascend GE backend, if dim of `input` equals to 0.
 
         Supported Platforms:
             ``Ascend`` ``GPU`` ``CPU``
@@ -2801,7 +2830,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
             return []
         return self._tolist()
 
-
     def unsorted_segment_min(self, segment_ids, num_segments):
         r"""
         For details, please refer to :func:`mindspore.ops.unsorted_segment_min`.
@@ -2824,7 +2852,7 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         For details, please refer to :func:`mindspore.ops.unique_consecutive`.
         """
-        output, idx, counts =\
+        output, idx, counts = \
             tensor_operator_registry.get("unique_consecutive")(return_inverse, return_counts, dim)(self)
         if return_inverse and return_counts:
             return output, idx, counts
@@ -2963,7 +2991,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
             return str(self.dtype)
         return self.astype(dtype)
 
-
     def type_as(self, other):
         r"""
         Returns self tensor cast to the type of the with the input other tensor.
@@ -3006,7 +3033,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
             return self
         return TensorPy_.type_as(self, other)
 
-
     def bool(self):
         r"""
         Converts input tensor dtype to `bool`.
@@ -3502,49 +3528,6 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('zero_')(self)
 
-    def new_empty(self, size, *, dtype=None, device=None):
-        r"""
-        Returns an uninitialized Tensor of `size`. Its dtype is specified by `dtype` and its
-        device is specified by `device`.
-
-        .. warning::
-            This is an experimental API that is subject to change or deletion.
-
-        Args:
-            size (Union[tuple[int], list[int], int]): The specified shape of output tensor. Only positive integer or
-                tuple or list containing positive integers are allowed.
-
-        Keyword Args:
-            dtype (:class:`mindspore.dtype`, optional): The specified dtype of the output tensor. If `dtype = None`,
-                the tensor will have the same dtype as `self`. Default ``None``.
-            device (string, optional): The specified device of the output tensor. Support ``CPU`` and ``Ascend``. If
-                `device = None`, the tensor will have the same device as `self` and if the device of `self` is not
-                defined, the value set by :func:`mindspore.set_device` will be used. Default ``None``.
-
-        Returns:
-            Tensor, the shape, dtype and device is defined above but with uninitialized data (May be a random value).
-
-        Raises:
-            TypeError: If `size` is neither an int nor a tuple or list of int.
-
-        Supported Platforms:
-            ``Ascend``
-
-        Examples:
-            >>> import mindspore
-            >>> from mindspore import Tensor
-            >>> x = Tensor([[1, 2, 3], [4, 5, 6]])
-            >>> output1 = x.new_empty((2, 3))
-            >>> print(output1)
-            [[0 0 0]
-             [0 0 0]]
-            >>> output2 = x.new_empty((2, 3), dtype=mindspore.float64)
-            >>> print(output2)
-            [[0. 0. 0.]
-             [0. 0. 0.]]
-        """
-        return tensor_operator_registry.get('new_empty')(self, size, dtype, device)
-
     def sign(self):
         r"""
         For details, please refer to :func:`mindspore.ops.sign`.
@@ -3833,6 +3816,23 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return TensorPy_._data_ptr(self)
 
+    def data_ptr(self):
+        r"""
+        Get the data ptr address of tensor, for CPU is host address, GPU/NPU is device address.
+        User should know how to use the data ptr address.
+        Note: this api is an experimental api, users need understatnd it before use.
+
+        Supported Platforms:
+            ``CPU/GPU/Ascend``
+
+        Examples:
+            >>> import mindspore as ms
+            >>> from mindspore import Tensor
+            >>> x = ms.Tensor([1, 2, 3], ms.int64)
+            >>> data_ptr = x.data_ptr()
+        """
+        return TensorPy_._data_ptr(self)
+
     def normal_(self, mean=0, std=1, *, generator=None):
         r"""
         Update the `self` tensor in place by generating random numbers sampled from the normal
@@ -3872,13 +3872,13 @@ class Tensor(TensorPy_, metaclass=_TensorMeta):
         """
         return tensor_operator_registry.get('normal_')(self, mean=mean, std=std, generator=generator)
 
-
     def triangular_solve(self, A, upper=True, transpose=False, unitriangular=False):
         r"""
         For details, please refer to :func:`mindspore.mint.triangular_solve`.
         """
         return tensor_operator_registry.get('triangular_solve')(self, A, upper, transpose, unitriangular)
 
+
 def _vm_compare(*args):
     """Implement `vm_compare` for tensor."""
     if args:

mindspore/communication/_comm_helper.py

@@ -25,7 +25,7 @@ from mindspore import context
 from mindspore.parallel._ps_context import _is_role_sched, _is_ps_mode,\
                                            _get_ps_context
 from mindspore import log as logger
-from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception, MSContext
+from mindspore._c_expression import CollectiveManager, set_cluster_exit_with_exception, MSContext, GroupOptions
 from mindspore.common._utils import load_lib
 
 HCCL_LIB = 'libhccl_plugin.so'
@@ -470,14 +470,25 @@ def _get_group_ranks(group):
 
 
 @check_parameter_available
-def _create_group_helper(group, rank_ids):
+def _create_group_helper(group, rank_ids, options=None):
     """
     The Helper to do create_group.
 
     Args:
         group (str): The communication group.
         rank_ids (list): Rank ids in the group.
-        backend (str): The backend, like "hccl".
+        options (GroupOptions, optional): Additional communication group configuration parameters.
+            The backend will automatically select supported parameters and apply them during group
+            initialization. i.e. for the ``HCCL`` backend, ``hccl_config`` can be specified so that
+            group initialization configurations can be applied. Default is ``None``.
+
+            `GroupOptions` is defined as a class that can be instantiated as a python object.
+
+            .. code-block::
+
+                GroupOptions {
+                    hccl_config(dict)
+                }
 
     Raises:
         TypeError: If rank_ids is not a list.
@@ -499,10 +510,15 @@ def _create_group_helper(group, rank_ids):
                          "but got 'rank_ids' size : {}.".format(len(rank_ids)))
     if len(rank_ids) - len(list(set(rank_ids))) > 0:
         raise ValueError("List rank_ids in Group {} has duplicate data!".format(group))
+    if options is None:
+        options = GroupOptions()
+    if not isinstance(options, GroupOptions):
+        raise TypeError("For 'create_group', the argument 'options' must be type of GroupOptions, "
+                        "but got 'options' type : {}.".format(type(options)))
     if _hccl_test():
         hccl.create_group(group, rank_size, rank_ids)
     else:
-        result = CollectiveManager.get_instance().create_group(group, rank_ids)
+        result = CollectiveManager.get_instance().create_group(group, rank_ids, options)
         if not result:
             raise RuntimeError("Failed to create communication group for {} with rank ids {}. "
                                "If NCCL is used, 'export NCCL_DEBUG=INFO' "
@@ -554,3 +570,29 @@ def _get_group_map():
 def _wait_all_comm_init():
     """Wait for all communicators to be initialized."""
     return CollectiveManager.get_instance().wait_all_comm_init()
+
+
+def _remove_group_info(group_name):
+    """
+    Remove group info after destroy group by user when using arf.
+
+    Args:
+        group_name (str): The user communication group name.
+
+    """
+    CollectiveManager.get_instance().remove_group_info(group_name)
+
+
+def _comm_switch_nic_helper(global_ranks: list, use_backup: list) -> bool:
+    """Switch network interface card between the primary and the secondary NIC.
+
+    Args:
+        global_ranks (list[int], tuple[int]): list of integers. The global rank ids that need switch network interface .
+        use_backup (list[bool], tuple[int]): list of bool. For each rank id in global_ranks, determine whether to use
+            the backup network interface card. True means use, False means not use.
+
+    Returns:
+        bool, whether the network card switch is successful.
+            If one fails, return False. If all are successful, return True.
+    """
+    return CollectiveManager.get_instance().comm_switch_nic(global_ranks, use_backup)

mindspore/communication/management.py

@@ -21,7 +21,7 @@ from mindspore.communication._comm_helper import Backend, _get_rank_helper, _get
     _get_world_rank_from_group_rank_helper, _get_group_rank_from_world_rank_helper, \
     _create_group_helper, _destroy_group_helper, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, \
     MCCL_WORLD_COMM_GROUP, DEVICE_TO_BACKEND, _get_local_rank_helper, _get_local_size_helper, GlobalComm, \
-    _check_mpi_envs, _set_elegant_exit_handle, _get_group_ranks, _get_comm_name_helper
+    _check_mpi_envs, _set_elegant_exit_handle, _get_group_ranks, _get_comm_name_helper, _comm_switch_nic_helper
 from mindspore._c_expression import init_hccl, finalize_hccl, init_cluster, MSContext, ms_ctx_param
 from mindspore.hal.device import is_initialized
 
@@ -116,7 +116,7 @@ def _check_hccl():
     except Exception as e:
         logger.error(f"Check hccl failed: {e}")
         raise RuntimeError("\"hccl\" wheel was not installed correctly. For details, refer to the installation "
-                           "guidelines: https://www.mindspore.cn/install")
+                           "guidelines: https://www.mindspore.cn/install") from e
 
 
 def init(backend_name=None):
@@ -545,7 +545,7 @@ def get_group_rank_from_world_rank(world_rank_id, group):
     return _get_group_rank_from_world_rank_helper(world_rank_id=world_rank_id, group=group)
 
 
-def create_group(group, rank_ids):
+def create_group(group, rank_ids, options=None):
     """
     Create a user collective communication group.
 
@@ -558,6 +558,18 @@ def create_group(group, rank_ids):
     Args:
         group (str): The name of the communication group to be created.
         rank_ids (list): A list of device IDs.
+        options (GroupOptions, optional): Additional communication group configuration parameters.
+            The backend will automatically select supported parameters and apply them during group
+            initialization. i.e. for the ``HCCL`` backend, ``hccl_config`` can be specified so that
+            group initialization configurations can be applied. Default is ``None``.
+
+            `GroupOptions` is defined as a class that can be instantiated as a python object.
+
+            .. code-block::
+
+                GroupOptions {
+                    hccl_config(dict)
+                }
 
     Raises:
         TypeError: If group is not a string or `rank_ids` is not a list.
@@ -578,22 +590,24 @@ def create_group(group, rank_ids):
             for more details.
 
         >>> import mindspore as ms
-        >>> from mindspore import set_context
-        >>> from mindspore import ops
+        >>> from mindspore import set_context, ops
+        >>> from mindspore._c_expression import GroupOptions
         >>> from mindspore.communication import init, create_group, get_rank
         >>> set_context(mode=ms.GRAPH_MODE)
         >>> ms.set_device(device_target="Ascend")
         >>> init()
         >>> group = "0-7"
         >>> rank_ids = [0,7]
+        >>> options = GroupOptions()
+        >>> options.hccl_config = {"hccl_buffer_size": 400}
         >>> if get_rank() in rank_ids:
-        ...     create_group(group, rank_ids)
+        ...     create_group(group, rank_ids, options)
         ...     allreduce = ops.AllReduce(group)
     """
     if not isinstance(group, str):
         raise TypeError("For 'create_group', the argument 'group' must be type of string, "
                         "but got 'group' type : {}.".format(type(group)))
-    _create_group_helper(group, rank_ids)
+    _create_group_helper(group, rank_ids, options)
 
 
 def destroy_group(group):
@@ -740,3 +754,61 @@ def get_process_group_ranks(group=GlobalComm.WORLD_COMM_GROUP):
 
     """
     return _get_group_ranks(group=_get_group(group))
+
+
+def _comm_switch_nic(global_ranks, use_backup):
+    """Switch network interface card between the primary and the secondary NIC.
+
+    Args:
+        global_ranks (list[int], tuple[int]): list of integers. The global rank ids that need switch network interface .
+        use_backup (list[bool], tuple[int]): list of bool. For each rank id in global_ranks, determine whether to use
+            the backup network interface card. True means use, False means not use.
+
+    Returns:
+        bool, whether the network card switch is successful.
+            If one fails, return False. If all are successful, return True.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+
+            Please see the `msrun start up
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
+            for more details.
+
+            This example should be run with 4 devices.
+
+        >>> import numpy as np
+        >>> from mindspore.communication import init, _comm_switch_nic
+        >>> from mindspore.communication.management import _comm_switch_nic
+        >>>
+        >>> init()
+        >>> ret = _comm_switch_nic([0, 1], [True, False])
+        >>> print(ret)
+        True
+
+    """
+    max_rank = get_group_size() - 1
+    if not all(isinstance(i, (list, tuple)) for i in (global_ranks, use_backup)):
+        raise ValueError(f"For _comm_switch_nic, the args 'global_ranks' and 'use_backup' should be list or tuple, "
+                         f"but got 'global_ranks' type {type(global_ranks)}, 'use_backup' type {type(use_backup)}")
+    if not all(isinstance(rank, int) and not isinstance(rank, bool) and rank <= max_rank for rank in global_ranks):
+        raise ValueError(f"For _comm_switch_nic, the all elements  in 'global_ranks' should be int number, and less "
+                         f"than {get_group_size()}, but got 'global_ranks' : {global_ranks}.")
+    if not all(isinstance(ub, bool) for ub in use_backup):
+        raise ValueError(f"For _comm_switch_nic, the all elements  in 'use_backup' should be bool, but got "
+                         f"'use_backup' : {use_backup}.")
+    if len(set(global_ranks)) != len(global_ranks):
+        raise ValueError(f"For _comm_switch_nic, the all elements  in 'global_ranks' should be different, but got "
+                         f"'global_ranks' : {global_ranks}.")
+    if len(global_ranks) != len(use_backup):
+        raise ValueError(f"For _comm_switch_nic, the elements number in 'global_ranks' should be equal to 'use_backup',"
+                         f" but got 'global_ranks' {len(global_ranks)} elements: {global_ranks},"
+                         f" 'use_backup' {len(use_backup)} elements:  {use_backup},.")
+    return _comm_switch_nic_helper(global_ranks, use_backup)