mindspore 2.3.0__cp39-none-any.whl → 2.3.0rc2__cp39-none-any.whl

mindspore/common/parameter.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2024 Huawei Technologies Co., Ltd
+# Copyright 2020-2022 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -188,9 +188,6 @@ class Parameter(Tensor_):
         parallel_optimizer (bool): It is used to filter the weight shard operation in `SEMI_AUTO_PARALLEL` or
             `AUTO_PARALLEL` mode. It works only when enable parallel optimizer in
             `mindspore.set_auto_parallel_context()`. Default: ``True`` .
-        storage_format (str): Only Ascend device target is supported. It is used to specify the format of the weight
-            loaded to the device. By default, the format is not changed. The optional values are ``"FRACTAL_NZ"`` ,
-            ``"NC1HWC0"`` , ``"FRACTAL_Z"`` , etc. Default: ``""`` .
 
     Examples:
         >>> import numpy as np
@@ -226,7 +223,6 @@ class Parameter(Tensor_):
         # it's better to make the Initializer a kind of tensor.
         obj.init_mode = None
         obj.is_default_input_init = init_data_flag
-        obj.from_ckpt = False
         if obj.has_init:
             obj.init_mode = default_input
         else:
@@ -243,8 +239,7 @@ class Parameter(Tensor_):
         return (
             Parameter, (data, self.name, self.requires_grad, self.layerwise_parallel))
 
-    def __init__(self, default_input, name=None, requires_grad=True, layerwise_parallel=False, parallel_optimizer=True,
-                 storage_format=""):
+    def __init__(self, default_input, name=None, requires_grad=True, layerwise_parallel=False, parallel_optimizer=True):
         self.param_info = ParamInfo()
         self.init_in_server = False
         self.name = name
@@ -267,7 +262,6 @@ class Parameter(Tensor_):
         self.is_in_shard = False
         self._pipeline_stage_list = []
         self.slice_num = 1
-        self.from_ckpt = False
         if -1 in self.shape:
             raise ValueError(f"All shape elements of the Parameter must be positive. But got None.")
         if isinstance(default_input, (Tensor_, Tensor)):
@@ -295,7 +289,6 @@ class Parameter(Tensor_):
             raise TypeError(f"The type of the argument 'default_input' must be in ['Tensor', 'int', 'float',"
                             f" 'numpy.ndarray', 'list']. But got type {type(default_input)}.")
         self.param_info.parameter_shape = self.shape
-        self.param_info.storage_format = storage_format
 
         import mindspore.ops.operations.other_ops as other_ops
         self.load = other_ops.Load()
@@ -338,6 +331,11 @@ class Parameter(Tensor_):
                     # in other place, so we can make a Tensor without copy data.
                     return (Tensor, data)
                 # make a copy of Tensor to init the parameter.
+                if data.dtype == mstype.bfloat16:
+                    from mindspore.ops.operations import Cast
+                    cpu_cast = Cast().set_device("CPU")
+                    data = cpu_cast(data, mstype.float32)
+                    return (Tensor, data.asnumpy(), mstype.bfloat16)
                 if data.dtype == mstype.qint4x2:
                     return (Tensor, data.asnumpy(), mstype.qint4x2)
                 return (Tensor, data.asnumpy())
@@ -750,7 +748,6 @@ class Parameter(Tensor_):
     def requires_grad(self, value=True):
         if not isinstance(value, bool):
             raise TypeError("The argument `requires_grad` must be bool type")
-        Tensor_.wait_pipeline(self)
         self.param_info.requires_grad = value
 
     @property
@@ -792,10 +789,14 @@ class Parameter(Tensor_):
         return new_param
 
     @_LogActionOnce(logger=logger, key='add_pipeline_stage')
+    @deprecated("2.3", "add_pipeline_stage")
     def add_pipeline_stage(self, stage):
         """
         Add a pipeline stage to the parameter.
 
+        Note:
+            This interface is deprecated in 2.3, and will be deleted in the future.
+
         Args:
             stage(int): The pipeline stage to be added.
 
@@ -813,9 +814,9 @@ class Parameter(Tensor_):
                         f"Use .set_dtype(xxx) to change the dtype.")
 
     @staticmethod
-    def _set_data_check_input_valid(current_shape, data_shape, current_tensor_is_init, incoming_tensor_is_init,
-                                    from_ckpt, slice_shape=False, slice_num=1):
-        if not from_ckpt and incoming_tensor_is_init and not current_tensor_is_init:
+    def _set_data_check_input_valid(current_shape, data_shape, current_tensor_is_init,
+                                    incoming_tensor_is_init, slice_shape=False, slice_num=1):
+        if incoming_tensor_is_init and not current_tensor_is_init:
             raise TypeError("The original tensor data is initialized, but the argument 'data' is not initialized."
                             "Please initialize 'data' before call this method.")
         if tuple(current_shape) != tuple(data_shape):
@@ -870,7 +871,7 @@ class Parameter(Tensor_):
         incoming_tensor_is_init = isinstance(data, Tensor) and not data.has_init
         current_tensor_is_init = isinstance(self, Tensor) and not self.has_init
         Parameter._set_data_check_input_valid(self.shape, data.shape, current_tensor_is_init, incoming_tensor_is_init,
-                                              self.from_ckpt, slice_shape, self.slice_num)
+                                              slice_shape, self.slice_num)
         if self.dtype != data.dtype:
             if mstype.implicit_conversion_seq.get(self.dtype) < mstype.implicit_conversion_seq.get(data.dtype):
                 self._raise_type_error(data.dtype)

mindspore/common/recompute.py

@@ -20,11 +20,10 @@ from mindspore import log as logger
 from mindspore.nn.cell import Cell
 from mindspore import context
 from mindspore.common.tensor import Tensor
-from mindspore import ops
 from mindspore.ops.composite import GradOperation
 from mindspore.common._register_for_recompute import recompute_registry
 from mindspore.common.api import _pynative_executor
-from mindspore.common.generator import get_rng_state, set_rng_state
+from mindspore.nn.generator import get_rng_state, set_rng_state
 
 
 class _WrapCell(Cell):
@@ -52,11 +51,10 @@ class _RecomputeCell(Cell):
 
     def __init__(self, block):
         """Initialize Recompute cell."""
-        super(_RecomputeCell, self).__init__(auto_prefix=False)
+        super(_RecomputeCell, self).__init__()
         self.args = []
         self.kwargs = []
         self.wrap_cell = _WrapCell(block)
-
         self.net = block
         self.internal_params = []
         self.save_rng_state = False
@@ -87,15 +85,13 @@ class _RecomputeCell(Cell):
         self.kwargs.pop()
         if kwargs:
             input_args = list(input_args) + list(kwargs.values())
-        # To detach inputs to avoid erasing auto grad meta info of origin inputs.
-        input_args = _detach_input(input_args)
         try:
             pre_rng_state = get_rng_state()
-            set_rng_state(self.cpu_rng_state)
+            set_rng_state(*self.cpu_rng_state)
             _pynative_executor.set_is_run_recompute(True)
             grads = self.grad(self.net, self.internal_params)(*input_args, grad_input)
             _pynative_executor.set_is_run_recompute(False)
-            set_rng_state(pre_rng_state)
+            set_rng_state(*pre_rng_state)
         except Exception as err:
             _pynative_executor.clear_res()
             raise err
@@ -167,17 +163,6 @@ def _padding_input_grads(args, input_grads):
                          "but got {}".format(len(args), len(input_grads)))
 
 
-def _detach_input(input_arg):
-    if isinstance(input_arg, Tensor):
-        return ops.stop_gradient(input_arg)
-    if isinstance(input_arg, (list, tuple)):
-        detach_inputs = []
-        for arg in input_arg:
-            detach_inputs.append(_detach_input(arg))
-        return detach_inputs if isinstance(input_arg, list) else tuple(detach_inputs)
-    return input_arg
-
-
 def _check_validation(block):
     if not isinstance(block, Cell):
         raise TypeError("Recompute function now only support block which inherited from Cell!")
@@ -218,7 +203,7 @@ def recompute(block, *args, **kwargs):
     Examples:
         >>> import numpy as np
         >>> import mindspore.nn as nn
-        >>> from mindspore import ops
+        >>> import mindspore.ops as ops
         >>> from mindspore import Tensor, recompute
         >>> class MyCell(nn.Cell):
         ...     def __init__(self):

mindspore/common/sparse_tensor.py

@@ -642,29 +642,14 @@ class CSRTensor(CSRTensor_):
     values indicated by `values` and row and column positions indicated by `indptr`
     and `indices`.
 
-    For example, if indptr is [0, 2, 5, 6], indices is [0, 3, 1, 2, 4, 2], values is
-    [1., 2., 3., 4., 5., 6.], shape is (3, 5), then the dense representation of the sparse tensor will be:
+    For example, if indptr is [0, 1, 2, 2], indices is [1, 2], values is [1., 2.], shape is
+    (3, 4), then the dense representation of the sparse tensor will be:
 
     .. code-block::
-        [[1., 0., 0., 2., 0.],
-         [0., 3., 4., 0., 5.],
-         [0., 0., 6., 0., 0.]]
-
-    The length of `indptr` should equal to `shape[0]+1`, where the elements should be equal or monotonically
-    increasing and the maximum value should be equal to the number of non-zero values in the tensor. The length
-    of `indices` and `values` should be equal to the number of non-zero values in the tensor. To be concrete, get
-    the query indices of none-zero elements in every line according to `indptr`. Then get the column positions of
-    none-zero elements in every line by looking up query indices in `indices`. Finally, get the actual values of
-    none-zero elements in every line by looking up query indices in `values`. In the former example, 'indptr' of
-    [0, 2, 5, 6] represents that the indices of 0th row of the tensor origins from [0, 2), the indices of
-    the 1st row of the tensor origins from [2, 5) and the 2nd row of the tensor origins from [5, 6). For example,
-    the column positions of the non-zero elements of the 0th row in the tensor are provided by the [0, 2) elements in
-    `indices` (i.e. [0, 3]) and the corresponding values are provided by the [0, 2) elements in `values`
-    (i.e. [1., 2.]). The column positions of the non-zero elements of the 1st row in the tensor are provided by the
-    [2, 5) elements in `indices` (i.e. [1, 2, 4]) and the corresponding values are provided by the [2, 5) elements in
-    `values` (i.e. [3., 4., 5.]). The column positions of the non-zero elements of the 2nd row in the tensor are
-    provided by the [5, 6) elements in `indices` (i.e. [2]) and the corresponding values are provided by the [5, 6)
-    elements in `values` (i.e. [6.]).
+
+        [[0., 1., 0., 0.],
+         [0., 0., 2., 0.],
+         [0., 0., 0., 0.]]
 
     Common arithmetic operations include: addition (+), subtraction (-), multiplication (*),
     and division (/). For details about operations supported by `CSRTensor`, see

mindspore/common/tensor.py

@@ -50,8 +50,7 @@ def _check_input_data_type(input_data):
     valid_dtypes = (np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64,
                     np.float16, np.float32, np.float64, np.bool_, np.str_, np.complex64, np.complex128)
     if isinstance(input_data, np.ndarray) and input_data.dtype not in valid_dtypes and \
-            input_data.dtype.kind != 'U' and input_data.dtype.kind != 'S' and \
-            input_data.dtype.kind != 'T':  # Support dtype np.str_ and npy_bfloat16
+            input_data.dtype.kind != 'U' and input_data.dtype.kind != 'S':  # Support dtype np.str_
         new_line = '\n'
         for index, x in np.ndenumerate(input_data):
             if np.array(x).dtype not in valid_dtypes:
@@ -344,7 +343,10 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         return out
 
     def __bool__(self):
-        data = self.asnumpy()
+        if self.dtype == mstype.bfloat16:
+            data = self.float().asnumpy()
+        else:
+            data = self.asnumpy()
         if data.shape == ():
             return bool(data)
         if data.shape == (1,):
@@ -360,15 +362,24 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         raise ValueError(message)
 
     def __int__(self):
-        data = self.asnumpy()
+        if self.dtype == mstype.bfloat16:
+            data = self.float().asnumpy()
+        else:
+            data = self.asnumpy()
         return self._convert_scalar_(data, int, "Only one element tensors can be converted to Python scalars")
 
     def __float__(self):
-        data = self.asnumpy()
+        if self.dtype == mstype.bfloat16:
+            data = self.float().asnumpy()
+        else:
+            data = self.asnumpy()
         return self._convert_scalar_(data, float, "Only one element tensors can be converted to Python scalars")
 
     def __index__(self):
-        data = self.asnumpy()
+        if self.dtype == mstype.bfloat16:
+            data = self.float().asnumpy()
+        else:
+            data = self.asnumpy()
         if data.dtype not in ["int8", "int16", "int32", "int64", "bool"]:
             raise ValueError("Only integer tensors of a single element can be converted to an index.")
         return self._convert_scalar_(data, int,
@@ -505,6 +516,8 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
     def __str__(self):
         if self.dtype == mstype.type_none:
             return "Unknown Tensor type!"
+        if self.dtype == mstype.bfloat16:
+            return str(self.float().asnumpy())
         return str(self.asnumpy())
 
     def __getstate__(self):
@@ -524,13 +537,6 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         """
         return self._shape
 
-    @shape.setter
-    def shape(self, shape_value):
-        r"""
-        Set the shape value.
-        """
-        self._shape = shape_value
-
     @property
     def dtype(self):
         """Return the dtype of the tensor (:class:`mindspore.dtype`)."""
@@ -2841,7 +2847,12 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         if slice_num_of_persistent_data > 1:
             self.assign_value(Tensor_.persistent_data_from_numpy(data, slice_num_of_persistent_data))
         else:
-            self.assign_value(Tensor_.from_numpy(data))
+            if self.dtype == mstype.bfloat16:
+                # The dtype of data is np.float32 when mstype is bfloat16,
+                # so we create tensor_ by init func instead of asnumpy
+                self.assign_value(Tensor_(data, self.dtype))
+            else:
+                self.assign_value(Tensor_.from_numpy(data))
         return self
 
     def resize(self, *new_shape):
@@ -3161,9 +3172,9 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
                 location found is given. If 'right', return the last such index. If there is
                 no suitable index, return either 0 or N (where N is the length of the tensor).
                 Default: ``left`` .
-            sorter (Union[int, list, tuple, Tensor]): optional tensor of
-                integer indices that sort the tensor into ascending order on the innermost dimension
-                and the type must be int64. They are typically the result of argsort. Default: ``None`` .
+            sorter (Union[int, float, bool, list, tuple, Tensor]): 1-D optional tensor of
+                integer indices that sort the tensor into ascending order. They are typically
+                the result of argsort. Default: ``None`` .
 
         Returns:
             Tensor, array of insertion points with the same shape as `v`.
@@ -3184,21 +3195,31 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         if side not in ('left', 'right'):
             raise ValueError(f"For 'Tensor.searchsorted', the argument 'side' should be one of in "
                              f"['left', 'right'], but got {side}.")
+        a = self.astype(mstype.float32)
         if not isinstance(v, Tensor):
             v = tensor_operator_registry.get('make_tensor')(v)
+        shape = v.shape
         if sorter is not None:
-            if not isinstance(sorter, (int, list, tuple, Tensor)):
+            if not isinstance(sorter, (int, float, bool, list, tuple, Tensor)):
                 raise TypeError("For Tensor.searchsorted, the type of the argument 'sorter' must be one of 'int', "
-                                "'list', 'tuple', 'Tensor', but got {}.".format(type(sorter)))
+                                "'float', 'bool', 'list', 'tuple', 'Tensor', but got {}.".format(type(sorter)))
             if not isinstance(sorter, Tensor):
                 sorter = tensor_operator_registry.get('make_tensor')(sorter)
-            if sorter.size != self.size:
-                raise ValueError('The size of sorter must be the same as the Tensor')
-
-        dtype = mstype.int32
-        right = (side == 'right')
-        search_sorted_ = tensor_operator_registry.get('searchsorted')(dtype, right)
-        return search_sorted_(self, v, sorter)
+            if sorter.ndim != 1 or sorter.size != a.size:
+                raise ValueError('sorter must be 1-D array with the same size as the Tensor')
+            sorter = sorter.reshape(sorter.shape + (1,))
+            a = tensor_operator_registry.get('gather_nd')(a, sorter)
+        less_op = tensor_operator_registry.get('__le__') if side == 'left' else tensor_operator_registry.get('__lt__')
+        i = tensor_operator_registry.get('fill')(mstype.int32, shape, 0)
+        j = tensor_operator_registry.get('fill')(mstype.int32, shape, a.size)
+
+        sort_range = tuple(range(math.ceil(math.log2(tensor_operator_registry.get('shape_mul')(a.shape) + 1))))
+        for _ in sort_range:
+            mid = (i - -j) // 2
+            mask = less_op(v, tensor_operator_registry.get('gather_nd')(a, mid.reshape(mid.shape + (1,))))
+            i = tensor_operator_registry.get('select')(mask, i, mid)
+            j = tensor_operator_registry.get('select')(mask, mid, j)
+        return j
 
     def gather_nd(self, indices):
         r"""
@@ -3215,35 +3236,6 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         validator.check_is_int(batch_dims, "batch_dims")
         return tensor_operator_registry.get('gather')(self, input_indices, axis, batch_dims)
 
-    def uniform(self, from_=0., to=1., generator=None):
-        r"""
-        Generates random numbers in the half-open interval [from_, to).
-
-        Args:
-            from_ (number): The lower bound of the interval.
-            to (number): The upper bound of the interval.
-            generator (Generator, optional): The random seed. Default: None.
-
-        Returns:
-            Tensor, with the same shape as tensor.
-
-        Raises:
-            TypeError: If `from_` is larger than `to`.
-
-        Supported Platforms:
-            ``Ascend``
-
-        Examples:
-            >>> import mindspore
-            >>> x = mindspore.ops.ones((4, 2))
-            >>> generator = mindspore.Generator()
-            >>> generator.manual_seed(100)
-            >>> output = x.uniform(1., 2., generator)
-            >>> print(output.shape)
-            (4, 2)
-        """
-        return tensor_operator_registry.get('uniform')(self, from_, to, generator)
-
     def var(self, axis=None, ddof=0, keepdims=False):
         """
         Compute the variance along the specified axis.
@@ -3364,12 +3356,8 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
             [10. 35.]
         """
         if initial is None:
-            res = tensor_operator_registry.get("sum")(self, axis, keepdims, dtype=dtype)
-        else:
-            res = tensor_operator_registry.get("sum")(self, axis, keepdims, dtype=dtype) + initial
-        if dtype is not None and (dtype == mstype.bool_):
-            res = res.astype(mstype.bool_)
-        return res
+            return tensor_operator_registry.get("sum")(self, axis, keepdims, dtype=dtype)
+        return tensor_operator_registry.get("sum")(self, axis, keepdims, dtype=dtype) + initial
 
     def sum_to_size(self, *size):
         r"""
@@ -3539,11 +3527,11 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         validator.check_value_type('index', index, (Tensor, Tensor_,), 'Tensor.gather_elements')
         return tensor_operator_registry.get('gather_elements')(self, dim, index)
 
-    def nonzero(self, as_tuple=False):
+    def nonzero(self):
         """
         For details, please refer to :func:`mindspore.ops.nonzero`.
         """
-        return tensor_operator_registry.get('nonzero')(self, as_tuple)
+        return tensor_operator_registry.get('nonzero')(self)
 
     def svd(self, full_matrices=False, compute_uv=True):
         """
@@ -4725,42 +4713,6 @@ class Tensor(Tensor_, metaclass=_TensorMeta):
         return _index_put(self, values, indices)
 
 
-    def move_to(self, to, blocking=True):
-        r"""
-        Copy Tensor to target device synchronously or asynchronously, default synchronously. only support PyNative mode.
-
-        Args:
-            to (str): a string type value, one of ``"Ascend"``, ``"GPU"``, ``"CPU"``.
-            blocking (bool): a bool type value, using synchronous copy or asynchronous copy.
-                Default: ``True`` , synchronous copy.
-
-        Returns:
-            New Tensor, storged on target device which with the same type and shape as the "self Tensor".
-
-        Raises:
-            ValueError: If the type of `blocking` is not bool type.
-            ValueError: If the value of `to` is not one of ``"Ascend"``, ``"GPU"``, ``"CPU"``.
-            ValueError: If the run mode is not PyNative mode.
-
-        Supported Platforms:
-            ``Ascend`` ``GPU`` ``CPU``
-
-        Examples:
-            >>> import mindspore as ms
-            >>> from mindspore import Tensor
-            >>> x = ms.Tensor([1, 2, 3], ms.int64)
-            >>> new_tensor = x.move_to("CPU")
-        """
-        if not isinstance(blocking, bool):
-            raise ValueError(f"The type of 'blocking' must be bool, but got {blocking}")
-        if to not in ("Ascend", "GPU", "CPU"):
-            raise ValueError(f"The value of 'to' must be one of ['Ascend', 'GPU', 'CPU'], but got {to}")
-        mode = context.get_context("mode")
-        if mode != context.PYNATIVE_MODE:
-            raise ValueError(f"The method of 'move_to' only supported in pynative mode, but got: {mode}.")
-        return Tensor(Tensor_.move_to(self, to, blocking))
-
-
     def _offload(self):
         r"""
         Offload tensor parameter to host. Currently, only support for pynative mode.
@@ -4856,4 +4808,4 @@ def _check_astype_and_convert(dtype):
     return dtype
 
 
-setattr(tensor_operator_registry, 'vm_compare', _vm_compare)
+tensor_operator_registry.register('vm_compare', _vm_compare)

mindspore/communication/__init__.py

@@ -17,21 +17,26 @@ Collective communication interface.
 
 Note that the APIs in the following list need to preset communication environment variables.
 
-For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
-without any third-party or configuration file dependencies.
-Please see the `msrun start up
-<https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+Please see the `rank table Startup
+<https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
 for more details.
+
+For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+<https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
+
+For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster Startup
+<https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
 """
 
 from mindspore.communication.management import GlobalComm, init, release, get_rank, \
     get_group_size, get_world_rank_from_group_rank, \
     get_group_rank_from_world_rank, create_group, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, \
-    MCCL_WORLD_COMM_GROUP, get_local_rank, get_local_rank_size, destroy_group, get_process_group_ranks
+    MCCL_WORLD_COMM_GROUP, get_local_rank, get_local_rank_size, destroy_group
 
 
 __all__ = [
     "GlobalComm", "init", "release", "get_rank", "get_group_size", "get_world_rank_from_group_rank",
     "get_group_rank_from_world_rank", "create_group", "HCCL_WORLD_COMM_GROUP", "NCCL_WORLD_COMM_GROUP",
-    "MCCL_WORLD_COMM_GROUP", "get_local_rank", "get_local_rank_size", "destroy_group", "get_process_group_ranks"
+    "MCCL_WORLD_COMM_GROUP", "get_local_rank", "get_local_rank_size", "destroy_group"
 ]