mindspore 2.4.0__cp311-cp311-win_amd64.whl → 2.4.10__cp311-cp311-win_amd64.whl

mindspore/ops/function/random_func.py

@@ -30,7 +30,8 @@ from mindspore.common.api import _function_forbid_reuse
 from mindspore.ops.auto_generate import randperm
 from mindspore.common.generator import default_generator
 from mindspore.ops.auto_generate import UniformExt, NormalTensorTensor, \
-    NormalTensorFloat, NormalFloatTensor, NormalFloatFloat, RandExt, RandLikeExt, MultinomialExt
+    NormalTensorFloat, NormalFloatTensor, NormalFloatFloat, RandExt, RandLikeExt, MultinomialExt, \
+    Randn, RandnLike, RandInt, RandIntLike, RandpermExt
 
 normal_tensor_tensor_op = NormalTensorTensor()
 normal_tensor_float_op = NormalTensorFloat()
@@ -42,10 +43,15 @@ real_div_ = P.RealDiv()
 reshape_ = P.Reshape()
 shape_ = P.Shape()
 top_k_ = P.TopK()
+randperm_ext_ = RandpermExt()
 uniform_ = UniformExt()
 rand_ext_ = RandExt()
 rand_like_ext_ = RandLikeExt()
 multinomial_ext_ = MultinomialExt()
+randn_ = Randn()
+randn_like_ = RandnLike()
+randint_ = RandInt()
+randint_like_ = RandIntLike()
 generator_step_ = Tensor(10, mstype.int64)
 
 
@@ -287,7 +293,8 @@ def uniform_ext(tensor, a, b, generator=None):
     """
     if generator is None:
         generator = default_generator
-    seed, offset = generator._step(generator_step_)  # pylint: disable=protected-access
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
     return uniform_(tensor, a, b, seed, offset)
 
 
@@ -755,7 +762,8 @@ def normal_ext(mean=0.0, std=1.0, size=None, generator=None):
     """
     if generator is None:
         generator = default_generator
-    seed, offset = generator._step(generator_step_)  # pylint: disable=protected-access
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
 
     is_mean_tensor = isinstance(mean, Tensor)
     is_std_tensor = isinstance(std, Tensor)
@@ -1129,7 +1137,8 @@ def rand_ext(*size, generator=None, dtype=None):
     """
     if not generator:
         generator = default_generator
-    seed, offset = generator._step(generator_step_)  # pylint: disable=protected-access
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
     return rand_ext_(size, seed, offset, dtype)
 
 
@@ -1163,10 +1172,174 @@ def rand_like_ext(input, *, dtype=None):
         >>> print(ops.function.random_func.rand_like_ext(a, dtype=ms.float32).shape)
         (2, 3)
     """
-    seed, offset = default_generator._step(generator_step_)  # pylint: disable=protected-access
+    seed, offset = default_generator._step(  # pylint: disable=protected-access
+        generator_step_)
     return rand_like_ext_(input, seed, offset, dtype)
 
 
+@_function_forbid_reuse
+def randn_ext(*size, generator=None, dtype=None):
+    r"""
+    Returns a new tensor filled with numbers from the normal distribution over an interval :math:`[0, 1)`
+    based on the given shape and dtype.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        size (Union[int, tuple(int), list(int)]): Shape of the new tensor, e.g. :math:`(2, 3)` or :math:`2`.
+
+    Keyword Args:
+        generator (:class:`mindspore.Generator`, optional): a pseudorandom number generator.
+            Default: ``None``, uses the default pseudorandom number generator.
+        dtype (:class:`mindspore.dtype`, optional): Designated tensor dtype, it must be float type. If None,
+            `mindspore.float32` will be applied. Default: ``None`` .
+
+    Returns:
+        Tensor, with the designated shape and dtype, filled with random numbers from the normal distribution on
+        the interval :math:`[0, 1)`.
+
+    Raises:
+        ValueError: If `dtype` is not a `mstype.float_type` type.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore import ops
+        >>> print(ops.function.random_func.randn_ext(2, 3).shape)
+        (2, 3)
+    """
+    if not generator:
+        generator = default_generator
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
+    return randn_(size, seed, offset, dtype)
+
+
+@_function_forbid_reuse
+def randn_like_ext(input, *, dtype=None):
+    r"""
+    Returns a new tensor filled with numbers from the normal distribution over an interval :math:`[0, 1)`
+    based on the given dtype and shape of the input tensor.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        input (Tensor): Input Tensor to specify the output shape and its default dtype.
+
+    Keyword Args:
+        dtype (:class:`mindspore.dtype`, optional): Designated tensor dtype, it must be float type. If None,
+            the same dtype of `input` will be applied. Default: ``None`` .
+
+    Returns:
+        Tensor, with the designated shape and dtype, filled with random numbers from the normal distribution on
+        the interval :math:`[0, 1)`.
+
+    Raises:
+        ValueError: If `dtype` is not a `mstype.float_type` type.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import Tensor, ops
+        >>> a = Tensor([[2, 3, 4], [1, 2, 3]])
+        >>> print(ops.function.random_func.randn_like_ext(a, dtype=ms.float32).shape)
+        (2, 3)
+    """
+    seed, offset = default_generator._step(  # pylint: disable=protected-access
+        generator_step_)
+    return randn_like_(input, seed, offset, dtype)
+
+
+@_function_forbid_reuse
+def randint_ext(low, high, size, *, generator=None, dtype=None):
+    r"""
+    Returns a new tensor filled with integer numbers from the uniform distribution over an interval :math:`[low, high)`
+    based on the given shape and dtype.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        low (int): the lower bound of the generated random number
+        high (int): the upper bound of the generated random number
+        size (Union[tuple(int), list(int)]): Shape of the new tensor, e.g. :math:`(2, 3)`.
+
+    Keyword Args:
+        generator (:class:`mindspore.Generator`, optional): a pseudorandom number generator.
+            Default: ``None``, uses the default pseudorandom number generator.
+        dtype (:class:`mindspore.dtype`, optional): Designated tensor dtype. If None,
+            `mindspore.int64` will be applied. Default: ``None`` .
+
+    Returns:
+        Tensor, with the designated shape and dtype, filled with random numbers from the uniform distribution on
+        the interval :math:`[low, high)`.
+
+    Raises:
+        TypeError: If `size` is not a tuple.
+        TypeError: If `low` or `high` is not integer.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore import ops
+        >>> print(ops.function.random_func.randint_ext(0, 5, (2, 3)).shape)
+        (2, 3)
+    """
+    if not generator:
+        generator = default_generator
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
+    return randint_(low, high, size, seed, offset, dtype)
+
+
+@_function_forbid_reuse
+def randint_like_ext(input, low, high, *, dtype=None):
+    r"""
+    Returns a new tensor filled with integer numbers from the uniform distribution over an interval :math:`[low, high)`
+    based on the given dtype and shape of the input tensor.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        input (Tensor): Input Tensor to specify the output shape and its default dtype.
+        low (int): the lower bound of the generated random number
+        high (int): the upper bound of the generated random number
+
+    Keyword Args:
+        dtype (:class:`mindspore.dtype`, optional): Designated tensor dtype. If None,
+            the same dtype of `input` will be applied. Default: ``None`` .
+
+    Returns:
+        Tensor, with the designated shape and dtype, filled with random numbers from the uniform distribution on
+        the interval :math:`[low, high)`.
+
+    Raises:
+        TypeError: If `low` or `high` is not integer.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import Tensor, ops
+        >>> a = Tensor([[2, 3, 4], [1, 2, 3]])
+        >>> low = 0
+        >>> high = 5
+        >>> print(ops.function.random_func.randint_like_ext(a, low, high, dtype=ms.int32).shape)
+        (2, 3)
+    """
+    seed, offset = default_generator._step(  # pylint: disable=protected-access
+        generator_step_)
+    return randint_like_(input, low, high, seed, offset, dtype)
+
+
 @_function_forbid_reuse
 def randn(*size, dtype=None, seed=None):
     r"""
@@ -1395,6 +1568,47 @@ def randint_like(input, low, high, seed=None, *, dtype=None):
     return cast_(output, dtype)
 
 
+def randperm_ext(n, *, generator=None, dtype=mstype.int64):
+    r"""
+    Generates random permutation of integers from 0 to n-1.
+
+    .. warning::
+        - This is an experimental API that is subject to change or deletion.
+
+
+    Args:
+        n (Union[Tensor, int]): size of the permutation. int or Tensor with shape: () or (1,) and
+            data type int64. The value of `n` must be greater than zero.
+        generator (:class:`mindspore.Generator`, optional): a pseudorandom number generator.
+            Default: ``None``, uses the default pseudorandom number generator.
+        dtype (mindspore.dtype, optional): The type of output. Default: mstype.int64.
+
+    Returns:
+        Tensor with shape (n,) and type `dtype`.
+
+    Raises:
+        TypeError: If `dtype` is not supported.
+        ValueError: If `n` is a negative or 0 element.
+        ValueError: If `n` is larger than the maximal data of the set dtype.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore import ops
+        >>> from mindspore import dtype as mstype
+        >>> n = 4
+        >>> output = ops.randperm_ext(n, dtype=mstype.int64)
+        >>> print(output.shape)
+        (4,)
+    """
+    if not generator:
+        generator = default_generator
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
+    return randperm_ext_(n, seed, offset, dtype)
+
+
 @_function_forbid_reuse
 def poisson(shape, mean, seed=None):
     r"""
@@ -1675,10 +1889,10 @@ def multinomial_ext(input, num_samples, replacement=False, *, generator=None):
         >>> # [[0 0 0 0 0 0 0 0 1 0]
         >>> #  [1 1 1 1 1 0 1 1 1 1]]
     """
-
     if generator is None:
         generator = default_generator
-    seed, offset = generator._step(generator_step_)  # pylint: disable=protected-access
+    seed, offset = generator._step(  # pylint: disable=protected-access
+        generator_step_)
     return multinomial_ext_(input, num_samples, replacement, seed, offset)
 
 

mindspore/ops/operations/__init__.py

@@ -55,7 +55,7 @@ from .comm_ops import (AllGather, AllReduce, Reduce, NeighborExchange, NeighborE
                        _MirrorOperator, _MirrorMiniStepOperator, _MiniStepAllGather, ReduceOp, _VirtualDataset,
                        _VirtualOutput, _VirtualDiv, _GetTensorSlice, _VirtualAdd, _VirtualAssignAdd, _VirtualAccuGrad,
                        _HostAllGather, _HostReduceScatter, _MirrorMicroStepOperator, _MicroStepAllGather,
-                       _VirtualPipelineEnd, AlltoAllV, ReduceScatter)
+                       _VirtualPipelineEnd, AlltoAllV, ReduceScatter, _VirtualAssignKvCache)
 from .control_ops import GeSwitch, Merge
 from .custom_ops import (Custom)
 from .debug_ops import (ImageSummary, InsertGradientOf, HookBackward, ScalarSummary,

mindspore/ops/operations/array_ops.py

@@ -771,12 +771,14 @@ class Padding(Primitive):
 class UniqueWithPad(Primitive):
     """
     'ops.UniqueWithPad' is deprecated from version 2.4 and will be removed in a future version.
+    Please use the :func:`mindspore.ops.unique` combined with :func:`mindspore.ops.pad` to realize
+    the same function.
 
     Supported Platforms:
         Deprecated
     """
 
-    @deprecated("2.4", "ops.Unique and ops.PadV3", False)
+    @deprecated("2.4", "ops.unique and ops.pad", False)
     @prim_attr_register
     def __init__(self):
         """init UniqueWithPad"""

mindspore/ops/operations/comm_ops.py

@@ -988,6 +988,9 @@ class NeighborExchangeV2(Primitive):
         in the same subnet, please check the `details \
         <https://www.mindspore.cn/docs/en/master/api_python/samples/ops/communicate_ops.html#notes>`_.
 
+        Users need to ensure that the length of the received data `recv_lens` is consistent with that of
+        the sent data `send_lens`.
+
     Args:
         send_rank_ids (list(int)): Ranks which the data is sent to. 8 rank_ids represents 8 directions, if one
                                    direction is not send to , set it -1.
@@ -1393,7 +1396,7 @@ class Send(PrimitiveWithInfer):
         >>>     def __init__(self):
         >>>         super(SendNet, self).__init__()
         >>>         self.depend = ops.Depend()
-        >>>         self.send = ops.Send(st_tag=0, dest_rank=8, group="hccl_world_group")
+        >>>         self.send = ops.Send(sr_tag=0, dest_rank=8, group="hccl_world_group")
         >>>
         >>>     def construct(self, x):
         >>>         out = self.depend(x, self.send(x))
@@ -1682,6 +1685,27 @@ class _VirtualAssignAdd(PrimitiveWithInfer):
 virtual_assign_add = _VirtualAssignAdd()
 
 
+class _VirtualAssignKvCache(PrimitiveWithInfer):
+    """
+    Auto parallel virtual operator. Do nothing in forward, do Assign kv cache in backward. It is only for
+    internal use of parallel modules and cannot be called by users.
+
+    """
+
+    @prim_attr_register
+    def __init__(self):
+        """Initialize _VirtualAssignAdd."""
+        self.add_prim_attr('order_enforce_skip', True)
+        self.add_prim_attr('side_effect_backprop_mem', True)
+
+    def infer_shape(self, x_shape, y_shape, kv_equal_shape):
+        return x_shape
+
+    def infer_dtype(self, x_dtype, y_dtype, kv_equal_dtype):
+        return x_dtype
+virtual_assign_kv_cache = _VirtualAssignKvCache()
+
+
 class _VirtualAccuGrad(PrimitiveWithInfer):
     """
     Auto parallel virtual operator. Do nothing in forward, return y in backward. It is only for

mindspore/ops/operations/custom_ops.py

@@ -251,11 +251,13 @@ class Custom(ops.PrimitiveWithInfer):
 
                  - "xxx.so" file generation:
 
-                   1) GPU Platform: Given user defined "xxx.cu" file (ex. "{path}/add.cu"), use nvcc command to compile
-                   it.(ex. "nvcc --shared -Xcompiler -fPIC -o add.so add.cu")
+                   1) GPU Platform: Given user defined "xxx.cu" file (ex. "{path}/add.cu"),
+                   use nvcc command to compile
+                   it.(ex. :code:`nvcc --shared -Xcompiler -fPIC -o add.so add.cu`)
 
-                   2) CPU Platform: Given user defined "xxx.cc" file (ex. "{path}/add.cc"), use g++/gcc command to
-                   compile it.(ex. "g++ --shared -fPIC  -o add.so add.cc")
+                   2) CPU Platform: Given user defined "xxx.cc" file (ex. "{path}/add.cc"),
+                   use g++/gcc command to
+                   compile it.(ex. :code:`g++ --shared -fPIC -o add.so add.cc`)
 
                  - Define a "xxx.cc"/"xxx.cu" file:
 

mindspore/ops/operations/manually_defined/ops_def.py

@@ -1171,17 +1171,15 @@ class Cast(Primitive):
         self.init_prim_io_names(inputs=['x', 'dst_type'], outputs=['output'])
 
     def check_elim(self, x, dtype):
-        if isinstance(x, (Tensor, numbers.Number, Parameter)):
-            if isinstance(x, Parameter):
-                data = x.data
-                if data.dtype == dtype:
-                    return (True, x)
-            if isinstance(x, Tensor) and x.dtype == dtype:
-                x = Tensor(x)
-                x.set_cast_dtype()
+        if isinstance(x, Parameter):
+            data = x.data
+            if data.dtype == dtype:
                 return (True, x)
-            if isinstance(x, numbers.Number):
-                return (True, Tensor(x, dtype=dtype))
+        if isinstance(x, Tensor) and x.dtype == dtype:
+            x.set_cast_dtype()
+            return (True, x)
+        if isinstance(x, numbers.Number):
+            return (True, Tensor(x, dtype=dtype))
         return (False, None)
 
     def __call__(self, input_x, dtype):

mindspore/ops/operations/nn_ops.py

@@ -1430,6 +1430,9 @@ class MaxPool3D(Primitive):
         \max_{l=0, \ldots, d_{ker}-1} \max_{m=0, \ldots, h_{ker}-1} \max_{n=0, \ldots, w_{ker}-1}
         \text{input}(N_i, C_j, s_0 \times d + l, s_1 \times h + m, s_2 \times w + n)
 
+    .. note::
+        For Atlas training series products, this primitive is not supported.
+
     Args:
         kernel_size (Union[int, tuple[int]]): The size of kernel used to take the maximum value,
             is an int number that represents depth, height and width of the kernel, or a tuple
@@ -4759,7 +4762,8 @@ class SparseApplyAdagradV2(Primitive):
         - **grad** (Tensor) - Gradients has the same shape as `var` and
           :math:`grad.shape[1:] = var.shape[1:]` if var.shape > 1.
         - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`.
-          The type must be int32 and :math:`indices.shape[0] = grad.shape[0]`.
+          The type must be int32 and :math:`indices.shape[0] = grad.shape[0]`. The value of indices
+          must be unique. Otherwise, the result is unpredictable.
 
     Outputs:
         Tuple of 2 tensors, the updated parameters.
@@ -7158,7 +7162,8 @@ class Conv3DTranspose(Primitive):
         \times (\text{kernel_size}[2] - 1) + \text{output_padding}[2] + 1
 
     Note:
-        In Ascend, only support :math:`group=1`.
+        - In Ascend, only support :math:`group=1`.
+        - For Atlas A2 training series products, `output_padding` is currently not supported.
 
     Args:
         in_channel (int): The channel of the input x.

mindspore/parallel/_auto_parallel_context.py

@@ -76,6 +76,7 @@ class _PipelineConfig:
 class _PipelineScheduler:
     PIPELINE_1F1B = "1f1b"
     PIPELINE_GPIPE = "gpipe"
+    PIPELINE_SEQPIPE = "seqpipe"
 
 
 class _AutoParallelContext:
@@ -186,6 +187,25 @@ class _AutoParallelContext:
         self.check_context_handle()
         return self._context_handle.get_dump_local_norm()
 
+    def set_dump_device_local_norm(self, dump_device_local_norm):
+        """
+        Set dump device local norm for auto parallel.
+
+        Args:
+            dump_device_local_norm (bool): User need to specify if he want to dump device local norm.  Default: False
+
+        Raises:
+            ValueError: If the dump_device_local_norm in not a bool value.
+        """
+        self.check_context_handle()
+        self._context_handle.set_dump_device_local_norm(dump_device_local_norm)
+
+    def get_dump_device_local_norm(self):
+        """Get dump device local norm."""
+        self.check_context_handle()
+        return self._context_handle.get_dump_device_local_norm()
+
+
     def set_fusion_threshold_mb(self, fusion_threshold=64, comm_type="allreduce"):
         """
         Set fusion threshold (MB) for auto parallel.
@@ -914,7 +934,8 @@ class _AutoParallelContext:
             pipeline_config[pp_interleave])
 
         Validator.check_string(pipeline_config[pp_scheduler], [_PipelineScheduler.PIPELINE_1F1B,
-                                                               _PipelineScheduler.PIPELINE_GPIPE])
+                                                               _PipelineScheduler.PIPELINE_GPIPE,
+                                                               _PipelineScheduler.PIPELINE_SEQPIPE])
         if not pipeline_config[pp_interleave] and pipeline_config[pp_scheduler] != _PipelineScheduler.PIPELINE_1F1B:
             raise ValueError(f"When pipeline_interleave is False, {pp_scheduler} is not supported")
 
@@ -1285,7 +1306,8 @@ _set_auto_parallel_context_func_map = {
     "enable_alltoall": auto_parallel_context().set_enable_alltoall,
     "strategy_ckpt_config": auto_parallel_context().set_strategy_ckpt_config,
     "comm_fusion": auto_parallel_context().set_comm_fusion,
-    "dump_local_norm": auto_parallel_context().set_dump_local_norm}
+    "dump_local_norm": auto_parallel_context().set_dump_local_norm,
+    "dump_device_local_norm": auto_parallel_context().set_dump_device_local_norm}
 
 _get_auto_parallel_context_func_map = {
     "device_num": auto_parallel_context().get_device_num,
@@ -1318,7 +1340,8 @@ _get_auto_parallel_context_func_map = {
     "comm_fusion": auto_parallel_context().get_comm_fusion,
     "strategy_ckpt_config": auto_parallel_context().get_strategy_ckpt_config,
     "full_batch_is_set": auto_parallel_context().get_full_batch_is_set,
-    "dump_local_norm": auto_parallel_context().get_dump_local_norm}
+    "dump_local_norm": auto_parallel_context().get_dump_local_norm,
+    "dump_device_local_norm": auto_parallel_context().get_dump_device_local_norm}
 
 
 @args_type_check(device_num=int, global_rank=int, gradients_mean=bool, gradient_fp32_sync=bool,
@@ -1429,8 +1452,6 @@ def _set_auto_parallel_context(**kwargs):
                     - reducescatter: If communication fusion type is `reducescatter`. The `mode` contains: `auto`
                         and `size`. Config is same as `allgather`.
 
-
-
     Raises:
         ValueError: If input key is not attribute in auto parallel context.
     """

mindspore/parallel/_cell_wrapper.py

@@ -24,7 +24,8 @@ from mindspore.ops import operations as P
 from mindspore.ops.operations.comm_ops import AllGather
 from mindspore.communication import GlobalComm
 from mindspore.common import jit
-from mindspore.communication import create_group
+from mindspore.communication import create_group, destroy_group
+from mindspore.communication._comm_helper import _get_group_map
 from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy
 
 _ALLGATHER_CELL = None
@@ -126,9 +127,26 @@ def _restore_parallel_context(origin_parallel_mode, origin_dataset_strategy):
     if context.get_context("mode") == context.GRAPH_MODE:
         context.set_auto_parallel_context(parallel_mode=origin_parallel_mode)
         if origin_dataset_strategy != "data_parallel":
+            if origin_dataset_strategy is not None and isinstance(origin_dataset_strategy, list):
+                origin_dataset_strategy = tuple(tuple(ds_item) for ds_item in origin_dataset_strategy)
             context.set_auto_parallel_context(dataset_strategy=origin_dataset_strategy)
 
 
+def _get_group_name(group_map, group):
+    """get group name"""
+    group_name = str(group)
+    is_manual_communication_group = True
+    if group_map:
+        for name, rank_list in group_map.items():
+            if list(group) == rank_list:
+                group_name = name
+                is_manual_communication_group = False
+                break
+    if is_manual_communication_group:
+        create_group(str(group), list(group))
+    return group_name, is_manual_communication_group
+
+
 def _single_parameter_broadcast(net, layout, cur_rank=0, initial_rank=0):
     """
     Broadcast single parameter to other rank in data parallel dimension.
@@ -156,8 +174,9 @@ def _single_parameter_broadcast(net, layout, cur_rank=0, initial_rank=0):
         return
     net_param_dict = net.parameters_dict()
     _chang_parallel_context(origin_dataset_strategy)
+    group_map = _get_group_map()
     for group, params in param_redundancy_reversed.items():
-        create_group(str(group), list(group))
+        group_name, is_manual_communication_group = _get_group_name(group_map, group)
         allreduce_input = []
         for param in params:
             if param not in net_param_dict:
@@ -168,7 +187,9 @@ def _single_parameter_broadcast(net, layout, cur_rank=0, initial_rank=0):
             allreduce_input.append(real_param)
         if not allreduce_input:
             continue
-        communicator = SingleCommunicator(str(group))
+        communicator = SingleCommunicator(group_name)
         for real_param in allreduce_input:
             real_param.set_data(communicator(real_param), real_param.sliced)
+        if is_manual_communication_group:
+            destroy_group(group_name)
     _restore_parallel_context(origin_parallel_mode, origin_dataset_strategy)

mindspore/parallel/_tensor.py

@@ -590,6 +590,8 @@ def _apply_operator(operator_name):
         Returns:
             The data of tensor after apply operator.
         """
+        if str(type(numpy_data)) == "<class 'builtins.PySafeSlice'>":
+            numpy_data = numpy_data[:]
         if not isinstance(numpy_data, np.ndarray):
             raise TypeError("The data should be a numpy.ndarray.")
         _check_operator(reshape_op)
@@ -629,8 +631,6 @@ def _apply_operator(operator_name):
         Returns:
             The data of tensor after apply operator.
         """
-        if not isinstance(numpy_data, np.ndarray):
-            raise TypeError("The data should be a numpy.ndarray.")
         _check_operator(slice_op)
         if len(slice_op[1]) % 3 != 0:
             raise ValueError("The slice operator information is wrong.")
@@ -701,6 +701,50 @@ def _load_tensor_shape(dev_mat, tensor_map, full_shape=None, rank_id=-1):
     return tuple(res)
 
 
+def _count_tensor_shape(dev_mat, tensor_map, full_shape=None, rank_id=-1):
+    """get tensor shape"""
+    if rank_id == -1:
+        rank = get_rank()
+    else:
+        rank = rank_id
+    tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map)
+    tensor_slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, rank)
+    np_tensor_list = _chunk_shape_by_strategy(full_shape, tensor_strategy)
+    np_tensor_slice_index = np_tensor_list[int(tensor_slice_index)]
+    res = []
+    for index in np_tensor_slice_index:
+        res.append(index[1] - index[0])
+    return res
+
+
+def _load_tensor_shape_by_layout(tensor, layout, rank_id):
+    """get tensor shape by layout"""
+    if not isinstance(layout, tuple):
+        raise TypeError("The layout should be tuple! layout is {}".format(layout))
+    if len(layout) < 7:
+        raise ValueError("The length of layout must be larger than 6! layout is {}".format(layout))
+    slice_shape = layout[2]
+    if slice_shape:
+        return slice_shape
+    tensor_map = layout[1]
+    if not tensor_map:
+        return tensor.shape
+    dev_mat = layout[0]
+    uniform_split = layout[4]
+    group = layout[5]
+    full_shape = layout[6]
+    if not full_shape:
+        full_shape = tensor.shape
+    if uniform_split == 0:
+        raise RuntimeError("The load tensor only support uniform split now")
+    tensor_slice_shape = _count_tensor_shape(dev_mat, tensor_map, full_shape, rank_id)
+    if group:
+        # get a totally shard tensor slice for parallel optimizer
+        size = get_group_size(group)
+        tensor_slice_shape[0] //= size
+    return tensor_slice_shape
+
+
 def _chunk_shape_by_strategy(full_shape, strategy):
     """chunk shape by strategy"""
     shape = []