mindspore 2.2.11__cp37-cp37m-manylinux1_x86_64.whl → 2.3.0rc1__cp37-cp37m-manylinux1_x86_64.whl

mindspore/nn/reinforcement/_tensors_queue.py

@@ -37,7 +37,7 @@ class TensorsQueue(Cell):
         name (string): the name of this TensorsQueue. Default: "TQ".
 
     Raises:
-        TypeError: If `dtype` is not mindpore number type.
+        TypeError: If `dtype` is not mindspore number type.
         ValueError: If `size` is less than 0.
         ValueError: If `shapes` size is less than 1.
 
@@ -69,6 +69,7 @@ class TensorsQueue(Cell):
         self.tensors_q_clear = rl_ops.TensorsQueueClear()
         self.tensors_q_close = rl_ops.TensorsQueueClose()
         self.tensors_q_size = rl_ops.TensorsQueueSize()
+        self.__is_tensors_queue__ = True
 
     def put(self, element):
         """
@@ -103,6 +104,17 @@ class TensorsQueue(Cell):
         element = self.tensors_q_pop(self.handle_)
         return element
 
+    def __graph_pop__(self):
+        """
+        Get one element int the front of the TensorsQueue, and remove it.
+        This is only used in graph mode.
+
+        Returns:
+            tuple(Tensors), the element in TensorsQueue.
+        """
+        element = self.tensors_q_pop(self.handle_)
+        return self.handle_, element
+
     def size(self):
         """
         Get the used/available size of the TensorsQueue, and remove it.

mindspore/nn/wrap/__init__.py

@@ -24,7 +24,7 @@ from mindspore.nn.wrap.cell_wrapper import ForwardValueAndGrad, TrainOneStepCell
     GradAccumulationCell
 from mindspore.nn.wrap.loss_scale import TrainOneStepWithLossScaleCell,\
     DynamicLossScaleUpdateCell, FixedLossScaleUpdateCell
-from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
+from mindspore.nn.wrap.grad_reducer import DistributedGradReducer, PipelineGradReducer
 from mindspore.nn.layer.timedistributed import TimeDistributed
 
 
@@ -40,6 +40,7 @@ __all__ = [
     "GetNextSingleOp",
     "TrainOneStepWithLossScaleCell",
     "DistributedGradReducer",
+    "PipelineGradReducer",
     "ParameterUpdate",
     "DynamicLossScaleUpdateCell",
     "FixedLossScaleUpdateCell",

mindspore/nn/wrap/cell_wrapper.py

@@ -99,7 +99,7 @@ class WithLossCell(Cell):
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
         >>> net_with_criterion = nn.WithLossCell(net, loss_fn)
@@ -132,7 +132,7 @@ class WithLossCell(Cell):
         Examples:
             >>> from mindspore import nn
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
             >>> net_with_criterion = nn.WithLossCell(net, loss_fn)
@@ -175,7 +175,7 @@ class WithGradCell(Cell):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>> # Defined a network without loss function, taking LeNet5 as an example.
-        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> grad_net = nn.WithGradCell(net, loss_fn)
@@ -346,7 +346,7 @@ class TrainOneStepCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
@@ -454,7 +454,7 @@ class GetNextSingleOp(Cell):
         queue_name (str): Queue name to fetch the data.
 
     Outputs:
-        tuple[Tensor], the data get from Dataset.
+        tuple[Tensor], the data gets from Dataset.
 
     Supported Platforms:
         ``Ascend`` ``GPU``
@@ -586,7 +586,7 @@ class MicroBatchInterleaved(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.MicroBatchInterleaved(net, 2)
     """
@@ -619,7 +619,7 @@ class MicroBatchInterleaved(Cell):
 
 class PipelineCell(Cell):
     """
-    Wrap the network with Micro Batch.
+    Slice MiniBatch into finer-grained MicroBatch for use in pipeline-parallel training.
 
     Note:
         micro_size must be greater or equal to pipeline stages.
@@ -634,7 +634,7 @@ class PipelineCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.PipelineCell(net, 4)
     """
@@ -685,7 +685,7 @@ class GradAccumulationCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.GradAccumulationCell(net, 4)
     """
@@ -726,7 +726,7 @@ class GradAccumulationCell(Cell):
 
 def _pipeline_clear_grad(accu_grad, grad):
     accu_grad = F.depend(accu_grad, grad)
-    zeros = F.tensor_mul(accu_grad, 0.0)
+    zeros = F.zeros_like(accu_grad)
     return F.assign(accu_grad, zeros)
 
 
@@ -772,6 +772,27 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         return loss
 
 
+class AllreduceGraph(Cell):
+    """
+    A allreduce graph to broadcast parameters.
+    """
+    def __init__(self, inputs, group_name):
+        super(AllreduceGraph, self).__init__()
+        self.input_num = len(inputs)
+        self.inputs = inputs
+        self.allreduces = []
+        self.assigns = []
+        for _ in range(self.input_num):
+            self.allreduces.append(ops.AllReduce(op="sum", group=group_name))
+            self.assigns.append(ops.Assign())
+
+    def construct(self):
+        for i in range(self.input_num):
+            res = self.allreduces[i](self.inputs[i])
+            self.assigns[i](self.inputs[i], res)
+        return self.inputs
+
+
 class VirtualDatasetCellTriple(Cell):
     """
     Wrap the network with virtual dataset to convert data parallel layout to model parallel layout.
@@ -790,7 +811,7 @@ class VirtualDatasetCellTriple(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.VirtualDatasetCellTriple(net)
     """
@@ -833,7 +854,7 @@ class WithEvalCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define a forward network without loss function, taking LeNet5 as an example.
-        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.3.q1/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> eval_net = nn.WithEvalCell(net, loss_fn)

mindspore/nn/wrap/grad_reducer.py

@@ -18,17 +18,40 @@ from __future__ import absolute_import
 from mindspore import context
 from mindspore import log as logger
 from mindspore.nn.cell import Cell
+from mindspore.nn.layer import Identity
 from mindspore.communication.management import GlobalComm, get_group_size
 from mindspore.common.sparse_tensor import RowTensorInner
-from mindspore.ops import functional as F, composite as C
+from mindspore.ops import functional as F, composite as C, operations as P
 from mindspore.ops.operations.comm_ops import AllReduce, AllGather
 from mindspore.parallel._auto_parallel_context import auto_parallel_context
 import mindspore.common.dtype as mstype
 from mindspore.common.sparse_tensor import Tensor
 from mindspore.common.api import jit
-
+from mindspore.common.parameter import Parameter
+from mindspore.parallel._utils import _get_enable_parallel_optimizer
 
 reduce_opt = C.MultitypeFuncGraph("reduce_opt")
+grad_scale = C.MultitypeFuncGraph("grad_scale")
+shard_grad_scale = C.MultitypeFuncGraph("shard_grad_scale")
+reciprocal = P.Reciprocal()
+
+
+@grad_scale.register("Tensor", "Tensor", "Tensor")
+def tensor_grad_scale_pipeline(scale, grad, accu_grad):
+    accu_grad = F.depend(accu_grad, grad)
+    new_grad = accu_grad * reciprocal(scale)
+    accu_grad = F.depend(accu_grad, new_grad)
+    zeros = F.tensor_mul(accu_grad, 0.0)
+    new_grad = F.depend(new_grad, F.assign(accu_grad, zeros))
+    return new_grad
+
+
+@shard_grad_scale.register("Tensor", "Tensor", "Tensor")
+def tensor_shard_grad_scale_pipeline(scale, grad, accu_grad):
+    new_grad = grad * reciprocal(scale)
+    accu_grad = F.depend(accu_grad, new_grad)
+    new_grad = F.depend(new_grad, F.assign(accu_grad, F.zeros_like(accu_grad)))
+    return new_grad
 
 
 def _init_allreduce_operators(length, split_indices, group=GlobalComm.WORLD_COMM_GROUP):
@@ -53,9 +76,6 @@ def _init_allreduce_operators(length, split_indices, group=GlobalComm.WORLD_COMM
     for i in range(length):
         op = AllReduce('sum', group)
         op_fusion_id = fusion[i]
-        # When running in ge and enabled all_reduce_fusion_config, hccl will check the allreduce' fusion id to be -1
-        if context.get_context("enable_ge") and context.get_auto_parallel_context("all_reduce_fusion_config"):
-            op_fusion_id = -1
         op.add_prim_attr('fusion', op_fusion_id)
         op.add_prim_attr('index', index[i])
         op_list = op_list + (op,)
@@ -315,14 +335,14 @@ class DistributedGradReducer(Cell):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table Startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/tutorials/experts/en/r2.3.q1/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/tutorials/experts/en/r2.3.q1/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.3.q1/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with multiple devices.
 
@@ -466,3 +486,123 @@ class DistributedGradReducer(Cell):
         if context.get_context('mode') == context.GRAPH_MODE and parallel_mode in (
                 context.ParallelMode.SEMI_AUTO_PARALLEL, context.ParallelMode.AUTO_PARALLEL):
             raise RuntimeError("{} can not use DistributedGradReducer in graph mode".format(parallel_mode))
+
+
+class PipelineGradReducer(Cell):
+    """
+    PipelineGradReducer is a gradient reducer for pipeline parallelism.
+
+    Args:
+        parameters (list): the parameters to be updated.
+        scale_sense (float): the scale sense of the gradient. Default: 1.0.
+
+    Raise:
+        RuntimeError: If the mode is not graph mode.
+        RuntimeError: If the parallel mode is not semi auto parallel or auto parallel.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.3.q1/parallel/rank_table.html>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/r2.3.q1/parallel/mpirun.html>`_ .
+
+            This example should be run with multiple devices.
+
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import nn, ops, Tensor
+        >>> from mindspore.communication import init
+        >>>
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
+        >>> ms.reset_auto_parallel_context()
+        >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.SEMI_AUTO_PARALLEL, pipeline_stages=2)
+        >>> init()
+        >>> ms.set_seed(1)
+        >>>
+        >>> class Network(nn.Cell):
+        ...     def __init__(self, in_features, out_features, sens=1.0):
+        ...         super().__init__()
+        ...         self.layer1 = nn.Dense(in_features, 16)
+        ...         self.relu1 = nn.ReLU()
+        ...         self.layer2 = nn.Dense(16, 16)
+        ...         self.relu2 = nn.ReLU()
+        ...         self.layer3 = nn.Dense(16, out_features)
+        ...
+        ...     def construct(self, x):
+        ...         x = self.layer1(x)
+        ...         x = self.relu1(x)
+        ...         x = self.layer2(x)
+        ...         x = self.relu2(x)
+        ...         logits = self.layer3(x)
+        ...         return logits
+        >>>
+        >>> size, in_features, out_features = 16, 32, 10
+        >>> net = Network(in_features, out_features)
+        >>> net.layer1.pipeline_stage = 0
+        >>> net.relu1.pipeline_stage = 0
+        >>> net.layer2.pipeline_stage = 0
+        >>> net.relu2.pipeline_stage = 1
+        >>> net.layer3.pipeline_stage = 1
+        >>> loss_fn = nn.CrossEntropyLoss()
+        >>> optimizer = nn.SGD(net.trainable_params(), 1e-2)
+        >>> net_with_loss = nn.PipelineCell(nn.WithLossCell(net, loss_fn), 2)
+        >>> net_with_loss.set_train()
+        >>> def forward_fn(inputs, target):
+        ...     loss = net_with_loss(inputs, target)
+        ...     return loss
+        >>>
+        >>> grad_fn = ops.value_and_grad(forward_fn, None, net_with_loss.trainable_params())
+        >>> pp_grad_reducer = nn.PipelineGradReducer(optimizer.parameters)
+        >>>
+        >>> @ms.jit
+        >>> def train_one_step(inputs, target):
+        ...     loss, grads = grad_fn(inputs, target)
+        ...     grads = pp_grad_reducer(grads)
+        ...     optimizer(grads)
+        ...     return loss, grads
+        >>>
+        >>> inputs = Tensor(np.ones([size, in_features]).astype(np.float32))
+        >>> label = Tensor(np.ones([size, out_features]).astype(np.float32))
+        >>> loss, _ = train_one_step(inputs, label)
+        >>> print(loss)
+        46.36721
+    """
+    def __init__(self, parameters, scale_sense=1.0):
+        super(PipelineGradReducer, self).__init__(auto_prefix=False)
+        self._check_mode()
+        self.accu_grads = parameters.clone(prefix="accu_grads", init="zeros")
+        self.grad_reducer = Identity()
+        self.degree = Tensor(1, mstype.float32)
+        self.scale_sense = Parameter(scale_sense, name='scale_sense')
+        self.hyper_map = C.HyperMap()
+        self.opt_shard = _get_enable_parallel_optimizer()
+
+    @jit
+    def construct(self, grads):
+        new_grads = None
+        if self.opt_shard:
+            grads = self.grad_reducer(grads)
+            new_grads = self.hyper_map(F.partial(shard_grad_scale, self.scale_sense * self.degree),
+                                       grads, self.accu_grads)
+        else:
+            accu_grads = self.grad_reducer(self.accu_grads)
+            new_grads = self.hyper_map(F.partial(grad_scale, self.scale_sense * self.degree), grads, accu_grads)
+        return new_grads
+
+    def _check_mode(self):
+        """check parallel mode"""
+        mode = context.get_context('mode')
+        if mode != context.GRAPH_MODE:
+            raise RuntimeError(f"PipelineGradReducer only support graph mode, but get {mode}")
+        parallel_mode = context.get_auto_parallel_context('parallel_mode')
+        if parallel_mode not in (context.ParallelMode.SEMI_AUTO_PARALLEL, context.ParallelMode.AUTO_PARALLEL):
+            raise RuntimeError(f"{parallel_mode} can not use PipelineGradReducer in graph mode")

mindspore/nn/wrap/loss_scale.py

@@ -71,7 +71,7 @@ def _tensor_ascend_grad_overflow(grad):
     status = ascend_grad_overflow(grad)
     base = Tensor(1.0, dtype=mstype.float32)
     output = base - status.all()
-    output = P.Reshape()(output, ((1,)))
+    output = P.Reshape()(output, ((-1,)))
     return output
 
 
@@ -368,9 +368,9 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
         self.gpu_target = (context.get_context("device_target") == "GPU")
         self.ascend_910a_target = (MSContext.get_instance().get_ascend_soc_version() == 'ascend910')
-        self.ascend_910b_target = (MSContext.get_instance().get_ascend_soc_version() == 'ascend910b')
+        self.ascend_910bc_target = (MSContext.get_instance().get_ascend_soc_version() in ['ascend910b', 'ascend910c'])
         self.loss_scaling_manager = None
-        self._ascend910b_check_overflow_status_mode = os.environ.get('MS_ASCEND_CHECK_OVERFLOW_MODE')
+        self._ascend_check_overflow_mode = os.environ.get('MS_ASCEND_CHECK_OVERFLOW_MODE')
 
 
         if isinstance(scale_sense, Cell):
@@ -450,8 +450,8 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
             is cleaned up when the function returns.
         """
         status = Tensor([0] * 8, mstype.int32)
-        if self.ascend_910a_target or (self.ascend_910b_target and \
-                                       self._ascend910b_check_overflow_status_mode != "INFNAN_MODE"):
+        if self.ascend_910a_target or (self.ascend_910bc_target and \
+                                       self._ascend_check_overflow_mode == "SATURATION_MODE"):
             status = F.depend(status, pre_cond)
             # clear overflow buffer
             clear_status = NPUClearFloatStatusV2()(status)
@@ -531,8 +531,8 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         """
         if self.gpu_target:
             overflow = self._get_gpu_overflow_status(compute_output)
-        elif self.ascend_910b_target:
-            if self._ascend910b_check_overflow_status_mode != "INFNAN_MODE":
+        elif self.ascend_910bc_target:
+            if self._ascend_check_overflow_mode == "SATURATION_MODE":
                 overflow = self._get_ascend_overflow_status_on_saturation_mode(status, compute_output)
             else:
                 overflow = self._get_ascend_overflow_status_on_infnan_mode(compute_output)

mindspore/numpy/__init__.py

@@ -64,6 +64,8 @@ from mindspore.numpy.logic_ops import (not_equal, less_equal, less, greater_equa
                                        logical_or, logical_xor, in1d, isin, isclose, signbit, sometrue,
                                        array_equal, array_equiv, setdiff1d)
 
+from . import fft
+
 mod = remainder
 fabs = absolute
 round = around # pylint: disable=redefined-builtin

mindspore/numpy/array_creations.py

@@ -134,6 +134,8 @@ def asarray_const(a, dtype=None):
                 dtype = mstype.float32
             elif dtype == mstype.int64:
                 dtype = mstype.int32
+        if a.size == 0:
+            a = Tensor_(a)
 
     if isinstance(a, onp.ndarray) and dtype is None:
         if a.dtype is onp.dtype('object'):

mindspore/numpy/array_ops.py

@@ -1137,7 +1137,7 @@ def _get_moved_perm(ndim, source, destination):
 @_primexpr
 def _get_moved_shape(shape, perm):
     """
-    Helper function for moveaxis, returns the permuated shape after
+    Helper function for moveaxis, returns the permuted shape after
     applying perm.
     """
     return tuple([shape[i] for i in perm])
@@ -1186,10 +1186,6 @@ def moveaxis(a, source, destination):
         _raise_value_error('`source` and `destination` arguments must have the same number of elements')
     perm = _get_moved_perm(ndim, source, destination)
 
-    shape = F.shape(a)
-    if _is_shape_empty(shape):
-        return _empty(F.dtype(a), _get_moved_shape(shape, perm))
-
     return F.transpose(a, perm)