mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/nn/probability/distribution/poisson.py

@@ -229,11 +229,11 @@ class Poisson(Distribution):
         rate = self._check_param_type(rate)
         log_rate = self.log(rate)
         zeros = F.fill(self.dtypeop(value), self.shape(value), 0.0)
-        inf = F.fill(self.dtypeop(value), self.shape(value), np.inf)
+        neginf = F.fill(self.dtypeop(value), self.shape(value), -np.inf)
         safe_x = self.select(self.less(value, zeros), zeros, value)
         y = log_rate * safe_x - self.lgamma(safe_x + 1.)
         comp = self.equal(value, safe_x)
-        log_unnormalized_prob = self.select(comp, y, (-1) * inf)
+        log_unnormalized_prob = self.select(comp, y, neginf)
         log_normalization = self.exp(log_rate)
         return log_unnormalized_prob - log_normalization
 

mindspore/nn/probability/distribution/uniform.py

@@ -31,9 +31,9 @@ class Uniform(Distribution):
     and the probability density function:
 
     .. math::
-        f(x, a, b) = 1 / (b - a),
+        f(x, a, b) = 1 / (b - a)
 
-    where :math:`a, b` are the lower and upper bound respectively.
+    Where :math:`a, b` are the lower and upper bound respectively.
 
     Args:
         low (int, float, list, numpy.ndarray, Tensor): The lower bound of the distribution. Default: ``None`` .

mindspore/nn/reinforcement/_tensors_queue.py

@@ -37,7 +37,7 @@ class TensorsQueue(Cell):
         name (string): the name of this TensorsQueue. Default: "TQ".
 
     Raises:
-        TypeError: If `dtype` is not mindpore number type.
+        TypeError: If `dtype` is not mindspore number type.
         ValueError: If `size` is less than 0.
         ValueError: If `shapes` size is less than 1.
 
@@ -69,6 +69,7 @@ class TensorsQueue(Cell):
         self.tensors_q_clear = rl_ops.TensorsQueueClear()
         self.tensors_q_close = rl_ops.TensorsQueueClose()
         self.tensors_q_size = rl_ops.TensorsQueueSize()
+        self.__is_tensors_queue__ = True
 
     def put(self, element):
         """
@@ -103,6 +104,17 @@ class TensorsQueue(Cell):
         element = self.tensors_q_pop(self.handle_)
         return element
 
+    def __graph_pop__(self):
+        """
+        Get one element int the front of the TensorsQueue, and remove it.
+        This is only used in graph mode.
+
+        Returns:
+            tuple(Tensors), the element in TensorsQueue.
+        """
+        element = self.tensors_q_pop(self.handle_)
+        return self.handle_, element
+
     def size(self):
         """
         Get the used/available size of the TensorsQueue, and remove it.

mindspore/nn/wrap/__init__.py

@@ -24,7 +24,7 @@ from mindspore.nn.wrap.cell_wrapper import ForwardValueAndGrad, TrainOneStepCell
     GradAccumulationCell
 from mindspore.nn.wrap.loss_scale import TrainOneStepWithLossScaleCell,\
     DynamicLossScaleUpdateCell, FixedLossScaleUpdateCell
-from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
+from mindspore.nn.wrap.grad_reducer import DistributedGradReducer, PipelineGradReducer
 from mindspore.nn.layer.timedistributed import TimeDistributed
 
 
@@ -40,6 +40,7 @@ __all__ = [
     "GetNextSingleOp",
     "TrainOneStepWithLossScaleCell",
     "DistributedGradReducer",
+    "PipelineGradReducer",
     "ParameterUpdate",
     "DynamicLossScaleUpdateCell",
     "FixedLossScaleUpdateCell",

mindspore/nn/wrap/cell_wrapper.py

@@ -17,16 +17,18 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import os
 from types import FunctionType, MethodType
 
 from mindspore import log as logger
 from mindspore.parallel._utils import _get_device_num, _get_gradients_mean,\
     _get_parallel_mode, _get_enable_parallel_optimizer, _is_pynative_parallel
-from mindspore.context import ParallelMode
+from mindspore.context import ParallelMode, GRAPH_MODE, get_context
 from mindspore import _checkparam as validator
 from mindspore import ops, nn
 from mindspore.common import dtype as mstype
 from mindspore.common.parameter import Parameter, ParameterTuple
+from mindspore.common.tensor import Tensor
 from mindspore.ops.primitive import _primexpr
 from mindspore.ops import composite as C
 from mindspore.ops import functional as F
@@ -99,7 +101,7 @@ class WithLossCell(Cell):
         >>> from mindspore import Tensor, nn
         >>> import numpy as np
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
         >>> net_with_criterion = nn.WithLossCell(net, loss_fn)
@@ -132,7 +134,7 @@ class WithLossCell(Cell):
         Examples:
             >>> from mindspore import nn
             >>> # Define the network structure of LeNet5. Refer to
-            >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+            >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
             >>> net = LeNet5()
             >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False)
             >>> net_with_criterion = nn.WithLossCell(net, loss_fn)
@@ -175,7 +177,7 @@ class WithGradCell(Cell):
         >>> import mindspore as ms
         >>> from mindspore import nn
         >>> # Defined a network without loss function, taking LeNet5 as an example.
-        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # Refer to https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> grad_net = nn.WithGradCell(net, loss_fn)
@@ -346,7 +348,7 @@ class TrainOneStepCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
@@ -454,7 +456,7 @@ class GetNextSingleOp(Cell):
         queue_name (str): Queue name to fetch the data.
 
     Outputs:
-        tuple[Tensor], the data get from Dataset.
+        tuple[Tensor], the data gets from Dataset.
 
     Supported Platforms:
         ``Ascend`` ``GPU``
@@ -586,7 +588,7 @@ class MicroBatchInterleaved(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.MicroBatchInterleaved(net, 2)
     """
@@ -619,7 +621,7 @@ class MicroBatchInterleaved(Cell):
 
 class PipelineCell(Cell):
     """
-    Wrap the network with Micro Batch.
+    Slice MiniBatch into finer-grained MicroBatch for use in pipeline-parallel training.
 
     Note:
         micro_size must be greater or equal to pipeline stages.
@@ -634,7 +636,7 @@ class PipelineCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.PipelineCell(net, 4)
     """
@@ -685,7 +687,7 @@ class GradAccumulationCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.GradAccumulationCell(net, 4)
     """
@@ -726,7 +728,7 @@ class GradAccumulationCell(Cell):
 
 def _pipeline_clear_grad(accu_grad, grad):
     accu_grad = F.depend(accu_grad, grad)
-    zeros = F.tensor_mul(accu_grad, 0.0)
+    zeros = F.zeros_like(accu_grad)
     return F.assign(accu_grad, zeros)
 
 
@@ -740,6 +742,18 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         self.hyper_map = ops.HyperMap()
         self.opt_shard = _get_enable_parallel_optimizer()
         self._get_attr_from_cell(network)
+        self.enable_mindio = False
+        mode = get_context("mode")
+        device_type = get_context("device_target")
+        if device_type != "Ascend" or mode != GRAPH_MODE:
+            return
+        graceful_exit = os.getenv("MS_ENABLE_MINDIO_GRACEFUL_EXIT")
+        ttp_lib_path = os.getenv("MS_MINDIO_TTP_LIB_PATH")
+        ttp_path_check = ttp_lib_path is not None and os.path.isfile(ttp_lib_path)
+        if graceful_exit == "true" and ttp_path_check:
+            self.g_one = Tensor([0.1])
+            self.allreduce_sum = ops.AllReduce()
+            self.enable_mindio = True
 
     def construct(self, *inputs):
         if not self.sense_flag:
@@ -748,6 +762,11 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         sens = ops.fill(ops.DType()(loss), ops.Shape()(loss), self.sens)
         grads = self.grad(self.network, self.weights)(*inputs, sens)
         accu_grads = ops.depend(self.accu_grads, grads)
+        if self.enable_mindio:
+            g_one = ops.depend(self.g_one, accu_grads)
+            g_one_res = self.allreduce_sum(g_one)
+            accu_grads = ops.depend(accu_grads, g_one_res)
+            grads = ops.depend(grads, g_one_res)
         if self.opt_shard:
             succ = self.optimizer(grads)
         else:
@@ -762,6 +781,11 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         loss = self.network(*inputs)
         grads = self.grad_no_sens(self.network, self.weights)(*inputs)
         accu_grads = ops.depend(self.accu_grads, grads)
+        if self.enable_mindio:
+            g_one = ops.depend(self.g_one, accu_grads)
+            g_one_res = self.allreduce_sum(g_one)
+            accu_grads = ops.depend(accu_grads, g_one_res)
+            grads = ops.depend(grads, g_one_res)
         if self.opt_shard:
             succ = self.optimizer(grads)
         else:
@@ -772,6 +796,27 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         return loss
 
 
+class AllreduceGraph(Cell):
+    """
+    A allreduce graph to broadcast parameters.
+    """
+    def __init__(self, inputs, group_name):
+        super(AllreduceGraph, self).__init__()
+        self.input_num = len(inputs)
+        self.inputs = inputs
+        self.allreduces = []
+        self.assigns = []
+        for _ in range(self.input_num):
+            self.allreduces.append(ops.AllReduce(op="sum", group=group_name))
+            self.assigns.append(ops.Assign())
+
+    def construct(self):
+        for i in range(self.input_num):
+            res = self.allreduces[i](self.inputs[i])
+            self.assigns[i](self.inputs[i], res)
+        return self.inputs
+
+
 class VirtualDatasetCellTriple(Cell):
     """
     Wrap the network with virtual dataset to convert data parallel layout to model parallel layout.
@@ -790,7 +835,7 @@ class VirtualDatasetCellTriple(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> net = nn.VirtualDatasetCellTriple(net)
     """
@@ -833,7 +878,7 @@ class WithEvalCell(Cell):
     Examples:
         >>> import mindspore.nn as nn
         >>> # Define a forward network without loss function, taking LeNet5 as an example.
-        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # Refer to https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
         >>> eval_net = nn.WithEvalCell(net, loss_fn)

mindspore/nn/wrap/grad_reducer.py

@@ -18,17 +18,40 @@ from __future__ import absolute_import
 from mindspore import context
 from mindspore import log as logger
 from mindspore.nn.cell import Cell
+from mindspore.nn.layer import Identity
 from mindspore.communication.management import GlobalComm, get_group_size
 from mindspore.common.sparse_tensor import RowTensorInner
-from mindspore.ops import functional as F, composite as C
+from mindspore.ops import functional as F, composite as C, operations as P
 from mindspore.ops.operations.comm_ops import AllReduce, AllGather
 from mindspore.parallel._auto_parallel_context import auto_parallel_context
 import mindspore.common.dtype as mstype
 from mindspore.common.sparse_tensor import Tensor
 from mindspore.common.api import jit
-
+from mindspore.common.parameter import Parameter
+from mindspore.parallel._utils import _get_enable_parallel_optimizer
 
 reduce_opt = C.MultitypeFuncGraph("reduce_opt")
+grad_scale = C.MultitypeFuncGraph("grad_scale")
+shard_grad_scale = C.MultitypeFuncGraph("shard_grad_scale")
+reciprocal = P.Reciprocal()
+
+
+@grad_scale.register("Tensor", "Tensor", "Tensor")
+def tensor_grad_scale_pipeline(scale, grad, accu_grad):
+    accu_grad = F.depend(accu_grad, grad)
+    new_grad = accu_grad * reciprocal(scale)
+    accu_grad = F.depend(accu_grad, new_grad)
+    zeros = F.tensor_mul(accu_grad, 0.0)
+    new_grad = F.depend(new_grad, F.assign(accu_grad, zeros))
+    return new_grad
+
+
+@shard_grad_scale.register("Tensor", "Tensor", "Tensor")
+def tensor_shard_grad_scale_pipeline(scale, grad, accu_grad):
+    new_grad = grad * reciprocal(scale)
+    accu_grad = F.depend(accu_grad, new_grad)
+    new_grad = F.depend(new_grad, F.assign(accu_grad, F.zeros_like(accu_grad)))
+    return new_grad
 
 
 def _init_allreduce_operators(length, split_indices, group=GlobalComm.WORLD_COMM_GROUP):
@@ -53,9 +76,6 @@ def _init_allreduce_operators(length, split_indices, group=GlobalComm.WORLD_COMM
     for i in range(length):
         op = AllReduce('sum', group)
         op_fusion_id = fusion[i]
-        # When running in ge and enabled all_reduce_fusion_config, hccl will check the allreduce' fusion id to be -1
-        if context.get_context("enable_ge") and context.get_auto_parallel_context("all_reduce_fusion_config"):
-            op_fusion_id = -1
         op.add_prim_attr('fusion', op_fusion_id)
         op.add_prim_attr('index', index[i])
         op_list = op_list + (op,)
@@ -315,14 +335,14 @@ class DistributedGradReducer(Cell):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table Startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with multiple devices.
 
@@ -466,3 +486,123 @@ class DistributedGradReducer(Cell):
         if context.get_context('mode') == context.GRAPH_MODE and parallel_mode in (
                 context.ParallelMode.SEMI_AUTO_PARALLEL, context.ParallelMode.AUTO_PARALLEL):
             raise RuntimeError("{} can not use DistributedGradReducer in graph mode".format(parallel_mode))
+
+
+class PipelineGradReducer(Cell):
+    """
+    PipelineGradReducer is a gradient reducer for pipeline parallelism.
+
+    Args:
+        parameters (list): the parameters to be updated.
+        scale_sense (float): the scale sense of the gradient. Default: 1.0.
+
+    Raise:
+        RuntimeError: If the mode is not graph mode.
+        RuntimeError: If the parallel mode is not semi auto parallel or auto parallel.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
+            Please see the `rank table Startup
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
+            for more details.
+
+            For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
+
+            This example should be run with multiple devices.
+
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import nn, ops, Tensor
+        >>> from mindspore.communication import init
+        >>>
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
+        >>> ms.reset_auto_parallel_context()
+        >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.SEMI_AUTO_PARALLEL, pipeline_stages=2)
+        >>> init()
+        >>> ms.set_seed(1)
+        >>>
+        >>> class Network(nn.Cell):
+        ...     def __init__(self, in_features, out_features, sens=1.0):
+        ...         super().__init__()
+        ...         self.layer1 = nn.Dense(in_features, 16)
+        ...         self.relu1 = nn.ReLU()
+        ...         self.layer2 = nn.Dense(16, 16)
+        ...         self.relu2 = nn.ReLU()
+        ...         self.layer3 = nn.Dense(16, out_features)
+        ...
+        ...     def construct(self, x):
+        ...         x = self.layer1(x)
+        ...         x = self.relu1(x)
+        ...         x = self.layer2(x)
+        ...         x = self.relu2(x)
+        ...         logits = self.layer3(x)
+        ...         return logits
+        >>>
+        >>> size, in_features, out_features = 16, 32, 10
+        >>> net = Network(in_features, out_features)
+        >>> net.layer1.pipeline_stage = 0
+        >>> net.relu1.pipeline_stage = 0
+        >>> net.layer2.pipeline_stage = 0
+        >>> net.relu2.pipeline_stage = 1
+        >>> net.layer3.pipeline_stage = 1
+        >>> loss_fn = nn.CrossEntropyLoss()
+        >>> optimizer = nn.SGD(net.trainable_params(), 1e-2)
+        >>> net_with_loss = nn.PipelineCell(nn.WithLossCell(net, loss_fn), 2)
+        >>> net_with_loss.set_train()
+        >>> def forward_fn(inputs, target):
+        ...     loss = net_with_loss(inputs, target)
+        ...     return loss
+        >>>
+        >>> grad_fn = ops.value_and_grad(forward_fn, None, net_with_loss.trainable_params())
+        >>> pp_grad_reducer = nn.PipelineGradReducer(optimizer.parameters)
+        >>>
+        >>> @ms.jit
+        >>> def train_one_step(inputs, target):
+        ...     loss, grads = grad_fn(inputs, target)
+        ...     grads = pp_grad_reducer(grads)
+        ...     optimizer(grads)
+        ...     return loss, grads
+        >>>
+        >>> inputs = Tensor(np.ones([size, in_features]).astype(np.float32))
+        >>> label = Tensor(np.ones([size, out_features]).astype(np.float32))
+        >>> loss, _ = train_one_step(inputs, label)
+        >>> print(loss)
+        46.36721
+    """
+    def __init__(self, parameters, scale_sense=1.0):
+        super(PipelineGradReducer, self).__init__(auto_prefix=False)
+        self._check_mode()
+        self.accu_grads = parameters.clone(prefix="accu_grads", init="zeros")
+        self.grad_reducer = Identity()
+        self.degree = Tensor(1, mstype.float32)
+        self.scale_sense = Parameter(scale_sense, name='scale_sense')
+        self.hyper_map = C.HyperMap()
+        self.opt_shard = _get_enable_parallel_optimizer()
+
+    @jit
+    def construct(self, grads):
+        new_grads = None
+        if self.opt_shard:
+            grads = self.grad_reducer(grads)
+            new_grads = self.hyper_map(F.partial(shard_grad_scale, self.scale_sense * self.degree),
+                                       grads, self.accu_grads)
+        else:
+            accu_grads = self.grad_reducer(self.accu_grads)
+            new_grads = self.hyper_map(F.partial(grad_scale, self.scale_sense * self.degree), grads, accu_grads)
+        return new_grads
+
+    def _check_mode(self):
+        """check parallel mode"""
+        mode = context.get_context('mode')
+        if mode != context.GRAPH_MODE:
+            raise RuntimeError(f"PipelineGradReducer only support graph mode, but get {mode}")
+        parallel_mode = context.get_auto_parallel_context('parallel_mode')
+        if parallel_mode not in (context.ParallelMode.SEMI_AUTO_PARALLEL, context.ParallelMode.AUTO_PARALLEL):
+            raise RuntimeError(f"{parallel_mode} can not use PipelineGradReducer in graph mode")

mindspore/nn/wrap/loss_scale.py

@@ -29,6 +29,7 @@ from mindspore.ops.operations.math_ops import NPUGetFloatStatusV2, NPUClearFloat
 from mindspore.ops import functional as F
 from mindspore.ops import composite as C
 from mindspore.ops import operations as P
+from mindspore.ops.operations.nn_ops import AllFinite
 from mindspore.common import dtype as mstype
 from mindspore.common.api import jit
 from mindspore._c_expression import MSContext
@@ -71,7 +72,7 @@ def _tensor_ascend_grad_overflow(grad):
     status = ascend_grad_overflow(grad)
     base = Tensor(1.0, dtype=mstype.float32)
     output = base - status.all()
-    output = P.Reshape()(output, ((1,)))
+    output = P.Reshape()(output, ((-1,)))
     return output
 
 
@@ -368,10 +369,19 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
         self.gpu_target = (context.get_context("device_target") == "GPU")
         self.ascend_910a_target = (MSContext.get_instance().get_ascend_soc_version() == 'ascend910')
-        self.ascend_910b_target = (MSContext.get_instance().get_ascend_soc_version() == 'ascend910b')
+        self.ascend_910bc_target = (MSContext.get_instance().get_ascend_soc_version() in ['ascend910b', 'ascend910c'])
         self.loss_scaling_manager = None
-        self._ascend910b_check_overflow_status_mode = os.environ.get('MS_ASCEND_CHECK_OVERFLOW_MODE')
-
+        self._ascend_check_overflow_mode = os.environ.get('MS_ASCEND_CHECK_OVERFLOW_MODE')
+
+        self.enable_allfinite = False
+        runtime_conf = os.environ.get('MS_DEV_RUNTIME_CONF')
+        global_jit_config = context.get_jit_config()
+        if runtime_conf is not None and ("all_finite:True" in runtime_conf or "all_finite:true" in runtime_conf):
+            self.enable_allfinite = True
+        elif runtime_conf is not None and ("all_finite:False" in runtime_conf or "all_finite:false" in runtime_conf):
+            self.enable_allfinite = False
+        elif global_jit_config:
+            self.enable_allfinite = global_jit_config["jit_level"] == "O0" or global_jit_config["jit_level"] == "O1"
 
         if isinstance(scale_sense, Cell):
             self.loss_scaling_manager = scale_sense
@@ -450,8 +460,8 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
             is cleaned up when the function returns.
         """
         status = Tensor([0] * 8, mstype.int32)
-        if self.ascend_910a_target or (self.ascend_910b_target and \
-                                       self._ascend910b_check_overflow_status_mode != "INFNAN_MODE"):
+        if self.ascend_910a_target or (self.ascend_910bc_target and \
+                                       self._ascend_check_overflow_mode == "SATURATION_MODE"):
             status = F.depend(status, pre_cond)
             # clear overflow buffer
             clear_status = NPUClearFloatStatusV2()(status)
@@ -478,6 +488,15 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
             overflow = self.less_equal(self.base, flag_sum)
         return overflow
 
+    def _get_distributed_overflow_status_on_infnan_enable_allfinite(self, compute_output):
+        """check overflow status on infnan kernel mode."""
+        overflow = AllFinite()(compute_output)
+
+        if self.is_distributed:
+            overflow = P.Cast()(overflow, mstype.int8)
+            overflow = P.Cast()(self.allreduce(overflow), mstype.bool_)
+        return overflow
+
     def _get_gpu_overflow_status(self, compute_output):
         """get overflow status of gpu."""
         overflow = self._get_distributed_overflow_status_on_infnan_mode(_grad_overflow, compute_output)
@@ -485,7 +504,11 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
 
     def _get_ascend_overflow_status_on_infnan_mode(self, compute_output):
         """get overflow status of ascend on infnan mode."""
-        overflow = self._get_distributed_overflow_status_on_infnan_mode(_ascend_grad_overflow, compute_output)
+        overflow = False
+        if self.enable_allfinite:
+            overflow = self._get_distributed_overflow_status_on_infnan_enable_allfinite(compute_output)
+        else:
+            overflow = self._get_distributed_overflow_status_on_infnan_mode(_ascend_grad_overflow, compute_output)
         return overflow
 
     def _get_ascend_overflow_status_on_saturation_mode(self, status, compute_output):
@@ -531,8 +554,8 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         """
         if self.gpu_target:
             overflow = self._get_gpu_overflow_status(compute_output)
-        elif self.ascend_910b_target:
-            if self._ascend910b_check_overflow_status_mode != "INFNAN_MODE":
+        elif self.ascend_910bc_target:
+            if self._ascend_check_overflow_mode == "SATURATION_MODE":
                 overflow = self._get_ascend_overflow_status_on_saturation_mode(status, compute_output)
             else:
                 overflow = self._get_ascend_overflow_status_on_infnan_mode(compute_output)

mindspore/numpy/__init__.py

@@ -64,6 +64,8 @@ from mindspore.numpy.logic_ops import (not_equal, less_equal, less, greater_equa
                                        logical_or, logical_xor, in1d, isin, isclose, signbit, sometrue,
                                        array_equal, array_equiv, setdiff1d)
 
+from . import fft
+
 mod = remainder
 fabs = absolute
 round = around # pylint: disable=redefined-builtin

mindspore/numpy/array_creations.py

@@ -134,6 +134,8 @@ def asarray_const(a, dtype=None):
                 dtype = mstype.float32
             elif dtype == mstype.int64:
                 dtype = mstype.int32
+        if a.size == 0:
+            a = Tensor_(a)
 
     if isinstance(a, onp.ndarray) and dtype is None:
         if a.dtype is onp.dtype('object'):

mindspore/numpy/array_ops.py

@@ -1137,7 +1137,7 @@ def _get_moved_perm(ndim, source, destination):
 @_primexpr
 def _get_moved_shape(shape, perm):
     """
-    Helper function for moveaxis, returns the permuated shape after
+    Helper function for moveaxis, returns the permuted shape after
     applying perm.
     """
     return tuple([shape[i] for i in perm])
@@ -1186,10 +1186,6 @@ def moveaxis(a, source, destination):
         _raise_value_error('`source` and `destination` arguments must have the same number of elements')
     perm = _get_moved_perm(ndim, source, destination)
 
-    shape = F.shape(a)
-    if _is_shape_empty(shape):
-        return _empty(F.dtype(a), _get_moved_shape(shape, perm))
-
     return F.transpose(a, perm)
 
 
@@ -2610,7 +2606,11 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False):
         array1 = ar1.ravel()
         array2 = ar2.ravel()
     concat_array = concatenate((array1, array2))
-    concat_array, concat_sort_indices = concat_array.sort()
+    if return_indices:
+        concat_sort_indices = F.argsort(concat_array)
+        concat_array = concat_array[concat_sort_indices]
+    else:
+        concat_array, concat_sort_indices = concat_array.sort()
 
     mask_res = concat_array[1:] == concat_array[:-1]
     res = F.masked_select(concat_array[1:], mask_res)

mindspore/numpy/dtypes.py

@@ -86,7 +86,7 @@ dtype_map = {
 }
 
 all_types = [
-    'np.int',
+    'np.int_',
     'np.int8',
     'np.int16',
     'np.int32',
@@ -96,11 +96,11 @@ all_types = [
     'np.uint16',
     'np.uint32',
     'np.uint64',
-    'np.float',
+    'np.float_',
     'np.float16',
     'np.float32',
     'np.float64',
-    'np.bool']
+    'np.bool_']
 
 promotion_rule = {
     (uint8, uint16): uint16,