mindspore 2.3.0__cp310-cp310-win_amd64.whl → 2.4.0__cp310-cp310-win_amd64.whl

mindspore/nn/layer/transformer.py

@@ -16,6 +16,7 @@
 Transformer Cells module, include TransformerEncoderLayer, TransformerDecoderLayer,
 TransformerEncoder, TransformerDecoder, Transformer.
 """
+import copy
 import math
 from typing import Union, Optional
 import mindspore
@@ -31,7 +32,6 @@ from .basic import Dense, Dropout
 from .activation import ReLU, GELU
 from .normalization import LayerNorm
 from .container import CellList
-
 __all__ = ['MultiheadAttention', 'TransformerEncoderLayer', 'TransformerDecoderLayer',
            'TransformerEncoder', 'TransformerDecoder', 'Transformer']
 
@@ -588,7 +588,7 @@ class TransformerEncoder(Cell):
                                          encoder_layer.dropout_num, encoder_layer.activation1,
                                          encoder_layer.layernorm_eps, encoder_layer.batch_first,
                                          encoder_layer.norm_first, dtype=encoder_layer.dtype)
-        self.layers = CellList([layers for _ in range(num_layers)])
+        self.layers = CellList([copy.deepcopy(layers) for _ in range(num_layers)])
         self.num_layers = num_layers
         self.norm = norm
 
@@ -663,7 +663,7 @@ class TransformerDecoder(Cell):
                                          decoder_layer.dropout_num, decoder_layer.activation1,
                                          decoder_layer.layernorm_eps, decoder_layer.batch_first,
                                          decoder_layer.norm_first, dtype=decoder_layer.dtype)
-        self.layers = CellList([layers for _ in range(num_layers)])
+        self.layers = CellList([copy.deepcopy(layers) for _ in range(num_layers)])
         self.num_layers = num_layers
         self.norm = norm
 

mindspore/nn/loss/__init__.py

@@ -25,7 +25,7 @@ from mindspore.nn.loss.loss import LossBase, L1Loss, CTCLoss, MSELoss, SmoothL1L
     SampledSoftmaxLoss, TripletMarginWithDistanceLoss,\
     PoissonNLLLoss, MultiLabelSoftMarginLoss, DiceLoss, BCEWithLogitsLoss, MultiClassDiceLoss, \
     RMSELoss, MAELoss, HuberLoss, CrossEntropyLoss, NLLLoss, KLDivLoss, MarginRankingLoss, GaussianNLLLoss, \
-    HingeEmbeddingLoss, MultilabelMarginLoss, TripletMarginLoss
+    HingeEmbeddingLoss, MultilabelMarginLoss, TripletMarginLoss, L1LossExt
 
 
 __all__ = ['LossBase', 'L1Loss', 'CTCLoss', 'MSELoss', 'SmoothL1Loss', 'SoftMarginLoss', 'FocalLoss',
@@ -33,4 +33,4 @@ __all__ = ['LossBase', 'L1Loss', 'CTCLoss', 'MSELoss', 'SmoothL1Loss', 'SoftMarg
            'CosineEmbeddingLoss', 'SampledSoftmaxLoss', 'TripletMarginWithDistanceLoss', 'PoissonNLLLoss',
            'MultiLabelSoftMarginLoss', 'DiceLoss', 'MultiClassDiceLoss', 'MultilabelMarginLoss',
            'RMSELoss', 'MAELoss', 'HuberLoss', 'CrossEntropyLoss', 'NLLLoss', 'KLDivLoss', 'MarginRankingLoss',
-           'GaussianNLLLoss', 'HingeEmbeddingLoss', 'TripletMarginLoss']
+           'GaussianNLLLoss', 'HingeEmbeddingLoss', 'TripletMarginLoss', 'L1LossExt']

mindspore/nn/loss/loss.py

@@ -33,6 +33,7 @@ from mindspore.nn.cell import Cell
 from mindspore.nn.layer.activation import get_activation
 from mindspore import _checkparam as validator
 from mindspore import context
+from mindspore.ops.auto_generate import l1_loss_ext_op
 
 
 class LossBase(Cell):
@@ -247,6 +248,80 @@ class L1Loss(LossBase):
         return F.l1_loss(logits, labels, self.reduction)
 
 
+class L1LossExt(LossBase):
+    r"""
+    L1Loss is used to calculate the mean absolute error between the predicted value and the target value.
+
+    Assuming that the :math:`x` and :math:`y` are 1-D Tensor, length :math:`N`, then calculate the loss of :math:`x` and
+    :math:`y` without dimensionality reduction (the reduction parameter is set to ``'none'`` ). The formula is as
+    follows:
+
+    .. math::
+        \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad \text{with } l_n = \left| x_n - y_n \right|,
+
+    where :math:`N` is the batch size. If `reduction` is not ``'none'`` , then:
+
+    .. math::
+        \ell(x, y) =
+        \begin{cases}
+            \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
+            \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
+        \end{cases}
+
+    Args:
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
+    Inputs:
+        - **logits** (Tensor) - Predicted value, Tensor of any dimension.
+        - **labels** (Tensor) - Target value, same shape as the `logits` in common cases.
+          However, it supports the shape of `logits` is different from the shape of `labels`
+          and they should be broadcasted to each other.
+
+    Outputs:
+        Tensor, data type is float.
+
+    Raises:
+        ValueError: If `reduction` is not one of ``'none'`` , ``'mean'`` or ``'sum'`` .
+        ValueError: If `logits` and `labels` have different shapes and cannot be broadcasted to each other.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> # Case 1: logits.shape = labels.shape = (3,)
+        >>> loss = nn.L1LossExt()
+        >>> logits = Tensor(np.array([1, 2, 3]), mindspore.float32)
+        >>> labels = Tensor(np.array([1, 2, 2]), mindspore.float32)
+        >>> output = loss(logits, labels)
+        >>> print(output)
+        0.33333334
+        >>> # Case 2: logits.shape = (3,), labels.shape = (2, 3)
+        >>> loss = nn.L1LossExt(reduction='none')
+        >>> logits = Tensor(np.array([1, 2, 3]), mindspore.float32)
+        >>> labels = Tensor(np.array([[1, 1, 1], [1, 2, 2]]), mindspore.float32)
+        >>> output = loss(logits, labels)
+        >>> print(output)
+        [[0. 1. 2.]
+         [0. 0. 1.]]
+    """
+
+    def __init__(self, reduction='mean'):
+        """Initialize L1LossExt."""
+        super(L1LossExt, self).__init__(reduction)
+        self.reduction = reduction
+
+    def construct(self, logits, labels):
+        return l1_loss_ext_op(logits, labels, self.reduction)
+
+
 class MSELoss(LossBase):
     r"""
     Calculates the mean squared error between the predicted value and the label value.
@@ -287,6 +362,7 @@ class MSELoss(LossBase):
     Raises:
         ValueError: If `reduction` is not one of ``'none'``, ``'mean'`` or ``'sum'``.
         ValueError: If `logits` and `labels` have different shapes and cannot be broadcasted.
+        TypeError: if `logits` and `labels` have different data types.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -1580,7 +1656,7 @@ class BCELoss(LossBase):
     The formula is as follow:
 
     .. math::
-        L = \{l_1,\dots,l_N\}^\top, \quad
+        L = \{l_1,\dots,l_n,\dots,l_N\}^\top, \quad
         l_n = - w_n \left[ y_n \cdot \log x_n + (1 - y_n) \cdot \log (1 - x_n) \right]
 
     where N is the batch size. Then,
@@ -1850,14 +1926,16 @@ class BCEWithLogitsLoss(LossBase):
 
         weight (Tensor, optional): A rescaling weight applied to the loss of each batch element.
             If not None, it can be broadcast to a tensor with shape of `input`,
-            data type must be float16 or float32. Default: ``None`` .
+            data type must be float16, float32 or bfloat16(only Atlas A2 series products are supported).
+            Default: ``None`` .
         pos_weight (Tensor, optional): A weight of positive examples. Must be a vector with length equal to the
             number of classes. If not None, it must be broadcast to a tensor with shape of `input`, data type
-            must be float16 or float32. Default: ``None`` .
+            must be float16, float32 or bfloat16(only Atlas A2 series products are supported). Default: ``None`` .
 
     Inputs:
         - **input** (Tensor) - Input `input` with shape :math:`(N, *)` where :math:`*` means, any number
-          of additional dimensions. The data type must be float16 or float32.
+          of additional dimensions. The data type must be float16, float32 or bfloat16(only Atlas A2 series products
+          are supported).
         - **target** (Tensor) - Ground truth label with shape :math:`(N, *)` where :math:`*` means, any number
           of additional dimensions. The same shape and data type as `input`.
 
@@ -1867,9 +1945,9 @@ class BCEWithLogitsLoss(LossBase):
 
     Raises:
         TypeError: If input `input` or `target` is not Tensor.
-        TypeError: If data type of `input` or `target` is neither float16 nor float32.
+        TypeError: If data type of `input` or `target` is not float16, float32 or bfloat16.
         TypeError: If `weight` or `pos_weight` is a parameter.
-        TypeError: If data type of `weight` or `pos_weight` is neither float16 nor float32.
+        TypeError: If data type of `weight` or `pos_weight` is not float16 , float32 or bfloat16.
         TypeError: If data type of `reduction` is not string.
         ValueError: If `weight` or `pos_weight` can not be broadcast to a tensor with shape of `input`.
         ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.

mindspore/nn/optim/__init__.py

@@ -38,7 +38,8 @@ from mindspore.nn.optim.adafactor import AdaFactor
 from mindspore.nn.optim.adasum import AdaSumByDeltaWeightWrapCell, AdaSumByGradWrapCell
 from mindspore.nn.optim.adamax import AdaMax
 from mindspore.nn.optim.adadelta import Adadelta
+from mindspore.nn.optim.tft_wrapper import OptTFTWrapper
 
 __all__ = ['Optimizer', 'Momentum', 'LARS', 'Adam', 'AdamWeightDecay', 'LazyAdam', 'AdamOffload',
            'Lamb', 'SGD', 'ASGD', 'Rprop', 'FTRL', 'RMSProp', 'ProximalAdagrad', 'Adagrad', 'thor', 'AdaFactor',
-           'AdaSumByDeltaWeightWrapCell', 'AdaSumByGradWrapCell', 'AdaMax', 'Adadelta']
+           'AdaSumByDeltaWeightWrapCell', 'AdaSumByGradWrapCell', 'AdaMax', 'Adadelta', 'OptTFTWrapper']

mindspore/nn/optim/adadelta.py

@@ -55,7 +55,7 @@ class Adadelta(Optimizer):
             w_{t} = w_{t-1} - \gamma * update_{t}
         \end{array}
 
-    where :math:`g` represents `grads`, :math:`\gamma` represents `learning_rate`, :math:`p` represents `rho`,
+    where :math:`g` represents `grads`, :math:`\gamma` represents `learning_rate`, :math:`\rho` represents `rho`,
     :math:`\epsilon` represents `epsilon`, :math:`w` represents `params`,
     :math:`accum` represents accumulation, :math:`accum\_update` represents accumulation update,
     :math:`t` represents current step.

mindspore/nn/optim/adam.py

@@ -906,7 +906,7 @@ class AdamWeightDecay(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means, refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/master/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without

mindspore/nn/optim/lamb.py

@@ -132,7 +132,7 @@ class Lamb(Optimizer):
         There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
         and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
         As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means. Refer
-        document `LossScale <https://www.mindspore.cn/tutorials/en/master/advanced/mixed_precision.html>`_ to
+        document `LossScale <https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html>`_ to
         process `loss_scale` correctly.
 
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without

mindspore/nn/optim/tft_wrapper.py

@@ -0,0 +1,127 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""OptTFTWrapper"""
+from __future__ import absolute_import
+
+import os
+from mindspore.common.tensor import Tensor
+from mindspore.nn.optim.optimizer import Optimizer
+from mindspore.ops.operations.manually_defined._inner import TensorReport
+from mindspore import ops, context
+
+class OptTFTWrapper(Optimizer):
+    r"""
+    Implements TFT optimizer wrapper, this wrapper is used to report status to MindIO TFT before optimizer updating.
+
+    Note:
+        This optimizer is depend on MindIO TFT feature. Currently only support ascend graph mode and
+        sink_size must be less than 1.
+
+    Args:
+        opt (Optimizer): Must be sub-class of Optimizer.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of opt's `params`, the shape is the same as opt's `params`.
+
+    Outputs:
+        Tensor, result of executing optimizer 'opt'.
+
+    Raises:
+        TypeError: If the parameter opt is not an subclass of Optimizer.
+        ValueError: If the platform is not Ascend graph mode, or customer doesn't switch on TFT feature.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import nn
+        >>>
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> #1) All parameters use the same learning rate and weight decay
+        >>> optim = nn.SGD(params=net.trainable_params())
+        >>> optim_wrapper = nn.OptTFTWrapper(optim)
+        >>>
+        >>> loss = nn.SoftmaxCrossEntropyWithLogits()
+        >>> model = ms.train.Model(net, loss_fn=loss, optimizer=optim)
+    """
+
+    def __init__(self, opt, **kwargs):
+        super(OptTFTWrapper, self).__init__(opt.learning_rate, opt._parameters) # pylint: disable=W0212
+        if not isinstance(opt, Optimizer):
+            raise TypeError(f"For 'OptTFTWrapper', the argument 'opt' must be Optimizer type, " f"but got {type(opt)}.")
+        tft_env = os.getenv("MS_ENABLE_TFT", "")
+        if ("TTP:1" not in tft_env) and ("UCE:1" not in tft_env):
+            raise ValueError("MindIO TFT regitster need custom switch on[MS_ENABLE_TFT='{TTP:1,UCE:1}']!")
+        mode = context.get_context("mode")
+        device_target = context.get_context("device_target")
+        if device_target != "Ascend" or mode != context.GRAPH_MODE:
+            raise ValueError("MindIO adataper only support on Ascend device with GRAPH Mode!")
+        self.opt = opt
+        self.report = TensorReport()
+        self.depend = ops.Depend()
+        self.g_one = Tensor([0.1])
+        # enable consistent check by default, only disable when enable_consistent_check is False
+        self.use_allreduce = kwargs.get("enable_consistent_check", True)
+
+        if self.use_allreduce:
+            self.allreduce_sum = ops.AllReduce()
+            self.allreduce_sum.add_prim_attr("tft_report_before", True)
+
+        self.param_rank = opt.param_rank
+        self.optim_filter = opt.optim_filter
+        self.loss_scale = opt.loss_scale
+        self.dynamic_weight_decay = opt.dynamic_weight_decay
+        self.grad_centralization = opt.grad_centralization
+
+        self.dynamic_lr = opt.dynamic_lr
+        self.global_step = opt.global_step
+        self.is_group = opt.is_group
+        self.is_group_lr = opt.is_group_lr
+        self.is_group_params_ordered = opt.is_group_params_ordered
+        self.use_parallel = opt.use_parallel
+        if self.is_group:
+            self.group_params = opt.group_params
+            self.group_lr = opt.group_lr
+            self.group_weight_decay = opt.group_weight_decay
+            self.group_grad_centralization = opt.group_grad_centralization
+            self.grad_centralization_flags = opt.grad_centralization_flags
+
+        self.skip_auto_parallel_compile = opt.skip_auto_parallel_compile
+
+        self.learning_rate = opt.learning_rate
+        self.parameters = opt.parameters
+        self.decay_flags = opt.decay_flags
+        self.dynamic_decay_flags = opt.dynamic_decay_flags
+        self.weight_decay = opt.weight_decay
+        self.exec_weight_decay = opt.exec_weight_decay
+        self.ps_parameters = opt.ps_parameters
+        self.cache_enable = opt.cache_enable
+        self.reciprocal_scale = opt.reciprocal_scale
+        self.need_scale = opt.need_scale
+        self.global_step_increase_tensor = opt.global_step_increase_tensor
+        self.param_length = opt.param_length
+        self.enable_tuple_broaden = opt.enable_tuple_broaden
+
+    def construct(self, gradients):
+        g_one = self.depend(self.g_one, gradients)
+        if self.use_allreduce is True:
+            g_one_res = self.allreduce_sum(g_one)
+        else:
+            g_one_res = g_one
+        self.report("tft_report", g_one_res)
+        return self.opt(gradients)

mindspore/nn/wrap/cell_wrapper.py

@@ -23,7 +23,7 @@ from types import FunctionType, MethodType
 from mindspore import log as logger
 from mindspore.parallel._utils import _get_device_num, _get_gradients_mean,\
     _get_parallel_mode, _get_enable_parallel_optimizer, _is_pynative_parallel
-from mindspore.context import ParallelMode, GRAPH_MODE, get_context
+from mindspore.context import ParallelMode
 from mindspore import _checkparam as validator
 from mindspore import ops, nn
 from mindspore.common import dtype as mstype
@@ -36,6 +36,7 @@ from mindspore.ops import operations as P
 from mindspore.ops.operations.comm_ops import _VirtualDataset
 from mindspore.nn.cell import Cell
 from mindspore.nn.wrap.grad_reducer import DistributedGradReducer
+from mindspore.utils import ExitByRequest
 
 _get_datatype = C.MultitypeFuncGraph("_get_datatype")
 
@@ -414,6 +415,11 @@ class TrainOneStepCell(Cell):
                 group = server_group_name
             self.grad_reducer = DistributedGradReducer(self.weights, self.mean, self.degree, group=group)
         self._get_attr_from_cell(network)
+        self.use_graceful_exit = os.environ.get("MS_ENABLE_GRACEFUL_EXIT") == "1"
+        if self.use_graceful_exit:
+            self.graceful_exit = ExitByRequest()
+            self.exit_param = Parameter(Tensor(False, mstype.bool_), name="graceful_exit")  # update by reduce value
+            self.init_param = Parameter(Tensor([0], mstype.int32), name="graceful_init")  # update by config file
 
     def construct(self, *inputs):
         if not self.sense_flag:
@@ -422,6 +428,8 @@ class TrainOneStepCell(Cell):
         sens = F.fill(loss.dtype, loss.shape, self.sens)
         grads = self.grad(self.network, self.weights)(*inputs, sens)
         grads = self.grad_reducer(grads)
+        if self.use_graceful_exit:
+            grads = self.graceful_exit.exit_by_request(grads, self.init_param, self.exit_param)
         loss = F.depend(loss, self.optimizer(grads))
         if self.return_grad:
             grad_with_param_name = {}
@@ -435,6 +443,8 @@ class TrainOneStepCell(Cell):
         loss = self.network(*inputs)
         grads = self.grad_no_sens(self.network, self.weights)(*inputs)
         grads = self.grad_reducer(grads)
+        if self.use_graceful_exit:
+            grads = self.graceful_exit.exit_by_request(grads, self.init_param, self.exit_param)
         loss = F.depend(loss, self.optimizer(grads))
         if self.return_grad:
             grad_with_param_name = {}
@@ -742,18 +752,7 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         self.hyper_map = ops.HyperMap()
         self.opt_shard = _get_enable_parallel_optimizer()
         self._get_attr_from_cell(network)
-        self.enable_mindio = False
-        mode = get_context("mode")
-        device_type = get_context("device_target")
-        if device_type != "Ascend" or mode != GRAPH_MODE:
-            return
-        graceful_exit = os.getenv("MS_ENABLE_MINDIO_GRACEFUL_EXIT")
-        ttp_lib_path = os.getenv("MS_MINDIO_TTP_LIB_PATH")
-        ttp_path_check = ttp_lib_path is not None and os.path.isfile(ttp_lib_path)
-        if graceful_exit == "true" and ttp_path_check:
-            self.g_one = Tensor([0.1])
-            self.allreduce_sum = ops.AllReduce()
-            self.enable_mindio = True
+        self.enable_tft = False
 
     def construct(self, *inputs):
         if not self.sense_flag:
@@ -762,11 +761,6 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         sens = ops.fill(ops.DType()(loss), ops.Shape()(loss), self.sens)
         grads = self.grad(self.network, self.weights)(*inputs, sens)
         accu_grads = ops.depend(self.accu_grads, grads)
-        if self.enable_mindio:
-            g_one = ops.depend(self.g_one, accu_grads)
-            g_one_res = self.allreduce_sum(g_one)
-            accu_grads = ops.depend(accu_grads, g_one_res)
-            grads = ops.depend(grads, g_one_res)
         if self.opt_shard:
             succ = self.optimizer(grads)
         else:
@@ -781,11 +775,6 @@ class _TrainGradAccuStepCell(TrainOneStepCell):
         loss = self.network(*inputs)
         grads = self.grad_no_sens(self.network, self.weights)(*inputs)
         accu_grads = ops.depend(self.accu_grads, grads)
-        if self.enable_mindio:
-            g_one = ops.depend(self.g_one, accu_grads)
-            g_one_res = self.allreduce_sum(g_one)
-            accu_grads = ops.depend(accu_grads, g_one_res)
-            grads = ops.depend(grads, g_one_res)
         if self.opt_shard:
             succ = self.optimizer(grads)
         else:

mindspore/nn/wrap/grad_reducer.py

@@ -335,14 +335,14 @@ class DistributedGradReducer(Cell):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table Startup
-            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
-            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/docs/en/master/model_train/parallel/dynamic_cluster.html>`_ .
 
             This example should be run with multiple devices.
 
@@ -509,11 +509,11 @@ class PipelineGradReducer(Cell):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table Startup
-            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun Startup
-            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/mpirun.html>`_ .
 
             This example should be run with multiple devices.
 

mindspore/nn/wrap/loss_scale.py

@@ -33,6 +33,8 @@ from mindspore.ops.operations.nn_ops import AllFinite
 from mindspore.common import dtype as mstype
 from mindspore.common.api import jit
 from mindspore._c_expression import MSContext
+from mindspore.run_check._check_version import AscendEnvChecker
+from mindspore import log as logger
 
 _grad_scale = C.MultitypeFuncGraph("grad_scale")
 reciprocal = P.Reciprocal()
@@ -49,6 +51,7 @@ def tensor_grad_scale_row_tensor(scale, grad):
                           grad.values * F.cast(reciprocal(scale), F.dtype(grad.values)),
                           grad.dense_shape)
 
+
 _grad_overflow = C.MultitypeFuncGraph("_grad_overflow")
 grad_overflow = P.FloatStatus()
 
@@ -355,6 +358,7 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         >>> train_network.set_sense_scale(scaling_sens)
         >>> output = train_network(inputs, label)
     """
+
     def __init__(self, network, optimizer, scale_sense):
         super(TrainOneStepWithLossScaleCell, self).__init__(network, optimizer, sens=None)
         self.hyper_map = C.HyperMap()
@@ -369,7 +373,7 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
         self.gpu_target = (context.get_context("device_target") == "GPU")
         self.ascend_910a_target = (MSContext.get_instance().get_ascend_soc_version() == 'ascend910')
-        self.ascend_910bc_target = (MSContext.get_instance().get_ascend_soc_version() in ['ascend910b', 'ascend910c'])
+        self.ascend_910b_target = (MSContext.get_instance().get_ascend_soc_version() in ['ascend910b', 'ascend910_93'])
         self.loss_scaling_manager = None
         self._ascend_check_overflow_mode = os.environ.get('MS_ASCEND_CHECK_OVERFLOW_MODE')
 
@@ -377,12 +381,21 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         runtime_conf = os.environ.get('MS_DEV_RUNTIME_CONF')
         global_jit_config = context.get_jit_config()
         if runtime_conf is not None and ("all_finite:True" in runtime_conf or "all_finite:true" in runtime_conf):
+            logger.debug("Enable AllFinite through the environment variable MS_DEV_RUNTIME_CONF.")
             self.enable_allfinite = True
         elif runtime_conf is not None and ("all_finite:False" in runtime_conf or "all_finite:false" in runtime_conf):
+            logger.debug("Disable AllFinite through the environment variable MS_DEV_RUNTIME_CONF.")
             self.enable_allfinite = False
         elif global_jit_config:
+            logger.debug("Current global jit config is: {}".format(global_jit_config["jit_level"]))
             self.enable_allfinite = global_jit_config["jit_level"] == "O0" or global_jit_config["jit_level"] == "O1"
 
+        if self.ascend_910b_target:
+            checker = AscendEnvChecker(None)
+            if not checker.check_custom_version():
+                logger.debug("Disable AllFinite due to version check failure.")
+                self.enable_allfinite = False
+
         if isinstance(scale_sense, Cell):
             self.loss_scaling_manager = scale_sense
             self.scale_sense = Parameter(Tensor(scale_sense.get_loss_scale(), dtype=mstype.float32),
@@ -460,7 +473,7 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
             is cleaned up when the function returns.
         """
         status = Tensor([0] * 8, mstype.int32)
-        if self.ascend_910a_target or (self.ascend_910bc_target and \
+        if self.ascend_910a_target or (self.ascend_910b_target and \
                                        self._ascend_check_overflow_mode == "SATURATION_MODE"):
             status = F.depend(status, pre_cond)
             # clear overflow buffer
@@ -554,7 +567,7 @@ class TrainOneStepWithLossScaleCell(TrainOneStepCell):
         """
         if self.gpu_target:
             overflow = self._get_gpu_overflow_status(compute_output)
-        elif self.ascend_910bc_target:
+        elif self.ascend_910b_target:
             if self._ascend_check_overflow_mode == "SATURATION_MODE":
                 overflow = self._get_ascend_overflow_status_on_saturation_mode(status, compute_output)
             else:
@@ -613,6 +626,7 @@ class _TrainGradAccuWithLossScaleCell(TrainOneStepCell):
         optimizer (Optimizer): Optimizer for updating the weights.
         scale_sense (Cell): Cell to do the loss scale.
     """
+
     def __init__(self, network, optimizer, scale_sense):
         super(_TrainGradAccuWithLossScaleCell, self).__init__(network, optimizer, sens=None)
         self.network = network

mindspore/numpy/__init__.py

@@ -64,7 +64,7 @@ from mindspore.numpy.logic_ops import (not_equal, less_equal, less, greater_equa
                                        logical_or, logical_xor, in1d, isin, isclose, signbit, sometrue,
                                        array_equal, array_equiv, setdiff1d)
 
-from . import fft
+from mindspore.numpy import fft
 
 mod = remainder
 fabs = absolute