mindspore 2.4.10__cp310-cp310-win_amd64.whl → 2.6.0rc1__cp310-cp310-win_amd64.whl

mindspore/nn/layer/transformer.py

@@ -26,12 +26,12 @@ from mindspore.common.tensor import Tensor
 from mindspore.common.parameter import Parameter
 from mindspore.common.initializer import initializer, XavierNormal, XavierUniform, \
     HeUniform, Uniform, _calculate_fan_in_and_fan_out
-from mindspore.ops.function.nn_func import multi_head_attention_forward
 from mindspore.nn.cell import Cell
 from .basic import Dense, Dropout
 from .activation import ReLU, GELU
 from .normalization import LayerNorm
 from .container import CellList
+
 __all__ = ['MultiheadAttention', 'TransformerEncoderLayer', 'TransformerDecoderLayer',
            'TransformerEncoder', 'TransformerDecoder', 'Transformer']
 
@@ -54,16 +54,16 @@ class MultiheadAttention(Cell):
         embed_dim (int): Total dimension of MultiheadAttention.
         num_heads (int): Number of attention heads. Note that `embed_dim` will be split
             across `num_heads` (i.e. each head will have dimension `embed_dim // num_heads`).
-        dropout (float): Dropout probability of `attn_output_weights`. Default: ``0.0``.
-        has_bias (bool): Whether adds bias to input / output projection layers. Default: ``True``.
-        add_bias_kv (bool): Whether adds bias to the key and value sequences at axis=0. Default: ``False``.
-        add_zero_attn (bool): Whether adds a new batch of zeros to the key and value sequences at axis=1.
+        dropout (float, optional): Dropout probability of `attn_output_weights`. Default: ``0.0``.
+        has_bias (bool, optional): Whether adds bias to input / output projection layers. Default: ``True``.
+        add_bias_kv (bool, optional): Whether adds bias to the key and value sequences at axis=0. Default: ``False``.
+        add_zero_attn (bool, optional): Whether adds a new batch of zeros to the key and value sequences at axis=1.
             Default: ``False``.
-        kdim (int): Total number of features for keys. Default: ``None`` (`kdim=embed_dim`).
-        vdim (int): Total number of features for values. Default: ``None`` (`vdim=embed_dim`).
-        batch_first (bool): If ``True``, then the input and output shape are :math:`(batch, seq, feature)` ,
+        kdim (int, optional): Total number of features for keys. Default: ``None`` (`kdim=embed_dim`).
+        vdim (int, optional): Total number of features for values. Default: ``None`` (`vdim=embed_dim`).
+        batch_first (bool, optional): If ``True``, then the input and output shape are :math:`(batch, seq, feature)` ,
             else :math:`(seq, batch, feature)` . Default: ``False``.
-        dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
+        dtype (:class:`mindspore.dtype`, optional): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
         - **query** (Tensor) - The query embeddings. If `query` is unbatched, the shape is :math:`(L, E_q)`,
@@ -85,7 +85,7 @@ class MultiheadAttention(Cell):
           For a binary mask, a ``True`` value indicates that the corresponding `key` value will be ignored for
           the purpose of attention. For a float mask, it will be directly added to the corresponding `key` value.
           Supported float types: float16, float32, float64. Default: ``None``.
-        - **need_weights** (bool) - Whether returns `attn_output_weights` in addition to `attn_outputs`.
+        - **need_weights** (bool, optional) - Whether returns `attn_output_weights` in addition to `attn_outputs`.
           Default: ``True``.
         - **attn_mask** (Tensor, optional) - If specified, a 2D or 3D mask preventing attention to certain positions.
           Must be of shape :math:`(L, S)` or :math:`(N\cdot\text{num_heads}, L, S)`, where :math:`N` is the
@@ -94,7 +94,8 @@ class MultiheadAttention(Cell):
           in the batch. For a binary mask, a ``True`` value indicates that the corresponding position is not allowed
           to attend. For a float mask, the mask values will be added to the attention weight.
           Supported float types: float16, float32, float64. Default: ``None``.
-        - **average_attn_weights** (bool) - If true, indicates that the returned `attn_weights` should be averaged
+        - **average_attn_weights** (bool, optional) - If true, indicates that
+          the returned `attn_weights` should be averaged
           across heads. Otherwise, `attn_weights` are provided separately per head. Note that this flag only
           has an effect when `need_weights=True`. Default: ``True`` (i.e. average weights across heads)
 
@@ -212,7 +213,7 @@ class MultiheadAttention(Cell):
                 query, key, value = [x.swapaxes(1, 0) for x in (query, key, value)]
 
         if not self._qkv_same_embed_dim:
-            attn_output, attn_output_weights = multi_head_attention_forward(
+            attn_output, attn_output_weights = ops.function.nn_func.multi_head_attention_forward(
                 query, key, value, self.embed_dim, self.num_heads,
                 self.in_proj_weight, self.in_proj_bias,
                 self.bias_k, self.bias_v, self.add_zero_attn,
@@ -224,7 +225,7 @@ class MultiheadAttention(Cell):
                 v_proj_weight=self.v_proj_weight, average_attn_weights=average_attn_weights,
                 k_is_v=self.k_is_v, q_is_k=self.q_is_k, dtype=self.dtype)
         else:
-            attn_output, attn_output_weights = multi_head_attention_forward(
+            attn_output, attn_output_weights = ops.function.nn_func.multi_head_attention_forward(
                 query, key, value, self.embed_dim, self.num_heads,
                 self.in_proj_weight, self.in_proj_bias,
                 self.bias_k, self.bias_v, self.add_zero_attn,
@@ -328,7 +329,7 @@ class TransformerEncoderLayer(Cell):
         self.activation1 = activation
 
         if not isinstance(activation, str) and not isinstance(activation, Cell) \
-            and not callable(activation):
+                and not callable(activation):
             raise ValueError(f"The argument 'activation' must be str, callable or Cell instance,"
                              f" but get {activation}.")
         if isinstance(activation, Cell) and (not isinstance(activation, ReLU) and \
@@ -360,15 +361,23 @@ class TransformerEncoderLayer(Cell):
                 raise AssertionError(
                     "only bool and floating types of key_padding_mask are supported")
 
-        x = src
+        input_data = src
+
         if self.norm_first:
-            x = x + self._sa_block(self.norm1(x), src_mask, src_key_padding_mask)
-            x = x + self._ff_block(self.norm2(x))
+            normed_input = self.norm1(input_data)
+            sa_block_result = self._sa_block(normed_input, src_mask, src_key_padding_mask)
+            input_data = input_data + sa_block_result
+            normed_updated_input = self.norm2(input_data)
+            ff_block_result = self._ff_block(normed_updated_input)
+            input_data = input_data + ff_block_result
         else:
-            x = self.norm1(x + self._sa_block(x, src_mask, src_key_padding_mask))
-            x = self.norm2(x + self._ff_block(x))
+            sa_block_result = self._sa_block(input_data, src_mask, src_key_padding_mask)
+            normed_sa_result = self.norm1(input_data + sa_block_result)
+            input_data = normed_sa_result
+            ff_block_result = self._ff_block(input_data)
+            input_data = self.norm2(input_data + ff_block_result)
 
-        return x
+        return input_data
 
     def _sa_block(self, x, attn_mask, key_padding_mask):
         x = self.self_attn(x, x, x,
@@ -480,7 +489,7 @@ class TransformerDecoderLayer(Cell):
         self.activation1 = activation
 
         if not isinstance(activation, str) and not isinstance(activation, Cell) \
-            and not callable(activation):
+                and not callable(activation):
             raise ValueError(f"The argument 'activation' must be str, callable or Cell instance,"
                              f" but get {activation}.")
         if isinstance(activation, Cell) and (not isinstance(activation, ReLU) and \
@@ -507,17 +516,29 @@ class TransformerDecoderLayer(Cell):
     def construct(self, tgt: Tensor, memory: Tensor, tgt_mask: Optional[Tensor] = None,
                   memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None,
                   memory_key_padding_mask: Optional[Tensor] = None):
-        x = tgt
+        input_data = tgt
+
         if self.norm_first:
-            x = x + self._sa_block(self.norm1(x), tgt_mask, tgt_key_padding_mask)
-            x = x + self._mha_block(self.norm2(x), memory, memory_mask, memory_key_padding_mask)
-            x = x + self._ff_block(self.norm3(x))
+            normed_input = self.norm1(input_data)
+            sa_block_result = self._sa_block(normed_input, tgt_mask, tgt_key_padding_mask)
+            input_data = input_data + sa_block_result
+            normed_updated_input_1 = self.norm2(input_data)
+            mha_block_result = self._mha_block(normed_updated_input_1, memory, memory_mask, memory_key_padding_mask)
+            input_data = input_data + mha_block_result
+            normed_updated_input_2 = self.norm3(input_data)
+            ff_block_result = self._ff_block(normed_updated_input_2)
+            input_data = input_data + ff_block_result
         else:
-            x = self.norm1(x + self._sa_block(x, tgt_mask, tgt_key_padding_mask))
-            x = self.norm2(x + self._mha_block(x, memory, memory_mask, memory_key_padding_mask))
-            x = self.norm3(x + self._ff_block(x))
+            sa_block_result = self._sa_block(input_data, tgt_mask, tgt_key_padding_mask)
+            normed_sa_result = self.norm1(input_data + sa_block_result)
+            input_data = normed_sa_result
+            mha_block_result = self._mha_block(input_data, memory, memory_mask, memory_key_padding_mask)
+            normed_mha_result = self.norm2(input_data + mha_block_result)
+            input_data = normed_mha_result
+            ff_block_result = self._ff_block(input_data)
+            input_data = self.norm3(input_data + ff_block_result)
 
-        return x
+        return input_data
 
     def _sa_block(self, x, attn_mask, key_padding_mask):
         x = self.self_attn(x, x, x,
@@ -670,17 +691,19 @@ class TransformerDecoder(Cell):
     def construct(self, tgt: Tensor, memory: Tensor, tgt_mask: Optional[Tensor] = None,
                   memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None,
                   memory_key_padding_mask: Optional[Tensor] = None):
-        output = tgt
+        processed_output = tgt
         for mod in self.layers:
-            output = mod(output, memory, tgt_mask=tgt_mask,
-                         memory_mask=memory_mask,
-                         tgt_key_padding_mask=tgt_key_padding_mask,
-                         memory_key_padding_mask=memory_key_padding_mask)
+            layer_output = mod(processed_output, memory,
+                               tgt_mask=tgt_mask,
+                               memory_mask=memory_mask,
+                               tgt_key_padding_mask=tgt_key_padding_mask,
+                               memory_key_padding_mask=memory_key_padding_mask)
+            processed_output = layer_output
 
         if self.norm is not None:
-            output = self.norm(output)
+            processed_output = self.norm(processed_output)
 
-        return output
+        return processed_output
 
 
 class Transformer(Cell):

mindspore/nn/learning_rate_schedule.py

@@ -80,7 +80,8 @@ class ExponentialDecayLR(LearningRateSchedule):
         learning_rate (float): The initial value of learning rate.
         decay_rate (float): The decay rate.
         decay_steps (int): Number of steps to decay over.
-        is_stair (bool): If true, learning rate is decayed once every `decay_steps` time. Default: ``False`` .
+        is_stair (bool, optional): If ``True``, learning rate is decayed once every `decay_steps` time.
+            Default: ``False`` .
 
     Inputs:
         - **global_step** (Tensor) - The current step number. :math:`current\_step` in the above formula.
@@ -223,7 +224,9 @@ class InverseDecayLR(LearningRateSchedule):
         learning_rate (float): The initial value of learning rate.
         decay_rate (float): The decay rate.
         decay_steps (int): Number of steps to decay over.
-        is_stair (bool): If true, learning rate decay once every `decay_steps` times. Default: ``False`` .
+        is_stair (bool, optional): If true, learning rate decay once every `decay_steps` times.
+            If False, the learning rate
+            decays for every step. Default: ``False`` .
 
     Inputs:
         - **global_step** (Tensor) - The current step number.
@@ -454,8 +457,9 @@ class WarmUpLR(LearningRateSchedule):
         tmp\_step= \min(current\_step, warmup\_steps)
 
     Args:
-        learning_rate (float): The initial value of learning rate.
-        warmup_steps (int): The warm up steps of learning rate.
+        learning_rate (float): The initial value of learning rate. The value of `learning_rate` must be greater than 0.
+        warmup_steps (int): The warm up steps of learning rate. The value of `warmup_steps` must be greater than
+            or equal to 1.
 
     Inputs:
         - **global_step** (Tensor) - The current step number. Shape is :math:`()`.

mindspore/nn/loss/loss.py

@@ -24,8 +24,6 @@ from mindspore.common.tensor import Tensor
 from mindspore.common.parameter import Parameter
 from mindspore.ops import operations as P
 from mindspore.ops.operations import _inner_ops as inner
-from mindspore.ops.operations.nn_ops import MultiMarginLoss as MultiMarginLossOp
-from mindspore.ops.operations.nn_ops import MultilabelMarginLoss as MultilabelMarginLossOp
 from mindspore.ops import functional as F
 from mindspore import nn
 from mindspore.ops.primitive import constexpr, _primexpr
@@ -33,7 +31,6 @@ from mindspore.nn.cell import Cell
 from mindspore.nn.layer.activation import get_activation
 from mindspore import _checkparam as validator
 from mindspore import context
-from mindspore.ops.auto_generate import l1_loss_ext_op
 
 
 class LossBase(Cell):
@@ -130,7 +127,8 @@ class LossBase(Cell):
         Args:
             x (Tensor): Tensor of shape :math:`(N, *)` where :math:`*` means, any number of
                 additional dimensions.
-            weights (Union[float, Tensor]): Optional `Tensor` whose rank is either 0, or the same rank as inputs,
+            weights (Union[float, Tensor], optional): Weights. When `weights` is a Tensor,
+                the rank is either 0, or the same rank as inputs,
                 and must be broadcastable to inputs (i.e., all dimensions must be either `1`,
                 or the same as the corresponding inputs dimension). Default: ``1.0`` .
 
@@ -319,7 +317,7 @@ class L1LossExt(LossBase):
         self.reduction = reduction
 
     def construct(self, logits, labels):
-        return l1_loss_ext_op(logits, labels, self.reduction)
+        return ops.auto_generate.l1_loss_ext_op(logits, labels, self.reduction)
 
 
 class MSELoss(LossBase):
@@ -620,7 +618,8 @@ class MarginRankingLoss(LossBase):
 
 class SmoothL1Loss(LossBase):
     r"""
-    SmoothL1 loss function, if the absolute error element-wise between the predicted value and the target value
+    SmoothL1 loss function. Compare the error value element-wise and
+    if the absolute error between the predicted value and the target value
     is less than the set threshold `beta`, the square term is used, otherwise the absolute error term is used.
 
     Given two input :math:`x,\  y`, the SmoothL1Loss can be described as follows:
@@ -628,11 +627,11 @@ class SmoothL1Loss(LossBase):
     .. math::
         L_{i} =
         \begin{cases}
-        \frac{0.5 (x_i - y_i)^{2}}{\beta}, & \text{if } |x_i - y_i| < {\beta} \\
-        |x_i - y_i| - 0.5 {\beta}, & \text{otherwise.}
+        \frac{0.5 (x_i - y_i)^{2}}{\text{beta}}, & \text{if } |x_i - y_i| < \text{beta} \\
+        |x_i - y_i| - 0.5 * {\text{beta}}, & \text{otherwise.}
         \end{cases}
 
-    Where :math:`{\beta}` represents the threshold `beta`.
+    Where :math:`{\text{beta}}` represents the threshold `beta`.
 
     If `reduction` is not `none`, then:
 
@@ -653,8 +652,11 @@ class SmoothL1Loss(LossBase):
           robust to outliers, and the loss function has better robustness.
 
     Args:
-        beta (float): The loss function calculates the threshold of the transformation between L1Loss and L2Loss.
-            Default: ``1.0`` .
+        beta (number, optional): The loss function calculates the threshold of the transformation
+            between L1Loss and L2Loss. Default: ``1.0`` .
+
+            - Ascend: The value should be equal to or greater than zero.
+            - CPU/GPU: The value should be greater than zero.
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'none'`` .
 
@@ -663,22 +665,28 @@ class SmoothL1Loss(LossBase):
             - ``'sum'``: the output elements will be summed.
 
     Inputs:
-        - **logits** (Tensor) - Predictive value. Tensor of any dimension. Data type must be one of float16 or
-          float32.
-        - **labels** (Tensor) - Ground truth data, same shape and dtype as the `logits`.
+        - **logits** (Tensor) - Predictive value. Tensor of any dimension. Supported dtypes:
+
+          - Ascend: float16, float32, bfloat16.
+          - CPU/GPU: float16, float32, float64.
+
+        - **labels** (Tensor) - Ground truth data.
+
+          - CPU/Ascend: has the same shape as the `logits`,
+            `logits` and `labels` comply with the implicit type conversion rules to make the data types consistent.
+          - GPU: has the same shape and dtype as the `logits`.
 
     Outputs:
         Tensor, if `reduction` is ``'none'``, then output is a tensor with the same shape as `logits`.
         Otherwise the shape of output tensor is :math:`()`.
 
     Raises:
-        TypeError: If `beta` is not a float.
-        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
-        TypeError: If `logits` or `labels` are not Tensor.
-        TypeError: If dtype of `logits` or `labels` is neither float16 not float32.
-        TypeError: If dtype of `logits` is not the same as `labels`.
-        ValueError: If `beta` is less than or equal to 0.
+        TypeError: If input `logits` or `labels` are not Tensor.
+        RuntimeError: If dtype of `logits` or `labels` is not one of float16, float32, float64, bfloat16.
         ValueError: If shape of `logits` is not the same as `labels`.
+        ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
+        TypeError: If `beta` is not a float, int or bool.
+        RuntimeError: If `beta` is less than or equal to 0.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -728,16 +736,19 @@ class SoftMarginLoss(LossBase):
             - ``'sum'``: the output elements will be summed.
 
     Inputs:
-        - **logits** (Tensor) - Predict data. Data type must be float16 or float32.
-        - **labels** (Tensor) - Ground truth data, with the same type and shape as `logits`.
+        - **logits** (Tensor) - Predict data. Data type must be float16, float32,
+          bfloat16 (Among them, the Atlas training series products do not support bfloat16).
+        - **labels** (Tensor) - Ground truth data, with the same shape as `logits`.
+          In GE mode, the data type should be the same as `logits`.
 
     Outputs:
-        Tensor or Scalar, if `reduction` is ``"none"``, its shape is the same as `logits`.
+        Tensor or Scalar, if `reduction` is ``'none'``, its shape is the same as `logits`.
         Otherwise, a scalar value will be returned.
 
     Raises:
         TypeError: If `logits` or `labels` is not a Tensor.
-        TypeError: If dtype of `logits` or `labels` is neither float16 nor float32.
+        TypeError: If dtype of `logits` or `labels` is not float16, float32,
+                   bfloat16 (Among them, the Atlas training series products do not support bfloat16).
         ValueError: If shape of `logits` is not the same as `labels`.
         ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
 
@@ -758,10 +769,10 @@ class SoftMarginLoss(LossBase):
 
     def __init__(self, reduction='mean'):
         super(SoftMarginLoss, self).__init__()
-        self.soft_margin_loss = P.SoftMarginLoss(reduction)
+        self.reduction = reduction
 
     def construct(self, logits, labels):
-        return self.soft_margin_loss(logits, labels)
+        return F.soft_margin_loss(logits, labels, self.reduction)
 
 
 class SoftmaxCrossEntropyWithLogits(LossBase):
@@ -809,8 +820,8 @@ class SoftmaxCrossEntropyWithLogits(LossBase):
 
     Raises:
         TypeError: If `sparse` is not a bool.
-        TypeError: If `sparse` is True and dtype of `labels` is neither int32 nor int64.
-        TypeError: If `sparse` is False and dtype of `labels` is neither float16 not float32.
+        TypeError: If `sparse` is ``True`` and dtype of `labels` is neither int32 nor int64.
+        TypeError: If `sparse` is ``False`` and dtype of `labels` is neither float16 not float32.
         ValueError: If `reduction` is not one of ``'none'``, ``'mean'``, ``'sum'``.
 
     Supported Platforms:
@@ -889,8 +900,8 @@ class DiceLoss(LossBase):
     :math:`pred` represent `logits`, :math:`true` represent `labels` .
 
     Args:
-        smooth (float): A term added to the denominator to improve numerical stability. Should be greater than 0.
-                        Default: ``1e-5`` .
+        smooth (float, optional): A term added to the denominator to improve numerical stability.
+            Should be greater than 0. Default: ``1e-5`` .
 
     Inputs:
         - **logits** (Tensor) - Input predicted value. The data type must be float16 or float32.
@@ -934,11 +945,12 @@ class DiceLoss(LossBase):
         if label.dtype == mstype.uint8:
             raise TypeError(f"For '{self.cls_name}', the dtype of 'labels' can not be uint8.")
         intersection = self.reduce_sum(self.mul(logits.view(-1), label.view(-1)))
-        unionset = self.reduce_sum(self.mul(logits.view(-1), logits.view(-1))) + \
-                   self.reduce_sum(self.mul(label.view(-1), label.view(-1)))
+        unionset_part1 = self.reduce_sum(self.mul(logits.view(-1), logits.view(-1)))
+        unionset_part2 = self.reduce_sum(self.mul(label.view(-1), label.view(-1)))
+        unionset = ops.add(unionset_part1, unionset_part2)
 
-        single_dice_coeff = (2 * intersection) / (unionset + self.smooth)
-        dice_loss = 1 - single_dice_coeff
+        single_dice_coeff = (2 * intersection) / ops.add(unionset, self.smooth)
+        dice_loss = ops.sub(1, single_dice_coeff)
 
         return dice_loss
 
@@ -1054,7 +1066,7 @@ class MultiClassDiceLoss(LossBase):
                 dice_loss = self.binarydiceloss(logits[:, i], label[:, i])
                 if self.weights is not None:
                     _check_weights(self.weights.shape[0], label.shape[1], self.cls_name)
-                    dice_loss *= self.weights[i]
+                    dice_loss = dice_loss * self.weights[i]
                 total_loss += dice_loss
 
         return total_loss / label.shape[1]
@@ -1631,7 +1643,7 @@ class MultiMarginLoss(LossBase):
     def __init__(self, p=1, margin=1.0, reduction='mean', weight=None):
         """Initialize MultiMarginLoss."""
         super(MultiMarginLoss, self).__init__()
-        self.multi_margin_loss = MultiMarginLossOp(p=p, margin=margin, reduction=reduction)
+        self.multi_margin_loss = ops.MultiMarginLoss(p=p, margin=margin, reduction=reduction)
         self.weight = weight
 
     def construct(self, x, target, weight=None):
@@ -1718,22 +1730,11 @@ class BCELoss(LossBase):
     def __init__(self, weight=None, reduction='mean'):
         """Initialize BCELoss."""
         super(BCELoss, self).__init__(reduction)
-        self.binary_cross_entropy = P.BinaryCrossEntropy(reduction=reduction)
-        self.weight_one = weight is None
-        if not self.weight_one:
-            self.weight = weight
-        else:
-            self.ones = P.OnesLike()
+        self.reduction = reduction
+        self.weight = weight
 
     def construct(self, logits, labels):
-        _check_is_tensor('logits', logits, self.cls_name)
-        _check_is_tensor('labels', labels, self.cls_name)
-        if self.weight_one:
-            weight = self.ones(logits)
-        else:
-            weight = self.weight
-        loss = self.binary_cross_entropy(logits, labels, weight)
-        return loss
+        return F.binary_cross_entropy(logits, labels, self.weight, self.reduction)
 
 
 class CosineEmbeddingLoss(LossBase):
@@ -1887,7 +1888,7 @@ class MultilabelMarginLoss(LossBase):
 
     def __init__(self, reduction='mean'):
         super(MultilabelMarginLoss, self).__init__()
-        self.multilabel_margin_loss = MultilabelMarginLossOp(reduction=reduction)
+        self.multilabel_margin_loss = ops.MultilabelMarginLoss(reduction=reduction)
 
     def construct(self, x, target):
         loss, _ = self.multilabel_margin_loss(x, target)
@@ -2265,7 +2266,8 @@ class TripletMarginLoss(LossBase):
             - ``'mean'``: compute and return the mean of elements in the output.
             - ``'sum'``: the output elements will be summed.
 
-        margin (Union[Tensor, float]): Make a margin between the positive pair and the negative pair.
+        margin (Union[Tensor, float]): Make a margin between the positive pair and the negative pair. The length of
+            shape of `margin` must be 0.
             Default: ``1.0`` .
 
     Inputs:
@@ -2275,7 +2277,8 @@ class TripletMarginLoss(LossBase):
           shape as `x`. :math:`p` in the above formula.
         - **negative** (Tensor) - A sample belonging to the different class from `x`, with the same type and shape
           as `x`. :math:`n` in the above formula.
-        - **margin** (Union[Tensor, float]) - Make a margin between the positive pair and the negative pair.
+        - **margin** (Union[Tensor, float]) - Make a margin between the positive pair and the negative pair. The length
+          of shape of `margin` must be 0.
           Default: ``1.0`` .
 
     Outputs:
@@ -2576,7 +2579,7 @@ class KLDivLoss(LossBase):
     the updating formulas of KLDivLoss algorithm are as follows,
 
     .. math::
-        L(x, target) = target \cdot (\log target - x)
+        L(x, target) = target \cdot (\log target - \log x)
 
     Then,
 
@@ -2870,7 +2873,7 @@ class HingeEmbeddingLoss(LossBase):
     where :math:`L = \{l_1,\dots,l_N\}^\top`.
 
     Args:
-        margin (float, int): Threshold defined by Hinge Embedding Loss :math:`margin`.
+        margin (float, int, optional): Threshold defined by Hinge Embedding Loss :math:`margin`.
             Represented as :math:`\Delta` in the formula. Default: ``1.0`` .
         reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
             ``'sum'`` . Default: ``'mean'`` .

mindspore/nn/optim/ada_grad.py

@@ -78,6 +78,7 @@ class Adagrad(Optimizer):
     :math:`state\_sum` stands for the accumulated squared sum of the gradients :math:`accum`.
     :math:`g` stands for `grads`, :math:`\lambda` stands for `weight_decay`.
     :math:`\gamma` stands for `learning_rate`, :math:`w` stands for `params`.
+    :math:`t` represents current `step`.
 
     Note:
         If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
@@ -112,8 +113,8 @@ class Adagrad(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        accum (float): The starting value for :math:`h`, must be zero or positive values. Default: ``0.1`` .
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``0.001`` .
+        accum (float, optional): The starting value for :math:`h`, must be zero or positive values. Default: ``0.1`` .
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule], optional): Default: ``0.001`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -129,13 +130,14 @@ class Adagrad(Optimizer):
               <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
               with step as the input to get the learning rate of current step.
 
-        update_slots (bool): Whether the :math:`h` will be updated. Default: ``True`` .
-        loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
+        update_slots (bool, optional): Whether the :math:`h` will be updated. Default: ``True`` .
+        loss_scale (float, optional): Value for the loss scale. It must be greater than 0.0. In general,
+            use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
             `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
             Default: ``1.0`` .
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
+        weight_decay (Union[float, int, Cell], optional): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 

mindspore/nn/optim/adadelta.py

@@ -68,8 +68,8 @@ class Adadelta(Optimizer):
 
     Args:
         params (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
-            `params` is a list of `dict`, the string "params", "lr", "weight_decay", "grad_centralization" and
-            "order_params" are the keys can be parsed.
+            `params` is a list of `dict`, the string `"params"`, `"lr"`, `"weight_decay"`, `"grad_centralization"` and
+            `"order_params"` are the keys can be parsed.
 
             - params: Required. Parameters in current group. The value must be a list of `Parameter`.
 
@@ -93,7 +93,7 @@ class Adadelta(Optimizer):
               If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
               one group of `params`.
 
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: ``1.0`` .
+        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule], optional): Default: ``1.0`` .
 
             - float: The fixed learning rate value. Must be equal to or greater than 0.
 
@@ -109,14 +109,16 @@ class Adadelta(Optimizer):
               <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_
               with step as the input to get the learning rate of current step.
 
-        rho (float): Decay rate, must be in range [0.0, 1.0]. Default: ``0.9`` .
-        epsilon (float):  A small value added for numerical stability, must be non-negative. Default: ``1e-6`` .
-        loss_scale (float): Value for the loss scale. It must be greater than 0.0. In general, use the default value.
+        rho (float, optional): Decay rate, must be in range [0.0, 1.0]. Default: ``0.9`` .
+        epsilon (float, optional):  A small value added for numerical stability, must be non-negative.
+            Default: ``1e-6`` .
+        loss_scale (float, optional): Value for the loss scale. It must be greater than 0.0. In general,
+            use the default value.
             Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
             `FixedLossScaleManager` is set to ``False`` , then this value needs to be the same as the `loss_scale` in
             `FixedLossScaleManager`. Refer to class :class:`mindspore.amp.FixedLossScaleManager` for more details.
             Default: ``1.0`` .
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: ``0.0`` .
+        weight_decay (Union[float, int, Cell], optional): Weight decay (L2 penalty). Default: ``0.0`` .
 
             - float: The fixed weight decay value. Must be equal to or greater than 0.
 
@@ -134,9 +136,9 @@ class Adadelta(Optimizer):
 
     Raises:
         TypeError: If `learning_rate` is not one of int, float, Tensor, Iterable, LearningRateSchedule.
-        TypeError: If element of `parameters` is neither Parameter nor dict.
+        TypeError: If element of `params` is neither Parameter nor dict.
         TypeError: If `rho`, `epsilon` or `loss_scale` is not a float.
-        TypeError: If `weight_decay` is neither float nor int.
+        TypeError: If `weight_decay` is not float, int or cell.
         ValueError: if `rho` is not in range [0.0, 1.0].
         ValueError: If `loss_scale` is less than or equal to 0.
         ValueError: If `learning_rate`, `epsilon` or `weight_decay` is less than 0.

mindspore/nn/optim/adafactor.py

@@ -406,7 +406,7 @@ class AdaFactor(Optimizer):
         """
         return False
 
-    @jit
+    @jit(backend="ms_backend")
     def construct(self, gradients):
         gradients = self.flatten_gradients(gradients)
         lr = self.get_lr()