mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/nn/layer/transformer.py

@@ -16,6 +16,7 @@
 Transformer Cells module, include TransformerEncoderLayer, TransformerDecoderLayer,
 TransformerEncoder, TransformerDecoder, Transformer.
 """
+import copy
 import math
 from typing import Union, Optional
 import mindspore
@@ -31,7 +32,6 @@ from .basic import Dense, Dropout
 from .activation import ReLU, GELU
 from .normalization import LayerNorm
 from .container import CellList
-
 __all__ = ['MultiheadAttention', 'TransformerEncoderLayer', 'TransformerDecoderLayer',
            'TransformerEncoder', 'TransformerDecoder', 'Transformer']
 
@@ -66,35 +66,35 @@ class MultiheadAttention(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **query** (Tensor): The query embeddings. If `query` is unbatched, the shape is :math:`(L, E_q)`,
+        - **query** (Tensor) - The query embeddings. If `query` is unbatched, the shape is :math:`(L, E_q)`,
           otherwise the shape is :math:`(L, N, E_q)` when `batch_first=False` or :math:`(N, L, E_q)` when
           `batch_first=True` , where :math:`L`is the target sequence length, :math:`N` is the batch size,
           and :math:`E_q` is the query embedding dimension `embed_dim`. Supported types: float16, float32,
           float64. Queries are compared against key-value pairs to produce the output.
-        - **key** (Tensor): The key embeddings. If `key` is unbatched, the shape is :math:`(S, E_k)`, otherwise
+        - **key** (Tensor) - The key embeddings. If `key` is unbatched, the shape is :math:`(S, E_k)`, otherwise
           the shape is :math:`(S, N, E_k)` when `batch_first=False` or :math:`(N, S, E_k)` when
           `batch_first=True` , where :math:`S` is the source sequence length, :math:`N` is the batch size,
           and :math:`E_k` is the key embedding dimension `kdim`. Supported types: float16, float32, float64.
-        - **value** (Tensor): The value embeddings. If `value` is unbatched, the shape is :math:`(S, E_v)`,
+        - **value** (Tensor) - The value embeddings. If `value` is unbatched, the shape is :math:`(S, E_v)`,
           otherwise the shape is :math:`(S, N, E_v)` when `batch_first=False` or :math:`(N, S, E_v)` when
           `batch_first=True` , where :math:`S` is the source sequence length, :math:`N` is the batch size,
           and :math:`E_v` is the value embedding dimension `vdim`. Supported types: float16, float32, float64.
-        - **key_padding_mask** (Tensor, optional): If specified, a mask of shape :math:`(N, S)` indicating which
+        - **key_padding_mask** (Tensor, optional) - If specified, a mask of shape :math:`(N, S)` indicating which
           elements within `key` to ignore for the purpose of attention (i.e. treat as "padding").
           For unbatched `query`, shape should be :math:`(S)`. Binary and float masks are supported.
           For a binary mask, a ``True`` value indicates that the corresponding `key` value will be ignored for
           the purpose of attention. For a float mask, it will be directly added to the corresponding `key` value.
           Supported float types: float16, float32, float64. Default: ``None``.
-        - **need_weights** (bool): Whether returns `attn_output_weights` in addition to `attn_outputs`.
+        - **need_weights** (bool) - Whether returns `attn_output_weights` in addition to `attn_outputs`.
           Default: ``True``.
-        - **attn_mask** (Tensor, optional): If specified, a 2D or 3D mask preventing attention to certain positions.
+        - **attn_mask** (Tensor, optional) - If specified, a 2D or 3D mask preventing attention to certain positions.
           Must be of shape :math:`(L, S)` or :math:`(N\cdot\text{num_heads}, L, S)`, where :math:`N` is the
           batch size, :math:`L` is the target sequence length, and :math:`S` is the source sequence length.
           A 2D mask will be broadcasted across the batch while a 3D mask allows for a different mask for each entry
           in the batch. For a binary mask, a ``True`` value indicates that the corresponding position is not allowed
           to attend. For a float mask, the mask values will be added to the attention weight.
           Supported float types: float16, float32, float64. Default: ``None``.
-        - **average_attn_weights** (bool): If true, indicates that the returned `attn_weights` should be averaged
+        - **average_attn_weights** (bool) - If true, indicates that the returned `attn_weights` should be averaged
           across heads. Otherwise, `attn_weights` are provided separately per head. Note that this flag only
           has an effect when `need_weights=True`. Default: ``True`` (i.e. average weights across heads)
 
@@ -265,14 +265,14 @@ class TransformerEncoderLayer(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **src** (Tensor): the sequence to the encoder layer. For unbatched input, the shape is
+        - **src** (Tensor) - the sequence to the encoder layer. For unbatched input, the shape is
           :math:`(S, E)` ; otherwise if `batch_first=False` , the shape is :math:`(S, N, E)` and if
-          `batch_first=True` , the shape is :math:`(S, N, E)`, where :math:`(S)` is the source sequence
+          `batch_first=True` , the shape is :math:`(N, S, E)`, where :math:`(S)` is the source sequence
           length, :math:`(N)` is the batch number and :math:`(E)` is the feature number.
           Supported types: float16, float32, float64.
-        - **src_mask** (Tensor, optional): the mask for the src sequence. The shape is :math:`(S, S)`
+        - **src_mask** (Tensor, optional) - the mask for the src sequence. The shape is :math:`(S, S)`
           or :math:`(N*nhead, S, S)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **src_key_padding_mask** (Tensor, optional): the mask for the src keys per batch. The shape is
+        - **src_key_padding_mask** (Tensor, optional) - the mask for the src keys per batch. The shape is
           :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
 
@@ -406,20 +406,20 @@ class TransformerDecoderLayer(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **tgt** (Tensor): The sequence to the decoder layer. For unbatched input, the shape is
+        - **tgt** (Tensor) - The sequence to the decoder layer. For unbatched input, the shape is
           :math:`(T, E)` ; otherwise if `batch_first=False` , the shape is :math:`(T, N, E)` and if
-          `batch_first=True` , the shape is :math:`(T, N, E)`, where :math:`(T)` is the target sequence
+          `batch_first=True` , the shape is :math:`(N, T, E)`, where :math:`(T)` is the target sequence
           length. Supported types: float16, float32, float64.
-        - **memory** (Tensor): The sequence from the last layer of the encoder. Supported types: float16,
+        - **memory** (Tensor) - The sequence from the last layer of the encoder. Supported types: float16,
           float32, float64.
-        - **tgt_mask** (Tensor, optional): The mask of the tgt sequence. The shape is :math:`(T, T)`
+        - **tgt_mask** (Tensor, optional) - The mask of the tgt sequence. The shape is :math:`(T, T)`
           or :math:`(N*nhead, T, T)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_mask** (Tensor, optional): The mask of the memory sequence. The shape is
+        - **memory_mask** (Tensor, optional) - The mask of the memory sequence. The shape is
           :math:`(T, S)` . Supported types: float16, float32, float64, bool. Default: ``None``.
         - **tgt_key_padding_mask** (Tensor, optional): The mask of the tgt keys per batch. The shape is
-          :math:`(T)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
+          :math:`(T)` for unbatched input, otherwise :math:`(N, T)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
-        - **memory_key_padding_mask** (Tensor, optional): The mask of the memory keys per batch. The shape
+        - **memory_key_padding_mask** (Tensor, optional) - The mask of the memory keys per batch. The shape
           is :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
 
@@ -540,8 +540,8 @@ class TransformerDecoderLayer(Cell):
 
 class TransformerEncoder(Cell):
     r"""
-    Transformer Encoder module with multi-layer stacked of `TransformerEncoderLayer`, including multihead
-    attention and feedforward layer. Users can build the
+    Transformer Encoder module with multi-layer stacked of :class:`mindspore.nn.TransformerEncoderLayer`,
+    including multihead attention and feedforward layer. Users can build the
     BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.
 
     Args:
@@ -550,15 +550,15 @@ class TransformerEncoder(Cell):
         norm (Cell, optional): The layer normalization module. Default: ``None``.
 
     Inputs:
-        - **src** (Tensor): The sequence to the encoder. For unbatched input, the shape is
-          :math:`(S, E)` ; otherwise if `batch_first=False` in TransformerEncoderLayer, the shape is
-          :math:`(S, N, E)` and if `batch_first=True` , the shape is :math:`(S, N, E)`, where :math:`(S)` is the
-          source sequence length, :math:`(N)` is the batch number and :math:`(E)` is the feature number.
-          Supported types: float16, float32, float64.
-        - **src_mask** (Tensor, optional): The mask of the src sequence. The shape is :math:`(S, S)`
-          or :math:`(N*nhead, S, S)` , where `nhead` is the arguent in TransformerDecoderLayer.
+        - **src** (Tensor) - The sequence to the encoder. For unbatched input, the shape is
+          :math:`(S, E)` ; otherwise if `batch_first=False` in :class:`mindspore.nn.TransformerEncoderLayer`,
+          the shape is :math:`(S, N, E)` and if `batch_first=True` , the shape is :math:`(N, S, E)`,
+          where :math:`(S)` is the source sequence length, :math:`(N)` is the batch number and :math:`(E)` is
+          the feature number. Supported types: float16, float32, float64.
+        - **src_mask** (Tensor, optional) - The mask of the src sequence. The shape is :math:`(S, S)`
+          or :math:`(N*nhead, S, S)` , where `nhead` is the arguent in :class:`mindspore.nn.TransformerEncoderLayer`.
           Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **src_key_padding_mask** (Tensor, optional): the mask of the src keys per batch. The shape is
+        - **src_key_padding_mask** (Tensor, optional) - the mask of the src keys per batch. The shape is
           :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool.  Default: ``None``.
 
@@ -588,7 +588,7 @@ class TransformerEncoder(Cell):
                                          encoder_layer.dropout_num, encoder_layer.activation1,
                                          encoder_layer.layernorm_eps, encoder_layer.batch_first,
                                          encoder_layer.norm_first, dtype=encoder_layer.dtype)
-        self.layers = CellList([layers for _ in range(num_layers)])
+        self.layers = CellList([copy.deepcopy(layers) for _ in range(num_layers)])
         self.num_layers = num_layers
         self.norm = norm
 
@@ -611,8 +611,8 @@ class TransformerEncoder(Cell):
 
 class TransformerDecoder(Cell):
     r"""
-    Transformer Decoder module with multi-layer stacked of `TransformerDecoderLayer`, including multihead self
-    attention, cross attention and feedforward layer.
+    Transformer Decoder module with multi-layer stacked of :class:`mindspore.nn.TransformerDecoderLayer`,
+    including multihead self attention, cross attention and feedforward layer.
 
     Args:
         decoder_layer (Cell): An instance of the :class:`mindspore.nn.TransformerDecoderLayer` class.
@@ -620,20 +620,22 @@ class TransformerDecoder(Cell):
         norm (Cell, optional): The layer normalization module. Default: ``None``.
 
     Inputs:
-        - **tgt** (Tensor): The sequence to the decoder. For unbatched input, the shape is
-          :math:`(T, E)` ; otherwise if `batch_first=False` in TransformerDecoderLayer, the shape is
-          :math:`(T, N, E)` and if `batch_first=True` , the shape is :math:`(T, N, E)`, where :math:`(T)` is the
-          target sequence length. Supported types: float16, float32, float64.
-        - **memory** (Tensor): The sequence from the last layer of the encoder. Supported types: float16,
+        - **tgt** (Tensor) - The sequence to the decoder. For unbatched input, the shape is
+          :math:`(T, E)` ; otherwise if `batch_first=False` in :class:`mindspore.nn.TransformerDecoderLayer`,
+          the shape is :math:`(T, N, E)` and if `batch_first=True` , the shape is :math:`(N, T, E)`,
+          where :math:`(T)` is the target sequence length, :math:`(N)` is the number of batches,
+          and :math:`(E)` is the number of features. Supported types: float16, float32, float64.
+        - **memory** (Tensor) - The sequence from the last layer of the encoder. Supported types: float16,
           float32, float64.
-        - **tgt_mask** (Tensor, optional): the mask of the tgt sequence. The shape is :math:`(T, T)`
-          or :math:`(N*nhead, T, T)` , where `nhead` is the arguent in TransformerDecoderLayer.
+        - **tgt_mask** (Tensor, optional) - the mask of the tgt sequence. The shape is :math:`(T, T)`
+          or :math:`(N*nhead, T, T)` , where `nhead` is the arguent in :class:`mindspore.nn.TransformerDecoderLayer`.
           Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_mask** (Tensor, optional): the mask of the memory sequence. The shape is
+        - **memory_mask** (Tensor, optional) - the mask of the memory sequence. The shape is
           :math:`(T, S)` . Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **tgt_key_padding_mask** (Tensor, optional): the mask of the tgt keys per batch. Supported
+        - **tgt_key_padding_mask** (Tensor, optional) - the mask of the tgt keys per batch.
+          Shape is :math:`(T)`. Supported
           types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_key_padding_mask** (Tensor, optional): the mask of the memory keys per batch. The shape
+        - **memory_key_padding_mask** (Tensor, optional) - the mask of the memory keys per batch. The shape
           is :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
 
@@ -661,7 +663,7 @@ class TransformerDecoder(Cell):
                                          decoder_layer.dropout_num, decoder_layer.activation1,
                                          decoder_layer.layernorm_eps, decoder_layer.batch_first,
                                          decoder_layer.norm_first, dtype=decoder_layer.dtype)
-        self.layers = CellList([layers for _ in range(num_layers)])
+        self.layers = CellList([copy.deepcopy(layers) for _ in range(num_layers)])
         self.num_layers = num_layers
         self.norm = norm
 
@@ -710,28 +712,28 @@ class Transformer(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **src** (Tensor): The source sequence to the encoder. For unbatched input, the shape is
+        - **src** (Tensor) - The source sequence to the encoder. For unbatched input, the shape is
           :math:`(S, E)` ; otherwise if `batch_first=False` , the shape is :math:`(S, N, E)` and if
-          `batch_first=True` , the shape is :math:`(S, N, E)`, where :math:`(S)` is the source sequence
+          `batch_first=True` , the shape is :math:`(N, S, E)`, where :math:`(S)` is the source sequence
           length, :math:`(N)` is the batch number and :math:`(E)` is the feature number. Supported
           types: float16, float32, float64.
-        - **tgt** (Tensor): The target sequence to the decoder. For unbatched input, the shape is
+        - **tgt** (Tensor) - The target sequence to the decoder. For unbatched input, the shape is
           :math:`(T, E)` ; otherwise if `batch_first=False` , the shape is :math:`(T, N, E)` and if
-          `batch_first=True` , the shape is :math:`(T, N, E)`, where :math:`(T)` is the target sequence
+          `batch_first=True` , the shape is :math:`(N, T, E)`, where :math:`(T)` is the target sequence
           length. Supported types: float16, float32, float64.
-        - **src_mask** (Tensor, optional): The mask of the src sequence. The shape is :math:`(S, S)`
+        - **src_mask** (Tensor, optional) - The mask of the src sequence. The shape is :math:`(S, S)`
           or :math:`(N*nhead, S, S)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **tgt_mask** (Tensor, optional): The mask of the tgt sequence. The shape is :math:`(T, T)`
+        - **tgt_mask** (Tensor, optional) - The mask of the tgt sequence. The shape is :math:`(T, T)`
           or :math:`(N*nhead, T, T)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_mask** (Tensor, optional): The additive mask of the encoder output. The shape is
+        - **memory_mask** (Tensor, optional) - The additive mask of the encoder output. The shape is
           :math:`(T, S)` . Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **src_key_padding_mask** (Tensor, optional): The mask of src keys per batch. The shape is
+        - **src_key_padding_mask** (Tensor, optional) - The mask of src keys per batch. The shape is
           :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
-        - **tgt_key_padding_mask** (Tensor, optional): The mask of tgt keys per batch. The shape is
+        - **tgt_key_padding_mask** (Tensor, optional) - The mask of tgt keys per batch. The shape is
           :math:`(T)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
-        - **memory_key_padding_mask** (Tensor, optional): The mask of memory keys per batch. The shape
+        - **memory_key_padding_mask** (Tensor, optional) - The mask of memory keys per batch. The shape
           is :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16,
           float32, float64, bool. Default: ``None``.
 

mindspore/nn/learning_rate_schedule.py

@@ -83,7 +83,7 @@ class ExponentialDecayLR(LearningRateSchedule):
         is_stair (bool): If true, learning rate is decayed once every `decay_steps` time. Default: ``False`` .
 
     Inputs:
-        - **global_step** (Tensor) - The current step number.
+        - **global_step** (Tensor) - The current step number. :math:`current\_step` in the above formula.
 
     Outputs:
         Tensor. The learning rate value for the current step with shape :math:`()`.
@@ -214,7 +214,7 @@ class InverseDecayLR(LearningRateSchedule):
     .. math::
         p = \frac{current\_step}{decay\_steps}
 
-    If `is_stair` is True, The formula is :
+    If `is_stair` is True, the formula is :
 
     .. math::
         p = floor(\frac{current\_step}{decay\_steps})
@@ -332,12 +332,13 @@ class CosineDecayLR(LearningRateSchedule):
         self.math_pi = math.pi
         self.delta = 0.5 * (max_lr - min_lr)
         self.cos = P.Cos()
+        self.sin = P.Sin()
         self.min = P.Minimum()
         self.cast = P.Cast()
 
     def construct(self, global_step):
         p = self.cast(self.min(global_step, self.decay_steps), mstype.float32)
-        return self.min_lr + self.delta * (1.0 + self.cos(self.math_pi * p / self.decay_steps))
+        return self.min_lr + self.delta * (1.0 + self.sin(self.math_pi * (p / self.decay_steps + 0.5)))
 
 
 class PolynomialDecayLR(LearningRateSchedule):
@@ -371,7 +372,7 @@ class PolynomialDecayLR(LearningRateSchedule):
             Default: ``False`` .
 
     Inputs:
-        - **global_step** (Tensor) - The current step number.
+        - **global_step** (Tensor) - The current step number. Shape is :math:`()`.
 
     Outputs:
         Tensor. The learning rate value for the current step with shape :math:`()`.
@@ -457,7 +458,7 @@ class WarmUpLR(LearningRateSchedule):
         warmup_steps (int): The warm up steps of learning rate.
 
     Inputs:
-        - **global_step** (Tensor) - The current step number.
+        - **global_step** (Tensor) - The current step number. Shape is :math:`()`.
 
     Outputs:
         Tensor. The learning rate value for the current step with shape :math:`()`.

mindspore/nn/loss/__init__.py

@@ -25,7 +25,7 @@ from mindspore.nn.loss.loss import LossBase, L1Loss, CTCLoss, MSELoss, SmoothL1L
     SampledSoftmaxLoss, TripletMarginWithDistanceLoss,\
     PoissonNLLLoss, MultiLabelSoftMarginLoss, DiceLoss, BCEWithLogitsLoss, MultiClassDiceLoss, \
     RMSELoss, MAELoss, HuberLoss, CrossEntropyLoss, NLLLoss, KLDivLoss, MarginRankingLoss, GaussianNLLLoss, \
-    HingeEmbeddingLoss, MultilabelMarginLoss, TripletMarginLoss
+    HingeEmbeddingLoss, MultilabelMarginLoss, TripletMarginLoss, L1LossExt
 
 
 __all__ = ['LossBase', 'L1Loss', 'CTCLoss', 'MSELoss', 'SmoothL1Loss', 'SoftMarginLoss', 'FocalLoss',
@@ -33,4 +33,4 @@ __all__ = ['LossBase', 'L1Loss', 'CTCLoss', 'MSELoss', 'SmoothL1Loss', 'SoftMarg
            'CosineEmbeddingLoss', 'SampledSoftmaxLoss', 'TripletMarginWithDistanceLoss', 'PoissonNLLLoss',
            'MultiLabelSoftMarginLoss', 'DiceLoss', 'MultiClassDiceLoss', 'MultilabelMarginLoss',
            'RMSELoss', 'MAELoss', 'HuberLoss', 'CrossEntropyLoss', 'NLLLoss', 'KLDivLoss', 'MarginRankingLoss',
-           'GaussianNLLLoss', 'HingeEmbeddingLoss', 'TripletMarginLoss']
+           'GaussianNLLLoss', 'HingeEmbeddingLoss', 'TripletMarginLoss', 'L1LossExt']