mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/nn/layer/rnns.py

@@ -588,13 +588,14 @@ class RNN(_RNNBase):
     Here :math:`h_t` is the hidden state at time `t`, :math:`x_t` is
     the input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the
     previous layer at time :math:`t-1` or the initial hidden state at time `0`.
-    If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used instead of :math:`\tanh`.
+    :math:`W_{ih}` is the learnable input-hidden weights, and :math:`b_{ih}` is the learnable input-hidden bias.
+    :math:`W_{hh}` is the learnable hidden-hidden weights, and :math:`b_{hh}` is the learnable hidden-hidden bias.
 
     Args:
         input_size (int): Number of features of input.
         hidden_size (int):  Number of features of hidden layer.
         num_layers (int): Number of layers of stacked RNN. Default: ``1`` .
-        nonlinearity (str): The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'``
+        nonlinearity (str): The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'``.
         has_bias (bool): Whether the cell has bias :math:`b_{ih}` and :math:`b_{hh}`. Default: ``True`` .
         batch_first (bool): Specifies whether the first dimension of input `x` is batch_size. Default: ``False`` .
         dropout (float): If not 0.0, append `Dropout` layer on the outputs of each
@@ -696,7 +697,7 @@ class GRU(_RNNBase):
         input_size (int): Number of features of input.
         hidden_size (int):  Number of features of hidden layer.
         num_layers (int): Number of layers of stacked GRU. Default: ``1`` .
-        has_bias (bool): Whether the cell has bias `b_in` and `b_hn`. Default: ``True`` .
+        has_bias (bool): Whether the cell has bias :math:`b_{in}` and :math:`b_{hn}`. Default: ``True`` .
         batch_first (bool): Specifies whether the first dimension of input `x` is batch_size. Default: ``False`` .
         dropout (float): If not 0.0, append `Dropout` layer on the outputs of each
             GRU layer except the last layer. Default ``0.0`` . The range of dropout is [0.0, 1.0).
@@ -706,7 +707,7 @@ class GRU(_RNNBase):
 
     Inputs:
         - **x** (Tensor) - Tensor of data type mindspore.float32 or mindspore.float16 and
-          shape (seq\_len, batch\_size, `input\_size`) or :math:`(batch\_size, seq\_len, input\_size)`.
+          shape :math:`(seq\_len, batch\_size, input\_size)` or :math:`(batch\_size, seq\_len, input\_size)`.
         - **hx** (Tensor) - Tensor of data type mindspore.float32 or mindspore.float16 and
           shape :math:`(num\_directions * num\_layers, batch\_size, hidden\_size)`.
         - **seq_length** (Tensor) - The length of each sequence in an input batch.
@@ -796,7 +797,7 @@ class LSTM(_RNNBase):
         input_size (int): Number of features of input.
         hidden_size (int):  Number of features of hidden layer.
         num_layers (int): Number of layers of stacked LSTM . Default: ``1`` .
-        has_bias (bool): Whether the cell has bias :math:`b_{ih}` and :math:`b_{hh}`. Default: ``True`` .
+        has_bias (bool): Whether the cell has bias :math:`b_{ih}` and :math:`b_{fh}`. Default: ``True`` .
         batch_first (bool): Specifies whether the first dimension of input `x` is batch_size. Default: ``False`` .
         dropout (float, int): If not 0, append `Dropout` layer on the outputs of each
             LSTM layer except the last layer. Default ``0`` . The range of dropout is [0.0, 1.0).

mindspore/nn/layer/thor_layer.py

@@ -449,7 +449,7 @@ class Conv2dThor(_ConvThor):
             weight_shape = [1, self.in_channels, *self.kernel_size]
             self.weight_init = weight_init
             if isinstance(weight_init, Tensor):
-                self.weight_init = Tensor(weight_init.asnumpy().swapaxes(0, 1), weight_init.dtype)
+                self.weight_init = weight_init.swapaxes(0, 1)
             if isinstance(weight_init, Initializer):
                 self.weight_init.shape = weight_shape
             self.weight = Parameter(initializer(self.weight_init, weight_shape), name='weight')
@@ -596,7 +596,6 @@ class EmbeddingThor(Cell):
         if padding_idx is not None:
             self.padding_idx = Validator.check_int_range(padding_idx, 0, vocab_size, Validator.INC_BOTH,
                                                          "padding_idx", self.cls_name)
-            self.init_tensor = self.init_tensor.init_data().asnumpy()
             self.init_tensor[self.padding_idx] = 0
         self.embedding_table = Parameter(self.init_tensor, name='embedding_table')
         self.expand = P.ExpandDims()

mindspore/nn/layer/timedistributed.py

@@ -85,7 +85,7 @@ class TimeDistributed(Cell):
           where :math:`*` means any number of additional dimensions.
 
     Outputs:
-        Tensor of shape :math:`(N, T, *)`
+        Tensor of shape :math:`(N, T, *)`.
 
     Raises:
         TypeError: If layer is not a Cell or Primitive.

mindspore/nn/layer/transformer.py

@@ -66,35 +66,35 @@ class MultiheadAttention(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **query** (Tensor): The query embeddings. If `query` is unbatched, the shape is :math:`(L, E_q)`,
+        - **query** (Tensor) - The query embeddings. If `query` is unbatched, the shape is :math:`(L, E_q)`,
           otherwise the shape is :math:`(L, N, E_q)` when `batch_first=False` or :math:`(N, L, E_q)` when
           `batch_first=True` , where :math:`L`is the target sequence length, :math:`N` is the batch size,
           and :math:`E_q` is the query embedding dimension `embed_dim`. Supported types: float16, float32,
           float64. Queries are compared against key-value pairs to produce the output.
-        - **key** (Tensor): The key embeddings. If `key` is unbatched, the shape is :math:`(S, E_k)`, otherwise
+        - **key** (Tensor) - The key embeddings. If `key` is unbatched, the shape is :math:`(S, E_k)`, otherwise
           the shape is :math:`(S, N, E_k)` when `batch_first=False` or :math:`(N, S, E_k)` when
           `batch_first=True` , where :math:`S` is the source sequence length, :math:`N` is the batch size,
           and :math:`E_k` is the key embedding dimension `kdim`. Supported types: float16, float32, float64.
-        - **value** (Tensor): The value embeddings. If `value` is unbatched, the shape is :math:`(S, E_v)`,
+        - **value** (Tensor) - The value embeddings. If `value` is unbatched, the shape is :math:`(S, E_v)`,
           otherwise the shape is :math:`(S, N, E_v)` when `batch_first=False` or :math:`(N, S, E_v)` when
           `batch_first=True` , where :math:`S` is the source sequence length, :math:`N` is the batch size,
           and :math:`E_v` is the value embedding dimension `vdim`. Supported types: float16, float32, float64.
-        - **key_padding_mask** (Tensor, optional): If specified, a mask of shape :math:`(N, S)` indicating which
+        - **key_padding_mask** (Tensor, optional) - If specified, a mask of shape :math:`(N, S)` indicating which
           elements within `key` to ignore for the purpose of attention (i.e. treat as "padding").
           For unbatched `query`, shape should be :math:`(S)`. Binary and float masks are supported.
           For a binary mask, a ``True`` value indicates that the corresponding `key` value will be ignored for
           the purpose of attention. For a float mask, it will be directly added to the corresponding `key` value.
           Supported float types: float16, float32, float64. Default: ``None``.
-        - **need_weights** (bool): Whether returns `attn_output_weights` in addition to `attn_outputs`.
+        - **need_weights** (bool) - Whether returns `attn_output_weights` in addition to `attn_outputs`.
           Default: ``True``.
-        - **attn_mask** (Tensor, optional): If specified, a 2D or 3D mask preventing attention to certain positions.
+        - **attn_mask** (Tensor, optional) - If specified, a 2D or 3D mask preventing attention to certain positions.
           Must be of shape :math:`(L, S)` or :math:`(N\cdot\text{num_heads}, L, S)`, where :math:`N` is the
           batch size, :math:`L` is the target sequence length, and :math:`S` is the source sequence length.
           A 2D mask will be broadcasted across the batch while a 3D mask allows for a different mask for each entry
           in the batch. For a binary mask, a ``True`` value indicates that the corresponding position is not allowed
           to attend. For a float mask, the mask values will be added to the attention weight.
           Supported float types: float16, float32, float64. Default: ``None``.
-        - **average_attn_weights** (bool): If true, indicates that the returned `attn_weights` should be averaged
+        - **average_attn_weights** (bool) - If true, indicates that the returned `attn_weights` should be averaged
           across heads. Otherwise, `attn_weights` are provided separately per head. Note that this flag only
           has an effect when `need_weights=True`. Default: ``True`` (i.e. average weights across heads)
 
@@ -265,14 +265,14 @@ class TransformerEncoderLayer(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **src** (Tensor): the sequence to the encoder layer. For unbatched input, the shape is
+        - **src** (Tensor) - the sequence to the encoder layer. For unbatched input, the shape is
           :math:`(S, E)` ; otherwise if `batch_first=False` , the shape is :math:`(S, N, E)` and if
-          `batch_first=True` , the shape is :math:`(S, N, E)`, where :math:`(S)` is the source sequence
+          `batch_first=True` , the shape is :math:`(N, S, E)`, where :math:`(S)` is the source sequence
           length, :math:`(N)` is the batch number and :math:`(E)` is the feature number.
           Supported types: float16, float32, float64.
-        - **src_mask** (Tensor, optional): the mask for the src sequence. The shape is :math:`(S, S)`
+        - **src_mask** (Tensor, optional) - the mask for the src sequence. The shape is :math:`(S, S)`
           or :math:`(N*nhead, S, S)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **src_key_padding_mask** (Tensor, optional): the mask for the src keys per batch. The shape is
+        - **src_key_padding_mask** (Tensor, optional) - the mask for the src keys per batch. The shape is
           :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
 
@@ -406,20 +406,20 @@ class TransformerDecoderLayer(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **tgt** (Tensor): The sequence to the decoder layer. For unbatched input, the shape is
+        - **tgt** (Tensor) - The sequence to the decoder layer. For unbatched input, the shape is
           :math:`(T, E)` ; otherwise if `batch_first=False` , the shape is :math:`(T, N, E)` and if
-          `batch_first=True` , the shape is :math:`(T, N, E)`, where :math:`(T)` is the target sequence
+          `batch_first=True` , the shape is :math:`(N, T, E)`, where :math:`(T)` is the target sequence
           length. Supported types: float16, float32, float64.
-        - **memory** (Tensor): The sequence from the last layer of the encoder. Supported types: float16,
+        - **memory** (Tensor) - The sequence from the last layer of the encoder. Supported types: float16,
           float32, float64.
-        - **tgt_mask** (Tensor, optional): The mask of the tgt sequence. The shape is :math:`(T, T)`
+        - **tgt_mask** (Tensor, optional) - The mask of the tgt sequence. The shape is :math:`(T, T)`
           or :math:`(N*nhead, T, T)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_mask** (Tensor, optional): The mask of the memory sequence. The shape is
+        - **memory_mask** (Tensor, optional) - The mask of the memory sequence. The shape is
           :math:`(T, S)` . Supported types: float16, float32, float64, bool. Default: ``None``.
         - **tgt_key_padding_mask** (Tensor, optional): The mask of the tgt keys per batch. The shape is
-          :math:`(T)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
+          :math:`(T)` for unbatched input, otherwise :math:`(N, T)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
-        - **memory_key_padding_mask** (Tensor, optional): The mask of the memory keys per batch. The shape
+        - **memory_key_padding_mask** (Tensor, optional) - The mask of the memory keys per batch. The shape
           is :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
 
@@ -540,8 +540,8 @@ class TransformerDecoderLayer(Cell):
 
 class TransformerEncoder(Cell):
     r"""
-    Transformer Encoder module with multi-layer stacked of `TransformerEncoderLayer`, including multihead
-    attention and feedforward layer. Users can build the
+    Transformer Encoder module with multi-layer stacked of :class:`mindspore.nn.TransformerEncoderLayer`,
+    including multihead attention and feedforward layer. Users can build the
     BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.
 
     Args:
@@ -550,15 +550,15 @@ class TransformerEncoder(Cell):
         norm (Cell, optional): The layer normalization module. Default: ``None``.
 
     Inputs:
-        - **src** (Tensor): The sequence to the encoder. For unbatched input, the shape is
-          :math:`(S, E)` ; otherwise if `batch_first=False` in TransformerEncoderLayer, the shape is
-          :math:`(S, N, E)` and if `batch_first=True` , the shape is :math:`(S, N, E)`, where :math:`(S)` is the
-          source sequence length, :math:`(N)` is the batch number and :math:`(E)` is the feature number.
-          Supported types: float16, float32, float64.
-        - **src_mask** (Tensor, optional): The mask of the src sequence. The shape is :math:`(S, S)`
-          or :math:`(N*nhead, S, S)` , where `nhead` is the arguent in TransformerDecoderLayer.
+        - **src** (Tensor) - The sequence to the encoder. For unbatched input, the shape is
+          :math:`(S, E)` ; otherwise if `batch_first=False` in :class:`mindspore.nn.TransformerEncoderLayer`,
+          the shape is :math:`(S, N, E)` and if `batch_first=True` , the shape is :math:`(N, S, E)`,
+          where :math:`(S)` is the source sequence length, :math:`(N)` is the batch number and :math:`(E)` is
+          the feature number. Supported types: float16, float32, float64.
+        - **src_mask** (Tensor, optional) - The mask of the src sequence. The shape is :math:`(S, S)`
+          or :math:`(N*nhead, S, S)` , where `nhead` is the arguent in :class:`mindspore.nn.TransformerEncoderLayer`.
           Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **src_key_padding_mask** (Tensor, optional): the mask of the src keys per batch. The shape is
+        - **src_key_padding_mask** (Tensor, optional) - the mask of the src keys per batch. The shape is
           :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool.  Default: ``None``.
 
@@ -611,8 +611,8 @@ class TransformerEncoder(Cell):
 
 class TransformerDecoder(Cell):
     r"""
-    Transformer Decoder module with multi-layer stacked of `TransformerDecoderLayer`, including multihead self
-    attention, cross attention and feedforward layer.
+    Transformer Decoder module with multi-layer stacked of :class:`mindspore.nn.TransformerDecoderLayer`,
+    including multihead self attention, cross attention and feedforward layer.
 
     Args:
         decoder_layer (Cell): An instance of the :class:`mindspore.nn.TransformerDecoderLayer` class.
@@ -620,20 +620,22 @@ class TransformerDecoder(Cell):
         norm (Cell, optional): The layer normalization module. Default: ``None``.
 
     Inputs:
-        - **tgt** (Tensor): The sequence to the decoder. For unbatched input, the shape is
-          :math:`(T, E)` ; otherwise if `batch_first=False` in TransformerDecoderLayer, the shape is
-          :math:`(T, N, E)` and if `batch_first=True` , the shape is :math:`(T, N, E)`, where :math:`(T)` is the
-          target sequence length. Supported types: float16, float32, float64.
-        - **memory** (Tensor): The sequence from the last layer of the encoder. Supported types: float16,
+        - **tgt** (Tensor) - The sequence to the decoder. For unbatched input, the shape is
+          :math:`(T, E)` ; otherwise if `batch_first=False` in :class:`mindspore.nn.TransformerDecoderLayer`,
+          the shape is :math:`(T, N, E)` and if `batch_first=True` , the shape is :math:`(N, T, E)`,
+          where :math:`(T)` is the target sequence length, :math:`(N)` is the number of batches,
+          and :math:`(E)` is the number of features. Supported types: float16, float32, float64.
+        - **memory** (Tensor) - The sequence from the last layer of the encoder. Supported types: float16,
           float32, float64.
-        - **tgt_mask** (Tensor, optional): the mask of the tgt sequence. The shape is :math:`(T, T)`
-          or :math:`(N*nhead, T, T)` , where `nhead` is the arguent in TransformerDecoderLayer.
+        - **tgt_mask** (Tensor, optional) - the mask of the tgt sequence. The shape is :math:`(T, T)`
+          or :math:`(N*nhead, T, T)` , where `nhead` is the arguent in :class:`mindspore.nn.TransformerDecoderLayer`.
           Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_mask** (Tensor, optional): the mask of the memory sequence. The shape is
+        - **memory_mask** (Tensor, optional) - the mask of the memory sequence. The shape is
           :math:`(T, S)` . Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **tgt_key_padding_mask** (Tensor, optional): the mask of the tgt keys per batch. Supported
+        - **tgt_key_padding_mask** (Tensor, optional) - the mask of the tgt keys per batch.
+          Shape is :math:`(T)`. Supported
           types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_key_padding_mask** (Tensor, optional): the mask of the memory keys per batch. The shape
+        - **memory_key_padding_mask** (Tensor, optional) - the mask of the memory keys per batch. The shape
           is :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
 
@@ -710,28 +712,28 @@ class Transformer(Cell):
         dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
-        - **src** (Tensor): The source sequence to the encoder. For unbatched input, the shape is
+        - **src** (Tensor) - The source sequence to the encoder. For unbatched input, the shape is
           :math:`(S, E)` ; otherwise if `batch_first=False` , the shape is :math:`(S, N, E)` and if
-          `batch_first=True` , the shape is :math:`(S, N, E)`, where :math:`(S)` is the source sequence
+          `batch_first=True` , the shape is :math:`(N, S, E)`, where :math:`(S)` is the source sequence
           length, :math:`(N)` is the batch number and :math:`(E)` is the feature number. Supported
           types: float16, float32, float64.
-        - **tgt** (Tensor): The target sequence to the decoder. For unbatched input, the shape is
+        - **tgt** (Tensor) - The target sequence to the decoder. For unbatched input, the shape is
           :math:`(T, E)` ; otherwise if `batch_first=False` , the shape is :math:`(T, N, E)` and if
-          `batch_first=True` , the shape is :math:`(T, N, E)`, where :math:`(T)` is the target sequence
+          `batch_first=True` , the shape is :math:`(N, T, E)`, where :math:`(T)` is the target sequence
           length. Supported types: float16, float32, float64.
-        - **src_mask** (Tensor, optional): The mask of the src sequence. The shape is :math:`(S, S)`
+        - **src_mask** (Tensor, optional) - The mask of the src sequence. The shape is :math:`(S, S)`
           or :math:`(N*nhead, S, S)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **tgt_mask** (Tensor, optional): The mask of the tgt sequence. The shape is :math:`(T, T)`
+        - **tgt_mask** (Tensor, optional) - The mask of the tgt sequence. The shape is :math:`(T, T)`
           or :math:`(N*nhead, T, T)`. Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **memory_mask** (Tensor, optional): The additive mask of the encoder output. The shape is
+        - **memory_mask** (Tensor, optional) - The additive mask of the encoder output. The shape is
           :math:`(T, S)` . Supported types: float16, float32, float64, bool. Default: ``None``.
-        - **src_key_padding_mask** (Tensor, optional): The mask of src keys per batch. The shape is
+        - **src_key_padding_mask** (Tensor, optional) - The mask of src keys per batch. The shape is
           :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
-        - **tgt_key_padding_mask** (Tensor, optional): The mask of tgt keys per batch. The shape is
+        - **tgt_key_padding_mask** (Tensor, optional) - The mask of tgt keys per batch. The shape is
           :math:`(T)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16, float32,
           float64, bool. Default: ``None``.
-        - **memory_key_padding_mask** (Tensor, optional): The mask of memory keys per batch. The shape
+        - **memory_key_padding_mask** (Tensor, optional) - The mask of memory keys per batch. The shape
           is :math:`(S)` for unbatched input, otherwise :math:`(N, S)` . Supported types: float16,
           float32, float64, bool. Default: ``None``.
 

mindspore/nn/learning_rate_schedule.py

@@ -83,7 +83,7 @@ class ExponentialDecayLR(LearningRateSchedule):
         is_stair (bool): If true, learning rate is decayed once every `decay_steps` time. Default: ``False`` .
 
     Inputs:
-        - **global_step** (Tensor) - The current step number.
+        - **global_step** (Tensor) - The current step number. :math:`current\_step` in the above formula.
 
     Outputs:
         Tensor. The learning rate value for the current step with shape :math:`()`.
@@ -214,7 +214,7 @@ class InverseDecayLR(LearningRateSchedule):
     .. math::
         p = \frac{current\_step}{decay\_steps}
 
-    If `is_stair` is True, The formula is :
+    If `is_stair` is True, the formula is :
 
     .. math::
         p = floor(\frac{current\_step}{decay\_steps})
@@ -332,12 +332,13 @@ class CosineDecayLR(LearningRateSchedule):
         self.math_pi = math.pi
         self.delta = 0.5 * (max_lr - min_lr)
         self.cos = P.Cos()
+        self.sin = P.Sin()
         self.min = P.Minimum()
         self.cast = P.Cast()
 
     def construct(self, global_step):
         p = self.cast(self.min(global_step, self.decay_steps), mstype.float32)
-        return self.min_lr + self.delta * (1.0 + self.cos(self.math_pi * p / self.decay_steps))
+        return self.min_lr + self.delta * (1.0 + self.sin(self.math_pi * (p / self.decay_steps + 0.5)))
 
 
 class PolynomialDecayLR(LearningRateSchedule):
@@ -371,7 +372,7 @@ class PolynomialDecayLR(LearningRateSchedule):
             Default: ``False`` .
 
     Inputs:
-        - **global_step** (Tensor) - The current step number.
+        - **global_step** (Tensor) - The current step number. Shape is :math:`()`.
 
     Outputs:
         Tensor. The learning rate value for the current step with shape :math:`()`.
@@ -457,7 +458,7 @@ class WarmUpLR(LearningRateSchedule):
         warmup_steps (int): The warm up steps of learning rate.
 
     Inputs:
-        - **global_step** (Tensor) - The current step number.
+        - **global_step** (Tensor) - The current step number. Shape is :math:`()`.
 
     Outputs:
         Tensor. The learning rate value for the current step with shape :math:`()`.