mindspore 2.6.0__cp310-cp310-win_amd64.whl → 2.7.0__cp310-cp310-win_amd64.whl

mindspore/nn/layer/rnns.py

@@ -19,10 +19,9 @@ import math
 import numpy as np
 
 import mindspore.nn as nn
-import mindspore.ops as P
+import mindspore.ops as ops
 import mindspore.context as context
 import mindspore.common.dtype as mstype
-from mindspore.ops import functional as F
 from mindspore.ops.primitive import constexpr, _primexpr
 from mindspore.common.tensor import Tensor
 from mindspore.common.parameter import ParameterTuple, Parameter
@@ -36,8 +35,8 @@ __all__ = ['LSTM', 'GRU', 'RNN']
 
 
 def _init_state(shape, dtype, is_lstm):
-    hx = P.zeros(shape, dtype)
-    cx = P.zeros(shape, dtype)
+    hx = ops.zeros(shape, dtype)
+    cx = ops.zeros(shape, dtype)
     if is_lstm:
         return (hx, cx)
     return hx
@@ -57,17 +56,17 @@ def _check_input_dtype_same_and_valid(args_name, args_value, valid_values, cls_n
 @constexpr(check=False)
 def _check_is_tensor(param_name, input_data, cls_name):
     """Internal function, used to check whether the input data is Tensor."""
-    if input_data is not None and not isinstance(P.typeof(input_data), mstype.TensorType):
+    if input_data is not None and not isinstance(ops.typeof(input_data), mstype.TensorType):
         raise TypeError(f"For '{cls_name}', the '{param_name}' must be '{mstype.TensorType}', "
-                        f"but got '{P.typeof(input_data)}'")
+                        f"but got '{ops.typeof(input_data)}'")
 
 
 @constexpr
 def _check_is_tuple(param_name, input_data, cls_name):
     """Internal function, used to check whether the input data is Tensor."""
-    if input_data is not None and not isinstance(P.typeof(input_data), mstype.Tuple):
+    if input_data is not None and not isinstance(ops.typeof(input_data), mstype.Tuple):
         raise TypeError(f"For '{cls_name}', the '{param_name}' must be '{mstype.Tuple}', "
-                        f"but got '{P.typeof(input_data)}'")
+                        f"but got '{ops.typeof(input_data)}'")
 
 
 @constexpr
@@ -87,7 +86,7 @@ def _check_seq_length_size(batch_size_x, seq_length_size, cls_name):
 
 def sequence_mask(lengths, maxlen):
     """generate mask matrix by seq_length"""
-    range_vector = P.arange(start=0, end=maxlen, step=1, dtype=lengths.dtype)
+    range_vector = ops.arange(start=0, end=maxlen, step=1, dtype=lengths.dtype)
     result = range_vector < lengths.view(lengths.shape + (1,))
     return result.astype(mstype.int32)
 
@@ -100,9 +99,9 @@ def select_by_mask(inputs, mask):
 
 def get_hidden(output, seq_length):
     """get hidden state by seq_length"""
-    batch_index = P.arange(start=0, end=seq_length.shape[0], step=1, dtype=seq_length.dtype)
-    indices = P.Concat(1)((seq_length.view(-1, 1) - 1, batch_index.view(-1, 1)))
-    return P.GatherNd()(output, indices)
+    batch_index = ops.arange(start=0, end=seq_length.shape[0], step=1, dtype=seq_length.dtype)
+    indices = ops.Concat(1)((seq_length.view(-1, 1) - 1, batch_index.view(-1, 1)))
+    return ops.GatherNd()(output, indices)
 
 
 class _DynamicRNNBase(Cell):
@@ -131,14 +130,14 @@ class _DynamicRNNBase(Cell):
         h = h_0
         while t < time_step:
             x_t = x[t:t + 1:1]
-            x_t = P.Squeeze(0)(x_t)
+            x_t = ops.Squeeze(0)(x_t)
             h = self.cell(x_t, h, w_ih, w_hh, b_ih, b_hh)
             if self.is_lstm:
                 outputs.append(h[0])
             else:
                 outputs.append(h)
             t += 1
-        outputs = P.Stack()(outputs)
+        outputs = ops.Stack()(outputs)
         return outputs, h
 
     def variable_recurrent(self, x, h, seq_length, w_ih, w_hh, b_ih, b_hh):
@@ -147,34 +146,34 @@ class _DynamicRNNBase(Cell):
         h_t = h
         if self.is_lstm:
             hidden_size = h[0].shape[-1]
-            zero_output = P.ZerosLike()(h_t[0])
+            zero_output = ops.ZerosLike()(h_t[0])
         else:
             hidden_size = h.shape[-1]
-            zero_output = P.ZerosLike()(h_t)
-        seq_length = P.Cast()(seq_length, mstype.float32)
-        seq_length = F.broadcast_to(seq_length, (hidden_size, -1))
-        seq_length = P.Cast()(seq_length, mstype.int32)
-        seq_length = P.Transpose()(seq_length, (1, 0))
+            zero_output = ops.ZerosLike()(h_t)
+        seq_length = ops.Cast()(seq_length, mstype.float32)
+        seq_length = ops.broadcast_to(seq_length, (hidden_size, -1))
+        seq_length = ops.Cast()(seq_length, mstype.int32)
+        seq_length = ops.Transpose()(seq_length, (1, 0))
 
         outputs = []
         state_t = h_t
         t = 0
         while t < time_step:
             x_t = x[t:t + 1:1]
-            x_t = P.Squeeze(0)(x_t)
+            x_t = ops.Squeeze(0)(x_t)
             h_t = self.cell(x_t, state_t, w_ih, w_hh, b_ih, b_hh)
             seq_cond = seq_length > t
             if self.is_lstm:
-                state_t_0 = P.Select()(seq_cond, h_t[0], state_t[0])
-                state_t_1 = P.Select()(seq_cond, h_t[1], state_t[1])
-                output = P.Select()(seq_cond, h_t[0], zero_output)
+                state_t_0 = ops.Select()(seq_cond, h_t[0], state_t[0])
+                state_t_1 = ops.Select()(seq_cond, h_t[1], state_t[1])
+                output = ops.Select()(seq_cond, h_t[0], zero_output)
                 state_t = (state_t_0, state_t_1)
             else:
-                state_t = P.Select()(seq_cond, h_t, state_t)
-                output = P.Select()(seq_cond, h_t, zero_output)
+                state_t = ops.Select()(seq_cond, h_t, state_t)
+                output = ops.Select()(seq_cond, h_t, zero_output)
             outputs.append(output)
             t += 1
-        outputs = P.Stack()(outputs)
+        outputs = ops.Stack()(outputs)
         return outputs, state_t
 
     def construct(self, x, h, seq_length, w_ih, w_hh, b_ih, b_hh):
@@ -210,7 +209,7 @@ class _DynamicGRUCPUGPU(Cell):
 
     def __init__(self):
         super().__init__()
-        self.concat = P.Concat()
+        self.concat = ops.Concat()
         self.is_gpu = context.get_context("device_target") == "GPU"
 
     def construct(self, x, h_0, seq_length, w_ih, w_hh, b_ih, b_hh):
@@ -253,15 +252,16 @@ class _DynamicGRUAscend(Cell):
 
     def __init__(self):
         super().__init__()
-        self.gru = P.DynamicGRUV2(gate_order='rzh')
-        self.transpose = P.Transpose()
+        self.gru = ops.DynamicGRUV2(gate_order='rzh')
+        self.transpose = ops.Transpose()
         self.dtype = mstype.float16
+        self.cast = ops.Cast()
 
     def construct(self, x, h_0, seq_length, w_ih, w_hh, b_ih, b_hh):
         '''Dynamic GRU module on Ascend'''
         if b_ih is None:
-            b_ih = P.zeros(w_ih.shape[0], w_ih.dtype)
-            b_hh = P.zeros(w_ih.shape[0], w_ih.dtype)
+            b_ih = ops.zeros(w_ih.shape[0], w_ih.dtype)
+            b_hh = ops.zeros(w_ih.shape[0], w_ih.dtype)
         outputs, _, _, _, _, _ = self.gru(self.cast(x, self.dtype), \
                                           self.cast(self.transpose(w_ih, (1, 0)), self.dtype), \
                                           self.cast(self.transpose(w_hh, (1, 0)), self.dtype), \
@@ -282,7 +282,7 @@ class _DynamicLSTMCPUGPU(Cell):
 
     def __init__(self):
         super().__init__()
-        self.concat = P.Concat()
+        self.concat = ops.Concat()
         self.is_gpu = context.get_context("device_target") == "GPU"
 
     def construct(self, x, h_0, seq_length, w_ih, w_hh, b_ih, b_hh):
@@ -314,10 +314,10 @@ class _DynamicLSTMCPUGPU(Cell):
                         w_hh.view(-1, 1, 1),
                         bias.view(-1, 1, 1)
                     ))
-            output, h_n, c_n, _, _ = P.LSTM(input_size, hidden_size, 1, has_bias, False, 0.0)(
+            output, h_n, c_n, _, _ = ops.LSTM(input_size, hidden_size, 1, has_bias, False, 0.0)(
                 x,
-                P.ExpandDims()(h_0[0], 0),
-                P.ExpandDims()(h_0[1], 0),
+                ops.ExpandDims()(h_0[0], 0),
+                ops.ExpandDims()(h_0[1], 0),
                 weights.astype(x.dtype)
             )
         return output, (h_n, c_n)
@@ -328,12 +328,12 @@ class _DynamicLSTMAscend(Cell):
 
     def __init__(self):
         super().__init__()
-        self.lstm = P.DynamicRNN()
-        self.concat_dim1 = P.Concat(axis=1)
-        self.concat_dim0 = P.Concat(axis=0)
-        self.transpose = P.Transpose()
-        self.cast = P.Cast()
-        self.split = P.Split(axis=0, output_num=4)
+        self.lstm = ops.DynamicRNN()
+        self.concat_dim1 = ops.Concat(axis=1)
+        self.concat_dim0 = ops.Concat(axis=0)
+        self.transpose = ops.Transpose()
+        self.cast = ops.Cast()
+        self.split = ops.Split(axis=0, output_num=4)
         self.dtype = mstype.float16
 
     def construct(self, x, h_0, seq_length, w_ih, w_hh, b_ih, b_hh):
@@ -344,7 +344,7 @@ class _DynamicLSTMAscend(Cell):
         w_hh = self.concat_dim0((w_hh_i, w_hh_g, w_hh_f, w_hh_o))
         weight = self.concat_dim1((w_ih, w_hh))
         if b_ih is None:
-            bias = P.zeros(w_ih.shape[0], w_ih.dtype)
+            bias = ops.zeros(w_ih.shape[0], w_ih.dtype)
         else:
             b_ih_i, b_ih_f, b_ih_g, b_ih_o = self.split(b_ih)
             b_hh_i, b_hh_f, b_hh_g, b_hh_o = self.split(b_hh)
@@ -356,8 +356,8 @@ class _DynamicLSTMAscend(Cell):
         outputs, h, c, _, _, _, _, _ = self.lstm(self.cast(x, self.dtype), \
                                                  self.cast(self.transpose(weight, (1, 0)), self.dtype), \
                                                  self.cast(bias, self.dtype), None, \
-                                                 self.cast(P.ExpandDims()(h_0[0], 0), self.dtype), \
-                                                 self.cast(P.ExpandDims()(h_0[1], 0), self.dtype))
+                                                 self.cast(ops.ExpandDims()(h_0[0], 0), self.dtype), \
+                                                 self.cast(ops.ExpandDims()(h_0[1], 0), self.dtype))
         if seq_length is not None:
             h = get_hidden(h, seq_length)
             c = get_hidden(c, seq_length)
@@ -413,8 +413,8 @@ class _RNNBase(Cell):
             raise ValueError(f"For '{self.cls_name}', the 'mode' must be in ['RNN_RELU', 'RNN_TANH', 'LSTM', 'GRU'], "
                              f"but got {mode}.")
 
-        self.reverse = P.ReverseV2([0])
-        self.reverse_sequence = P.ReverseSequence(0, 1)
+        self.reverse = ops.ReverseV2([0])
+        self.reverse_sequence = ops.ReverseSequence(0, 1)
         self.hidden_size = hidden_size
         self.batch_first = batch_first
         self.num_layers = num_layers
@@ -488,7 +488,7 @@ class _RNNBase(Cell):
                 output_b = self.reverse(output_b)
             else:
                 output_b = self.reverse_sequence(output_b, seq_length)
-            output = P.Concat(2)((output_f, output_b))
+            output = ops.Concat(2)((output_f, output_b))
             pre_layer = self.dropout_op(output) if (self.dropout != 0 and i < self.num_layers - 1) else output
             if self.is_lstm:
                 h_n += (h_t_f[0], h_t_b[0],)
@@ -496,14 +496,14 @@ class _RNNBase(Cell):
             else:
                 h_n += (h_t_f, h_t_b,)
         if self.is_lstm:
-            h_n = P.Concat(0)(h_n)
-            c_n = P.Concat(0)(c_n)
+            h_n = ops.Concat(0)(h_n)
+            c_n = ops.Concat(0)(c_n)
             h0_shape = h[0].shape
             h1_shape = h[1].shape
             h_n = h_n.view(h0_shape)
             c_n = c_n.view(h1_shape)
             return output, (h_n.view(h0_shape), c_n.view(h1_shape))
-        h_n = P.Concat(0)(h_n)
+        h_n = ops.Concat(0)(h_n)
         return output, h_n.view(h.shape)
 
     def _stacked_dynamic_rnn(self, x, h, seq_length):
@@ -530,14 +530,14 @@ class _RNNBase(Cell):
             else:
                 h_n += (h_t,)
         if self.is_lstm:
-            h_n = P.Concat(0)(h_n)
-            c_n = P.Concat(0)(c_n)
+            h_n = ops.Concat(0)(h_n)
+            c_n = ops.Concat(0)(c_n)
             h0_shape = h[0].shape
             h1_shape = h[1].shape
             h_n = h_n.view(h0_shape)
             c_n = c_n.view(h1_shape)
             return output, (h_n.view(h0_shape), c_n.view(h1_shape))
-        h_n = P.Concat(0)(h_n)
+        h_n = ops.Concat(0)(h_n)
         return output, h_n.view(h.shape)
 
     def construct(self, x, hx=None, seq_length=None):
@@ -565,13 +565,13 @@ class _RNNBase(Cell):
             _check_input_dtype(seq_length.dtype, "seq_length", [mstype.int32, mstype.int64], self.cls_name)
             _check_seq_length_size(max_batch_size, seq_length.shape[0], self.cls_name)
         if self.batch_first:
-            x = P.Transpose()(x, (1, 0, 2))
+            x = ops.Transpose()(x, (1, 0, 2))
         if self.bidirectional:
             x_n, hx_n = self._stacked_bi_dynamic_rnn(x, hx, seq_length)
         else:
             x_n, hx_n = self._stacked_dynamic_rnn(x, hx, seq_length)
         if self.batch_first:
-            x_n = P.Transpose()(x_n, (1, 0, 2))
+            x_n = ops.Transpose()(x_n, (1, 0, 2))
         if not self.is_lstm:
             return x_n.astype(x_dtype), hx_n.astype(x_dtype)
         return x_n.astype(x_dtype), (hx_n[0].astype(x_dtype), hx_n[1].astype(x_dtype))

mindspore/nn/layer/thor_layer.py

@@ -17,12 +17,12 @@ from __future__ import absolute_import
 
 import numpy as np
 
+import mindspore.ops as ops
 import mindspore.common.dtype as mstype
 import mindspore.log as logger
 from mindspore.common.tensor import Tensor
 from mindspore.common.initializer import initializer, Initializer
 from mindspore.communication.management import get_group_size, get_rank
-from mindspore.ops import operations as P
 from mindspore.ops.operations._thor_ops import ThorIm2Col
 from mindspore.common.parameter import Parameter
 from mindspore import _checkparam as Validator
@@ -34,7 +34,6 @@ from mindspore.parallel._ps_context import _is_role_worker, _get_ps_context, \
     _set_rank_id, _insert_hash_table_size, _set_cache_enable
 from mindspore.parallel._utils import _get_parallel_mode, _get_full_batch
 from mindspore.context import ParallelMode
-from mindspore.ops import functional as F
 from mindspore.nn.layer.basic import ClipByNorm
 from mindspore.ops.primitive import constexpr
 
@@ -61,9 +60,11 @@ class DenseThor(Cell):
         in_channels (int): The number of the input channels.
         out_channels (int): The number of the output channels.
         weight_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable weight_init parameter. The dtype
-            is same as `x`. The values of str refer to the function `initializer`. Default: ``'normal'`` .
+            is same as `x`. The values of str refer to the function :func:`mindspore.common.initializer.initializer`.
+            Default: ``'normal'`` .
         bias_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable bias_init parameter. The dtype is
-            same as `x`. The values of str refer to the function `initializer`. Default: ``'zeros'`` .
+            same as `x`. The values of str refer to the function :func:`mindspore.common.initializer.initializer`.
+            Default: ``'zeros'`` .
         has_bias (bool): Specifies whether the layer uses a bias vector. Default: ``True`` .
         activation (str): activate function applied to the output of the fully connected layer, eg. 'ReLU'.
             Default: ``None`` .
@@ -120,9 +121,9 @@ class DenseThor(Cell):
                                      f"be equal to 1, and the first dim must be equal to 'out_channels'. But got "
                                      f"'bias_init': {bias_init}, 'out_channels': {out_channels}.")
             self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias")
-            self.bias_add = P.BiasAdd()
+            self.bias_add = ops.BiasAdd()
 
-        self.matmul = P.MatMul(transpose_b=True)
+        self.matmul = ops.MatMul(transpose_b=True)
         self.activation = get_activation(activation)
         self.activation_flag = self.activation is not None
 
@@ -130,25 +131,25 @@ class DenseThor(Cell):
                                   name='matrix_a', requires_grad=False)
         self.matrix_g = Parameter(Tensor(np.eye(out_channels).astype(np.float32)),
                                   name="matrix_g", requires_grad=False)
-        self.shape = P.Shape()
-        self.reshape = P.Reshape()
-        self.transpose = P.Transpose()
-        self.mul = P.Mul()
+        self.shape = ops.Shape()
+        self.reshape = ops.Reshape()
+        self.transpose = ops.Transpose()
+        self.mul = ops.Mul()
         self.is_ascend = True
         self.split_dim = 128
         if context.get_context("device_target") == "Ascend":
             self._process_ascend_dense_thor(out_channels, in_channels)
         else:
             self.is_ascend = False
-            self.cube_matmul = P.MatMul(transpose_a=True)
-        self.getG = P.InsertGradientOf(self.save_gradient)
+            self.cube_matmul = ops.MatMul(transpose_a=True)
+        self.getG = ops.InsertGradientOf(self.save_gradient)
 
     def _process_ascend_dense_thor(self, out_channels, in_channels):
         """process ascend dense thor"""
-        self.matmul = P.MatMul(transpose_b=True)
-        self.cube_matmul = P.CusMatMulCube(transpose_a=True)
-        self.cast = P.Cast()
-        self.is_nsp_layer = (out_channels == 2)
+        self.matmul = ops.MatMul(transpose_b=True)
+        self.cube_matmul = ops.CusMatMulCube(transpose_a=True)
+        self.cast = ops.Cast()
+        self.is_nsp_layer = out_channels == 2
 
     def save_gradient(self, dout):
         """
@@ -195,7 +196,7 @@ class DenseThor(Cell):
             x = self.activation(x)
         # We use Depend to make 'self.matrix_g' as primal graph's weight parameter,
         # for it's used in 'save_gradient' gradient procedure.
-        return F.depend(x, self.matrix_g)
+        return ops.depend(x, self.matrix_g)
 
     def extend_repr(self):
         s = 'input_channels={}, output_channels={}'.format(self.in_channels, self.out_channels)
@@ -388,19 +389,19 @@ class Conv2dThor(_ConvThor):
         dilation = twice(dilation)
         super(Conv2dThor, self).__init__(in_channels, out_channels, kernel_size,
                                          stride, pad_mode, padding, dilation, group, has_bias, weight_init, bias_init)
-        self.conv2d = P.Conv2D(out_channel=self.out_channels, kernel_size=self.kernel_size,
-                               mode=1, pad_mode=self.pad_mode, pad=self.padding,
-                               stride=self.stride, dilation=self.dilation, group=self.group)
+        self.conv2d = ops.Conv2D(out_channel=self.out_channels, kernel_size=self.kernel_size,
+                                 mode=1, pad_mode=self.pad_mode, pad=self.padding,
+                                 stride=self.stride, dilation=self.dilation, group=self.group)
         self._init_depthwise_conv2d(weight_init)
-        self.bias_add = P.BiasAdd()
+        self.bias_add = ops.BiasAdd()
         self.thor = True
         self.hw = kernel_size[0] * kernel_size[1]
         self.matrix_a_dim = self.in_channels * self.kernel_size[0] * self.kernel_size[1]
         self.matrix_g_dim = self.out_channels
-        self.shape = P.Shape()
-        self.reshape = P.Reshape()
-        self.mul = P.Mul()
-        self.cast = P.Cast()
+        self.shape = ops.Shape()
+        self.reshape = ops.Reshape()
+        self.mul = ops.Mul()
+        self.cast = ops.Cast()
         self.a_normalizer = Parameter(initializer(1, [1], mstype.float32), name="a_normalizer", requires_grad=False)
         self.g_normalizer = Parameter(initializer(1, [1], mstype.float32), name="g_normalizer", requires_grad=False)
         self.is_ascend = True
@@ -409,30 +410,30 @@ class Conv2dThor(_ConvThor):
         else:
             self.is_ascend = False
             self.img2col = ThorIm2Col(kernel_size=kernel_size, stride=stride, pad_mode="same")
-            self.matmul = P.MatMul(transpose_b=True)
-            self.reduce_mean = P.ReduceMean(keep_dims=False)
+            self.matmul = ops.MatMul(transpose_b=True)
+            self.reduce_mean = ops.ReduceMean(keep_dims=False)
             self.matrix_a_cov = Parameter(Tensor(np.zeros([self.matrix_a_dim, self.matrix_a_dim]).astype(np.float32)),
                                           name='matrix_a', requires_grad=False)
             self.matrix_g_cov = Parameter(Tensor(np.zeros([self.matrix_g_dim, self.matrix_g_dim]).astype(np.float32)),
                                           name='matrix_g', requires_grad=False)
-        self.getG = P.InsertGradientOf(self.save_gradient)
+        self.getG = ops.InsertGradientOf(self.save_gradient)
 
     def _process_ascend_conv2d_thor(self, kernel_size, stride):
         """process ascend conv2d thor"""
         ksizes = (1, kernel_size[0], kernel_size[1], 1)
         strides = (1, stride[0], stride[1], 1)
         ksizes_tbe = (kernel_size[0], kernel_size[1])
-        self.img2col = P.CusImg2Col(ksizes=ksizes, strides=strides)
-        self.transpose = P.Transpose()
-        self.reshape = P.Reshape()
-        self.cube_matmul = P.CusMatMulCube(transpose_a=True)
+        self.img2col = ops.CusImg2Col(ksizes=ksizes, strides=strides)
+        self.transpose = ops.Transpose()
+        self.reshape = ops.Reshape()
+        self.cube_matmul = ops.CusMatMulCube(transpose_a=True)
         self.diag_block_dim = 128
         self.matrix_a_cov = Parameter(Tensor(np.eye(self.matrix_a_dim).astype(np.float32)),
                                       name='matrix_a', requires_grad=False)
         self.matrix_g_cov = Parameter(Tensor(np.eye(self.matrix_g_dim).astype(np.float32)),
                                       name='matrix_g', requires_grad=False)
-        self.slice = P.Slice()
-        self.im2col = P.NewIm2Col(ksizes=ksizes_tbe, strides=stride[0], padding_mode="SAME")
+        self.slice = ops.Slice()
+        self.im2col = ops.NewIm2Col(ksizes=ksizes_tbe, strides=stride[0], padding_mode="SAME")
 
     def _init_depthwise_conv2d(self, weight_init):
         """Initialize depthwise conv2d op"""
@@ -440,12 +441,12 @@ class Conv2dThor(_ConvThor):
             self.dilation = self._dilation
             Validator.check_int('group', self.group, self.in_channels, Validator.EQ, self.cls_name)
             Validator.check_int('group', self.group, self.out_channels, Validator.EQ, self.cls_name)
-            self.conv2d = P.DepthwiseConv2dNative(channel_multiplier=1,
-                                                  kernel_size=self.kernel_size,
-                                                  pad_mode=self.pad_mode,
-                                                  pad=self.padding,
-                                                  stride=self.stride,
-                                                  dilation=self.dilation)
+            self.conv2d = ops.DepthwiseConv2dNative(channel_multiplier=1,
+                                                    kernel_size=self.kernel_size,
+                                                    pad_mode=self.pad_mode,
+                                                    pad=self.padding,
+                                                    stride=self.stride,
+                                                    dilation=self.dilation)
             weight_shape = [1, self.in_channels, *self.kernel_size]
             self.weight_init = weight_init
             if isinstance(weight_init, Tensor):
@@ -598,29 +599,29 @@ class EmbeddingThor(Cell):
                                                          "padding_idx", self.cls_name)
             self.init_tensor[self.padding_idx] = 0
         self.embedding_table = Parameter(self.init_tensor, name='embedding_table')
-        self.expand = P.ExpandDims()
-        self.reshape_flat = P.Reshape()
+        self.expand = ops.ExpandDims()
+        self.reshape_flat = ops.Reshape()
         self.shp_flat = (-1,)
-        self.gather = P.Gather()
-        self.one_hot = P.OneHot()
+        self.gather = ops.Gather()
+        self.one_hot = ops.OneHot()
         self.on_value = Tensor(1.0, self.dtype)
         self.off_value = Tensor(0.0, self.dtype)
-        self.array_mul = P.MatMul()
-        self.reshape = P.Reshape()
-        self.get_shp = P.Shape()
+        self.array_mul = ops.MatMul()
+        self.reshape = ops.Reshape()
+        self.get_shp = ops.Shape()
         self.thor = True
         self.matrix_a = Parameter(Tensor(np.zeros([vocab_size]).astype(np.float32)),
                                   name='matrix_a', requires_grad=False)
         self.matrix_g = Parameter(Tensor(np.zeros([embedding_size, embedding_size]).astype(np.float32)),
                                   name="matrix_g", requires_grad=False)
-        self.reduce_sum = P.ReduceSum(keep_dims=False)
-        self.getG = P.InsertGradientOf(self.save_gradient)
-        self.cast = P.Cast()
+        self.reduce_sum = ops.ReduceSum(keep_dims=False)
+        self.getG = ops.InsertGradientOf(self.save_gradient)
+        self.cast = ops.Cast()
         if context.get_context("device_target") == "Ascend":
-            self.cube_matmul = P.CusMatMulCube(transpose_a=True)
+            self.cube_matmul = ops.CusMatMulCube(transpose_a=True)
         else:
-            self.cube_matmul = P.MatMul(transpose_a=True)
-        self.mul = P.Mul()
+            self.cube_matmul = ops.MatMul(transpose_a=True)
+        self.mul = ops.Mul()
 
     def save_gradient(self, dout):
         """
@@ -656,7 +657,7 @@ class EmbeddingThor(Cell):
         output = self.reshape(output_for_reshape, out_shape)
         # We use Depend to make 'self.matrix_g' as primal graph's weight parameter,
         # for it's used in 'save_gradient' gradient procedure.
-        return F.depend(output, self.matrix_g)
+        return ops.depend(output, self.matrix_g)
 
     def extend_repr(self):
         s = 'vocab_size={}, embedding_size={}, use_one_hot={}, embedding_table={}, dtype={}, padding_idx={}'.format(
@@ -752,10 +753,10 @@ class EmbeddingLookupThor(Cell):
             raise ValueError(f"For '{self.cls_name}', embedding_lookup must be sparse when 'target' is CPU, but got "
                              f"'sparse': {sparse}, 'target': {target}.")
         if sparse:
-            self.gatherv2 = P.SparseGatherV2()
+            self.gatherv2 = ops.SparseGatherV2()
         else:
-            self.gatherv2 = P.Gather()
-        self.embeddinglookup = P.EmbeddingLookup().set_device('CPU')
+            self.gatherv2 = ops.Gather()
+        self.embeddinglookup = ops.EmbeddingLookup().set_device('CPU')
         enable_ps = _get_ps_context("enable_ps")
         if enable_ps:
             self._process_vocab_cache(slice_mode)
@@ -764,13 +765,13 @@ class EmbeddingLookupThor(Cell):
                                                      mstype.float16), name='embedding_table')
         parallel_mode = _get_parallel_mode()
         is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL)
-        self.gather_revert = P.Gather()
-        self.reshape_first = P.Reshape()
-        self.reshape = P.Reshape()
-        self.unique = P.Unique()
-        self.shape = P.Shape()
+        self.gather_revert = ops.Gather()
+        self.reshape_first = ops.Reshape()
+        self.reshape = ops.Reshape()
+        self.unique = ops.Unique()
+        self.shape = ops.Shape()
         if is_auto_parallel:
-            self.unique = P.Unique().shard(((1,),))
+            self.unique = ops.Unique().shard(((1,),))
         if self.cache_enable and enable_ps:
             self._set_voacb_cache_enable_for_ps(vocab_cache_size, embedding_size, vocab_size)
             if is_auto_parallel:
@@ -833,14 +834,14 @@ class EmbeddingLookupThor(Cell):
                                   name='matrix_a', requires_grad=False)
         self.matrix_g = Parameter(Tensor(np.zeros([embedding_size, embedding_size]).astype(np.float32)),
                                   name="matrix_g", requires_grad=False)
-        self.reduce_sum = P.ReduceSum(keep_dims=False)
-        self.getG = P.InsertGradientOf(self.save_gradient)
-        self.cast = P.Cast()
-        self.cube_matmul = P.MatMul(transpose_a=True)
-        self.mul = P.Mul()
+        self.reduce_sum = ops.ReduceSum(keep_dims=False)
+        self.getG = ops.InsertGradientOf(self.save_gradient)
+        self.cast = ops.Cast()
+        self.cube_matmul = ops.MatMul(transpose_a=True)
+        self.mul = ops.Mul()
         self.on_value = Tensor(1.0, self.dtype)
         self.off_value = Tensor(0.0, self.dtype)
-        self.one_hot = P.OneHot()
+        self.one_hot = ops.OneHot()
 
 
     def save_gradient(self, dout):
@@ -872,11 +873,11 @@ class EmbeddingLookupThor(Cell):
 
         logger.info("EmbeddingLookup cache enable takes effect.")
         self.forward_unique = True
-        self.unique = P.Unique().set_device('CPU')
+        self.unique = ops.Unique().set_device('CPU')
         self.unique.add_prim_attr('cache_enable', True)
         self.embedding_table.cache_enable = self.cache_enable
         self.embedding_table.cache_shape = (self.vocab_cache_size, self.embedding_size)
-        self.reshape_first = P.Reshape().set_device('CPU')
+        self.reshape_first = ops.Reshape().set_device('CPU')
 
     def _process_vocab_cache(self, slice_mode):
         """PS embeddingLookup cache check and process."""
@@ -955,9 +956,9 @@ class EmbeddingLookupThor(Cell):
                 else:
                     out = self.gatherv2(self.embedding_table, indices, 0)
         if self.max_norm is not None:
-            axis = _make_axis_range(F.rank(indices), F.rank(out))
+            axis = _make_axis_range(ops.rank(indices), ops.rank(out))
             clip_by_norm = ClipByNorm(axis)
             out = clip_by_norm(out, self.max_norm)
         # We use Depend to make 'self.matrix_g' as primal graph's weight parameter,
         # for it's used in 'save_gradient' gradient procedure.
-        return F.depend(out, self.matrix_g)
+        return ops.depend(out, self.matrix_g)

mindspore/nn/layer/transformer.py

@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
+# The implementation of the classes `MultiheadAttention`, `TransformerEncoderLayer`, `TransformerDecoderLayer`,
+# `TransformerEncoder`, `TransformerDecoder` and `Transformer` is referenced from the PyTorch project.
+# - https://github.com/pytorch/pytorch/blob/release/2.0/torch/nn/modules/transformer.py
+# - https://github.com/pytorch/pytorch/blob/release/2.0/torch/nn/modules/activation.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2022.
 """
 Transformer Cells module, include TransformerEncoderLayer, TransformerDecoderLayer,
 TransformerEncoder, TransformerDecoder, Transformer.
@@ -245,7 +250,12 @@ class MultiheadAttention(Cell):
 class TransformerEncoderLayer(Cell):
     r"""
     Transformer Encoder Layer. This is an implementation of the single layer of the transformer
-    encoder layer, including multihead attention and feedward layer.
+    encoder layer, mainly including Multi-Head Attention, Feed Forward, Add and LayerNorm layer.
+
+    The TransformerEncoderLayer structure is shown in the following figure:
+
+    .. image:: ../images/TransformerEncoderLayer.png
+        :align: center
 
     Args:
         d_model (int): The number of features in the input tensor.