mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/nn/layer/basic.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2021 Huawei Technologies Co., Ltd
+# Copyright 2020-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ from mindspore.common.tensor import Tensor
 from mindspore.common.initializer import initializer, HeUniform, Uniform
 from mindspore.ops import operations as P
 from mindspore.ops import functional as F
+from mindspore.ops.function.nn_func import interpolate_ext
+from mindspore.ops.auto_generate import unfold_ext
 from mindspore.ops.operations import _inner_ops as inner
 from mindspore.ops.primitive import constexpr, Primitive, _primexpr
 from mindspore.common.parameter import Parameter
@@ -35,10 +37,12 @@ from mindspore import _checkparam as Validator
 from mindspore.nn.cell import Cell
 from mindspore.nn.layer.activation import get_activation
 from mindspore.common._decorator import deprecated
+from mindspore.ops.auto_generate import dropout_ext_op, fold_ext
+from mindspore.common.generator import default_generator
 
-__all__ = ['Dropout', 'Flatten', 'Dense', 'ClipByNorm', 'Norm', 'OneHot', 'Pad', 'Unfold', 'Tril', 'Triu',
-           'ResizeBilinear', 'MatrixDiag', 'MatrixDiagPart', 'MatrixSetDiag', 'L1Regularizer', 'Dropout1d',
-           'Dropout2d', 'Dropout3d', 'Upsample', 'Roll', 'Identity', 'Unflatten']
+__all__ = ['Dropout', 'Flatten', 'Dense', 'Linear', 'ClipByNorm', 'Norm', 'OneHot', 'Pad', 'Unfold', 'Tril', 'Triu',
+           'MatrixDiag', 'MatrixDiagPart', 'MatrixSetDiag', 'L1Regularizer', 'Dropout1d',
+           'Dropout2d', 'Dropout3d', 'Upsample', 'Roll', 'Identity', 'Unflatten', 'DropoutExt']
 
 
 class L1Regularizer(Cell):
@@ -201,6 +205,69 @@ class Dropout(Cell):
         return f'p={self.p}'
 
 
+class DropoutExt(Cell):
+    r"""
+    Dropout layer for the input.
+
+    Dropout is a means of regularization that reduces overfitting by preventing correlations between neuronal nodes.
+    The operator randomly sets some neurons output to 0 according to `p`, which means the probability of discarding
+    during training. And the return will be multiplied by :math:`\frac{1}{1-p}` during training.
+    During the reasoning, this layer returns the same Tensor as the `x`.
+
+    This technique is proposed in paper `Dropout: A Simple Way to Prevent Neural Networks from Overfitting
+    <http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf>`_ and proved to be effective to reduce
+    over-fitting and prevents neurons from co-adaptation. See more details in `Improving neural networks by
+    preventing co-adaptation of feature detectors
+    <https://arxiv.org/pdf/1207.0580.pdf>`_.
+
+    Note:
+        - Each channel will be zeroed out independently on every construct call.
+        - Parameter `p` means the probability of the element of the input tensor to be zeroed.
+
+    Args:
+        p (float): The dropout rate of input neurons, E.g. `p` =0.9, dropping out 90% of input neurons.
+            Default: ``0.5`` .
+
+    Inputs:
+        - **x** (Tensor) - The input of Dropout.
+
+    Outputs:
+        Tensor, output tensor with the same shape as the `x`.
+
+    Raises:
+        TypeError: If the dtype of `p` is not float.
+        ValueError: If length of shape of `x` is less than 1.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> x = Tensor(np.ones([2, 2, 3]), mindspore.float32)
+        >>> net = nn.DropoutExt(p=0.2)
+        >>> net.set_train()
+        >>> output = net(x)
+        >>> print(output.shape)
+        (2, 2, 3)
+    """
+
+    def __init__(self, p=0.5):
+        """Initialize DropoutExt."""
+        super(DropoutExt, self).__init__()
+        self.p = p
+        self.generator_step = Tensor(1, mstype.int64)
+
+    def construct(self, x):
+        if not self.training or self.p == 0:
+            return x
+
+        seed, offset = default_generator._step(self.generator_step)  # pylint: disable=protected-access
+        out, _ = dropout_ext_op(x, self.p, seed, offset)
+        return out
+
+
 class Dropout1d(Cell):
     r"""
     During training, randomly zeroes entire channels of the input tensor with probability `p`
@@ -236,7 +303,7 @@ class Dropout1d(Cell):
         TypeError: If `x` is not a Tensor.
         TypeError: If the data type of `p` is not float.
         ValueError: If `p` is out of the range `[0.0, 1.0]`.
-        ValueError: If `x` shape is not `2D` or `3D`.
+        ValueError: If the shape of `x` is not `2D` or `3D`.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -408,6 +475,47 @@ class Upsample(Cell):
         return out
 
 
+class UpsampleExt(Cell):
+    r"""
+    For details, please refer to :func:`mindspore.mint.nn.functional.interpolate`.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore import nn
+        >>> x = ms.Tensor([[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]]]])
+        >>> upsample = nn.UpsampleExt(size=(5, 5))
+        >>> out = upsample(x)
+        >>> print(x.asnumpy())
+        [[[[1. 2. 3. 4.]
+           [5. 6. 7. 8.]]]]
+        >>> print(out.asnumpy())
+        [[[[1. 1. 2. 3. 4.]
+           [1. 1. 2. 3. 4.]
+           [1. 1. 2. 3. 4.]
+           [5. 5. 6. 7. 8.]
+           [5. 5. 6. 7. 8.]]]]
+        >>> print(out.shape)
+        (1, 1, 5, 5)
+    """
+
+    def __init__(self, size=None, scale_factor=None, mode="nearest", align_corners=None, recompute_scale_factor=None):
+        """Initialize Upsample."""
+        super(UpsampleExt, self).__init__()
+        self.size = size
+        self.scale_factor = scale_factor
+        self.mode = mode
+        self.align_corners = align_corners
+        self.recompute_scale_factor = recompute_scale_factor
+
+    def construct(self, input):
+        out = interpolate_ext(input, self.size, self.scale_factor, self.mode,
+                              self.align_corners, self.recompute_scale_factor)
+        return out
+
+
 class Flatten(Cell):
     r"""
     Flatten the input Tensor along dimensions from `start_dim` to `end_dim`.
@@ -427,7 +535,8 @@ class Flatten(Cell):
         TypeError: If `x` is not a Tensor.
         TypeError: If `start_dim` or `end_dim` is not int.
         ValueError: If `start_dim` is greater than `end_dim` after canonicalized.
-        ValueError: If `start_dim` or `end_dim` is not in range of [-x.dim, x.dim-1].
+        ValueError: If `start_dim` or `end_dim` is not in range of [-x.dim, x.dim-1]. For example, the default values
+                    are used for the args and the input is a 0-dimensional or 1-dimensional Tensor.
 
     Supported Platforms:
         ``Ascend`` ``GPU`` ``CPU``
@@ -466,14 +575,6 @@ class Flatten(Cell):
         return F.flatten(x, start_dim=self.start_dim, end_dim=self.end_dim)
 
 
-@_primexpr
-def check_dense_input_shape(x, prim_name=None):
-    """ check the shape of inputs"""
-    msg_prefix = f"For '{prim_name}', the" if prim_name else "The"
-    if len(x) < 2:
-        raise ValueError(f"{msg_prefix} dimension of 'x' should not be less than 2, but got {len(x)}.")
-
-
 class Identity(Cell):
     r"""
     A placeholder identity operator that returns the same as input.
@@ -621,7 +722,6 @@ class Dense(Cell):
 
     def construct(self, x):
         x_shape = self.shape_op(x)
-        check_dense_input_shape(x_shape, self.cls_name)
         if len(x_shape) != 2:
             x = self.reshape(x, (-1, x_shape[-1]))
         x = self.matmul(x, self.weight)
@@ -643,6 +743,116 @@ class Dense(Cell):
         return s
 
 
+class Linear(Cell):
+    r"""
+    The linear connected layer.
+
+    Applies linear connected layer for the input. This layer implements the operation as:
+
+    .. math::
+        \text{outputs} = X * kernel + bias
+
+    where :math:`X` is the input tensors, :math:`\text{kernel}` is a weight matrix with the same
+    data type as the :math:`X` created by the layer, and :math:`\text{bias}` is a bias vector
+    with the same data type as the :math:`X` created by the layer (only if has_bias is True).
+
+    Args:
+        in_features (int): The number of features in the input space.
+        out_features (int): The number of features in the output space.
+        bias (bool): Specifies whether the layer uses a bias vector :math:`\text{bias}`. Default: ``True``.
+        weight_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable weight_init parameter. The dtype
+            is same as `x`. The values of str refer to the function `initializer`. Default: ``None`` ,
+            weight will be initialized using HeUniform.
+        bias_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable bias_init parameter. The dtype is
+            same as `x`. The values of str refer to the function `initializer`. Default: ``None`` ,
+            bias will be initialized using Uniform.
+        dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``None`` .
+
+    Inputs:
+        - **x** (Tensor) - Tensor of shape :math:`(*, in\_features)`. The `in_features` in `Args` should be equal
+          to :math:`in\_features` in `Inputs`.
+
+    Outputs:
+        Tensor of shape :math:`(*, out\_features)`.
+
+    Raises:
+        TypeError: If `in_features` or `out_features` is not an int.
+        TypeError: If `bias` is not a bool.
+        ValueError: If length of shape of `weight_init` is not equal to 2 or shape[0] of `weight_init`
+                    is not equal to `out_features` or shape[1] of `weight_init` is not equal to `in_features`.
+        ValueError: If length of shape of `bias_init` is not equal to 1
+                    or shape[0] of `bias_init` is not equal to `out_features`.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor
+        >>> from mindspore import nn
+        >>> import numpy as np
+        >>> x = Tensor(np.array([[180, 234, 154], [244, 48, 247]]), mindspore.float32)
+        >>> net = nn.mint.nn.Linear(3, 4)
+        >>> output = net(x)
+        >>> print(output.shape)
+        (2, 4)
+    """
+
+    @cell_attr_register(attrs=['has_bias'])
+    def __init__(self,
+                 in_features,
+                 out_features,
+                 bias=True,
+                 weight_init=None,
+                 bias_init=None,
+                 dtype=None):
+        """Initialize Linear."""
+        super(Linear, self).__init__()
+        self.in_features = Validator.check_positive_int(
+            in_features, "in_features", self.cls_name)
+        self.out_features = Validator.check_positive_int(
+            out_features, "out_features", self.cls_name)
+        self.has_bias = Validator.check_bool(
+            bias, "has_bias", self.cls_name)
+        self.dense = P.Dense()
+        if dtype is None:
+            dtype = mstype.float32
+        if isinstance(weight_init, Tensor):
+            if weight_init.ndim != 2 or weight_init.shape[0] != out_features or \
+                    weight_init.shape[1] != in_features:
+                raise ValueError(f"For '{self.cls_name}', weight init shape error. The ndim of 'weight_init' must "
+                                 f"be equal to 2, and the first dim must be equal to 'out_features', and the "
+                                 f"second dim must be equal to 'in_features'. But got 'weight_init': {weight_init}, "
+                                 f"'out_features': {out_features}, 'in_features': {in_features}.")
+        if weight_init is None:
+            weight_init = HeUniform(math.sqrt(5))
+        self.weight = Parameter(initializer(
+            weight_init, [out_features, in_features], dtype=dtype), name="weight")
+
+        self.bias = None
+        if self.has_bias:
+            if isinstance(bias_init, Tensor):
+                if bias_init.ndim != 1 or bias_init.shape[0] != out_features:
+                    raise ValueError(f"For '{self.cls_name}', bias init shape error. The ndim of 'bias_init' must "
+                                     f"be equal to 1, and the first dim must be equal to 'out_features'. But got "
+                                     f"'bias_init': {bias_init}, 'out_features': {out_features}.")
+            if bias_init is None:
+                bound = 1 / math.sqrt(in_features)
+                bias_init = Uniform(scale=bound)
+            self.bias = Parameter(initializer(
+                bias_init, [out_features], dtype=dtype), name="bias")
+
+    def construct(self, x):
+        x = self.dense(x, self.weight, self.bias)
+        return x
+
+    def extend_repr(self):
+        s = f'input_features={self.in_features}, output_features={self.out_features}'
+        if self.has_bias:
+            s += f', has_bias={self.has_bias}'
+        return s
+
+
 @constexpr
 def _is_equal_one(x):
     if x is None:
@@ -941,72 +1151,6 @@ class Pad(Cell):
         return x
 
 
-@_primexpr
-def bilinear(shape, size, scale, align_corners, prim_name=None):
-    """Check input and calculate shape"""
-    msg_prefix = f"For '{prim_name}', the" if prim_name else "The"
-    if not isinstance(align_corners, bool):
-        raise TypeError(
-            f"{msg_prefix} type of 'align_corners' must be bool, but got {type(align_corners).__name__}.")
-    if size is None and scale is None:
-        raise ValueError(f"{msg_prefix} 'size' and 'scale' both none.")
-    if size is not None and scale is not None:
-        raise ValueError(f"{msg_prefix} 'size' and 'scale' both not none.")
-    if size is not None:
-        if not isinstance(size, (tuple, list)):
-            raise ValueError(
-                f"{msg_prefix} 'size' must be tuple or list or None, but got {type(size).__name__}.")
-        Validator.check_int(len(size), 2, Validator.EQ, "size", "bilinear")
-        Validator.check_int(size[0], 1, Validator.GE, "size[0]", "bilinear")
-        Validator.check_int(size[1], 1, Validator.GE, "size[1]", "bilinear")
-        return size
-    Validator.check_int(scale, 1, Validator.GE, "scale factor", "bilinear")
-    ret = (scale * shape[2], scale * shape[3])
-    return ret
-
-
-class ResizeBilinear(Cell):
-    r"""
-    'nn.ResizeBilinear' is deprecated from version 2.0 and will be removed in a future version,
-    use :class:`mindspore.ops.ResizeBilinearV2` or :func:`mindspore.ops.interpolate` instead.
-
-    Supported Platforms:
-        Deprecated
-
-    Examples:
-        >>> import mindspore
-        >>> from mindspore import Tensor, nn
-        >>> x = Tensor([[[[1, 2, 3, 4], [5, 6, 7, 8]]]], mindspore.float32)
-        >>> resize_bilinear = nn.ResizeBilinear()
-        >>> result = resize_bilinear(x, size=(5,5))
-        >>> print(x)
-        [[[[1. 2. 3. 4.]
-           [5. 6. 7. 8.]]]]
-        >>> print(result)
-        [[[[1.        1.8       2.6       3.4       4.       ]
-           [2.6       3.4       4.2000003 5.        5.6000004]
-           [4.2       5.0000005 5.8       6.6       7.2      ]
-           [5.        5.8       6.6       7.4       8.       ]
-           [5.        5.8       6.6       7.4000006 8.       ]]]]
-        >>> print(result.shape)
-        (1, 1, 5, 5)
-    """
-
-    def __init__(self, half_pixel_centers=False):
-        """Initialize ResizeBilinear."""
-        super(ResizeBilinear, self).__init__()
-        logger.warning("'nn.ResizeBilinear' is deprecated from version 2.0 and will be removed in a "
-                       "future version, use 'ops.ResizeBilinearV2' or 'ops.interpolate' instead.")
-        self.half_pixel_centers = half_pixel_centers
-
-    def construct(self, x, size=None, scale_factor=None, align_corners=False):
-        shape = bilinear(x.shape, size, scale_factor,
-                         align_corners, self.cls_name)
-        resize_bilinear = P.ResizeBilinear(
-            shape, align_corners, self.half_pixel_centers)
-        return resize_bilinear(x)
-
-
 class Unfold(Cell):
     r"""
     Extracts patches from images.
@@ -1097,6 +1241,68 @@ class Unfold(Cell):
         return result
 
 
+class UnfoldExt(Cell):
+    r"""
+    Extracts sliding local blocks from a batched input tensor.
+
+    For details, please refer to :func:`mindspore.mint.nn.functional.unfold`.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> import numpy as np
+        >>> from mindspore import Tensor, nn
+        >>> input = Tensor(np.random.rand(4, 4, 32, 32), mindspore.float64)
+        >>> unfold = nn.UnfoldExt(kernel_size=3, dilation=1, stride=1)
+        >>> output = unfold(input)
+        >>> print(output.shape)
+        (4, 36, 900)
+    """
+    def __init__(self, kernel_size, dilation=1, padding=0, stride=1):
+        super(UnfoldExt, self).__init__()
+        self.kernel_size = kernel_size
+        self.dilation = dilation
+        self.padding = padding
+        self.stride = stride
+
+    def construct(self, input):
+        return unfold_ext(input, self.kernel_size, self.dilation, self.padding, self.stride)
+
+
+class Fold(Cell):
+    r"""
+    Combines an array of sliding local blocks into a large containing tensor.
+
+    For details, please refer to :func:`mindspore.mint.nn.functional.fold`.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import numpy as np
+        >>> from mindspore import Tensor, nn
+        >>> from mindspore import dtype as mstype
+        >>> fold = nn.Fold([8, 8], [2, 2], [2, 2], [2, 2], [2, 2])
+        >>> input = Tensor(input_data=np.random.rand(16, 64, 25), dtype=mstype.float32)
+        >>> output = fold(input)
+        >>> print(output.shape)
+        (16, 16, 8, 8)
+    """
+    def __init__(self, output_size, kernel_size, dilation=1, padding=0, stride=1):
+        super(Fold, self).__init__()
+        self.output_size = output_size
+        self.kernel_size = kernel_size
+        self.dilation = dilation
+        self.padding = padding
+        self.stride = stride
+
+    def construct(self, input):
+        return fold_ext(input, self.output_size, self.kernel_size,
+                        self.dilation, self.padding, self.stride)
+
+
 @_primexpr
 def tril(x_shape, x_dtype, k):
     Validator.check_int(len(x_shape), 1, Validator.GE, "x rank", "tril")
@@ -1365,8 +1571,7 @@ class Roll(Cell):
 
 class Unflatten(Cell):
     r"""
-    Summary:
-        Unflattens a Tensor dim according to `axis` and `unflattened_size`.
+    Unflattens a Tensor dim according to `axis` and `unflattened_size`.
 
     Args:
         axis (int): specifies the dimension of the input Tensor to be unflattened.

mindspore/nn/layer/channel_shuffle.py

@@ -15,7 +15,6 @@
 """channel shuffle"""
 from mindspore.ops import operations as P
 from mindspore.nn.cell import Cell
-from mindspore.ops.primitive import _primexpr
 
 __all__ = ['ChannelShuffle']
 
@@ -27,7 +26,8 @@ class ChannelShuffle(Cell):
     :math:`\frac{g}{}` and :math:`g` to restore Tensor to the original shape.
 
     Args:
-        groups (int): Number of groups to divide channels in, must be greater than 0. Refer to :math:`g`.
+        groups (int): Number of groups to divide channels in, must be greater than 0.
+            Refer to :math:`g` in the above formula.
 
     Inputs:
         - **x** (Tensor) - Tensor of shape :math:`(*, C_{in}, H_{in}, W_{in})`.
@@ -36,7 +36,7 @@ class ChannelShuffle(Cell):
         Tensor, with the same type and shape as the `x`.
 
     Raises:
-        TypeError: If `groups` is not an int.
+        TypeError: If `groups` is not a positive integer.
         ValueError: If `groups` is less than 1.
         ValueError: If dims of `x` is less than 3.
         ValueError: If number of channels can not be divisible by groups.
@@ -82,22 +82,9 @@ class ChannelShuffle(Cell):
         self.reshape = P.Reshape()
         self.transpose = P.Transpose()
 
-    @staticmethod
-    @_primexpr
-    def _check_input_dim(shape, channels, groups, cls_name):
-        """check input dim"""
-        dim = len(shape)
-        if dim < 3:
-            raise ValueError(f"For {cls_name}, the in_shape must have more than 2 dims, but got {dim}.")
-
-        if channels % groups != 0:
-            raise ValueError(f"For {cls_name}, number of channels must be divisible by groups, "
-                             f"but got {channels} channels and {groups} groups.")
-
     def construct(self, x):
         x_shape = self.shape(x)
         n, c = x_shape[0], x_shape[1]
-        self._check_input_dim(x_shape, c, self.groups, self.cls_name)
         out = self.reshape(x, (n, self.groups, c // self.groups, -1))
         out = self.transpose(out, (0, 2, 1, 3))
         return self.reshape(out, x_shape)

mindspore/nn/layer/container.py

@@ -123,7 +123,7 @@ class _CellListBase:
 class SequentialCell(Cell):
     """
     Sequential Cell container. For more details about Cell, please refer to
-    `Cell <https://www.mindspore.cn/docs/en/r2.2/api_python/nn/mindspore.nn.Cell.html#mindspore.nn.Cell>`_.
+    `Cell <https://www.mindspore.cn/docs/en/master/api_python/nn/mindspore.nn.Cell.html#mindspore.nn.Cell>`_.
 
     A list of Cells will be added to it in the order they are passed in the constructor.
     Alternatively, an ordered dict of cells can also be passed in.
@@ -325,7 +325,7 @@ class SequentialCell(Cell):
 class CellList(_CellListBase, Cell):
     """
     Holds Cells in a list. For more details about Cell, please refer to
-    `Cell <https://www.mindspore.cn/docs/en/r2.2/api_python/nn/mindspore.nn.Cell.html#mindspore.nn.Cell>`_.
+    `Cell <https://www.mindspore.cn/docs/en/master/api_python/nn/mindspore.nn.Cell.html#mindspore.nn.Cell>`_.
 
     CellList can be used like a regular Python list, the Cells it contains have been initialized and
     the types of Cells it contains can not be CellDict.
@@ -366,7 +366,7 @@ class CellList(_CellListBase, Cell):
     def __getitem__(self, index):
         cls_name = self.__class__.__name__
         if isinstance(index, slice):
-            return self.__class__(list(self._cells.values())[index])
+            return CellList(list(self._cells.values())[index])
         if isinstance(index, int):
             index = _valid_index(len(self), index, cls_name)
             return self._cells[str(index)]