mindspore 2.1.0__cp38-none-any.whl → 2.2.0__cp38-none-any.whl

mindspore/nn/dynamic_lr.py

@@ -26,7 +26,8 @@ def piecewise_constant_lr(milestone, learning_rates):
 
     Calculate learning rate by the given `milestone` and `learning_rates`. Let the value of `milestone` be
     :math:`(M_1, M_2, ..., M_t, ..., M_N)` and the value of `learning_rates` be :math:`(x_1, x_2, ..., x_t, ..., x_N)`.
-    N is the length of `milestone`. Let the output learning rate be `y`, then for the i-th step, the formula of
+    N is the length of `milestone`.
+    Let the output learning rate be :math:`y`, then for the :math:`i`-th step, the formula of
     computing decayed_learning_rate[i] is:
 
     .. math::
@@ -59,15 +60,14 @@ def piecewise_constant_lr(milestone, learning_rates):
         >>> # learning_rates = 0.05  if 2 < step <= 5
         >>> # learning_rates = 0.01  if 5 < step <= 10
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     validator.check_value_type('milestone', milestone, (tuple, list))
     validator.check_value_type('learning_rates', learning_rates, (tuple, list))
     if len(milestone) != len(learning_rates):
-        raise ValueError("For 'piecewise_constant_lr', "
-                         "the size of 'milestone' must be same with the size of 'learning_rates', "
-                         "but got 'milestone' size: {}, 'learning_rates' size: {}."
-                         .format(len(milestone), len(learning_rates)))
+        raise ValueError(f"For 'piecewise_constant_lr', "
+                         f"the size of 'milestone' must be same with the size of 'learning_rates', "
+                         f"but got 'milestone' size: {len(milestone)}, 'learning_rates' size: {len(learning_rates)}.")
     lr = []
     last_item = 0
     for i, item in enumerate(milestone):
@@ -137,7 +137,7 @@ def exponential_decay_lr(learning_rate, decay_rate, total_step, step_per_epoch,
         >>> decay_epoch = 1
         >>> lr = nn.exponential_decay_lr(learning_rate, decay_rate, total_step, step_per_epoch, decay_epoch)
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     _check_inputs(learning_rate, decay_rate, total_step, step_per_epoch, decay_epoch, is_stair)
 
@@ -192,7 +192,7 @@ def natural_exp_decay_lr(learning_rate, decay_rate, total_step, step_per_epoch,
         >>> decay_epoch = 2
         >>> lr = nn.natural_exp_decay_lr(learning_rate, decay_rate, total_step, step_per_epoch, decay_epoch, True)
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     _check_inputs(learning_rate, decay_rate, total_step, step_per_epoch, decay_epoch, is_stair)
 
@@ -248,7 +248,7 @@ def inverse_decay_lr(learning_rate, decay_rate, total_step, step_per_epoch, deca
         >>> decay_epoch = 1
         >>> lr = nn.inverse_decay_lr(learning_rate, decay_rate, total_step, step_per_epoch, decay_epoch, True)
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     _check_inputs(learning_rate, decay_rate, total_step, step_per_epoch, decay_epoch, is_stair)
 
@@ -303,11 +303,11 @@ def cosine_decay_lr(min_lr, max_lr, total_step, step_per_epoch, decay_epoch):
         >>> decay_epoch = 2
         >>> lr = nn.cosine_decay_lr(min_lr, max_lr, total_step, step_per_epoch, decay_epoch)
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     if not isinstance(min_lr, float):
-        raise TypeError("For 'cosine_decay_lr', the argument 'min_lr' must be type of float, "
-                        "but got 'min_lr' type: {}.".format(type(min_lr)))
+        raise TypeError(f"For 'cosine_decay_lr', the argument 'min_lr' must be type of float, "
+                        f"but got 'min_lr' type: {type(min_lr)}.")
     validator.check_non_negative_float(min_lr, "min_lr", None)
     validator.check_positive_float(max_lr, 'max_lr')
     validator.check_is_float(max_lr, 'max_lr')
@@ -315,8 +315,8 @@ def cosine_decay_lr(min_lr, max_lr, total_step, step_per_epoch, decay_epoch):
     validator.check_positive_int(step_per_epoch, 'step_per_epoch')
     validator.check_positive_int(decay_epoch, 'decay_epoch')
     if min_lr >= max_lr:
-        raise ValueError("For 'cosine_decay_lr', the 'max_lr' must be greater than the 'min_lr', "
-                         "but got 'max_lr' value: {}, 'min_lr' value: {}.".format(max_lr, min_lr))
+        raise ValueError(f"For 'cosine_decay_lr', the 'max_lr' must be greater than the 'min_lr', "
+                         f"but got 'max_lr' value: {max_lr}, 'min_lr' value: {min_lr}.")
     delta = 0.5 * (max_lr - min_lr)
     lr = []
     for i in range(total_step):
@@ -348,7 +348,7 @@ def polynomial_decay_lr(learning_rate, end_learning_rate, total_step, step_per_e
     .. math::
         tmp\_decay\_epoch = decay\_epoch
 
-    If `update_decay_epoch` is true, update the value of `tmp_decay_epoch` every epoch. The formula is:
+    If `update_decay_epoch` is true, update the value of :math:`tmp\_decay\_epoch` every epoch. The formula is:
 
     .. math::
         tmp\_decay\_epoch = decay\_epoch * ceil(current\_epoch / decay\_epoch)
@@ -385,13 +385,13 @@ def polynomial_decay_lr(learning_rate, end_learning_rate, total_step, step_per_e
         >>> power = 0.5
         >>> lr = nn.polynomial_decay_lr(lr, end_learning_rate, total_step, step_per_epoch, decay_epoch, power)
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     validator.check_positive_float(learning_rate, 'learning_rate')
     validator.check_is_float(learning_rate, 'learning_rate')
     if not isinstance(end_learning_rate, float):
-        raise TypeError("For 'polynomial_decay_lr', the argument 'end_learning_rate' must be type of float, "
-                        "but got 'end_learning_rate' type: {}.".format(type(end_learning_rate)))
+        raise TypeError(f"For 'polynomial_decay_lr', the argument 'end_learning_rate' must be type of float, "
+                        f"but got 'end_learning_rate' type: {type(end_learning_rate)}.")
     validator.check_non_negative_float(end_learning_rate, "end_learning_rate", None)
     validator.check_positive_float(power, 'power')
     validator.check_is_float(power, 'power')
@@ -451,11 +451,11 @@ def warmup_lr(learning_rate, total_step, step_per_epoch, warmup_epoch):
         >>> warmup_epoch = 2
         >>> lr = nn.warmup_lr(learning_rate, total_step, step_per_epoch, warmup_epoch)
         >>> net = nn.Dense(2, 3)
-        >>> optim = nn.SGD(net.trainable_params, learning_rate=lr)
+        >>> optim = nn.SGD(net.trainable_params(), learning_rate=lr)
     """
     if not isinstance(learning_rate, float):
-        raise TypeError("For 'warmup_lr', the argument 'learning_rate' must be type of float, "
-                        "but got 'learning_rate' type: {}.".format(type(learning_rate)))
+        raise TypeError(f"For 'warmup_lr', the argument 'learning_rate' must be type of float, "
+                        f"but got 'learning_rate' type: {type(learning_rate)}.")
     validator.check_non_negative_float(learning_rate, "learning_rate", None)
     validator.check_positive_int(warmup_epoch, 'warmup_epoch')
     validator.check_positive_int(total_step, 'total_step')

mindspore/nn/layer/activation.py

@@ -179,6 +179,7 @@ class Softmax2d(Cell):
 
     Inputs:
         - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` or :math:`(C_{in}, H_{in}, W_{in})`.
+          The input of Softmax with data type of float16 or float32.
 
     Outputs:
         Tensor, which has the same type and shape as `x` with values in the range[0,1].
@@ -287,7 +288,7 @@ class LogSoftmax(Cell):
 
     .. math::
 
-        \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right),
+        \text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right)
 
     Args:
         axis (int): The axis to apply LogSoftmax operation, -1 means the last dimension. Default: ``-1`` .
@@ -393,8 +394,11 @@ class ReLU(Cell):
 
         \text{ReLU}(x) = (x)^+ = \max(0, x),
 
-    It returns element-wise :math:`\max(0, x)`. Specially, the neurons with the negative output
-    will be suppressed and the active neurons will stay the same.
+    It returns element-wise :math:`\max(0, x)`.
+
+    .. note::
+        The neurons with the negative output
+        will be suppressed and the active neurons will stay the same.
 
     ReLU Activation Function Graph:
 
@@ -403,7 +407,7 @@ class ReLU(Cell):
 
     Inputs:
         - **x** (Tensor) - The input of ReLU is a Tensor of any dimension. The data type is `number <https://www.mind
-          spore.cn/docs/en/r2.1/api_python/mindspore.html#mindspore.dtype>`_ .
+          spore.cn/docs/en/r2.2/api_python/mindspore.html#mindspore.dtype>`_ .
 
     Outputs:
         Tensor, with the same type and shape as the `x`.
@@ -444,9 +448,7 @@ class ReLU6(Cell):
 
     .. math::
 
-        Y = \min(\max(0, x), 6).
-
-    The input is a Tensor of any valid shape.
+        Y = \min(\max(0, x), 6)
 
     ReLU6 Activation Function Graph:
 
@@ -454,7 +456,8 @@ class ReLU6(Cell):
         :align: center
 
     Inputs:
-        - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32.
+        - **x** (Tensor) - The input of ReLU6 with data type of float16 or float32 and that
+          is a Tensor of any valid shape.
 
     Outputs:
         Tensor, which has the same type as `x`.
@@ -535,17 +538,10 @@ class LeakyReLU(Cell):
     def __init__(self, alpha=0.2):
         """Initialize LeakyReLU."""
         super(LeakyReLU, self).__init__()
-        validator.check_value_type('alpha', alpha, [float, int], self.cls_name)
-        self.greater_equal = P.GreaterEqual()
-        self.mul = P.Mul()
         self.alpha = alpha
-        self.select_op = P.Maximum()
-        if self.alpha > 1:
-            self.select_op = P.Minimum()
 
     def construct(self, x):
-        alpha_array = P.Cast()(F.scalar_to_tensor(self.alpha), P.DType()(x))
-        out = self.select_op(alpha_array * x, x)
+        out = ops.leaky_relu(x, self.alpha)
         return out
 
 
@@ -937,20 +933,13 @@ class GELU(Cell):
         super(GELU, self).__init__()
         validator.check_bool(approximate, 'approximate', self.cls_name)
         self.approximate = approximate
-        if self.approximate:
-            self.gelu = P.GeLU()
+        if approximate:
+            self.approximate = 'tanh'
         else:
-            self.erf = P.Erf()
-            self.sqrt = P.Sqrt()
-            self.const0 = Tensor(0.5, mstype.float32)
-            self.const1 = Tensor(1.0, mstype.float32)
-            self.const2 = Tensor(2.0, mstype.float32)
+            self.approximate = 'none'
 
     def construct(self, x):
-        if self.approximate:
-            return self.gelu(x)
-        return x * F.cast(self.const0, x.dtype) * (F.cast(self.const1, x.dtype) + \
-                                                   self.erf(x / self.sqrt(F.cast(self.const2, x.dtype))))
+        return ops.gelu(x, approximate=self.approximate)
 
 
 class FastGelu(Cell):
@@ -1344,10 +1333,14 @@ class LRN(Cell):
     r"""
     Local Response Normalization.
 
+    .. warning::
+        LRN is deprecated on Ascend due to potential accuracy problem. It's recommended to use other
+        normalization methods, e.g. :class:`mindspore.nn.BatchNorm`.
+
     Refer to :func:`mindspore.ops.lrn` for more details.
 
     Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+        ``GPU`` ``CPU``
 
     Examples:
         >>> import mindspore

mindspore/nn/layer/basic.py

@@ -230,7 +230,7 @@ class Dropout1d(Cell):
           int64, float16, float32 or float64.
 
     Outputs:
-        Tensor, output, with the same shape and data type as `x`.
+        Tensor, has the same shape and data type as `x`.
 
     Raises:
         TypeError: If `x` is not a Tensor.
@@ -314,7 +314,7 @@ class Dropout2d(Cell):
         return out
 
     def extend_repr(self):
-        return 'p={}'.format(self.keep_prob)
+        return f"p={self.keep_prob}"
 
 
 class Dropout3d(Cell):
@@ -365,7 +365,7 @@ class Dropout3d(Cell):
         return out
 
     def extend_repr(self):
-        return 'p={}'.format(self.keep_prob)
+        return f'p={self.keep_prob}'
 
 
 class Upsample(Cell):
@@ -533,6 +533,7 @@ class Dense(Cell):
         activation (Union[str, Cell, Primitive, None]): activate function applied to the output of the fully connected
             layer. Both activation name, e.g. 'relu', and mindspore activation function, e.g. mindspore.ops.ReLU(),
             are supported. Default: ``None`` .
+        dtype (:class:`mindspore.dtype`): Data type of Parameter. Default: ``mstype.float32`` .
 
     Inputs:
         - **x** (Tensor) - Tensor of shape :math:`(*, in\_channels)`. The `in_channels` in `Args` should be equal
@@ -571,7 +572,8 @@ class Dense(Cell):
                  weight_init=None,
                  bias_init=None,
                  has_bias=True,
-                 activation=None):
+                 activation=None,
+                 dtype=mstype.float32):
         """Initialize Dense."""
         super(Dense, self).__init__()
         self.in_channels = Validator.check_positive_int(
@@ -593,7 +595,7 @@ class Dense(Cell):
         if weight_init is None:
             weight_init = HeUniform(math.sqrt(5))
         self.weight = Parameter(initializer(
-            weight_init, [out_channels, in_channels]), name="weight")
+            weight_init, [out_channels, in_channels], dtype=dtype), name="weight")
 
         self.bias = None
         if self.has_bias:
@@ -606,7 +608,7 @@ class Dense(Cell):
                 bound = 1 / math.sqrt(in_channels)
                 bias_init = Uniform(scale=bound)
             self.bias = Parameter(initializer(
-                bias_init, [out_channels]), name="bias")
+                bias_init, [out_channels], dtype=dtype), name="bias")
             self.bias_add = P.BiasAdd()
 
         self.matmul = P.MatMul(transpose_b=True)
@@ -633,12 +635,11 @@ class Dense(Cell):
         return x
 
     def extend_repr(self):
-        s = 'input_channels={}, output_channels={}'.format(
-            self.in_channels, self.out_channels)
+        s = f'input_channels={self.in_channels}, output_channels={self.out_channels}'
         if self.has_bias:
-            s += ', has_bias={}'.format(self.has_bias)
+            s += f', has_bias={self.has_bias}'
         if self.activation_flag:
-            s += ', activation={}'.format(self.activation)
+            s += f', activation={self.activation}'
         return s
 
 
@@ -751,7 +752,7 @@ class Norm(Cell):
         return x
 
     def extend_repr(self):
-        return 'axis={}, keep_dims={}'.format(self.axis, self.keep_dims)
+        return f'axis={self.axis}, keep_dims={self.keep_dims}'
 
 
 class OneHot(Cell):
@@ -1041,7 +1042,7 @@ class Unfold(Cell):
 
     Raises:
         TypeError: If `ksizes`, `strides` or `rates` is neither a tuple nor list.
-        ValueError: If shape of `ksizes`, `strides` or `rates` is not :math:`(1, x_row, x_col, 1)`.
+        ValueError: If shape of `ksizes`, `strides` or `rates` is not :math:`(1, x\_row, x\_col, 1)`.
         ValueError: If the second and third element of `ksizes`, `strides` or `rates` is less than 1.
 
     Supported Platforms:
@@ -1076,7 +1077,8 @@ class Unfold(Cell):
             if len(arg_val) != 4 or arg_val[0] != 1 or arg_val[3] != 1:
                 raise ValueError(f"For '{prim_name}' the format of '{arg_name}s' must be [1, {arg_name}_row, "
                                  f"{arg_name}_col, 1], but got {arg_val}.")
-            if not isinstance(arg_val[1], int) or not isinstance(arg_val[2], int) or arg_val[1] < 1 or arg_val[2] < 1:
+            is_int = isinstance(arg_val[1], int) and isinstance(arg_val[2], int)
+            if not is_int or arg_val[1] < 1 or arg_val[2] < 1:
                 raise ValueError(f"For '{prim_name}' the {arg_name}_row and {arg_name}_col in '{arg_name}s' must be "
                                  f"an positive integer number, but got {arg_name}_row is {arg_val[1]}, "
                                  f"{arg_name}_col is {arg_val[2]}")

mindspore/nn/layer/channel_shuffle.py

@@ -73,7 +73,7 @@ class ChannelShuffle(Cell):
         """Initialize ChannelShuffle."""
         super(ChannelShuffle, self).__init__()
         if not isinstance(groups, int):
-            raise TypeError("For ChannelShuffle, the param `groups` must be int, but got {}.".format(type(groups)))
+            raise TypeError(f"For ChannelShuffle, the param `groups` must be int, but got {type(groups)}.")
         if groups < 1:
             raise ValueError(f"For ChannelShuffle, the param `groups` must be larger than 0, but got {groups}.")