mindspore 2.3.0rc1__cp37-none-any.whl → 2.3.0rc2__cp37-none-any.whl

mindspore/mint/nn/functional.py

@@ -0,0 +1,424 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""mint nn functional."""
+from __future__ import absolute_import
+from mindspore.ops.extend import max_pool2d
+from mindspore.ops.functional import (
+    conv_transpose2d,
+    grid_sample
+)
+# 1
+
+# 2
+
+# 3
+
+# 4
+
+# 5
+from mindspore.ops.functional import pad_ext as pad
+# 6
+
+# 7
+
+# 8
+from mindspore.ops.functional import layer_norm
+# 9
+from mindspore.ops.function.nn_func import interpolate_ext as interpolate
+# 10
+
+# 11
+from mindspore.ops.functional import relu
+# 12
+
+# 13
+
+# 14
+from mindspore.ops.function.nn_func import dropout_ext as dropout
+# 15
+
+# 16
+
+# 17
+
+# 18
+
+# 19
+
+# 20
+
+# 21
+
+# 22
+
+# 23
+
+# 24
+
+# 25
+
+# 26
+
+# 27
+
+# 28
+
+# 29
+
+# 30
+
+# 31
+
+# 32
+
+# 33
+
+# 34
+
+# 35
+
+# 36
+from mindspore.ops.functional import gelu
+# 37
+
+# 38
+
+# 39
+from mindspore.ops.functional import group_norm
+# 40
+
+# 41
+
+# 42
+
+# 43
+
+# 44
+
+# 45
+
+# 46
+from mindspore.ops.functional import silu
+# 47
+
+# 48
+
+# 49
+from mindspore.ops.functional import sigmoid
+# 50
+
+# 51
+
+# 52
+from mindspore.ops.functional import embedding
+# 53
+
+# 54
+
+# 55
+
+# 56
+
+# 57
+
+# 58
+
+# 59
+
+# 60
+
+# 61
+
+# 62
+
+# 63
+
+# 64
+
+# 65
+
+# 66
+
+# 67
+
+# 68
+
+# 69
+
+# 70
+
+# 71
+
+# 72
+
+# 73
+
+# 74
+
+# 75
+
+# 76
+
+# 77
+
+# 78
+
+# 79
+
+# 80
+
+# 81
+
+# 82
+
+# 83
+
+# 84
+
+# 85
+
+# 86
+
+# 87
+
+# 88
+
+# 89
+
+# 90
+from mindspore.ops.function.nn_func import avg_pool2d_ext as avg_pool2d
+# 91
+
+# 92
+from mindspore.ops.extend import leaky_relu_ext as leaky_relu
+# 93
+from mindspore.ops.function.nn_func import softplus_ext as softplus
+# 94
+from mindspore.ops.function.math_func import tanh
+# 95
+
+# 96
+
+# 97
+
+# 98
+
+# 99
+
+# 100
+
+__all__ = [
+    'conv_transpose2d',
+    'max_pool2d',
+    # 1
+
+    # 2
+
+    # 3
+
+    # 4
+
+    # 5
+    'pad',
+    # 6
+
+    # 7
+
+    # 8
+    'layer_norm',
+    # 9
+    'interpolate',
+    # 10
+
+    # 11
+    'relu',
+    # 12
+
+    # 13
+
+    # 14
+    'dropout',
+    # 15
+
+    # 16
+
+    # 17
+
+    # 18
+
+    # 19
+
+    # 20
+
+    # 21
+
+    # 22
+
+    # 23
+
+    # 24
+
+    # 25
+
+    # 26
+
+    # 27
+
+    # 28
+
+    # 29
+
+    # 30
+
+    # 31
+
+    # 32
+
+    # 33
+
+    # 34
+
+    # 35
+
+    # 36
+    'gelu',
+    # 37
+
+    # 38
+
+    # 39
+    'group_norm',
+    # 40
+
+    # 41
+
+    # 42
+
+    # 43
+
+    # 44
+
+    # 45
+
+    # 46
+    'silu',
+    # 47
+
+    # 48
+
+    # 49
+    'sigmoid',
+    # 50
+
+    # 51
+
+    # 52
+    'embedding',
+    # 53
+
+    # 54
+
+    # 55
+
+    # 56
+
+    # 57
+
+    # 58
+
+    # 59
+
+    # 60
+
+    # 61
+
+    # 62
+
+    # 63
+
+    # 64
+
+    # 65
+
+    # 66
+
+    # 67
+
+    # 68
+
+    # 69
+
+    # 70
+
+    # 71
+
+    # 72
+
+    # 73
+
+    # 74
+
+    # 75
+
+    # 76
+
+    # 77
+
+    # 78
+
+    # 79
+
+    # 80
+
+    # 81
+
+    # 82
+
+    # 83
+
+    # 84
+
+    # 85
+
+    # 86
+
+    # 87
+
+    # 88
+
+    # 89
+
+    # 90
+    'avg_pool2d',
+    # 91
+    'grid_sample',
+    # 92
+    'leaky_relu',
+    # 93
+    'softplus',
+    # 94
+    'tanh',
+    # 95
+
+    # 96
+
+    # 97
+
+    # 98
+
+    # 99
+
+    # 100
+]

mindspore/mint/optim/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Optimizer.
+
+Provide common optimizers for training, such as AdamW.
+The optimizer is used to calculate and update the gradients.
+"""
+from __future__ import absolute_import
+from mindspore.mint.optim.adamw import AdamW
+
+__all__ = ['AdamW']

mindspore/mint/optim/adamw.py

@@ -0,0 +1,186 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""adamw"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common.parameter import Parameter
+from mindspore.common.tensor import Tensor
+from mindspore.common import dtype as mstype
+from mindspore.ops import auto_generate as gen
+from mindspore.experimental.optim.optimizer import Optimizer
+from mindspore import _checkparam as validator
+
+_optim_adamw_opt = C.MultitypeFuncGraph("optim_adamw_opt")
+hyper_map = C.HyperMap()
+
+
+@_optim_adamw_opt.register("Function", "Float", "Float", "Float", "Float", "Float", "Tensor", "Bool", "Bool", "Tensor",
+                           "Tensor", "Tensor", "Tensor", "Tensor")
+def _run_optim_adamw_opt(opt, beta1, beta2, lr, eps, weight_decay, step, amsgrad, maximize, parameters, grads, exp_avg,
+                         exp_avg_sq, max_exp_avg_sq):
+    """Apply adamw optimizer to the weight parameter."""
+    success = True
+    opt(parameters, exp_avg, exp_avg_sq, max_exp_avg_sq, P.Cast()(grads, F.dtype(parameters)), step, lr, beta1, beta2,
+        weight_decay, eps, amsgrad, maximize)
+    return success
+
+
+def _check_param_value(betas, eps, weight_decay, lr, amsgrad, maximize, prim_name):
+    """Check the type of inputs."""
+    validator.check_value_type('betas', betas, [tuple], prim_name)
+    validator.check("betas size", len(betas), "", [2], validator.IN, prim_name)
+    validator.check_value_type("betas[0]", betas[0], [float], prim_name)
+    validator.check_value_type("betas[1]", betas[1], [float], prim_name)
+    validator.check_value_type("eps", eps, [float], prim_name)
+    validator.check_value_type("weight_decay", weight_decay, [float], prim_name)
+    validator.check_value_type("lr", lr, [float], prim_name)
+    validator.check_value_type("amsgrad", amsgrad, [bool], prim_name)
+    validator.check_value_type("maximize", maximize, [bool], prim_name)
+
+
+class AdamW(Optimizer):
+    r"""
+    Implements Adam Weight Decay algorithm.
+
+    .. math::
+        \begin{aligned}
+            &\textbf{input}      : \gamma \text{(lr)}, \: \beta_1, \beta_2
+                \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)},
+                \: \epsilon \text{ (epsilon)}                                                    \\
+            &\hspace{13mm}      \lambda \text{(weight decay)},  \: \textit{amsgrad},
+                \: \textit{maximize}                                                             \\
+            &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0
+                \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0              \\[-1.ex]
+            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
+            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
+            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})          \\
+            &\hspace{5mm}\textbf{else}                                                           \\
+            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
+            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}         \\
+            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
+            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
+            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
+            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
+            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
+            &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max},
+                \widehat{v_t})                                                                   \\
+            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
+                \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big)                                 \\
+            &\hspace{5mm}\textbf{else}                                                           \\
+            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
+                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
+            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
+       \end{aligned}
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-3``.
+        betas (Tuple[float, float], optional): The exponential decay rate for the moment estimations.
+            Default: ``(0.9, 0.999)``.
+        eps (float, optional): term added to the denominator to improve
+            numerical stability. Default: ``1e-8``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
+        amsgrad (bool, optional): whether to use the AMSGrad algorithm. Default: ``False``.
+
+    Keyword Args:
+        maximize (bool, optional): maximize the params based on the objective, instead of minimizing.
+            Default: ``False``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `eps` is less than 0.0.
+        ValueError: If the `betas` not in the range of [0, 1).
+        ValueError: If the `weight_decay` is less than 0.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.mint import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.AdamW(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=1e-2, amsgrad=False, *, maximize=False):
+        _check_param_value(betas, eps, weight_decay, lr, amsgrad, maximize, self.cls_name)
+        if lr < 0.0:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if eps < 0.0:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        if weight_decay < 0.0:
+            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad,
+                        maximize=maximize)
+        super(AdamW, self).__init__(params, defaults)
+
+        self.exp_avg = self.parameters.clone(prefix="exp_avg", init='zeros')
+        self.exp_avg_sq = self.parameters.clone(prefix="exp_avg_sq", init='zeros')
+        self.max_exp_avg_sq = self.parameters.clone(prefix="max_exp_avg_sq", init='zeros')
+        self.state_step = Parameter(Tensor([0], mstype.float32), "state_step")
+        self.increase_tensor = Tensor(1, mstype.float32)
+        self.assignadd = P.AssignAdd()
+        self.op_cast = P.Cast()
+        self.adamw_opt = gen.AdamWeightDecayExt()
+
+    def construct(self, gradients):
+        self.assignadd(self.state_step, self.increase_tensor)
+        for group_id, group in enumerate(self.param_groups):
+            beta1, beta2 = group['betas']
+            maximize = group.get("maximize")
+            start_id = self.group_start_id[group_id]
+            end_id = self.group_start_id[group_id + 1]
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+            grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+
+            self.hyper_map(F.partial(_optim_adamw_opt, self.adamw_opt, beta1, beta2, float(lr),
+                                     group.get("eps"), group.get("weight_decay"), self.state_step,
+                                     group.get("amsgrad"), maximize),
+                           self.parameters[start_id: end_id], grads, self.exp_avg[start_id: end_id],
+                           self.exp_avg_sq[start_id: end_id], self.max_exp_avg_sq[start_id: end_id])
+        return True

mindspore/multiprocessing/__init__.py

@@ -16,6 +16,7 @@
 mindspore.multiprocessing is a wrapper around the native `multiprocessing` module.
 Some methods are overrode to support fork-based multiprocess.
 """
+import types
 import signal
 import multiprocessing as mp
 from multiprocessing import *
@@ -64,5 +65,8 @@ class Pool(mp.pool.Pool): # pylint: disable=function-redefined, abstract-method
     """
     def Process(self, *args, **kwds):
         if self._ctx.get_start_method() == "fork":
+            # Process() becomes a staticmethod function of Pool with first argument 'ctx' in python 3.8.0 and later
+            if isinstance(super().Process, types.FunctionType):
+                args = args[1:]
             return _MsProcess(*args, **kwds)
         return super().Process(*args, **kwds)

mindspore/nn/__init__.py

@@ -21,6 +21,7 @@ from __future__ import absolute_import
 
 from mindspore.nn import layer, loss, optim, wrap, grad, metrics, probability, sparse, dynamic_lr, reinforcement
 from mindspore.nn.learning_rate_schedule import *
+from mindspore.nn.generator import *
 from mindspore.nn.dynamic_lr import *
 from mindspore.nn.cell import Cell, GraphCell
 from mindspore.nn.layer import *
@@ -31,6 +32,7 @@ from mindspore.nn.wrap import *
 from mindspore.nn.grad import Jvp, Vjp
 from mindspore.nn.sparse import *
 from mindspore.nn.reinforcement import *
+from mindspore.nn import extend
 
 __all__ = ["Cell", "GraphCell"]
 __all__.extend(layer.__all__)
@@ -43,5 +45,6 @@ __all__.extend(sparse.__all__)
 __all__.extend(learning_rate_schedule.__all__)
 __all__.extend(dynamic_lr.__all__)
 __all__.extend(reinforcement.__all__)
+__all__.extend(generator.__all__)
 
 __all__.sort()