mindspore 2.3.0__cp39-cp39-win_amd64.whl → 2.4.1__cp39-cp39-win_amd64.whl

mindspore/mint/distributed/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Collective communication interface.
+
+Note that the APIs in the following list need to preset communication environment variables.
+
+For Ascend devices, it is recommended to use the msrun startup method
+without any third-party or configuration file dependencies.
+Please see the `msrun start up
+<https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
+for more details.
+"""
+from mindspore.mint.distributed.distributed import init_process_group, destroy_process_group, get_rank, get_world_size
+
+
+__all__ = [
+    "init_process_group", "destroy_process_group", "get_rank", "get_world_size"
+]

mindspore/mint/distributed/distributed.py

@@ -0,0 +1,254 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Communication management API"""
+from mindspore import log as logger
+from mindspore.communication._comm_helper import _destroy_group_helper, GlobalComm, _get_rank_helper, _get_size_helper
+from mindspore.communication import init, release, get_group_size
+
+
+def init_process_group(backend="hccl",
+                       init_method=None,
+                       timeout=None,
+                       world_size=-1,
+                       rank=-1,
+                       store=None,
+                       pg_options=None,
+                       device_id=None):
+    """
+    Init collective communication lib. And create a default collective communication group.
+
+    Note:
+        This method isn't supported in GPU and CPU versions of MindSpore.
+        In Ascend hardware platforms, this API should be set before the definition of any Tensor and Parameter,
+        and the instantiation and execution of any operation and net.
+
+    Args:
+        backend (str, optional): The backend to ues. default is hccl and now only support hccl.
+        init_method (str, invalid): URL specifying how to init collective communication group. Provides parameters
+                                    consistent with pytorch, but is not currently support, setting is invalid.
+        timeout (timedelta, invalid): Timeout for API executed. Provides parameters consistent with pytorch, but is not
+                                      currently support, setting is invalid.
+        world_size (int, optional): Number of the processes participating in the job.
+        rank (int, invalid): Rank of the current process. Provides parameters consistent with pytorch, but is not
+                             currently support, setting is invalid.
+        store (Store, invalid): Key/Value store accessible to all workers, used to exchange connection/address
+                                information. Provides parameters consistent with pytorch, but is not currently support,
+                                setting is invalid.
+        pg_options (ProcessGroupOptions, invalid): process group options specifying what additional options need to be
+                                                  passed in during the construction of specific process group. Provides
+                                                  parameters consistent with pytorch, but is not currently support,
+                                                  setting is invalid.
+        device_id (int, invalid): the device id to exeute. Provides parameters consistent with pytorch, but is not
+                                  currently support, setting is invalid.
+
+    Raises:
+        ValueError: If `backend` is not hccl.
+        ValueError: If `world_size` is not equal to -1 or process group number.
+        RuntimeError: If device target is invalid, or backend is invalid, or distributed initialization fails,
+                      or the environment variables RANK_ID/MINDSPORE_HCCL_CONFIG_PATH
+                      have not been exported when backend is HCCL.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start up
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
+            for more details.
+
+        >>> import mindspore as ms
+        >>> from mindspore import set_context
+        >>> from mindspore.mint.distributed import init_process_group, destroy_process_group
+        >>> set_context(device_target="Ascend")
+        >>> init_process_group()
+        >>> destroy_process_group()
+    """
+    if init_method is not None:
+        logger.warning("init_method is ignored, setting is invalid")
+    if timeout is not None:
+        logger.warning("timeout is ignored, setting is invalid")
+    if store is not None:
+        logger.warning("store is ignored, setting is invalid")
+    if pg_options is not None:
+        logger.warning("pg_options is ignored, setting is invalid")
+    if device_id is not None:
+        logger.warning("device_id is ignored, setting is invalid")
+    if rank != -1:
+        logger.warning("rank is ignored, setting is invalid")
+    if backend != "hccl":
+        raise ValueError("Only support hccl now, please setting backend to hccl or using default value")
+
+    #init hccl & create world group
+    init(backend)
+
+    if world_size != -1 and world_size != get_group_size():
+        raise ValueError("world_size is wrong, please using default value or setting: ", get_group_size())
+
+
+def destroy_process_group(group=None):
+    """
+    Destroy the user collective communication group.
+    If group is None or "hccl_world_group", Destroy all group and release collective communication lib.
+
+    Note:
+        This method isn't supported in GPU and CPU versions of MindSpore.
+        This method should be used after init_process_group().
+
+    Args:
+        group (str): The communication group to destroy, the group should be created by init_process_group or new_group.
+
+    Raises:
+        TypeError: If group is not a string.
+        RuntimeError: If HCCL is not available or MindSpore is GPU/CPU version.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start up
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
+            for more details.
+
+        >>> import mindspore as ms
+        >>> from mindspore import set_context
+        >>> from mindspore.mint.distributed import init_process_group, destroy_process_group
+        >>> set_context(device_target="Ascend")
+        >>> init_process_group()
+        >>> destroy_process_group()
+    """
+
+    if group == GlobalComm.WORLD_COMM_GROUP or group is None:
+        release()
+    elif not isinstance(group, str):
+        raise TypeError("For 'destroy_group', the argument 'group' must be type of string or None, "
+                        "but got 'group' type : {}.".format(type(group)))
+    else:
+        _destroy_group_helper(group)
+
+
+def get_rank(group=None):
+    """
+    Get the rank ID for the current device in the specified collective communication group.
+
+    Note:
+        This method should be used after init().
+
+    Args:
+        group (str): The communication group to work on. Normally, the group should be created by create_group,
+                     otherwise, using the default group. If None, ``GlobalComm.WORLD_COMM_GROUP`` will be used.
+
+    Returns:
+        int, the rank ID of the calling process within the group.
+        return -1, if not part of the group
+
+    Raises:
+        TypeError: If group is not a string.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start up
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
+            for more details.
+
+        >>> from mindspore import set_context
+        >>> from mindspore.mint.distributed import init_process_group, get_rank
+        >>> set_context(device_target="Ascend")
+        >>> init_process_group()
+        >>> rank_id = get_rank()
+        >>> print(rank_id)
+        >>> # the result is the rank_id in world_group
+    """
+    if group is None:
+        group = GlobalComm.WORLD_COMM_GROUP
+    if not isinstance(group, str):
+        raise TypeError("For 'get_rank', the argument 'group' must be type of string, "
+                        "but got 'group' type : {}.".format(type(group)))
+    try:
+        ret = _get_rank_helper(group)
+    except RuntimeError as e:
+        logger.warning(e)
+        ret = -1
+    return ret
+
+
+def get_world_size(group=None):
+    """
+    Get the rank size of the specified collective communication group.
+
+    Note:
+        This method should be used after init().
+
+    Args:
+        group (str): The communication group to work on. Normally, the group should be created by create_group,
+                     otherwise, using the default group. If None, ``GlobalComm.WORLD_COMM_GROUP`` will be used.
+
+    Returns:
+        int, the rank size of the group.
+        return -1, if the group is not available.
+
+    Raises:
+        TypeError: If group is not a string.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start up
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
+            for more details.
+
+        >>> import mindspore as ms
+        >>> from mindspore import set_context
+        >>> from mindspore.mint.distributed import init_process_group, get_world_size
+        >>> set_context(device_target="Ascend")
+        >>> init_process_group()
+        >>> group_size = get_world_size()
+        >>> print("group_size is: ", group_size)
+        group_size is: 8
+    """
+    ret = -1
+    if group is None:
+        group = GlobalComm.WORLD_COMM_GROUP
+    if not isinstance(group, str):
+        raise TypeError("For 'get_group_size', the argument 'group' must be type of string, "
+                        "but got 'group' type : {}.".format(type(group)))
+    try:
+        ret = _get_size_helper(group)
+    except RuntimeError as e:
+        logger.warning(e)
+        ret = -1
+    return ret

mindspore/mint/nn/__init__.py

@@ -18,16 +18,17 @@ Neural Networks Cells.
 Predefined building blocks or computing units to construct neural networks.
 """
 from __future__ import absolute_import
+import mindspore.ops as ops
+from mindspore.mint.nn import functional as F
 from mindspore.nn.cell import Cell
-from mindspore.nn.extend import *
-from mindspore.nn.extend import basic, embedding
-from mindspore.nn.extend import MaxPool2d
+from mindspore.nn import EmbeddingExt as Embedding, MaxPool2dExt as MaxPool2d, LayerNormExt as LayerNorm, Linear
+
 # 1
 
 # 2
 
 # 3
-
+from mindspore.nn.layer.basic import Identity
 # 4
 
 # 5
@@ -37,13 +38,13 @@ from mindspore.nn.layer.basic import UnfoldExt as Unfold
 # 7
 from mindspore.nn.layer.basic import Fold
 # 8
-from mindspore.nn.extend.layer import normalization
-from mindspore.nn.extend.layer.normalization import *
+from mindspore.nn.layer.activation import SoftmaxExt as Softmax
 # 9
 from mindspore.nn.layer.basic import UpsampleExt as Upsample
 # 10
 
 # 11
+from mindspore.nn.layer import ReLU
 
 # 12
 
@@ -54,11 +55,11 @@ from mindspore.nn.layer.basic import DropoutExt as Dropout
 # 15
 
 # 16
-
+from mindspore.nn.layer import LogSoftmaxExt as LogSoftmax
 # 17
 
 # 18
-
+from mindspore.nn.layer import PReLUExt as PReLU
 # 19
 
 # 20
@@ -98,11 +99,12 @@ from mindspore.nn.layer.basic import DropoutExt as Dropout
 # 37
 
 # 38
-from mindspore.nn.extend.basic import Linear
+
 # 39
 
 # 40
-
+from mindspore.mint.nn.layer.normalization import GroupNorm
+from mindspore.mint.nn.layer.normalization import LayerNorm
 # 41
 
 # 42
@@ -114,6 +116,7 @@ from mindspore.nn.extend.basic import Linear
 # 45
 
 # 46
+from mindspore.mint.nn.layer.activation import SiLU, LogSigmoid
 
 # 47
 
@@ -220,9 +223,38 @@ from mindspore.nn.extend.basic import Linear
 # 98
 
 # 99
-
+from mindspore.nn.layer import AvgPool2dExt as AvgPool2d
 # 100
-from mindspore.ops.auto_generate import BCEWithLogitsLoss as BCEWithLogitsLoss_prim
+from mindspore.nn.layer import SoftShrink as Softshrink
+# 159
+
+# 220
+from mindspore.nn.layer import HShrink as Hardshrink
+# 221
+from mindspore.nn.layer import HSigmoid as Hardsigmoid
+# 222
+from mindspore.nn.layer import HSwish as Hardswish
+# 238
+from mindspore.nn.loss import L1LossExt as L1Loss
+
+# 257
+
+# 258
+from mindspore.ops.function.nn_func import mse_loss_ext
+
+
+# 674
+from mindspore.mint.nn.layer.normalization import BatchNorm1d
+
+# 675
+from mindspore.mint.nn.layer.normalization import BatchNorm2d
+
+# 676
+from mindspore.mint.nn.layer.normalization import BatchNorm3d
+
+from mindspore.mint.nn.layer.pooling import AdaptiveAvgPool1d
+
+from mindspore.mint.nn.layer.pooling import AdaptiveAvgPool2d
 
 
 class BCEWithLogitsLoss(Cell):
@@ -294,9 +326,10 @@ class BCEWithLogitsLoss(Cell):
         >>> print(output)
         0.3463612
     """
+
     def __init__(self, weight=None, reduction='mean', pos_weight=None):
         super(BCEWithLogitsLoss, self).__init__()
-        self.bce_with_logits = BCEWithLogitsLoss_prim(reduction)
+        self.bce_with_logits = ops.auto_generate.BCEWithLogitsLoss(reduction)
         self.weight = weight
         self.pos_weight = pos_weight
 
@@ -304,14 +337,199 @@ class BCEWithLogitsLoss(Cell):
         out = self.bce_with_logits(input, target, self.weight, self.pos_weight)
         return out
 
+
+class SELU(Cell):
+    r"""
+    Activation function SELU (Scaled exponential Linear Unit).
+
+    Refer to :func:`mindspore.mint.nn.functional.selu` for more details.
+
+    SELU Activation Function Graph:
+
+    .. image:: ../images/SeLU.png
+        :align: center
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, mint
+        >>> import numpy as np
+        >>> input = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
+        >>> selu = mint.nn.SELU()
+        >>> output = selu(input)
+        >>> print(output)
+        [[-1.1113307 4.202804 -1.7575096]
+        [ 2.101402 -1.7462534 9.456309 ]]
+    """
+
+    def __init__(self):
+        """Initialize SELU"""
+        super(SELU, self).__init__()
+
+    def construct(self, input):
+        return F.selu(input)
+
+
+class GELU(Cell):
+    r"""
+    Activation function GELU (Gaussian Error Linear Unit).
+
+    Refer to :func:`mindspore.mint.nn.functional.gelu` for more details.
+
+    GELU Activation Function Graph:
+
+    .. image:: ../images/GELU.png
+        :align: center
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, mint
+        >>> import numpy as np
+        >>> input = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
+        >>> gelu = mint.nn.GELU()
+        >>> output = gelu(input)
+        >>> print(output)
+        [[-1.5880802e-01  3.9999299e+00 -3.1077917e-21]
+         [ 1.9545976e+00 -2.2918017e-07  9.0000000e+00]]
+        >>> gelu = mint.nn.GELU(approximate=False)
+        >>> # CPU not support "approximate=False", using "approximate=True" instead
+        >>> output = gelu(input)
+        >>> print(output)
+        [[-1.5865526e-01  3.9998732e+00 -0.0000000e+00]
+         [ 1.9544997e+00 -1.4901161e-06  9.0000000e+00]]
+    """
+
+    def __init__(self):
+        """Initialize GELU"""
+        super(GELU, self).__init__()
+
+    def construct(self, input):
+        return F.gelu(input)
+
+
+class Mish(Cell):
+    r"""
+    Computes MISH (A Self Regularized Non-Monotonic Neural Activation Function)
+    of input tensors element-wise.
+
+    Refer to :func:`mindspore.mint.nn.functional.mish` for more details.
+
+    Mish Activation Function Graph:
+
+    .. image:: ../images/Mish.png
+        :align: center
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, mint
+        >>> import numpy as np
+        >>> x = Tensor(np.array([[-1.1, 4.0, -8.0], [2.0, -5.0, 9.0]]), mindspore.float32)
+        >>> mish = mint.nn.Mish()
+        >>> output = mish(x)
+        >>> print(output)
+        [[-3.0764845e-01 3.9974124e+00 -2.6832507e-03]
+         [ 1.9439589e+00 -3.3576239e-02 8.9999990e+00]]
+    """
+
+    def __init__(self):
+        """Initialize Mish."""
+        super(Mish, self).__init__()
+
+    def construct(self, input):
+        return F.mish(input)
+
+
+class MSELoss(Cell):
+    r"""
+    Calculates the mean squared error between the predicted value and the label value.
+
+    For simplicity, let :math:`x` and :math:`y` be 1-dimensional Tensor with length :math:`N`,
+    the unreduced loss (i.e. with argument reduction set to 'none') of :math:`x` and :math:`y` is given as:
+
+    .. math::
+        \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad \text{with} \quad l_n = (x_n - y_n)^2.
+
+    where :math:`N` is the batch size. If `reduction` is not ``'none'``, then:
+
+    .. math::
+        \ell(x, y) =
+        \begin{cases}
+            \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
+            \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
+        \end{cases}
+
+    Args:
+        reduction (str, optional): Apply specific reduction method to the output: ``'none'`` , ``'mean'`` ,
+            ``'sum'`` . Default: ``'mean'`` .
+
+            - ``'none'``: no reduction will be applied.
+            - ``'mean'``: compute and return the mean of elements in the output.
+            - ``'sum'``: the output elements will be summed.
+
+    Inputs:
+        - **logits** (Tensor) - The predicted value of the input. Tensor of any dimension.
+          The data type needs to be consistent with the `labels`. It should also be broadcastable with the `labels`.
+        - **labels** (Tensor) - The input label. Tensor of any dimension.
+          The data type needs to be consistent with the `logits`. It should also be broadcastable with the `logits`.
+
+    Outputs:
+        - Tensor. If `reduction` is ``'mean'`` or ``'sum'``, the shape of output is `Tensor Scalar`.
+        - If reduction is ``'none'``, the shape of output is the broadcasted shape of `logits` and `labels` .
+
+    Raises:
+        ValueError: If `reduction` is not one of ``'mean'``, ``'sum'`` or ``'none'``.
+        ValueError: If `logits` and `labels` are not broadcastable.
+        TypeError: If `logits` and `labels` are in different data type.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import Tensor, nn
+        >>> import numpy as np
+        >>> # Case 1: logits.shape = labels.shape = (3,)
+        >>> loss = nn.MSELoss()
+        >>> logits = Tensor(np.array([1, 2, 3]), mindspore.float32)
+        >>> labels = Tensor(np.array([1, 1, 1]), mindspore.float32)
+        >>> output = loss(logits, labels)
+        >>> print(output)
+        1.6666667
+        >>> # Case 2: logits.shape = (3,), labels.shape = (2, 3)
+        >>> loss = nn.MSELoss(reduction='none')
+        >>> logits = Tensor(np.array([1, 2, 3]), mindspore.float32)
+        >>> labels = Tensor(np.array([[1, 1, 1], [1, 2, 2]]), mindspore.float32)
+        >>> output = loss(logits, labels)
+        >>> print(output)
+        [[0. 1. 4.]
+         [0. 0. 1.]]
+    """
+
+    def __init__(self, reduction='mean'):
+        super(MSELoss, self).__init__()
+        self.mse_loss = mse_loss_ext
+        self.reduction = reduction
+
+    def construct(self, input, target):
+        out = self.mse_loss(input, target, self.reduction)
+        return out
+
+
 __all__ = [
-    'MaxPool2d',
     # 1
     'BCEWithLogitsLoss',
     # 2
 
     # 3
-
+    'Identity',
     # 4
 
     # 5
@@ -321,12 +539,13 @@ __all__ = [
     # 7
     'Unfold',
     # 8
-
+    'Softmax',
     # 9
     'Upsample',
     # 10
 
     # 11
+    'ReLU',
 
     # 12
 
@@ -337,11 +556,11 @@ __all__ = [
     # 15
 
     # 16
-
+    'LogSoftmax',
     # 17
 
     # 18
-
+    'PReLU',
     # 19
 
     # 20
@@ -385,6 +604,7 @@ __all__ = [
     # 39
 
     # 40
+    'GroupNorm',
 
     # 41
 
@@ -397,6 +617,7 @@ __all__ = [
     # 45
 
     # 46
+    'SiLU',
 
     # 47
 
@@ -497,16 +718,40 @@ __all__ = [
     # 95
 
     # 96
+    'AdaptiveAvgPool1d',
 
     # 97
+    'AdaptiveAvgPool2d',
 
     # 98
 
     # 99
-
+    'AvgPool2d',
     # 100
+    'SELU',
+    # 159
+    'GELU',
+    # 220
+    'Hardshrink',
+    # 221
+    'Hardsigmoid',
+    # 222
+    'Hardswish',
+    # 238
+    'L1Loss',
+    # 267
+    'Mish',
+    # 258
+    'MSELoss',
+    # 259
+
+    # 556
+    'LogSigmoid',
+
+    # 674
+    'BatchNorm1d',
+    # 675
+    'BatchNorm2d',
+    # 676
+    'BatchNorm3d',
 ]
-
-__all__.extend(basic.__all__)
-__all__.extend(embedding.__all__)
-__all__.extend(normalization.__all__)