mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/experimental/optim/rprop.py

@@ -0,0 +1,164 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""rprop"""
+from __future__ import absolute_import
+
+from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore.common import Tensor, Parameter
+import mindspore.common.dtype as mstype
+from mindspore import _checkparam as validator
+from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than_without_equal
+from mindspore import ops
+from mindspore import jit
+
+_rprop_opt = C.MultitypeFuncGraph("rprop_opt")
+
+op_sign = P.Sign()
+op_fill = P.FillV2()
+op_assign = P.Assign()
+op_assignadd = P.AssignAdd()
+op_cast = P.Cast()
+op_select = P.Select()
+op_oneslike = P.OnesLike()
+
+
+@_rprop_opt.register("Tensor", "Tensor", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
+def _tensor_run_opt(etaminus, etaplus, step_size_min, step_size_max, step, lr, param, prev, step_size, gradient):
+    """Apply rprop optimizer to the weight parameter."""
+    if step == 1:
+        step_size_value = op_oneslike(step_size) * lr
+    else:
+        step_size_value = step_size.value()
+
+    sign = op_sign(gradient * prev)
+
+    sign[sign.gt(0)] = etaplus
+    sign[sign.lt(0)] = etaminus
+    sign[sign.eq(0)] = 1
+
+    step_size_clip = ops.clip_by_value(step_size_value * sign, step_size_min, step_size_max)
+    op_assign(step_size, step_size_clip)
+
+    gradient_update = op_select(sign == etaminus, op_fill(sign.shape, op_cast(0., mstype.float32)), gradient)
+
+    op_assign(prev, gradient_update)
+    next_param = param - op_sign(gradient_update) * step_size_clip
+
+    op_assign(param, next_param)
+
+    return True
+
+
+class Rprop(Optimizer):
+    r"""
+    Implements Rprop algorithm.
+
+    .. warning::
+        This is an experimental optimizer API that is subject to change.
+        This module must be used with lr scheduler module in `LRScheduler Class
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+
+    Args:
+        params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
+            parameter groups.
+        lr (Union[int, float, Tensor], optional): learning rate. Default: ``1e-2``.
+        etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that
+            are multiplicative increase and decrease factors. Default:``(0.5, 1.2)``
+        step_sizes (Tuple[float, float], optional): a pair of minimal and
+            maximal allowed step sizes. Default:``(1e-6, 50)``
+
+    Keyword Args:
+        maximize (bool, optional): maximize the params based on the objective, instead of minimizing.
+            Default: ``False``.
+
+    Inputs:
+        - **gradients** (tuple[Tensor]) - The gradients of `params`.
+
+    Raises:
+        ValueError: If the learning rate is not int, float or Tensor.
+        ValueError: If the learning rate is less than 0.
+        ValueError: If the `etas[1]` is less than or equal to 1.0.
+        ValueError: If the `etas[0]` not in the range of 0-1.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU`` ``CPU``
+
+    Examples:
+        >>> import mindspore
+        >>> from mindspore import nn
+        >>> from mindspore.experimental import optim
+        >>> # Define the network structure of LeNet5. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
+        >>> net = LeNet5()
+        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
+        >>> optimizer = optim.Rprop(net.trainable_params(), lr=0.1)
+        >>> def forward_fn(data, label):
+        ...     logits = net(data)
+        ...     loss = loss_fn(logits, label)
+        ...     return loss, logits
+        >>> grad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)
+        >>> def train_step(data, label):
+        ...     (loss, _), grads = grad_fn(data, label)
+        ...     optimizer(grads)
+        ...     return loss
+    """
+
+    def __init__(self, params, lr=1e-2, etas=(0.5, 1.2), step_sizes=(1e-6, 50), *, maximize=False):
+        check_not_less_than_without_equal(lr, "lr", self.cls_name)
+        check_not_less_than_without_equal(etas[1], "etas[1]", self.cls_name, 1.)
+        validator.check_float_range(etas[0], 0., 1., validator.INC_NEITHER, "etas[0]", self.cls_name)
+
+        defaults = dict(
+            lr=lr,
+            etas=etas,
+            step_sizes=step_sizes,
+            maximize=maximize,
+        )
+        super(Rprop, self).__init__(params, defaults)
+        self.prev = self.parameters.clone(prefix="prev", init='zeros')
+        self.step_size = self.parameters.clone(prefix="step_size", init='zeros')
+        self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
+        self.increase_tensor = Tensor(1, mstype.int32)
+        self.op_cast = P.Cast()
+
+    @jit
+    def implementation(self, etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max):
+        """Extract the common computing part for acceleration"""
+        etaminus, etaplus = op_cast(etaminus, mstype.float32), op_cast(etaplus, mstype.float32)
+        start_id = self.group_start_id[group_id]
+        end_id = self.group_start_id[group_id + 1]
+
+        params = self.parameters[start_id: end_id]
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        prev = self.prev[start_id: end_id]
+        step_size = self.step_size[start_id: end_id]
+        self.hyper_map(F.partial(_rprop_opt, etaminus, etaplus, step_size_min, step_size_max, self.step_t, lr),
+                       params, prev, step_size, grads)
+        return True
+
+    def construct(self, gradients):
+        op_assignadd(self.step_t, self.increase_tensor)
+        for group_id, group in enumerate(self.param_groups):
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
+                lr = self.op_cast(group.get("lr"), mstype.float32)
+            maximize = group.get("maximize")
+
+            etaminus, etaplus = group["etas"]
+            step_size_min, step_size_max = group["step_sizes"]
+
+            self.implementation(etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max)
+
+        return True

mindspore/experimental/optim/sgd.py

@@ -20,16 +20,16 @@ from mindspore.common.tensor import Tensor
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as Validator
 from mindspore.experimental.optim.optimizer import Optimizer
+from mindspore import jit
 
 _sgd_opt = C.MultitypeFuncGraph("sgd_opt")
 
 
-@_sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",)
+@_sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
 def _tensor_run_opt_ext(opt, momentum, learning_rate, gradient, weight, accum, stat):
     """Apply sgd optimizer to the weight parameter using Tensor."""
-    success = True
-    success = F.depend(success, opt(weight, gradient, learning_rate, accum, momentum, stat))
-    return success
+    opt(weight, gradient, learning_rate, accum, momentum, stat)
+    return True
 
 
 class SGD(Optimizer):
@@ -56,14 +56,14 @@ class SGD(Optimizer):
     .. warning::
         This is an experimental optimizer API that is subject to change.
         This module must be used with lr scheduler module in `LRScheduler Class
-        <https://www.mindspore.cn/docs/en/r2.2/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
+        <https://www.mindspore.cn/docs/en/master/api_python/mindspore.experimental.html#lrscheduler-class>`_ .
 
     Args:
         params (Union[list(Parameter), list(dict)]): list of parameters to optimize or dicts defining
             parameter groups.
         lr (Union[int, float, Tensor]): learning rate.
         momentum (Union[int, float], optional): momentum factor. Default: ``0``.
-        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0``.
+        weight_decay (float, optional): weight decay (L2 penalty). Default: ``0.``.
         dampening (Union[int, float], optional): dampening for momentum. Default: ``0``.
         nesterov (bool, optional): enables Nesterov momentum. Default: ``False``.
 
@@ -90,7 +90,7 @@ class SGD(Optimizer):
         >>> from mindspore import nn
         >>> from mindspore.experimental import optim
         >>> # Define the network structure of LeNet5. Refer to
-        >>> # https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
         >>> optimizer = optim.SGD(net.trainable_params(), lr=0.1)
@@ -104,7 +104,8 @@ class SGD(Optimizer):
         ...     optimizer(grads)
         ...     return loss
     """
-    def __init__(self, params, lr, momentum=0, dampening=0, weight_decay=0, nesterov=False, *,
+
+    def __init__(self, params, lr, momentum=0, dampening=0, weight_decay=0.0, nesterov=False, *,
                  maximize=False):
         Validator.check_value_type("lr", lr, [float, int, Tensor], self.cls_name)
         if lr < 0.0:
@@ -130,18 +131,26 @@ class SGD(Optimizer):
         self.stat = self.parameters.clone(prefix="stat", init='ones')
         self.op_cast = P.Cast()
 
+    @jit
+    def implementation(self, momentum, lr, group_id, gradients, maximize, dampening, weight_decay, nesterov):
+        """Extract the common computing part for acceleration"""
+        start_id = self.group_start_id[group_id]
+        end_id = self.group_start_id[group_id + 1]
+        momentum = self.op_cast(momentum, mstype.float32)
+        opt = P.SGD(dampening, weight_decay, nesterov)
+        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        self.hyper_map(F.partial(_sgd_opt, opt, momentum, lr), grads,
+                       self.parameters[start_id: end_id], self.accum[start_id: end_id],
+                       self.stat[start_id: end_id])
+        return True
+
     def construct(self, gradients):
         for group_id, group in enumerate(self.param_groups):
-            opt = P.SGD(group.get("dampening"), group.get("weight_decay"), group.get("nesterov"))
-            lr = group.get("lr")
-            if isinstance(lr, float):
+            lr = self.lrs[group_id]
+            if isinstance(group.get("lr"), float):
                 lr = self.op_cast(group.get("lr"), mstype.float32)
-            maximize = group.get("maximize")
-            momentum = self.op_cast(group.get("momentum"), mstype.float32)
-            start_id = self.group_start_id[group_id]
-            end_id = self.group_start_id[group_id+1]
-            grads = gradients[start_id: end_id] if not maximize else -gradients[start_id: end_id]
-            self.hyper_map(F.partial(_sgd_opt, opt, momentum, lr), grads,
-                           self.parameters[start_id: end_id], self.accum[start_id: end_id],
-                           self.stat[start_id: end_id])
+
+            self.implementation(group.get("momentum"), lr, group_id, gradients, group.get("maximize"),
+                                group.get("dampening"),
+                                group.get("weight_decay"), group.get("nesterov"))
         return True

mindspore/hal/__init__.py

@@ -0,0 +1,40 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""
+The HAL encapsulates interfaces for device management, flow management, and event management.
+MindSpore abstracts the preceding modules from different backends and allows users to schedule hardware
+resources at the Python layer. Currently, these interfaces take effect only in PyNative mode.
+"""
+
+from mindspore.hal.device import is_initialized, is_available, device_count, get_device_capability,\
+                                 get_device_properties, get_device_name, get_arch_list
+from mindspore.hal.stream import Stream, synchronize, set_cur_stream, current_stream, default_stream,\
+                                 StreamCtx
+from mindspore.hal.event import Event
+from mindspore.hal.memory import memory_stats, memory_reserved, max_memory_reserved, empty_cache,\
+                                 reset_peak_memory_stats, memory_summary, memory_allocated,\
+                                 max_memory_allocated, reset_max_memory_reserved, reset_max_memory_allocated
+
+__all__ = [
+    "is_initialized", "is_available", "device_count", "get_device_capability",
+    "get_device_properties", "get_device_name", "get_arch_list",
+    "Event", "Stream", "synchronize", "set_cur_stream", "current_stream", "default_stream", "StreamCtx",
+    "memory_stats", "memory_reserved", "max_memory_reserved", "empty_cache", "reset_peak_memory_stats",
+    "memory_summary", "memory_allocated", "max_memory_allocated", "reset_max_memory_reserved",
+    "reset_max_memory_allocated"
+]
+
+__all__.sort()

mindspore/hal/_ascend.py

@@ -0,0 +1,57 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Hal ascend interfaces."""
+from mindspore._c_expression import ascend_get_device_count, ascend_get_device_name, ascend_get_device_properties
+from mindspore import log as logger
+from ._base import _HalBase
+
+
+class _HalAscend(_HalBase):
+    """
+    Hal ascend interfaces.
+    """
+    def __init__(self):
+        super(_HalAscend, self).__init__("Ascend")
+
+    def device_count(self):
+        """
+        Return Ascend device count.
+        """
+        return ascend_get_device_count()
+
+    def get_device_capability(self, device_id):
+        """
+        Get Ascend capability of the specified device id. Not implemented for Ascend.
+        """
+        logger.warning("'get_device_capability' for Ascend is not implemented. Return None.")
+
+    def get_device_properties(self, device_id):
+        """
+        Get Ascend properties of the specified device id.
+        """
+        return ascend_get_device_properties(device_id)
+
+    def get_device_name(self, device_id):
+        """
+        Get Ascend name of the specified device id.
+        """
+        return ascend_get_device_name(device_id)
+
+    def get_arch_list(self):
+        """
+        Get the architecture list this MindSpore was compiled for. Not implemented for Ascend yet.
+        """
+        logger.warning("'get_arch_list' for Ascend is not implemented. Return None.")

mindspore/hal/_base.py

@@ -0,0 +1,57 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Hal base class."""
+from mindspore._c_expression import MSContext, DeviceContextManager
+
+_context_handle = MSContext.get_instance()
+_device_context_mgr = DeviceContextManager.get_instance()
+
+
+class _HalBase():
+    """
+    Hal base class. Different backends inherit from this class and wrap around hardware interfaces.
+    """
+    def __init__(self, backend=None):
+        valid_backends = ["CPU", "GPU", "Ascend"]
+        if backend not in valid_backends:
+            raise ValueError(f"For '_HalBase', the argument 'backend' must be one of "
+                             f"{valid_backends}, but got {backend}.")
+        self.backend = backend
+
+    def device_count(self):
+        """
+        Return device count of this backend.
+        """
+
+    def get_device_capability(self, device_id):
+        """
+        Get device capability of the specified device id.
+        """
+
+    def get_device_properties(self, device_id):
+        """
+        Get device properties of the specified device id.
+        """
+
+    def get_device_name(self, device_id):
+        """
+        Get device name of the specified device id.
+        """
+
+    def get_arch_list(self):
+        """
+        Get the architecture list this MindSpore was compiled for.
+        """

mindspore/hal/_cpu.py

@@ -0,0 +1,56 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Hal cpu interfaces."""
+from mindspore import log as logger
+from ._base import _HalBase
+
+
+class _HalCPU(_HalBase):
+    """
+    Hal cpu interfaces.
+    """
+    def __init__(self):
+        super(_HalCPU, self).__init__("CPU")
+
+    def device_count(self):
+        """
+        Return CPU device count. Always return 1 for CPU.
+        """
+        return 1
+
+    def get_device_capability(self, device_id):
+        """
+        Get CPU capability of the specified device id. Not implemented for CPU.
+        """
+        logger.warning("'get_device_capability' for CPU is not implemented. Return None.")
+
+    def get_device_properties(self, device_id):
+        """
+        Get CPU properties of the specified device id. Not implemented for CPU.
+        """
+        logger.warning("'get_device_properties' for CPU is not implemented. Return None.")
+
+    def get_device_name(self, device_id):
+        """
+        Get CPU name of the specified device id. Always return "CPU".
+        """
+        return "CPU"
+
+    def get_arch_list(self):
+        """
+        Get the architecture list this MindSpore was compiled for. Not implemented for CPU.
+        """
+        logger.warning("'get_arch_list' for CPU is not implemented. Return None.")

mindspore/hal/_gpu.py

@@ -0,0 +1,57 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Hal gpu interfaces."""
+from mindspore._c_expression import gpu_get_device_count, gpu_get_device_name, gpu_get_device_capability, \
+     gpu_get_device_properties, gpu_get_arch_list
+from ._base import _HalBase
+
+
+class _HalGPU(_HalBase):
+    """
+    Hal gpu interfaces.
+    """
+    def __init__(self):
+        super(_HalGPU, self).__init__("GPU")
+
+    def device_count(self):
+        """
+        Return GPU device count.
+        """
+        return gpu_get_device_count()
+
+    def get_device_capability(self, device_id):
+        """
+        Get GPU capability of the specified device id.
+        """
+        return gpu_get_device_capability(device_id)
+
+    def get_device_properties(self, device_id):
+        """
+        Get GPU properties of the specified device id.
+        """
+        return gpu_get_device_properties(device_id)
+
+    def get_device_name(self, device_id):
+        """
+        Get GPU name of the specified device id.
+        """
+        return gpu_get_device_name(device_id)
+
+    def get_arch_list(self):
+        """
+        Get the architecture list this MindSpore was compiled for.
+        """
+        return gpu_get_arch_list()