mindspore 2.6.0rc1__cp310-cp310-win_amd64.whl → 2.7.0rc1__cp310-cp310-win_amd64.whl

mindspore/experimental/optim/lr_scheduler.py

@@ -1,16 +1,6 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/lr_scheduler.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """LRScheduler."""
 from collections import Counter
@@ -20,8 +10,6 @@ from mindspore import ops, Tensor, Parameter
 from mindspore.experimental.optim.optimizer import Optimizer
 from mindspore.common.api import jit_class
 import mindspore.common.dtype as mstype
-from mindspore.ops import functional as F
-from mindspore.ops import operations as P
 from mindspore import _checkparam as Validator
 
 __all__ = ['StepLR', 'LinearLR', 'LRScheduler', 'ExponentialLR', 'PolynomialLR',
@@ -143,9 +131,12 @@ class LRScheduler:
 
 @jit_class
 class StepLR(LRScheduler):
-    """Decays the learning rate of each parameter group by gamma every
-    step_size epochs. Notice that such decay can happen simultaneously with
-    other changes to the learning rate from outside this scheduler.
+    """
+    During training, when calling `StepLR.step()` , if the current epoch number is an integer multiple of `step_size` ,
+    the learning rate will be decayed by multiplying it with `gamma` . The adjustment of the learning rate and
+    the parameter update of the optimizer are synergistically performed. The optimizer executes parameter optimization
+    operations based on the currently adjusted learning rate. The learning rate decay of StepLR may occur simultaneously
+    with external changes to the learning rate.
 
     .. warning::
         This is an experimental lr scheduler module that is subject to change.
@@ -431,8 +422,8 @@ class PolynomialLR(LRScheduler):
             raise TypeError(f"For 'PolynomialLR', the type of total_iters must be int, but got {type(total_iters)}.")
         self.total_iters = total_iters
         self.power = power
-        self.min = P.Minimum()
-        self.cast = P.Cast()
+        self.min = ops.Minimum()
+        self.cast = ops.Cast()
         super(PolynomialLR, self).__init__(optimizer, last_epoch)
 
     def get_lr(self):
@@ -804,7 +795,7 @@ class SequentialLR:
 
 @jit_class
 class ReduceLROnPlateau:
-    """
+    r"""
     Reduce learning rate when a metric has stopped improving.
     Models often benefit from reducing the learning rate by a factor
     of 2-10 once learning stagnates. The scheduler reads the metrics `metrics` during execution
@@ -886,7 +877,7 @@ class ReduceLROnPlateau:
         [Tensor(shape=[], dtype=Float32, value= 0.001)]
         [Tensor(shape=[], dtype=Float32, value= 0.001)]
         [Tensor(shape=[], dtype=Float32, value= 0.0001)]
-        """
+    """
 
     def __init__(self, optimizer, mode='min', factor=0.1, patience=10,
                  threshold=1e-4, threshold_mode='rel', cooldown=0,
@@ -915,8 +906,8 @@ class ReduceLROnPlateau:
         self.cooldown_counter = 0
         self.eps = eps
         self.mode_worse = None
-        self.assign = P.Assign()
-        self.cast = P.Cast()
+        self.assign = ops.Assign()
+        self.cast = ops.Cast()
         self.last_epoch = Parameter(Tensor(0, dtype=mstype.int32),
                                     name='last_epoch_' + self.__class__.__name__)
 
@@ -1079,17 +1070,8 @@ class CyclicLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.010225)]
     """
 
-    def __init__(self,
-                 optimizer,
-                 base_lr,
-                 max_lr,
-                 step_size_up=2000,
-                 step_size_down=None,
-                 mode='triangular',
-                 gamma=1.,
-                 scale_fn=None,
-                 scale_mode='cycle',
-                 last_epoch=-1):
+    def __init__(self, optimizer, base_lr, max_lr, step_size_up=2000, step_size_down=None, mode='triangular',
+                 gamma=1.0, scale_fn=None, scale_mode='cycle', last_epoch=-1):
 
         base_lrs = self._preprocess_input_param(optimizer, base_lr, 'base_lr')
 
@@ -1117,7 +1099,7 @@ class CyclicLR(LRScheduler):
         self._scale_fn_custom = scale_fn
         self.scale_mode = scale_mode
         self._init_scale_fn()
-        self.floor = P.Floor()
+        self.floor = ops.Floor()
 
         super(CyclicLR, self).__init__(optimizer, last_epoch)
         self.base_lrs = [Tensor(lr) for lr in base_lrs]
@@ -1252,12 +1234,12 @@ class CosineAnnealingWarmRestarts(LRScheduler):
         self.zero_tensor = Tensor(0, mstype.int32)
 
         self.math_pi = math.pi
-        self.cos = P.Cos()
-        self.cast = P.Cast()
-        self.log = P.Log()
-        self.cast = P.Cast()
-        self.assign = P.Assign()
-        self.floor = P.Floor()
+        self.cos = ops.Cos()
+        self.cast = ops.Cast()
+        self.log = ops.Log()
+        self.cast = ops.Cast()
+        self.assign = ops.Assign()
+        self.floor = ops.Floor()
         self._last_lr = [group["lr"] for group in optimizer.param_groups]
         super(CosineAnnealingWarmRestarts, self).__init__(optimizer, last_epoch)
 
@@ -1306,7 +1288,7 @@ class CosineAnnealingWarmRestarts(LRScheduler):
 
         for i, data in enumerate(zip(self.optimizer.param_groups, self.get_lr())):
             _, lr = data
-            F.assign(self.optimizer.param_groups[i]["lr"], lr)
+            ops.assign(self.optimizer.param_groups[i]["lr"], lr)
 
 
 @jit_class
@@ -1371,8 +1353,8 @@ class CosineAnnealingLR(LRScheduler):
         self.T_max = T_max
         self.eta_min = eta_min
         self.math_pi = math.pi
-        self.cos = P.Cos()
-        self.cast = P.Cast()
+        self.cos = ops.Cos()
+        self.cast = ops.Cast()
         super(CosineAnnealingLR, self).__init__(optimizer, last_epoch)
 
     def get_lr(self):

mindspore/experimental/optim/nadam.py

@@ -1,30 +1,20 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/nadam.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """nadam"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 from mindspore.common import Parameter, Tensor
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as validator
 from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
 from mindspore import jit
 
-_nadam_opt = C.MultitypeFuncGraph("nadam_opt")
+_nadam_opt = ops.MultitypeFuncGraph("nadam_opt")
 
-op_sqrt = P.Sqrt()
+op_sqrt = ops.Sqrt()
 
 
 @_nadam_opt.register("Number", "Number", "Number", "Number", "Tensor", "Tensor", "Tensor",
@@ -34,15 +24,15 @@ def _tensor_run_opt(beta1, beta2, momentum_decay, eps, step_t, lr, param, grad,
     bias_correction2 = 1 - beta2 ** step_t
     mu = beta1 * (1. - 0.5 * (0.96 ** (step_t * momentum_decay)))
     mu_next = beta1 * (1. - 0.5 * (0.96 ** ((step_t + 1) * momentum_decay)))
-    F.assign(mu_product, mu_product * mu)
-    F.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
-    F.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
+    ops.assign(mu_product, mu_product * mu)
+    ops.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
+    ops.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
 
     denom = op_sqrt(exp_avg_sq / bias_correction2) + eps
 
     mu_product_next = mu_product * mu_next
-    F.assign(param, param - lr * (1. - mu) / (1. - mu_product) * grad / denom)
-    F.assign(param, param - (lr * mu_next) / (1. - mu_product_next) * exp_avg / denom)
+    ops.assign(param, param - lr * (1. - mu) / (1. - mu_product) * grad / denom)
+    ops.assign(param, param - (lr * mu_next) / (1. - mu_product_next) * exp_avg / denom)
 
     return True
 
@@ -122,8 +112,8 @@ class NAdam(Optimizer):
         self.mu_product = [Parameter(Tensor(1.), "mu_product_" + param.name) for param in self.parameters]
 
         self.increase_tensor = Tensor(1, mstype.int32)
-        self.assignadd = P.AssignAdd()
-        self.op_cast = P.Cast()
+        self.assignadd = ops.AssignAdd()
+        self.op_cast = ops.Cast()
 
     @jit
     def implementation(self, lr, beta1, beta2, weight_decay, momentum_decay, eps, start_id, end_id, gradients):
@@ -135,7 +125,7 @@ class NAdam(Optimizer):
         exp_avg_sq = self.exp_avg_sq[start_id: end_id]
         mu_product = self.mu_product[start_id: end_id]
 
-        self.hyper_map(F.partial(_nadam_opt, beta1, beta2, momentum_decay, eps, self.step_t, lr),
+        self.hyper_map(ops.partial(_nadam_opt, beta1, beta2, momentum_decay, eps, self.step_t, lr),
                        params, grads, exp_avg, exp_avg_sq, mu_product)
         return True
 

mindspore/experimental/optim/optimizer.py

@@ -1,22 +1,12 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/optimizer.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """optimizer"""
 from __future__ import absolute_import
 from collections import defaultdict
 from typing import Iterable
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 
 from mindspore.nn.cell import Cell
 from mindspore.common.parameter import Parameter, ParameterTuple
@@ -98,7 +88,7 @@ class Optimizer(Cell):
         self.param_groups = []
         self.parameters = []
         self.lrs = []
-        self.map_ = C.Map()
+        self.map_ = ops.Map()
         self.group_start_id = [0]
         if not isinstance(param_groups[0], dict):
             param_groups = [{'params': param_groups}]
@@ -106,7 +96,7 @@ class Optimizer(Cell):
         for param_group in param_groups:
             self.add_param_group(param_group)
         self.parameters = ParameterTuple(self.parameters)
-        self.hyper_map = C.HyperMap()
+        self.hyper_map = ops.HyperMap()
         self.enable_tuple_broaden = True
 
     def __repr__(self):
@@ -167,7 +157,7 @@ class Optimizer(Cell):
         """Apply weight decay."""
         if weight_decay != 0.:
             weight_decay = Tensor(weight_decay, mstype.float32)
-            gradients = self.map_(F.partial(_apply_decay, weight_decay), params, gradients)
+            gradients = self.map_(ops.partial(_apply_decay, weight_decay), params, gradients)
         return gradients
 
     def _preprocess_param_group(self, param_group):
@@ -228,18 +218,18 @@ class Optimizer(Cell):
     def construct(self, *hyper_params):
         raise NotImplementedError
 
-op_add = P.AddN()
-op_gather = P.Gather()
-op_mul = P.Mul()
+op_add = ops.AddN()
+op_gather = ops.Gather()
+op_mul = ops.Mul()
 
-_apply_decay = C.MultitypeFuncGraph("apply_decay")
+_apply_decay = ops.MultitypeFuncGraph("apply_decay")
 
 
 @_apply_decay.register("Tensor", "Tensor", "RowTensor")
 def _tensor_apply_decay_with_sparse(weight_decay, weight, gradient):
     """Get grad with weight_decay."""
     indices = gradient.indices
-    values = op_add((op_gather(weight, indices, 0) * F.cast(weight_decay, F.dtype(weight)), gradient.values))
+    values = op_add((op_gather(weight, indices, 0) * ops.cast(weight_decay, ops.dtype(weight)), gradient.values))
     shape = gradient.dense_shape
     return RowTensorInner(indices, values, shape)
 
@@ -247,7 +237,7 @@ def _tensor_apply_decay_with_sparse(weight_decay, weight, gradient):
 @_apply_decay.register("Tensor", "Tensor", "Tensor")
 def _tensor_apply_decay(weight_decay, weight, gradient):
     """Get grad with weight_decay."""
-    return op_add((op_mul(weight, F.cast(weight_decay, F.dtype(weight))), gradient))
+    return op_add((op_mul(weight, ops.cast(weight_decay, ops.dtype(weight))), gradient))
 
 
 def check_not_less_than(arg_value, arg_name, prim, value=0.0):

mindspore/experimental/optim/radam.py

@@ -1,32 +1,22 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/radam.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """radam"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 from mindspore.common import Tensor, Parameter
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as validator
 from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
 from mindspore import jit
 
-_radam_opt = C.MultitypeFuncGraph("radam_opt")
+_radam_opt = ops.MultitypeFuncGraph("radam_opt")
 
-op_pow = P.Pow()
-op_sqrt = P.Sqrt()
-op_cast = P.Cast()
+op_pow = ops.Pow()
+op_sqrt = ops.Sqrt()
+op_cast = ops.Cast()
 
 
 @_radam_opt.register("Number", "Number", "Number", "Tensor", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
@@ -35,17 +25,17 @@ def _tensor_run_opt(beta1, beta2, eps, lr, rho_inf, rho_t, bias_correction1, bia
                     exp_avg_sq):
     """Apply radam optimizer to the weight parameter."""
 
-    F.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
-    F.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
+    ops.assign(exp_avg, exp_avg * beta1 + grad * (1 - beta1))
+    ops.assign(exp_avg_sq, exp_avg_sq * beta2 + grad * grad * (1 - beta2))
     bias_corrected_exp_avg = exp_avg / bias_correction1
 
     if rho_t > 5.0:
         rect = op_sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t))
         exp_avg_sq_sqrt = op_sqrt(exp_avg_sq) + eps
         adaptive_lr = op_sqrt(bias_correction2) / exp_avg_sq_sqrt
-        F.assign(param, param - bias_corrected_exp_avg * lr * adaptive_lr * rect)
+        ops.assign(param, param - bias_corrected_exp_avg * lr * adaptive_lr * rect)
     else:
-        F.assign(param, param - bias_corrected_exp_avg * lr)
+        ops.assign(param, param - bias_corrected_exp_avg * lr)
 
     return True
 
@@ -89,6 +79,9 @@ class RAdam(Optimizer):
             &\rule{180mm}{0.4pt}
         \end{align*}
 
+    For more details about RAdam algorithm, please refer to `On the Variance of the Adaptive Learning Rate and Beyond
+    <https://arxiv.org/abs/1908.03265>`_.
+
     .. warning::
         This is an experimental optimizer API that is subject to change.
         This module must be used with lr scheduler module in `LRScheduler Class
@@ -155,7 +148,7 @@ class RAdam(Optimizer):
         self.exp_avg = self.parameters.clone(prefix="exp_avg", init='zeros')
         self.exp_avg_sq = self.parameters.clone(prefix="exp_avg_sq", init='zeros')
         self.increase_tensor = Tensor(1, mstype.int32)
-        self.assignadd = P.AssignAdd()
+        self.assignadd = ops.AssignAdd()
 
     @jit(backend="ms_backend")
     def implementation(self, lr, beta1, beta2, weight_decay, eps, start_id, end_id, gradients):
@@ -175,7 +168,8 @@ class RAdam(Optimizer):
 
         rho_t = rho_inf - right
 
-        self.hyper_map(F.partial(_radam_opt, beta1, beta2, eps, lr, rho_inf, rho_t, bias_correction1, bias_correction2),
+        self.hyper_map(ops.partial(_radam_opt, beta1, beta2, eps, lr, rho_inf,
+                                   rho_t, bias_correction1, bias_correction2),
                        params, grads, exp_avg, exp_avg_sq)
         return True
 

mindspore/experimental/optim/rmsprop.py

@@ -1,48 +1,37 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/rmsprop.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """rmsprop"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
 import mindspore.common.dtype as mstype
 from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than, check_not_less_than_without_equal
 from mindspore import ops
 from mindspore import jit
 
-_rmsprop_opt = C.MultitypeFuncGraph("rmsprop_opt")
+_rmsprop_opt = ops.MultitypeFuncGraph("rmsprop_opt")
 
-op_mul = P.Mul()
-op_sqrt = P.Sqrt()
+op_mul = ops.Mul()
+op_sqrt = ops.Sqrt()
 
 
 @_rmsprop_opt.register("Bool", "Number", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
 def _run_rmsprop_opt(centered, alpha, eps, momentum, lr, weight, mean_square, mean_grad, mom, grad):
     """Apply rmsprop optimizer to the weight parameter using dynamic learning rate."""
-    F.assign(mean_square, ops.addcmul(op_mul(mean_square, alpha), grad, grad, 1 - alpha))
+    ops.assign(mean_square, ops.addcmul(op_mul(mean_square, alpha), grad, grad, 1 - alpha))
 
     if centered:
-        F.assign(mean_grad, op_mul(mean_grad, alpha) + op_mul(grad, 1 - alpha))
+        ops.assign(mean_grad, op_mul(mean_grad, alpha) + op_mul(grad, 1 - alpha))
         avg = op_sqrt(ops.addcmul(mean_square, mean_grad, mean_grad, -1.)) + eps
     else:
         avg = op_sqrt(mean_square) + eps
 
     if momentum > 0:
-        F.assign(mom, op_mul(mom, momentum) + grad / avg)
-        F.assign(weight, weight - mom * lr)
+        ops.assign(mom, op_mul(mom, momentum) + grad / avg)
+        ops.assign(weight, weight - mom * lr)
     else:
-        F.assign(weight, weight - lr * grad / avg)
+        ops.assign(weight, weight - lr * grad / avg)
     return True
 
 
@@ -124,7 +113,7 @@ class RMSprop(Optimizer):
         self.mean_grad = self.parameters.clone(prefix="mean_grad", init='zeros')
         self.mean_square = self.parameters.clone(prefix="mean_square", init='zeros')
         self.moment = self.parameters.clone(prefix="moment", init='zeros')
-        self.op_cast = P.Cast()
+        self.op_cast = ops.Cast()
 
     @jit
     def implementation(self, group_id, lr, gradients, maximize, weight_decay, centered, alpha, eps, momentum):
@@ -132,12 +121,12 @@ class RMSprop(Optimizer):
         start_id = self.group_start_id[group_id]
         end_id = self.group_start_id[group_id + 1]
         params = self.parameters[start_id: end_id]
-        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = tuple([grad if not maximize else ops.neg(grad) for grad in gradients[start_id: end_id]])
         grads = self._decay_weight(weight_decay, params, grads)
         mean_grad = self.mean_grad[start_id: end_id]
         mean_square = self.mean_square[start_id: end_id]
         moment = self.moment[start_id: end_id]
-        self.hyper_map(F.partial(_rmsprop_opt, centered, alpha, eps, momentum, lr),
+        self.hyper_map(ops.partial(_rmsprop_opt, centered, alpha, eps, momentum, lr),
                        params, mean_square, mean_grad, moment, grads)
         return True
 

mindspore/experimental/optim/rprop.py

@@ -1,37 +1,26 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/rprop.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """rprop"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 from mindspore.common import Tensor, Parameter
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as validator
 from mindspore.experimental.optim.optimizer import Optimizer, check_not_less_than_without_equal
-from mindspore import ops
 from mindspore import jit
 
-_rprop_opt = C.MultitypeFuncGraph("rprop_opt")
+_rprop_opt = ops.MultitypeFuncGraph("rprop_opt")
 
-op_sign = P.Sign()
-op_fill = P.FillV2()
-op_assign = P.Assign()
-op_assignadd = P.AssignAdd()
-op_cast = P.Cast()
-op_select = P.Select()
-op_oneslike = P.OnesLike()
+op_sign = ops.Sign()
+op_fill = ops.FillV2()
+op_assign = ops.Assign()
+op_assignadd = ops.AssignAdd()
+op_cast = ops.Cast()
+op_select = ops.Select()
+op_oneslike = ops.OnesLike()
 
 
 @_rprop_opt.register("Tensor", "Tensor", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
@@ -131,7 +120,7 @@ class Rprop(Optimizer):
         self.step_size = self.parameters.clone(prefix="step_size", init='zeros')
         self.step_t = Parameter(Tensor(0, mstype.int32), "step_t")
         self.increase_tensor = Tensor(1, mstype.int32)
-        self.op_cast = P.Cast()
+        self.op_cast = ops.Cast()
 
     @jit(backend="ms_backend")
     def implementation(self, etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max):
@@ -141,10 +130,10 @@ class Rprop(Optimizer):
         end_id = self.group_start_id[group_id + 1]
 
         params = self.parameters[start_id: end_id]
-        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
+        grads = tuple([grad if not maximize else ops.neg(grad) for grad in gradients[start_id: end_id]])
         prev = self.prev[start_id: end_id]
         step_size = self.step_size[start_id: end_id]
-        self.hyper_map(F.partial(_rprop_opt, etaminus, etaplus, step_size_min, step_size_max, self.step_t, lr),
+        self.hyper_map(ops.partial(_rprop_opt, etaminus, etaplus, step_size_min, step_size_max, self.step_t, lr),
                        params, prev, step_size, grads)
         return True
 

mindspore/experimental/optim/sgd.py

@@ -1,28 +1,18 @@
-# Copyright 2023 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# The code implementation refers to the following files from pytorch:
+# - https://github.com/pytorch/pytorch/blob/v1.13.0/torch/optim/sgd.py
+# Additional modifications are made by Huawei Technologies Co., Ltd in 2023.
 # ============================================================================
 """sgd"""
 from __future__ import absolute_import
 
-from mindspore.ops import functional as F, composite as C, operations as P
+from mindspore import ops
 from mindspore.common.tensor import Tensor
 import mindspore.common.dtype as mstype
 from mindspore import _checkparam as Validator
 from mindspore.experimental.optim.optimizer import Optimizer
 from mindspore import jit
 
-_sgd_opt = C.MultitypeFuncGraph("sgd_opt")
+_sgd_opt = ops.MultitypeFuncGraph("sgd_opt")
 
 
 @_sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor")
@@ -129,7 +119,7 @@ class SGD(Optimizer):
                              "equal to 0.0, but got 'momentum' {}, 'dampening' {}".format(momentum, dampening))
         self.accum = self.parameters.clone(prefix="accum", init='zeros')
         self.stat = self.parameters.clone(prefix="stat", init='ones')
-        self.op_cast = P.Cast()
+        self.op_cast = ops.Cast()
 
     @jit
     def implementation(self, momentum, lr, group_id, gradients, maximize, dampening, weight_decay, nesterov):
@@ -137,9 +127,9 @@ class SGD(Optimizer):
         start_id = self.group_start_id[group_id]
         end_id = self.group_start_id[group_id + 1]
         momentum = self.op_cast(momentum, mstype.float32)
-        opt = P.SGD(dampening, weight_decay, nesterov)
-        grads = tuple([grad if not maximize else F.neg(grad) for grad in gradients[start_id: end_id]])
-        self.hyper_map(F.partial(_sgd_opt, opt, momentum, lr), grads,
+        opt = ops.SGD(dampening, weight_decay, nesterov)
+        grads = tuple([grad if not maximize else ops.neg(grad) for grad in gradients[start_id: end_id]])
+        self.hyper_map(ops.partial(_sgd_opt, opt, momentum, lr), grads,
                        self.parameters[start_id: end_id], self.accum[start_id: end_id],
                        self.stat[start_id: end_id])
         return True

mindspore/hal/__init__.py

@@ -19,13 +19,13 @@ MindSpore abstracts the preceding modules from different backends and allows use
 resources at the Python layer. Currently, these interfaces take effect only in PyNative mode.
 """
 
-from mindspore.hal.device import is_initialized, is_available, device_count, get_device_capability,\
+from mindspore.hal.device import is_initialized, is_available, device_count, get_device_capability, \
                                  get_device_properties, get_device_name, get_arch_list
-from mindspore.hal.stream import Stream, synchronize, set_cur_stream, current_stream, default_stream,\
+from mindspore.hal.stream import Stream, synchronize, set_cur_stream, current_stream, default_stream, \
                                  communication_stream, StreamCtx
 from mindspore.hal.event import Event
-from mindspore.hal.memory import memory_stats, memory_reserved, max_memory_reserved, empty_cache,\
-                                 reset_peak_memory_stats, memory_summary, memory_allocated,\
+from mindspore.hal.memory import memory_stats, memory_reserved, max_memory_reserved, empty_cache, \
+                                 reset_peak_memory_stats, memory_summary, memory_allocated, \
                                  max_memory_allocated, reset_max_memory_reserved, reset_max_memory_allocated
 
 __all__ = [

mindspore/hal/contiguous_tensors_handle.py

@@ -27,7 +27,7 @@ def combine_tensor_list_contiguous(tensor_list, enable_mem_align=True):
     Return a contiguous memory handle where contiguous memory has been requested and slicing functionality is provided.
 
     Args:
-        tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
+        tensor_list (list[Tensor], tuple[Tensor]): The tensor list to be stored.
         enable_mem_align (bool, optional): Whether to enable the memory alignment function.
             False is not supported. Default ``True`` .
 
@@ -57,7 +57,7 @@ class ContiguousTensorsHandle:
     ContiguousTensorsHandle is a handle manage continuous memory.
 
     Args:
-        tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
+        tensor_list (list[Tensor], tuple[Tensor]): The tensor list to be stored.
         enable_mem_align (bool, optional): Whether to enable the memory alignment function.
             False is not supported. Default ``True`` .