mindspore 2.4.10__cp39-cp39-win_amd64.whl → 2.6.0rc1__cp39-cp39-win_amd64.whl

mindspore/experimental/optim/adamw.py

@@ -97,6 +97,7 @@ class AdamW(Optimizer):
 
     .. math::
         \begin{aligned}
+            &\rule{180mm}{0.4pt}                                                                 \\
             &\textbf{input}      : \gamma \text{(lr)}, \: \beta_1, \beta_2
                 \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)},
                 \: \epsilon \text{ (epsilon)}                                                    \\
@@ -104,6 +105,7 @@ class AdamW(Optimizer):
                 \: \textit{maximize}                                                             \\
             &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0
                 \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0              \\[-1.ex]
+            &\rule{180mm}{0.4pt}                                                                 \\
             &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
             &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
             &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})          \\
@@ -122,7 +124,9 @@ class AdamW(Optimizer):
             &\hspace{5mm}\textbf{else}                                                           \\
             &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                 \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
+            &\rule{180mm}{0.4pt}                                                          \\[-1.ex]
             &\bf{return} \:  \theta_t                                                     \\[-1.ex]
+            &\rule{180mm}{0.4pt}                                                          \\[-1.ex]
        \end{aligned}
 
     .. warning::

mindspore/experimental/optim/asgd.py

@@ -124,7 +124,7 @@ class ASGD(Optimizer):
         self.assignadd = P.AssignAdd()
         self.op_cast = P.Cast()
 
-    @jit
+    @jit(backend="ms_backend")
     def implementation(self, lambd, alpha, t0, lr, group_id, maximize, gradients, weight_decay):
         """Extract the common computing part for acceleration"""
         start_id = self.group_start_id[group_id]

mindspore/experimental/optim/lr_scheduler.py

@@ -24,7 +24,6 @@ from mindspore.ops import functional as F
 from mindspore.ops import operations as P
 from mindspore import _checkparam as Validator
 
-
 __all__ = ['StepLR', 'LinearLR', 'LRScheduler', 'ExponentialLR', 'PolynomialLR',
            'MultiplicativeLR', 'ConstantLR', 'MultiStepLR', 'LambdaLR', 'SequentialLR', 'ReduceLROnPlateau',
            'CyclicLR', 'CosineAnnealingWarmRestarts', 'CosineAnnealingLR']
@@ -42,10 +41,11 @@ class LRScheduler:
 
     Args:
         optimizer (:class:`mindspore.experimental.optim.Optimizer`): The optimizer instance.
-        last_epoch (int, optional): The index of the last epoch. Default: ``-1``.
+        last_epoch (int, optional): The number of times the `step()` method of
+            the current learning rate adjustment strategy has been executed. Default: ``-1``.
 
     Raises:
-        TypeError: If `optimizer` is not an Optimizer.
+        TypeError: If `optimizer` does not satisfy the type requirement.
         KeyError: If `last_epoch` != -1 and ``'initial_lr'`` not in param groups.
         ValueError: if `last_epoch` is not int.
         ValueError: If `last_epoch` is not greater than -1.
@@ -82,6 +82,7 @@ class LRScheduler:
         [Tensor(shape=[], dtype=Float32, value= 0.01)]
         [Tensor(shape=[], dtype=Float32, value= 0.01)]
     """
+
     def __init__(self, optimizer, last_epoch=-1):
         if not isinstance(optimizer, Optimizer):
             raise TypeError('{} is not an Optimizer'.format(
@@ -192,6 +193,7 @@ class StepLR(LRScheduler):
         ...     scheduler.step()
         ...     current_lr = scheduler.get_last_lr()
     """
+
     def __init__(self, optimizer, step_size, gamma=0.1, last_epoch=-1):
         if not isinstance(step_size, int) and not isinstance(step_size, bool):
             raise TypeError(f"For 'StepLR', the 'step_size' must be int, but got {type(step_size)}.")
@@ -297,8 +299,8 @@ class LinearLR(LRScheduler):
         if self.last_epoch > self.total_iters:
             return [lr * 1. for lr in self._last_lr]
 
-        factor = 1. + (self.end_factor - self.start_factor) / (
-            self.total_iters * self.start_factor + (self.last_epoch - 1) * (self.end_factor - self.start_factor))
+        factor = 1. + (self.end_factor - self.start_factor) / \
+                 (self.total_iters * self.start_factor + (self.last_epoch - 1) * (self.end_factor - self.start_factor))
         return [lr * factor for lr in self._last_lr]
 
     def _get_closed_form_lr(self):
@@ -419,6 +421,7 @@ class PolynomialLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0)]
         [Tensor(shape=[], dtype=Float32, value= 0)]
     """
+
     def __init__(self, optimizer, total_iters=5, power=1.0, last_epoch=-1):
         if not isinstance(power, float):
             raise TypeError(f"For 'PolynomialLR', the 'power' must be float, but got {type(power)}.")
@@ -435,8 +438,8 @@ class PolynomialLR(LRScheduler):
     def get_lr(self):
         if self.last_epoch == 0 or self.last_epoch > self.total_iters:
             return [lr * 1. for lr in self._last_lr]
-        factor = ((1.0 - self.last_epoch / self.total_iters) / (
-            1.0 - (self.last_epoch - 1) / self.total_iters)) ** self.power
+        factor = ((1.0 - self.last_epoch / self.total_iters) /
+                  (1.0 - (self.last_epoch - 1) / self.total_iters)) ** self.power
         return [lr * factor for lr in self._last_lr]
 
     def _get_closed_form_lr(self):
@@ -483,14 +486,16 @@ class LambdaLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.0081)]
         [Tensor(shape=[], dtype=Float32, value= 0.00729)]
     """
+
     def __init__(self, optimizer, lr_lambda, last_epoch=-1):
-        if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple):
-            self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
-        else:
-            if len(lr_lambda) != len(optimizer.param_groups):
+        param_groups_length = len(optimizer.param_groups)
+        if isinstance(lr_lambda, (list, tuple)):
+            if len(lr_lambda) != param_groups_length:
                 raise ValueError("Expected {} lr_lambdas, but got {}".format(
-                    len(optimizer.param_groups), len(lr_lambda)))
+                    param_groups_length, len(lr_lambda)))
             self.lr_lambdas = list(lr_lambda)
+        else:
+            self.lr_lambdas = [lr_lambda] * param_groups_length
         super(LambdaLR, self).__init__(optimizer, last_epoch)
 
     def get_lr(self):
@@ -533,14 +538,16 @@ class MultiplicativeLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.009025)]
         [Tensor(shape=[], dtype=Float32, value= 0.00857375)]
     """
+
     def __init__(self, optimizer, lr_lambda, last_epoch=-1):
-        if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple):
-            self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
-        else:
-            if len(lr_lambda) != len(optimizer.param_groups):
+        if isinstance(lr_lambda, (list, tuple)):
+            if len(lr_lambda) == len(optimizer.param_groups):
+                self.lr_lambdas = list(lr_lambda)
+            else:
                 raise ValueError("Expected {} lr_lambdas, but got {}".format(
                     len(optimizer.param_groups), len(lr_lambda)))
-            self.lr_lambdas = list(lr_lambda)
+        else:
+            self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
         super(MultiplicativeLR, self).__init__(optimizer, last_epoch)
 
     def get_lr(self):
@@ -599,6 +606,7 @@ class MultiStepLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.0005)]
         [Tensor(shape=[], dtype=Float32, value= 0.0005)]
     """
+
     def __init__(self, optimizer, milestones, gamma=0.1, last_epoch=-1):
         Validator.check_value_type('milestones', milestones, [list])
         for milestone in milestones:
@@ -668,6 +676,7 @@ class ConstantLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.05)]
         [Tensor(shape=[], dtype=Float32, value= 0.05)]
     """
+
     def __init__(self, optimizer, factor=1.0 / 3, total_iters=5, last_epoch=-1):
         if factor > 1.0 or factor < 0:
             raise ValueError('Constant multiplicative factor expected to be between 0 and 1.')
@@ -692,9 +701,8 @@ class ConstantLR(LRScheduler):
 @jit_class
 class SequentialLR:
     r"""
-    Receives the list of schedulers that is expected to be called sequentially during
-    optimization process and milestone points that provides exact intervals to reflect
-    which scheduler is supposed to be called at a given epoch.
+    Concatenate multiple learning rate adjustment strategies in `schedulers` in sequence,
+    switching to the next learning rate adjustment strategy at `milestone`.
 
     .. warning::
         This is an experimental lr scheduler module that is subject to change.
@@ -705,8 +713,10 @@ class SequentialLR:
         optimizer (:class:`mindspore.experimental.optim.Optimizer`): Wrapped optimizer.
         schedulers (list[:class:`mindspore.experimental.optim.lr_scheduler.LRScheduler`]):
             List of learning rate schedulers.
-        milestones (list): List of integers that reflects milestone points.
-        last_epoch (int, optional): The index of the last epoch. Default: ``-1``.
+        milestones (list): List of integers of milestone points,
+            sets which learning rate adjustment strategy is invoked for each epoch.
+        last_epoch (int, optional): The number of times the `step()` method
+            of the current learning rate adjustment strategy has been executed. Default: ``-1``.
 
     Raises:
         ValueError: The optimizer in `schedulers` is different from the `optimizer` passed in.
@@ -735,6 +745,7 @@ class SequentialLR:
         [Tensor(shape=[], dtype=Float32, value= 0.0729)]
         [Tensor(shape=[], dtype=Float32, value= 0.06561)]
     """
+
     def __init__(self, optimizer, schedulers, milestones, last_epoch=-1):
         for sched_idx in range(len(schedulers)):
             if schedulers[sched_idx].optimizer != optimizer:
@@ -796,9 +807,8 @@ class ReduceLROnPlateau:
     """
     Reduce learning rate when a metric has stopped improving.
     Models often benefit from reducing the learning rate by a factor
-    of 2-10 once learning stagnates. This scheduler reads a metrics
-    quantity and if no improvement is seen for a 'patience' number
-    of epochs, the learning rate is reduced.
+    of 2-10 once learning stagnates. The scheduler reads the metrics `metrics` during execution
+    and adjusts the learning rate via the `step` method if the metrics do not improve within `patience` cycles.
 
     .. warning::
         This is an experimental lr scheduler module that is subject to change.
@@ -807,7 +817,8 @@ class ReduceLROnPlateau:
 
     Args:
         optimizer (:class:`mindspore.experimental.optim.Optimizer`): Wrapped optimizer.
-        mode (str, optional): One of `min`, `max`. In `min` mode, lr will
+        mode (str, optional): Trigger mode that triggers a reduction in learning rate
+            when the monitoring metrics are at their `min` / `max` point. In `min` mode, lr will
             be reduced when the quantity monitored has stopped
             decreasing; in `max` mode it will be reduced when the
             quantity monitored has stopped increasing. Default: ``'min'``.
@@ -821,12 +832,25 @@ class ReduceLROnPlateau:
             Default: ``10``.
         threshold (float, optional): Threshold for measuring the new optimum,
             to only focus on significant changes. Default: ``1e-4``.
-        threshold_mode (str, optional): One of `rel`, `abs`. Given dynamic_threshold is the benchmark to
-            define whether the current metric is improvement,
-            in ``'rel'`` mode, dynamic_threshold = best * ( 1 + threshold ) in ``'max'`` mode
-            or best * ( 1 - threshold ) in ``'min'`` mode.
-            In ``'abs'`` mode, dynamic_threshold = best + threshold in ``'max'`` mode or
-            best - threshold in ``'min'`` mode. Default: ``'rel'``.
+        threshold_mode (str, optional): A mode for measuring indicators of change for the better.
+            One of `rel`, `abs`. Default: ``'rel'``.
+
+            Assume that `best` represents the best value of the current performance metric.
+
+            - In ``'rel'`` mode, the indicator is compared to a `threshold` in proportional form:
+
+              - When `mode` is ``'max'``, the indicator is considered better if it exceeds best * ( 1 + threshold ).
+
+              - When `mode` is ``'min'``, the indicator is considered better
+                if it is lower than best * ( 1 - threshold ).
+
+            - In ``'abs'`` mode, the indicator is compared to `threshold` in absolute value form:
+
+              - When `mode` is ``'max'``, the indicator is considered better if it exceeds best + threshold.
+
+              - When `mode` is ``'min'``, the indicator is considered better
+                if it is lower than best - threshold.
+
         cooldown (int, optional): Number of epochs to wait before resuming
             normal operation after lr has been reduced. Default: ``0``.
         min_lr (Union(float, list), optional): A scalar or a list of scalars. A
@@ -863,6 +887,7 @@ class ReduceLROnPlateau:
         [Tensor(shape=[], dtype=Float32, value= 0.001)]
         [Tensor(shape=[], dtype=Float32, value= 0.0001)]
         """
+
     def __init__(self, optimizer, mode='min', factor=0.1, patience=10,
                  threshold=1e-4, threshold_mode='rel', cooldown=0,
                  min_lr=0, eps=1e-8):
@@ -1053,6 +1078,7 @@ class CyclicLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.01018)]
         [Tensor(shape=[], dtype=Float32, value= 0.010225)]
     """
+
     def __init__(self,
                  optimizer,
                  base_lr,
@@ -1127,12 +1153,12 @@ class CyclicLR(LRScheduler):
     def _triangular_scale_fn(self, x):
         return 1.
 
-    def _triangular2_scale_fn(self, x):
-        return 1 / (2. ** (x - 1))
-
     def _exp_range_scale_fn(self, x):
         return self.gamma ** (x)
 
+    def _triangular2_scale_fn(self, x):
+        return 1 / (2. ** (x - 1))
+
     def get_lr(self):
         cycle = self.floor(1 + self.last_epoch / self.total_step_size)
         x = 1. + self.last_epoch / self.total_step_size - cycle
@@ -1143,13 +1169,9 @@ class CyclicLR(LRScheduler):
         lrs = []
         for base_lr, max_lr in zip(self.base_lrs, self.max_lrs):
             base_height = (max_lr - base_lr) * scale_factor
-
-            if self.scale_mode == 'cycle':
-                lr = base_lr + base_height * self.scale_fn(cycle)
-            else:
-                lr = base_lr + base_height * self.scale_fn(self.last_epoch)
+            cycle_or_epoch = cycle if self.scale_mode == 'cycle' else self.last_epoch
+            lr = base_lr + base_height * self.scale_fn(cycle_or_epoch)
             lrs.append(lr)
-
         return lrs
 
 
@@ -1157,15 +1179,17 @@ class CyclicLR(LRScheduler):
 class CosineAnnealingWarmRestarts(LRScheduler):
     r"""
     Set the learning rate of each parameter group using a cosine annealing warm restarts
-    schedule. Where :math:`\eta_{max}` is set to the initial lr, :math:`\eta_{min}` is the minimum value
-    for learning rate, :math:`\eta_{t}` is the current learning rate, :math:`T_{0}` is the number of iterations for the
-    first restar, :math:`T_{i}` is the current number of iterations between two warm restarts in SGDR,
-    :math:`T_{cur}` is the number of epochs since the last restart in SGDR.
+    schedule.
 
     .. math::
         \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 +
         \cos\left(\frac{T_{cur}}{T_{i}}\pi\right)\right)
 
+    Where :math:`\eta_{max}` is set to the initial lr, :math:`\eta_{min}` is the minimum value
+    for learning rate, :math:`\eta_{t}` is the current learning rate, :math:`T_{0}` is the number of iterations for the
+    first restar, :math:`T_{i}` is the current number of iterations between two warm restarts in SGDR,
+    :math:`T_{cur}` is the number of epochs since the last restart in SGDR.
+
     When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`.
     When :math:`T_{cur}=0` after restart, set :math:`\eta_t=\eta_{max}`.
 
@@ -1182,7 +1206,8 @@ class CosineAnnealingWarmRestarts(LRScheduler):
         T_0 (int): Number of iterations for the first restart.
         T_mult (int, optional): A factor increases :math:`T_{i}` after a restart. Default: ``1``.
         eta_min (Union(float, int), optional): Minimum learning rate. Default: ``0``.
-        last_epoch (int, optional): The index of the last epoch. Default: ``-1``.
+        last_epoch (int, optional): The number of times the `step()` method of
+            the current learning rate adjustment strategy has been executed. Default: ``-1``.
 
     Raises:
         ValueError: `T_0` is less than or equal than 0 or not an int.
@@ -1211,6 +1236,7 @@ class CosineAnnealingWarmRestarts(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.025)]
         [Tensor(shape=[], dtype=Float32, value= 0.00669873)]
     """
+
     def __init__(self, optimizer, T_0, T_mult=1, eta_min=0, last_epoch=-1):
         if T_0 <= 0 or not isinstance(T_0, int):
             raise ValueError("T_0 should be an integer and equal or greater than 0, but got {}".format(T_0))
@@ -1336,6 +1362,7 @@ class CosineAnnealingLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.05)]
         [Tensor(shape=[], dtype=Float32, value= 0)]
     """
+
     def __init__(self, optimizer, T_max, eta_min=0.0, last_epoch=-1):
         if not isinstance(eta_min, (float, int)):
             raise TypeError(f"For 'CosineAnnealingLR', the 'eta_min' must be float or int, but got {type(eta_min)}.")

mindspore/experimental/optim/radam.py

@@ -55,36 +55,39 @@ class RAdam(Optimizer):
     Implements RAdam algorithm.
 
     .. math::
-        \begin{aligned}
-            &\rule{110mm}{0.4pt}                                                                 \\
-            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
-                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
-                \lambda \text{ (weightdecay)},                                                   \\
-            &\hspace{13mm} \epsilon \text{ (epsilon)}                                            \\
-            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
-                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
-            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
-            &\rule{110mm}{0.4pt}  \\
-            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
-            &\hspace{6mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
-            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
-            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
-            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
-            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
-            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
-            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
-                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                     \\[-1.ex]
-            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
-            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
-            &\hspace{12mm} r_t \leftarrow
-                \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
-            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t} r_t l_t        \\
-            &\hspace{6mm}\textbf{else}                                                           \\
-            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}                \\
-            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
-            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
-            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
-        \end{aligned}
+        \begin{align*}
+            &\rule{180mm}{0.4pt} \\
+            &\textbf{Input}:
+                \gamma \text{ (lr)}, \: \beta_1, \beta_2 \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta)
+                \text{ (objective)}, \:
+                \lambda \text{ (weightdecay)}, \: \epsilon \text{ (epsilon)} \\
+            &\textbf{Initialize}:
+                \begin{cases}
+                    m_0 \leftarrow 0 \text{ (first moment)} \\
+                    v_0 \leftarrow 0 \text{ (second moment)} \\
+                    \rho_{\infty} \xleftarrow{\text{def}} \dfrac{2}{1 - \beta_2} - 1
+                \end{cases} \\
+            &\rule{180mm}{0.4pt} \\
+            &\textbf{For } t = 1 \text{ to } \ldots \text{ do}: \\
+            &\quad g_t \leftarrow \nabla_{\theta} f_t(\theta_{t - 1}) \\
+            &\quad \text{If } \lambda \neq 0: \\
+            &\quad\quad g_t \leftarrow g_t + \lambda \theta_{t - 1} \\
+            &\quad m_t \leftarrow \beta_1 m_{t - 1} + (1 - \beta_1) g_t \\
+            &\quad v_t \leftarrow \beta_2 v_{t - 1} + (1 - \beta_2) g_t^2 \\
+            &\quad \widehat{m_t} \leftarrow \dfrac{m_t}{1 - \beta_1^t} \\
+            &\quad \text{Let } \rho_t' = 2 t \beta_2^t /(1 - \beta_2^t) \quad \text{(auxiliary variable)} \\
+            &\quad \rho_t \leftarrow \rho_{\infty} - \rho_t' \\
+            &\quad \text{If } \rho_t > 5: \\
+            &\quad\quad l_t \leftarrow \dfrac{\sqrt{1 - \beta_2^t}}{\sqrt{v_t} + \epsilon} \\
+            &\quad\quad r_t \leftarrow \sqrt{\dfrac{(\rho_t - 4)(\rho_t - 2)\rho_{\infty}}{(\rho_{\infty} - 4)
+            (\rho_{\infty} - 2) \rho_t}} \\
+            &\quad\quad \theta_t \leftarrow \theta_{t - 1} - \gamma \widehat{m_t} r_t l_t \\
+            &\quad \text{Else}: \\
+            &\quad\quad \theta_t \leftarrow \theta_{t - 1} - \gamma \widehat{m_t} \\
+            &\rule{180mm}{0.4pt} \\
+            &\bf{Return}: \theta_t \\
+            &\rule{180mm}{0.4pt}
+        \end{align*}
 
     .. warning::
         This is an experimental optimizer API that is subject to change.
@@ -154,7 +157,7 @@ class RAdam(Optimizer):
         self.increase_tensor = Tensor(1, mstype.int32)
         self.assignadd = P.AssignAdd()
 
-    @jit
+    @jit(backend="ms_backend")
     def implementation(self, lr, beta1, beta2, weight_decay, eps, start_id, end_id, gradients):
         """Extract the common computing part for acceleration"""
         params = self.parameters[start_id: end_id]

mindspore/experimental/optim/rprop.py

@@ -133,7 +133,7 @@ class Rprop(Optimizer):
         self.increase_tensor = Tensor(1, mstype.int32)
         self.op_cast = P.Cast()
 
-    @jit
+    @jit(backend="ms_backend")
     def implementation(self, etaminus, etaplus, group_id, lr, gradients, maximize, step_size_min, step_size_max):
         """Extract the common computing part for acceleration"""
         etaminus, etaplus = op_cast(etaminus, mstype.float32), op_cast(etaplus, mstype.float32)

mindspore/experimental/optim/sgd.py

@@ -51,7 +51,7 @@ class SGD(Optimizer):
 
     To be noticed, for the first step, :math:`v_{t+1} = gradient`.
 
-    Here : where p, v and u denote the parameters, accum, and momentum respectively.
+    Here: where :math:`p`, :math:`v` and :math:`u` denote the parameters, accum, and momentum respectively.
 
     .. warning::
         This is an experimental optimizer API that is subject to change.

mindspore/hal/contiguous_tensors_handle.py

@@ -19,7 +19,7 @@ from mindspore.common.tensor import Tensor
 from mindspore.common.api import _convert_python_data
 from mindspore.common.dtype import type_size_in_bytes
 from mindspore._c_expression import slice_by_tensor_index, slice_by_padding_shape, \
-    combine_tensor_list_contiguous as combine_tensor_list, Tensor as Tensor_
+    combine_tensor_list_contiguous as combine_tensor_list, TensorPy as Tensor_
 
 
 def combine_tensor_list_contiguous(tensor_list, enable_mem_align=True):
@@ -29,7 +29,7 @@ def combine_tensor_list_contiguous(tensor_list, enable_mem_align=True):
     Args:
         tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
         enable_mem_align (bool, optional): Whether to enable the memory alignment function.
-            False is not supported. Default: ``True`` .
+            False is not supported. Default ``True`` .
 
     Returns:
             ContiguousTensorsHandle, a manager with contiguous memory.
@@ -59,7 +59,7 @@ class ContiguousTensorsHandle:
     Args:
         tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
         enable_mem_align (bool, optional): Whether to enable the memory alignment function.
-            False is not supported. Default: ``True`` .
+            False is not supported. Default ``True`` .
 
     Returns:
         ContiguousTensorsHandle, a manager with contiguous memory.
@@ -137,15 +137,11 @@ class ContiguousTensorsHandle:
         Return the tensor which is sliced by tensor index.
 
         Args:
-            start(int, None): Starting position. Default：``None``.
-            end(int, None): Deadline position. Default：``None``.
+            start(int, None): Starting position. Default ``None``.
+            end(int, None): Deadline position. Default ``None``.
 
         Returns:
-            Tensor，is sliced by tensor index.
-
-        Raises:
-            TypeError: If `start` or `end`, is neither an 'int' nor a 'none'.
-            ValueError: If values of `start` or `end` are negative, or out of the list range, or start >= end.
+            Tensor
 
         Examples:
             >>> import numpy as np