mindspore 2.4.1__cp310-cp310-manylinux1_x86_64.whl → 2.5.0__cp310-cp310-manylinux1_x86_64.whl

mindspore/experimental/llm_boost/atb/qwen_boost.py

@@ -15,11 +15,14 @@
 """llm boost"""
 import json
 import mindspore.common.dtype as mstype
-from mindspore.experimental.llm_boost.atb.boost_base import AtbBoostBase
+from mindspore.experimental.llm_boost.atb.boost_base import AtbBoostBase, NormType
 from mindspore._c_expression import LlmBoostBinder
 from mindspore.experimental.llm_boost.register import LlmBoostRegister, LlmBoostType
 
 
+CPP_QWEN_MODEL_CLASS_NAME = "qwen_QwenDecoderModel"
+
+
 @LlmBoostRegister.register(LlmBoostType.BUILDIN, "Qwen")
 class QwenBoost(AtbBoostBase):
     """QwenBoost class"""
@@ -30,9 +33,11 @@ class QwenBoost(AtbBoostBase):
         self.acl_encoder_operation_inputs = [None] * self.in_tensor_length
         self.acl_decoder_operation_inputs = [None] * self.in_tensor_length
         self.atb_encoder_operation = LlmBoostBinder(
-            "ATB", "qwen_DecoderModel")
+            self.backend_name, CPP_QWEN_MODEL_CLASS_NAME
+        )
         self.atb_decoder_operation = LlmBoostBinder(
-            "ATB", "qwen_DecoderModel")
+            self.backend_name, CPP_QWEN_MODEL_CLASS_NAME
+        )
 
     def init(self):
         """set param"""
@@ -42,24 +47,43 @@ class QwenBoost(AtbBoostBase):
             "withEmbedding": True,
             "isEmbeddingParallel": True,
             "isLmHeadParallel": True,
-            "linearTransposeType": [[1, -1, -1, 1, 1, -1, 1] for i in range(self.num_layers)],
+            "linearTransposeType": [
+                [1, -1, -1, 1, 1, -1, 1] for i in range(self.num_layers)
+            ],
             "lmHeadTransposeType": 1,
-            "supportSwiGLU": not self.need_nz,
-            "rmsNormEps": self.config.rms_norm_eps,
+            "enableSwiGLU": not self.need_nz,
+            "normEps": self.config.rms_norm_eps,
+            "normType": NormType.RMS_NORM,
             "numAttentionHeadsPerRank": self.config.num_heads // self.device_num,
             "hiddenSizePerAttentionHead": self.head_dim,
             "numHiddenLayers": self.num_layers,
             "numKeyValueHeadsPerRank": self.n_kv_heads // self.device_num,
             "rank": self.rank_id,
             "worldSize": self.device_num,
-            "backend": "lccl",
+            "backend": self.config.communication_backend,
             "packQuantType": [[1, 1] for _ in range(self.num_layers)],
-            "linearQuantType": [[0, -1, -1, 0, 0, -1, 0] for _ in range(self.num_layers)],
-            "kvQuant": self.kv_quant is not None,
+            "linearQuantType": [
+                [0, -1, -1, 0, 0, -1, 0] for _ in range(self.num_layers)
+            ],
+            "linearHasBias": [[True, False, False, False]] * self.num_layers,
+            "enableKvQuant": self.kv_quant is not None,
+            "enableLora": False,
+            "isUnpadInputs": True,
+            "enableAddNorm": False,
+        }
+        encoder_param = {
+            **param_dict,
+            "isPrefill": True,
+            "enableLcoc": False,
+            "enableSplitFuse": False,
+        }
+        decoder_param = {
+            **param_dict,
+            "isPrefill": False,
+            "enableLcoc": False,
+            "enableSpeculate": False,
+            "enablePrefixCache": False,
         }
-        encoder_param = {**param_dict, "isPrefill": True, "supportLcoc": False}
-        decoder_param = {**param_dict, "isPrefill": False,
-                         "supportLcoc": False, "supportSpeculate": False}
         self.atb_encoder_operation.init(json.dumps({**encoder_param}))
         self.atb_decoder_operation.init(json.dumps({**decoder_param}))
 
@@ -79,13 +103,14 @@ class QwenBoost(AtbBoostBase):
             **kwargs
     ):
         """prepare inputs"""
-        self.acl_param = json.dumps({
-            "seqLen": seqLen,
-        })
-        self.acl_decoder_operation_inputs[0] = self.cast(
-            input_ids, mstype.int64)
-        self.acl_decoder_operation_inputs[1] = self.cast(
-            position_ids, mstype.int32)
+        self.acl_param = json.dumps(
+            {
+                "seqLen": seqLen,
+            }
+        )
+
+        self.acl_decoder_operation_inputs[0] = input_ids
+        self.acl_decoder_operation_inputs[1] = position_ids
         self.acl_decoder_operation_inputs[2] = cos_embed
         self.acl_decoder_operation_inputs[3] = sin_embed
         self.acl_decoder_operation_inputs[4] = attention_mask
@@ -93,9 +118,7 @@ class QwenBoost(AtbBoostBase):
         self.acl_decoder_operation_inputs[6] = slots
         self.acl_decoder_operation_inputs[7] = self.placeholder
         self.acl_decoder_operation_inputs[8] = self.placeholder
-        self.acl_decoder_operation_inputs[9] = self.cast(
-            input_lengths, mstype.int32)
-        self.acl_decoder_operation_inputs[10] = self.cast(
-            lm_head_indices, mstype.int64)
-        self.acl_decoder_operation_inputs[11] = self.placeholder
+        self.acl_decoder_operation_inputs[9] = self.placeholder
+        self.acl_decoder_operation_inputs[10] = input_lengths
+        self.acl_decoder_operation_inputs[11] = lm_head_indices
         return self.acl_decoder_operation_inputs, self.acl_param

mindspore/experimental/llm_boost/register.py

@@ -23,6 +23,7 @@ class LlmBoostType:
         pass
 
     BUILDIN = 'BuildIn'
+    ASCEND_NATIVE = 'LLMBoost'
 
 
 class LlmBoostRegister:

mindspore/experimental/optim/adadelta.py

@@ -37,28 +37,32 @@ class Adadelta(Optimizer):
     Implements Adadelta algorithm.
 
     .. math::
-       \begin{aligned}
-            &\rule{150mm}{0.4pt}                                                                 \\
-            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)},
-                \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)},
-                \: \lambda \text{ (weight decay)}                                                \\
-            &\textbf{initialize} :  v_0  \leftarrow 0 \: \text{ (square avg)},
-                \: u_0 \leftarrow 0 \: \text{ (accumulate variables)}                     \\[-1.ex]
-            &\rule{110mm}{0.4pt}                                                                 \\
-            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
-            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
-            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
-            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
-            &\hspace{5mm} v_t      \leftarrow v_{t-1} \rho + g^2_t (1 - \rho)                    \\
-            &\hspace{5mm}\Delta x_t    \leftarrow   \frac{\sqrt{u_{t-1} +
-                \epsilon }}{ \sqrt{v_t + \epsilon}  }g_t \hspace{21mm}                           \\
-            &\hspace{5mm} u_t  \leftarrow   u_{t-1}  \rho +
-                 \Delta x^2_t  (1 - \rho)                                                        \\
-            &\hspace{5mm}\theta_t      \leftarrow   \theta_{t-1} - \gamma  \Delta x_t            \\
-            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
-            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
-            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
-       \end{aligned}
+        \newcommand{\grad}[2]{\nabla_{#1} f_{#2}(#2_{#2 - 1})}
+        \newcommand{\updateVar}[3]{#1_{#2} \leftarrow #1_{#2 - 1} \rho + #3_{#2} (1 - \rho)}
+
+        \begin{align*}
+            &\rule{150mm}{0.4pt} \\
+            &\textbf{Input}:
+                \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)},
+                \: \rho \text{ (decay)}, \: \lambda \text{ (weight decay)} \\
+            &\textbf{Initialize}:
+                \begin{cases}
+                    v_0 \leftarrow 0 \text{ (square avg)} \\
+                    u_0 \leftarrow 0 \text{ (accumulate variables)}
+                \end{cases} \\
+            &\rule{110mm}{0.4pt} \\
+            &\textbf{For } t = 1 \text{ to } \ldots \text{ do}: \\
+            &\quad g_t \leftarrow \grad{\theta}{t} \\
+            &\quad \text{If } \lambda \neq 0: \\
+            &\quad\quad g_t \leftarrow g_t + \lambda \theta_{t - 1} \\
+            &\quad v_t \leftarrow \updateVar{v}{t}{g^2} \\
+            &\quad \Delta x_t \leftarrow \frac{\sqrt{u_{t - 1} + \epsilon}}{\sqrt{v_t + \epsilon}} g_t \\
+            &\quad u_t \leftarrow \updateVar{u}{t}{\Delta x^2} \\
+            &\quad \theta_t \leftarrow \theta_{t - 1} - \gamma \Delta x_t \\
+            &\rule{110mm}{0.4pt} \\
+            &\bf{Return}: \theta_t \\
+            &\rule{110mm}{0.4pt}
+        \end{align*}
 
     .. warning::
         This is an experimental optimizer API that is subject to change.

mindspore/experimental/optim/adam.py

@@ -78,6 +78,9 @@ class Adam(Optimizer):
        \end{aligned}
 
     .. warning::
+        The implementation formula of this optimizer interface is not completely consistent with that in the paper.
+        If you want to use an interface that is completely consistent, it is recommended to use
+        :class:`mindspore.mint.optim.Adam`, which currently only supports Ascend.
         This is an experimental optimizer API that is subject to change.
         This module must be used with lr scheduler module in `LRScheduler Class
         <https://www.mindspore.cn/docs/en/master/api_python/mindspore.nn.html#learningrateschedule-class>`_ .

mindspore/experimental/optim/lr_scheduler.py

@@ -24,7 +24,6 @@ from mindspore.ops import functional as F
 from mindspore.ops import operations as P
 from mindspore import _checkparam as Validator
 
-
 __all__ = ['StepLR', 'LinearLR', 'LRScheduler', 'ExponentialLR', 'PolynomialLR',
            'MultiplicativeLR', 'ConstantLR', 'MultiStepLR', 'LambdaLR', 'SequentialLR', 'ReduceLROnPlateau',
            'CyclicLR', 'CosineAnnealingWarmRestarts', 'CosineAnnealingLR']
@@ -82,6 +81,7 @@ class LRScheduler:
         [Tensor(shape=[], dtype=Float32, value= 0.01)]
         [Tensor(shape=[], dtype=Float32, value= 0.01)]
     """
+
     def __init__(self, optimizer, last_epoch=-1):
         if not isinstance(optimizer, Optimizer):
             raise TypeError('{} is not an Optimizer'.format(
@@ -192,6 +192,7 @@ class StepLR(LRScheduler):
         ...     scheduler.step()
         ...     current_lr = scheduler.get_last_lr()
     """
+
     def __init__(self, optimizer, step_size, gamma=0.1, last_epoch=-1):
         if not isinstance(step_size, int) and not isinstance(step_size, bool):
             raise TypeError(f"For 'StepLR', the 'step_size' must be int, but got {type(step_size)}.")
@@ -297,8 +298,8 @@ class LinearLR(LRScheduler):
         if self.last_epoch > self.total_iters:
             return [lr * 1. for lr in self._last_lr]
 
-        factor = 1. + (self.end_factor - self.start_factor) / (
-            self.total_iters * self.start_factor + (self.last_epoch - 1) * (self.end_factor - self.start_factor))
+        factor = 1. + (self.end_factor - self.start_factor) / \
+                 (self.total_iters * self.start_factor + (self.last_epoch - 1) * (self.end_factor - self.start_factor))
         return [lr * factor for lr in self._last_lr]
 
     def _get_closed_form_lr(self):
@@ -419,6 +420,7 @@ class PolynomialLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0)]
         [Tensor(shape=[], dtype=Float32, value= 0)]
     """
+
     def __init__(self, optimizer, total_iters=5, power=1.0, last_epoch=-1):
         if not isinstance(power, float):
             raise TypeError(f"For 'PolynomialLR', the 'power' must be float, but got {type(power)}.")
@@ -435,8 +437,8 @@ class PolynomialLR(LRScheduler):
     def get_lr(self):
         if self.last_epoch == 0 or self.last_epoch > self.total_iters:
             return [lr * 1. for lr in self._last_lr]
-        factor = ((1.0 - self.last_epoch / self.total_iters) / (
-            1.0 - (self.last_epoch - 1) / self.total_iters)) ** self.power
+        factor = ((1.0 - self.last_epoch / self.total_iters) /
+                  (1.0 - (self.last_epoch - 1) / self.total_iters)) ** self.power
         return [lr * factor for lr in self._last_lr]
 
     def _get_closed_form_lr(self):
@@ -483,14 +485,16 @@ class LambdaLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.0081)]
         [Tensor(shape=[], dtype=Float32, value= 0.00729)]
     """
+
     def __init__(self, optimizer, lr_lambda, last_epoch=-1):
-        if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple):
-            self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
-        else:
-            if len(lr_lambda) != len(optimizer.param_groups):
+        param_groups_length = len(optimizer.param_groups)
+        if isinstance(lr_lambda, (list, tuple)):
+            if len(lr_lambda) != param_groups_length:
                 raise ValueError("Expected {} lr_lambdas, but got {}".format(
-                    len(optimizer.param_groups), len(lr_lambda)))
+                    param_groups_length, len(lr_lambda)))
             self.lr_lambdas = list(lr_lambda)
+        else:
+            self.lr_lambdas = [lr_lambda] * param_groups_length
         super(LambdaLR, self).__init__(optimizer, last_epoch)
 
     def get_lr(self):
@@ -533,14 +537,16 @@ class MultiplicativeLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.009025)]
         [Tensor(shape=[], dtype=Float32, value= 0.00857375)]
     """
+
     def __init__(self, optimizer, lr_lambda, last_epoch=-1):
-        if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple):
-            self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
-        else:
-            if len(lr_lambda) != len(optimizer.param_groups):
+        if isinstance(lr_lambda, (list, tuple)):
+            if len(lr_lambda) == len(optimizer.param_groups):
+                self.lr_lambdas = list(lr_lambda)
+            else:
                 raise ValueError("Expected {} lr_lambdas, but got {}".format(
                     len(optimizer.param_groups), len(lr_lambda)))
-            self.lr_lambdas = list(lr_lambda)
+        else:
+            self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
         super(MultiplicativeLR, self).__init__(optimizer, last_epoch)
 
     def get_lr(self):
@@ -599,6 +605,7 @@ class MultiStepLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.0005)]
         [Tensor(shape=[], dtype=Float32, value= 0.0005)]
     """
+
     def __init__(self, optimizer, milestones, gamma=0.1, last_epoch=-1):
         Validator.check_value_type('milestones', milestones, [list])
         for milestone in milestones:
@@ -668,6 +675,7 @@ class ConstantLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.05)]
         [Tensor(shape=[], dtype=Float32, value= 0.05)]
     """
+
     def __init__(self, optimizer, factor=1.0 / 3, total_iters=5, last_epoch=-1):
         if factor > 1.0 or factor < 0:
             raise ValueError('Constant multiplicative factor expected to be between 0 and 1.')
@@ -735,6 +743,7 @@ class SequentialLR:
         [Tensor(shape=[], dtype=Float32, value= 0.0729)]
         [Tensor(shape=[], dtype=Float32, value= 0.06561)]
     """
+
     def __init__(self, optimizer, schedulers, milestones, last_epoch=-1):
         for sched_idx in range(len(schedulers)):
             if schedulers[sched_idx].optimizer != optimizer:
@@ -863,6 +872,7 @@ class ReduceLROnPlateau:
         [Tensor(shape=[], dtype=Float32, value= 0.001)]
         [Tensor(shape=[], dtype=Float32, value= 0.0001)]
         """
+
     def __init__(self, optimizer, mode='min', factor=0.1, patience=10,
                  threshold=1e-4, threshold_mode='rel', cooldown=0,
                  min_lr=0, eps=1e-8):
@@ -1053,6 +1063,7 @@ class CyclicLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.01018)]
         [Tensor(shape=[], dtype=Float32, value= 0.010225)]
     """
+
     def __init__(self,
                  optimizer,
                  base_lr,
@@ -1127,12 +1138,12 @@ class CyclicLR(LRScheduler):
     def _triangular_scale_fn(self, x):
         return 1.
 
-    def _triangular2_scale_fn(self, x):
-        return 1 / (2. ** (x - 1))
-
     def _exp_range_scale_fn(self, x):
         return self.gamma ** (x)
 
+    def _triangular2_scale_fn(self, x):
+        return 1 / (2. ** (x - 1))
+
     def get_lr(self):
         cycle = self.floor(1 + self.last_epoch / self.total_step_size)
         x = 1. + self.last_epoch / self.total_step_size - cycle
@@ -1143,13 +1154,9 @@ class CyclicLR(LRScheduler):
         lrs = []
         for base_lr, max_lr in zip(self.base_lrs, self.max_lrs):
             base_height = (max_lr - base_lr) * scale_factor
-
-            if self.scale_mode == 'cycle':
-                lr = base_lr + base_height * self.scale_fn(cycle)
-            else:
-                lr = base_lr + base_height * self.scale_fn(self.last_epoch)
+            cycle_or_epoch = cycle if self.scale_mode == 'cycle' else self.last_epoch
+            lr = base_lr + base_height * self.scale_fn(cycle_or_epoch)
             lrs.append(lr)
-
         return lrs
 
 
@@ -1211,6 +1218,7 @@ class CosineAnnealingWarmRestarts(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.025)]
         [Tensor(shape=[], dtype=Float32, value= 0.00669873)]
     """
+
     def __init__(self, optimizer, T_0, T_mult=1, eta_min=0, last_epoch=-1):
         if T_0 <= 0 or not isinstance(T_0, int):
             raise ValueError("T_0 should be an integer and equal or greater than 0, but got {}".format(T_0))
@@ -1336,6 +1344,7 @@ class CosineAnnealingLR(LRScheduler):
         [Tensor(shape=[], dtype=Float32, value= 0.05)]
         [Tensor(shape=[], dtype=Float32, value= 0)]
     """
+
     def __init__(self, optimizer, T_max, eta_min=0.0, last_epoch=-1):
         if not isinstance(eta_min, (float, int)):
             raise TypeError(f"For 'CosineAnnealingLR', the 'eta_min' must be float or int, but got {type(eta_min)}.")

mindspore/experimental/optim/radam.py

@@ -55,36 +55,39 @@ class RAdam(Optimizer):
     Implements RAdam algorithm.
 
     .. math::
-        \begin{aligned}
-            &\rule{110mm}{0.4pt}                                                                 \\
-            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
-                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
-                \lambda \text{ (weightdecay)},                                                   \\
-            &\hspace{13mm} \epsilon \text{ (epsilon)}                                            \\
-            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
-                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
-            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
-            &\rule{110mm}{0.4pt}  \\
-            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
-            &\hspace{6mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
-            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
-            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
-            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
-            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
-            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
-            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
-                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                     \\[-1.ex]
-            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
-            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
-            &\hspace{12mm} r_t \leftarrow
-                \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
-            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t} r_t l_t        \\
-            &\hspace{6mm}\textbf{else}                                                           \\
-            &\hspace{12mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}                \\
-            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
-            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
-            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
-        \end{aligned}
+        \begin{align*}
+            &\rule{110mm}{0.4pt} \\
+            &\textbf{Input}:
+                \gamma \text{ (lr)}, \: \beta_1, \beta_2 \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta)
+                \text{ (objective)}, \:
+                \lambda \text{ (weightdecay)}, \: \epsilon \text{ (epsilon)} \\
+            &\textbf{Initialize}:
+                \begin{cases}
+                    m_0 \leftarrow 0 \text{ (first moment)} \\
+                    v_0 \leftarrow 0 \text{ (second moment)} \\
+                    \rho_{\infty} \xleftarrow{\text{def}} \dfrac{2}{1 - \beta_2} - 1
+                \end{cases} \\
+            &\rule{110mm}{0.4pt} \\
+            &\textbf{For } t = 1 \text{ to } \ldots \text{ do}: \\
+            &\quad g_t \leftarrow \nabla_{\theta} f_t(\theta_{t - 1}) \\
+            &\quad \text{If } \lambda \neq 0: \\
+            &\quad\quad g_t \leftarrow g_t + \lambda \theta_{t - 1} \\
+            &\quad m_t \leftarrow \beta_1 m_{t - 1} + (1 - \beta_1) g_t \\
+            &\quad v_t \leftarrow \beta_2 v_{t - 1} + (1 - \beta_2) g_t^2 \\
+            &\quad \widehat{m_t} \leftarrow \dfrac{m_t}{1 - \beta_1^t} \\
+            &\quad \text{Let } \rho_t' = 2 t \beta_2^t /(1 - \beta_2^t) \quad \text{(auxiliary variable)} \\
+            &\quad \rho_t \leftarrow \rho_{\infty} - \rho_t' \\
+            &\quad \text{If } \rho_t > 5: \\
+            &\quad\quad l_t \leftarrow \dfrac{\sqrt{1 - \beta_2^t}}{\sqrt{v_t} + \epsilon} \\
+            &\quad\quad r_t \leftarrow \sqrt{\dfrac{(\rho_t - 4)(\rho_t - 2)\rho_{\infty}}{(\rho_{\infty} - 4)
+            (\rho_{\infty} - 2) \rho_t}} \\
+            &\quad\quad \theta_t \leftarrow \theta_{t - 1} - \gamma \widehat{m_t} r_t l_t \\
+            &\quad \text{Else}: \\
+            &\quad\quad \theta_t \leftarrow \theta_{t - 1} - \gamma \widehat{m_t} \\
+            &\rule{110mm}{0.4pt} \\
+            &\bf{Return}: \theta_t \\
+            &\rule{110mm}{0.4pt}
+        \end{align*}
 
     .. warning::
         This is an experimental optimizer API that is subject to change.

mindspore/hal/device.py

@@ -122,6 +122,9 @@ def is_initialized(device_target):
     """
     Returns whether specified backend is initialized.
 
+    Note:
+        - The api will be deprecated.
+
     Note:
         MindSpore's backends "CPU", "GPU" and "Ascend" will be initialized in the following scenarios:
 
@@ -162,6 +165,12 @@ def is_available(device_target):
     Returns whether specified backend is available.
     All dependent libraries should be successfully loaded if this backend is available.
 
+    Note:
+        - The api will be deprecated.
+        - CPU hardware, please use the interface :func:`mindspore.device_context.cpu.is_available`.
+        - GPU hardware, please use the interface :func:`mindspore.device_context.gpu.is_available`.
+        - Ascend hardware, please use the interface :func:`mindspore.device_context.ascend.is_available`.
+
     Args:
         device_target (str): The device name of backend, should be one of "CPU", "GPU" and "Ascend".
 
@@ -194,6 +203,13 @@ def device_count(device_target=None):
     """
     Returns device count of specified backend.
 
+    Note:
+        - The api will be deprecated.
+        - CPU hardware, please use the interface :func:`mindspore.device_context.cpu.device_count`.
+        - GPU hardware, please use the interface :func:`mindspore.device_context.gpu.device_count`.
+        - Ascend hardware, please use the interface :func:`mindspore.device_context.ascend.device_count`.
+
+
     Note:
         If `device_target` is not specified, get the device count of the current backend set by context.
         For CPU backend, this method always returns 1.
@@ -221,6 +237,9 @@ def get_device_capability(device_id, device_target=None):
     """
     Get specified device's capability.
 
+    Note:
+        - The api will be deprecated.
+
     Note:
         If `device_target` is not specified, get the device capability of the current backend set by context.
 
@@ -253,6 +272,9 @@ def get_device_properties(device_id, device_target=None):
     """
     Get specified device's properties.
 
+    Note:
+        - The api will be deprecated.
+
     Note:
         If `device_target` is not specified, get the device properties of the current backend set by context.
         For Ascend, backend must be initialized before calling this method,
@@ -308,6 +330,9 @@ def get_device_name(device_id, device_target=None):
     """
     Get specified device's name.
 
+    Note:
+        - The api will be deprecated.
+
     Note:
         If `device_target` is not specified, get the device name of the current backend set by context.
         This method always returns "CPU" for CPU backend.
@@ -335,6 +360,9 @@ def get_arch_list(device_target=None):
     """
     Get the architecture list this MindSpore was compiled for.
 
+    Note:
+        - The api will be deprecated.
+
     Note:
         If `device_target` is not specified, get the device name of the current backend set by context.
 

mindspore/hal/event.py

@@ -17,6 +17,9 @@ from mindspore._c_expression import Event as Event_
 from mindspore._c_expression import Stream as Stream_
 from mindspore._c_expression import current_stream as current_stream_
 from mindspore import _checkparam as Validator
+from mindspore import log as logger
+
+function_event_status = {'Event': False, 'wait': False}
 
 
 class Event(Event_):
@@ -28,6 +31,9 @@ class Event(Event_):
 
     The underlying device events are lazily initialized when the event is first recorded.
 
+    Note:
+        - The api will be deprecated, please use the api :func:`mindspore.runtime.Event` instead.
+
     Args:
         enable_timing (bool, optional): indicates if the event should measure time (default: ``False``)
         blocking (bool, optional): if ``True``, `wait` will be blocking (default: ``False``)
@@ -58,7 +64,13 @@ class Event(Event_):
          [5. 5.]]
         >>> elapsed_time = start.elapsed_time(end)
     """
+
     def __init__(self, enable_timing=False, blocking=False):
+        if not function_event_status['Event']:
+            function_event_status['Event'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.Event(enable_timing=True) is deprecated."
+                " Please use mindspore.runtime.Event(enable_timing=True)")
         # pylint: disable=useless-super-delegation
         Validator.check_bool(enable_timing, "enable_timing", "Event")
         Validator.check_bool(blocking, "blocking", "Event")
@@ -118,6 +130,11 @@ class Event(Event_):
             [[4. 4.]
              [4. 4.]]
         """
+        if not function_event_status['wait']:
+            function_event_status['wait'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.Event() is deprecated."
+                " Please use mindspore.runtime.Event()")
         if stream is None:
             stream = current_stream_()
         if not isinstance(stream, Stream_):