mindspore 2.1.0__cp39-cp39-win_amd64.whl → 2.2.11__cp39-cp39-win_amd64.whl

mindspore/context.py

@@ -268,6 +268,8 @@ class _Context:
                             "allow_mix_precision_fp16" and "allow_mix_precision_bf16".
                 - jit_compile (bool): ``False`` and ``True``.
                 - atomic_clean_policy (int): ``0`` and ``1``. Default: ``1`` .
+                - exception_dump (str): Enable exception dump for Ascend operators. ``"0"`` , ``"1"`` and ``"2"``.
+                  Default: ``"2"`` .
                 - op_precision_mode (str): config file path.
                 - parallel_speed_up_json_path(Union[str, None]): The path to the parallel speed up json file.
                   If its value is None or '', it does not take effect. Default None.
@@ -280,6 +282,7 @@ class _Context:
             'atomic_clean_policy': [0, 1],
             'matmul_allow_hf32': [True, False],
             'conv_allow_hf32': [True, False],
+            'exception_dump': ["0", "1", "2"],
             'op_precision_mode': (str,),
             'parallel_speed_up_json_path': (str, None)
         }
@@ -289,6 +292,7 @@ class _Context:
             'atomic_clean_policy': self._get_ascend_config_setter('atomic_clean_policy', str),
             'matmul_allow_hf32': self._get_ascend_config_setter('matmul_allow_hf32', lambda v: "1" if v else "0"),
             'conv_allow_hf32': self._get_ascend_config_setter('conv_allow_hf32', lambda v: "1" if v else "0"),
+            'exception_dump': self._get_ascend_config_setter('exception_dump'),
             'op_precision_mode': self._set_op_precision_mode,
             'parallel_speed_up_json_path': self._set_speedup_config_path
         }
@@ -302,8 +306,8 @@ class _Context:
                 raise ValueError(f"For 'ascend_config', the value of argument {ascend_key} must be one of "
                                  f"{supported_modes}, but got {ascend_value}.")
             if isinstance(supported_modes, tuple) and not isinstance(ascend_value, supported_modes):
-                raise ValueError(f"For 'ascend_config', the type of argument {ascend_key} must be one of "
-                                 f"{supported_modes}, but got {type(ascend_value)}.")
+                raise TypeError(f"For 'ascend_config', the type of argument {ascend_key} must be one of "
+                                f"{supported_modes}, but got {type(ascend_value)}.")
             cfg_setter = ascend_cfg_setters.get(ascend_key)
             cfg_setter(ascend_value)
 
@@ -317,6 +321,8 @@ class _Context:
                 - conv_fprop_algo (str): "normal", "performance" or user specifies conv forward algorithm directly.
                 - conv_dgrad_algo (str): "normal", "performance" or user specifies conv data grad algorithm directly.
                 - conv_wgrad_algo (str): "normal", "performance" or user specifies conv weight grad algorithm directly.
+                - conv_allow_tf32 (bool): ``False`` and ``True``.
+                - matmul_allow_tf32 (bool): ``False`` and ``True``.
         """
 
         gpu_cfgs = {'conv_fprop_algo': ["normal", "performance", "implicit_gemm", "precomp_gemm", "gemm", "direct",
@@ -324,7 +330,9 @@ class _Context:
                     'conv_dgrad_algo': ["normal", "performance", "algo_0", "algo_1", "fft", "fft_tiling", "winograd",
                                         "winograd_nonfused"],
                     'conv_wgrad_algo': ["normal", "performance", "algo_0", "algo_1", "fft", "algo_3", "fft_tiling",
-                                        "winograd_nonfused"]}
+                                        "winograd_nonfused"],
+                    'conv_allow_tf32': [True, False],
+                    'matmul_allow_tf32': [True, False]}
         for gpu_key in gpu_config:
             if gpu_key not in gpu_cfgs:
                 raise ValueError(f"For 'context.set_context', the key of argument 'gpu_config' must be one of "
@@ -339,6 +347,10 @@ class _Context:
                 self.set_param(ms_ctx_param.conv_dgrad_algo, gpu_config[gpu_key])
             if gpu_key == 'conv_wgrad_algo':
                 self.set_param(ms_ctx_param.conv_wgrad_algo, gpu_config[gpu_key])
+            if gpu_key == 'conv_allow_tf32':
+                self.set_param(ms_ctx_param.conv_allow_tf32, gpu_config[gpu_key])
+            if gpu_key == 'matmul_allow_tf32':
+                self.set_param(ms_ctx_param.matmul_allow_tf32, gpu_config[gpu_key])
 
     def set_backend_policy(self, policy):
         success = self._context_handle.set_backend_policy(policy)
@@ -388,6 +400,29 @@ class _Context:
             raise ValueError(f"For 'context.set_context', the argument 'aoe_tune_mode' must be in "
                              f"['online', 'offline'], but got {tune_mode}.")
 
+    def set_aoe_config(self, aoe_config):
+        """
+        Enable aoe config.
+
+        Args:
+            aoe_config (dict):
+                - job_type (str): ``"1"``, ``"2"``. Default: ``"2"`` .
+                  - ``"1"``: subgraph tuning.
+                  - ``"2"``: operator tuning.
+        """
+
+        aoe_cfgs = {'job_type': ["1", "2"]}
+        for aoe_config_key in aoe_config:
+            if aoe_config_key not in aoe_cfgs:
+                raise ValueError(f"For 'context.set_context', the key of argument 'aoe_config' must be one of "
+                                 f"{aoe_cfgs}, but got {aoe_config_key}.")
+            supported_value = aoe_cfgs.get(aoe_config_key)
+            if aoe_config[aoe_config_key] not in supported_value:
+                raise ValueError(f"For 'aoe_config', the value of argument {aoe_config_key} must be one of "
+                                 f"{supported_value}, but got {aoe_config[aoe_config_key]}.")
+            if aoe_config_key == 'job_type':
+                self.set_param(ms_ctx_param.aoe_job_type, aoe_config[aoe_config_key])
+
     def set_device_id(self, device_id):
         if device_id < 0 or device_id > 4095:
             raise ValueError(f"For 'context.set_context', the argument 'device_id' must be in range [0, 4095], "
@@ -484,7 +519,7 @@ class _Context:
         except (TypeError, ValueError) as exo:
             raise ValueError(str(exo) + "\nFor 'context.set_context', open or load the 'env_config_path' file {} "
                                         "failed, please check whether 'env_config_path' is json file and correct, "
-                                        "or may not have permission to read it.".format(env_config_path))
+                                        "or may not have permission to read it.".format(env_config_path)) from exo
         self.set_param(ms_ctx_param.env_config_path, env_config_path)
 
     def set_runtime_num_threads(self, runtime_num_threads):
@@ -527,6 +562,7 @@ class _Context:
         'ascend_config': set_ascend_config,
         'jit_syntax_level': set_jit_syntax_level,
         'gpu_config': set_gpu_config,
+        'aoe_config': set_aoe_config,
     }
 
     @property
@@ -595,7 +631,9 @@ class _Context:
             valid_option = {"recompute_comm_overlap": ms_ctx_param.recompute_comm_overlap,
                             "matmul_grad_comm_overlap": ms_ctx_param.matmul_grad_comm_overlap,
                             "enable_task_opt": ms_ctx_param.enable_task_opt,
+                            "enable_grad_comm_opt": ms_ctx_param.enable_grad_comm_opt,
                             "interleaved_matmul_comm": ms_ctx_param.interleaved_matmul_comm,
+                            "enable_opt_shard_comm_opt": ms_ctx_param.enable_opt_shard_comm_opt,
                             "interleaved_layernorm_comm": ms_ctx_param.interleaved_layernorm_comm}
             with open(speedup_config_real_path, 'r') as f:
                 speedup_config = json.load(f)
@@ -611,7 +649,8 @@ class _Context:
             raise ValueError(str(exo) + "\nFor 'context.set_context', "
                                         "open or load the 'speedup_config_path' file {} "
                                         "failed, please check whether 'speedup_config_path' is json file and correct, "
-                                        "or may not have permission to read it.".format(speedup_config_real_path))
+                                        "or may not have permission to read it.".format(speedup_config_real_path)) \
+                                        from exo
 
 
 def _context():
@@ -641,8 +680,9 @@ def _context():
 @args_type_check(device_num=int, global_rank=int, gradients_mean=bool, gradient_fp32_sync=bool, parallel_mode=str,
                  auto_parallel_search_mode=str, search_mode=str, parameter_broadcast=bool, strategy_ckpt_load_file=str,
                  strategy_ckpt_save_file=str, full_batch=bool, enable_parallel_optimizer=bool, enable_alltoall=bool,
-                 all_reduce_fusion_config=list, pipeline_stages=int, grad_accumulation_step=int,
-                 parallel_optimizer_config=dict, comm_fusion=dict, strategy_ckpt_config=dict)
+                 all_reduce_fusion_config=list, pipeline_stages=int, pipeline_segments=int,
+                 parallel_optimizer_config=dict,
+                 comm_fusion=dict, strategy_ckpt_config=dict)
 def set_auto_parallel_context(**kwargs):
     r"""
     Set auto parallel context, only data parallel supported on CPU.
@@ -663,11 +703,11 @@ def set_auto_parallel_context(**kwargs):
     device_num                   gradient_fp32_sync
     global_rank                  loss_repeated_mean
     gradients_mean               search_mode
-    parallel_mode                strategy_ckpt_load_file
-    all_reduce_fusion_config     strategy_ckpt_save_file
-    enable_parallel_optimizer    dataset_strategy
-    parallel_optimizer_config    pipeline_stages
-    enable_alltoall              grad_accumulation_step
+    parallel_mode                parameter_broadcast
+    all_reduce_fusion_config     strategy_ckpt_load_file
+    enable_parallel_optimizer    strategy_ckpt_save_file
+    parallel_optimizer_config    dataset_strategy
+    enable_alltoall              pipeline_stages
                \                 auto_parallel_search_mode
                \                 comm_fusion
                \                 strategy_ckpt_config
@@ -694,7 +734,7 @@ def set_auto_parallel_context(**kwargs):
 
                      - auto_parallel: Achieving parallelism automatically.
         search_mode (str): There are three kinds of shard strategy search modes: ``"recursive_programming"`` ,
-                     ``"dynamic_programming"`` and ``"sharding_propagation"`` . Default: ``"dynamic_programming"`` .
+                     ``"dynamic_programming"`` and ``"sharding_propagation"`` . Default: ``"recursive_programming"`` .
 
                      - recursive_programming: Recursive programming search mode. In order to obtain optimal performance,
                        it is recommended that users set the batch size to be greater than or equal to the product of
@@ -737,12 +777,9 @@ def set_auto_parallel_context(**kwargs):
                         distributed alone in the pipeline. The total devices will be divided into 'pipeline_stags'
                         stages.
                         Default: ``1`` .
-        grad_accumulation_step (int): Set the accumulation steps of gradients in auto and semi auto parallel mode.
-                        This should be a positive int. Default: ``1`` .
         parallel_optimizer_config (dict): A dict contains the keys and values for setting the parallel optimizer
                         configure. The configure provides more detailed behavior control about parallel training
-                        when parallel optimizer is enabled. Currently it supports the key `gradient_accumulation_shard`.
-                        The configure will be effective when we use
+                        when parallel optimizer is enabled. The configure will be effective when we use
                         mindspore.set_auto_parallel_context(enable_parallel_optimizer=True).
                         It supports the following keys.
 
@@ -760,6 +797,14 @@ def set_auto_parallel_context(**kwargs):
                           across the devices. Parameter size = shape[0] \* ... \* shape[n] \* size(dtype). Non-negative.
                           Unit: KB. Default: ``64`` .
 
+                        - optimizer_weight_shard_size(int): Set the optimizer weight shard group size, if you want to
+                          specific the maximum group size across devices when the parallel optimizer is enabled.
+                          The numerical range can be (0, device_num]. If pipeline parallel is enabled, the numerical
+                          range is (0, device_num/stage]. If the size of data parallel communication domain
+                          of the parameter cannot be divided by `optimizer_weight_shard_size`, then the specified
+                          communication group size will not take effect. Default value is ``-1`` , which means the
+                          optimizer weight shard group size will be the size of data parallel group of each parameter.
+
         comm_fusion (dict): A dict contains the types and configurations for setting the communication fusion. each
                         communication fusion config has two keys: "mode" and "config".
                         It supports following communication fusion types and configurations:
@@ -820,7 +865,8 @@ def set_auto_parallel_context(**kwargs):
         >>> ms.set_auto_parallel_context(enable_alltoall=False)
         >>> ms.set_auto_parallel_context(all_reduce_fusion_config=[8, 160])
         >>> ms.set_auto_parallel_context(pipeline_stages=2)
-        >>> parallel_config = {"gradient_accumulation_shard": True, "parallel_optimizer_threshold": 24}
+        >>> parallel_config = {"gradient_accumulation_shard": True, "parallel_optimizer_threshold": 24,
+        ...                    "optimizer_weight_shard_size": 2}
         >>> ms.set_auto_parallel_context(parallel_optimizer_config=parallel_config, enable_parallel_optimizer=True)
         >>> config = {"allreduce": {"mode": "size", "config": 32}, "allgather": {"mode": "size", "config": 32}}
         >>> ms.set_auto_parallel_context(comm_fusion=config)
@@ -860,8 +906,8 @@ def reset_auto_parallel_context():
     - gradients_mean: False.
     - gradient_fp32_sync: True.
     - parallel_mode: 'stand_alone'.
-    - search_mode: 'dynamic_programming'.
-    - auto_parallel_search_mode: 'dynamic_programming'.
+    - search_mode: 'recursive_programming'.
+    - auto_parallel_search_mode: 'recursive_programming'.
     - parameter_broadcast: False.
     - strategy_ckpt_load_file: ''.
     - strategy_ckpt_save_file: ''.
@@ -881,24 +927,31 @@ def reset_auto_parallel_context():
 @args_type_check(offload_config=dict)
 def set_offload_context(offload_config):
     r"""
-    Set offload context.
-    Some configurations are offload specific, see the below table for details:
+    Configure heterogeneous training detailed parameters to adjust the offload strategy.
+
+    Note:
+        The offload configuration is only used if the memory offload feature is enabled
+        via mindspore.set_context(memory_offload="ON").
 
     Args:
         offload_config (dict): A dict contains the keys and values for setting the offload context
             configure.It supports the following keys.
 
-            - offload_param (str):  The param for offload destination, cpu or disk.
-            - offload_path (str):  The path of offload.
+            - offload_path (str):  The path of offload, relative paths are supported. Default: ``"./offload"``.
             - offload_cpu_size (str):  The cpu memory size for offload. The format is "xxGB".
             - offload_disk_size (str): The disk size for offload. The format is "xxGB"
-            - hbm_ratio (float): The ratio that can be used based on the maximum device memory. The range is (0,1].
-            - cpu_ratio (float): The ratio that can be used based on the maximum host memory. The range is (0,1].
+            - hbm_ratio (float): The ratio that can be used based on the maximum device memory.
+              The range is (0,1], Default: ``1.0``.
+            - cpu_ratio (float): The ratio that can be used based on the maximum host memory.
+              The range is (0,1], Default: ``1.0``.
+            - enable_pinned_mem (bool): The flag of whether enabling Pinned Memory. Default: ``True``.
             - enable_aio (bool): The flag of whether enabling aio. Default: ``True``.
-            - aio_block_size (str): The size of aio block. The format is "xxGB"
+            - aio_block_size (str): The size of aio block. The format is "xxGB".
             - aio_queue_depth (int): The depth of aio queue.
-            - enable_pinned_mem (bool): The flag of whether enabling pinned memory.
-            - auto_offload (bool): The flag of whether auto offload.
+            - offload_param (str):  The param for offload destination, cpu or disk, Default: ``""``.
+            - offload_checkpoint (str):  The checkpoint for offload destination, only valid if recompute is turned on,
+              cpu or disk, Default: ``""``.
+            - auto_offload (bool): The flag of whether auto offload. Default: ``True``.
             - host_mem_block_size (str): The memory block size of host memory pool. The format is "xxGB"
 
     Raises:
@@ -906,14 +959,19 @@ def set_offload_context(offload_config):
 
     Examples:
         >>> from mindspore import context
-        >>> context.set_offload_context(offload_config={"offload_param"="cpu"})
+        >>> context.set_offload_context(offload_config={"offload_param":"cpu"})
     """
     _set_offload_context(offload_config)
 
 
 def get_offload_context():
     """
-    Get offload context.
+    Gets the offload configuration parameters. Configure through interface mindspore.set_offload_context().
+    If the user is not set, the default configuration is obtained.
+
+    Returns:
+        Dict, heterogeneous training offload detailed configuration parameters.
+
     Examples:
         >>> from mindspore import context
         >>> offload_config = context.get_offload_context()
@@ -948,7 +1006,7 @@ def _check_target_specific_cfgs(device, arg_key):
 
 
 @args_type_check(mode=int, precompile_only=bool, device_target=str, device_id=int, save_graphs=(bool, int),
-                 save_graphs_path=str, enable_dump=bool, aoe_tune_mode=str,
+                 save_graphs_path=str, enable_dump=bool, aoe_tune_mode=str, aoe_config=dict,
                  save_dump_path=str, enable_reduce_precision=bool, variable_memory_max_size=str,
                  enable_auto_mixed_precision=bool, inter_op_parallel_num=int,
                  enable_graph_kernel=bool, reserve_class_name_in_scope=bool, check_bprop=bool,
@@ -1004,7 +1062,7 @@ def set_context(**kwargs):
     |                         +------------------------------+----------------------------+
     |                         |  reserve_class_name_in_scope |  CPU/GPU/Ascend            |
     |                         +------------------------------+----------------------------+
-    |                         |  pynative_synchronize        |  GPU/Ascend                |
+    |                         |  pynative_synchronize        |  CPU/GPU/Ascend            |
     +-------------------------+------------------------------+----------------------------+
     | Executive Control       |   mode                       |   CPU/GPU/Ascend           |
     |                         +------------------------------+----------------------------+
@@ -1014,6 +1072,10 @@ def set_context(**kwargs):
     |                         +------------------------------+----------------------------+
     |                         |  enable_reduce_precision     |  Ascend                    |
     |                         +------------------------------+----------------------------+
+    |                         |  aoe_tune_mode               |  Ascend                    |
+    |                         +------------------------------+----------------------------+
+    |                         |  aoe_config                  |  Ascend                    |
+    |                         +------------------------------+----------------------------+
     |                         |  check_bprop                 |  CPU/GPU/Ascend            |
     |                         +------------------------------+----------------------------+
     |                         |  max_call_depth              |  CPU/GPU/Ascend            |
@@ -1050,7 +1112,7 @@ def set_context(**kwargs):
             If device target is not set, the version of MindSpore package is used.
         max_device_memory (str): Set the maximum memory available for devices. The format is "xxGB".
             Default: ``" 1024GB"`` . The actual used memory size is the minimum of the available memory of the device
-            and max_device_memory.
+            and max_device_memory. 'max_device_memory' should be set before the program runs.
         variable_memory_max_size (str): This parameter is deprecated, and will be removed in a future version.
             Please use parameter 'max_device_memory' instead.
         mempool_block_size (str): Set the size of the memory pool block in PyNative mode for devices.
@@ -1063,7 +1125,7 @@ def set_context(**kwargs):
             Available values are:
 
             - False or 0: disable saving of intermediate compilation graphs.
-            - 1: some intermediate files will be generated during graph compliation.
+            - 1: some intermediate files will be generated during graph compilation.
             - True or 2: Generate more ir files related to backend process.
             - 3: Generate visualization computing graphs and detailed frontend ir graphs.
 
@@ -1134,11 +1196,17 @@ def set_context(**kwargs):
             If enable_graph_kernel is set to ``True`` , acceleration can be enabled.
             For details of graph kernel fusion, please check
             `Enabling Graph Kernel Fusion
-            <https://www.mindspore.cn/tutorials/experts/en/r2.1/optimize/graph_fusion_engine.html>`_.
+            <https://www.mindspore.cn/tutorials/experts/en/r2.2/optimize/graph_fusion_engine.html>`_.
         graph_kernel_flags (str):
             Optimization options of graph kernel fusion, and the priority is higher when it conflicts
             with enable_graph_kernel. Only for experienced users.
-            For example, mindspore.set_context(graph_kernel_flags="--opt_level=2 --dump_as_text"). Some general options:
+            For example,
+
+            .. code-block::
+
+                mindspore.set_context(graph_kernel_flags="--opt_level=2 --dump_as_text")
+
+            Some general options:
 
             - opt_level: Set the optimization level.
               Default: ``2`` . Graph kernel fusion can be enabled equivalently by setting opt_level greater than 0.
@@ -1154,10 +1222,19 @@ def set_context(**kwargs):
 
             - dump_as_text: dumps detail info as text files. Default: ``False`` .
 
-            More options can refer to the implementation code.
         enable_reduce_precision (bool): Whether to enable precision reduction.
             If the operator does not support the user-specified precision, the precision will
             be changed automatically. Default: ``True`` .
+        aoe_tune_mode (str): AOE tuning mode setting, which is not set by default.
+            When set to ``"online"`` , the tuning in online function is turned on.
+            When set to ``"offline"`` , ge graph will be save for offline tuning.
+        aoe_config (dict): Set the parameters specific to Ascend Optimization Engine. It is not set by default.
+
+            - job_type (str): Mode type setting, default value is ``"2"``.
+
+              - ``"1"``: subgraph tuning;
+              - ``"2"``: operator tuning.
+
         check_bprop (bool): Whether to check back propagation nodes. The checking ensures that the shape and dtype
             of back propagation node outputs is the same as input parameters. Default: ``False`` .
         max_call_depth (int): Specify the maximum depth of function call. Must be positive integer. Default: ``1000`` .
@@ -1205,12 +1282,10 @@ def set_context(**kwargs):
               memory_optimize_level is set 'O1'.
             - OFF: Turn off the memory Offload function.
         ascend_config (dict): Set the parameters specific to Ascend hardware platform. It is not set by default.
-            Currently, configurations except `parallel_speed_up_json_path` are currently only supported on Ascend910B
-            hardware platform. The default value of `precision_mode`, `jit_compile` and
+            The default value of `precision_mode`, `jit_compile` and
             `atomic_clean_policy` are experimental parameters, may change in the future.
 
-            - precision_mode (str): Mixed precision mode setting, on Ascend910B hardware platform, the default
-              value of training network is based on the value of CANN, and the default value of inference network
+            - precision_mode (str): Mixed precision mode setting, and the default value of inference network
               is ``force_fp16`` . The value range is as follows:
 
               - force_fp16: When the operator supports both float16 and float32, select float16 directly.
@@ -1238,35 +1313,44 @@ def set_context(**kwargs):
                 When the memory of the network exceeds the limit, you may try this cleaning policy, but it may cause
                 performance loss.
             - matmul_allow_hf32 (bool): Whether to convert FP32 to HF32 for Matmul operators. Default value: ``False``.
+              This is an experimental prototype that is subject to change and/or deletion.
               For detailed information, please refer to `Ascend community <https://www.hiascend.com/>`_ .
             - conv_allow_hf32 (bool): Whether to convert FP32 to HF32 for Conv operators. Default value: ``True``.
+              This is an experimental prototype that is subject to change and/or deletion.
               For detailed information, please refer to `Ascend community <https://www.hiascend.com/>`_ .
+            - exception_dump (str): Enable exception dump for Ascend operators, providing the input and output data for
+              failing Ascend operators. The value can be ``"0"`` , ``"1"`` and ``"2"``. For ``"0"`` , exception dump is
+              turned off; for ``"1"``, all inputs and outputs will be dumped for AICore and AICPU exception operators;
+              for ``"2"``, inputs will be dumped for AICore exception operators. Default: ``"2"`` .
             - op_precision_mode (str): Path to config file of op precision mode. For detailed information, please refer
               to `Ascend community <https://www.hiascend.com/>`_ .
             - parallel_speed_up_json_path(Union[str, None]): The path to the parallel speed up json file, configuration
               can refer to `parallel_speed_up.json
-              <https://gitee.com/mindspore/mindspore/blob/r2.1/config/parallel_speed_up.json>`_ .
+              <https://gitee.com/mindspore/mindspore/blob/r2.2/config/parallel_speed_up.json>`_ .
               If its value is None or '', it does not take effect. Default None.
 
               - recompute_comm_overlap (bool): Enable overlap between recompute ops and communication ops if True.
                 Default: False.
               - matmul_grad_comm_overlap (bool): Enable overlap between grad ops and communication ops if True.
                 Default: False.
-              - enable_task_opt (bool): Enable the optimizaton of the number of tasks for each communication if True.
+              - enable_task_opt (bool): Enable the optimization of the number of tasks for each communication if True.
                 Default: False.
               - interleaved_matmul_comm (bool): Enable interleaved optimization of Matmul-Comm if True. Default: False.
               - interleaved_layernorm_comm (bool): Enable interleaved optimization of LayerNorm-Comm if True.
                 Default: False.
+
         jit_syntax_level (int): Set JIT syntax level for graph compiling, triggered by GRAPH_MODE and @jit decorator.
-            The value must be in [STRICT, LAX]. Default: LAX. All levels
-            support all backends.
+            The value must be ``STRICT`` or ``LAX`` . Default: ``LAX`` . All levels support all backends.
+
+            - ``STRICT`` : Only basic syntax is supported, and execution performance is optimal. Can be used for MindIR
+              load and export.
+            - ``LAX`` : Compatible with all Python syntax as much as possible. However, execution performance may be
+              affected and not optimal. Cannot be used for MindIR load and export due to some syntax that may not be
+              able to be exported.
 
-            - STRICT: Only basic syntax is supported, and execution performance is optimal.
-            - LAX: Compatible with all Python syntax as much as possible. However, execution performance may be
-              affected and not optimal.
         gpu_config (dict): Set the parameters specific to gpu hardware platform. It is not set by default.
-            Currently, only setting `conv_fprop_algo` and `conv_dgrad_algo` and `conv_wgrad_algo` are supported on GPU
-            hardware platform.
+            Currently, only setting `conv_fprop_algo` and `conv_dgrad_algo` and `conv_wgrad_algo` and `conv_allow_tf32`
+            and `matmul_allow_tf32` are supported on GPU hardware platform.
 
             - conv_fprop_algo (str): Specifies convolution forward algorithm and the default value is 'normal',
               The value range is as follows:
@@ -1330,6 +1414,10 @@ def set_context(**kwargs):
               - fft_tiling: This algorithm uses the Fast-Fourier Transform approach but splits the inputs into tiles.
                 A significant memory workspace is needed to store intermediate results but less than fft for large size
                 images. The results are deterministic.
+            - conv_allow_tf32 (bool): The flag below controls to allow Tensor core TF32 computation on CUDNN and the
+              default value is ``True``.
+            - matmul_allow_tf32 (bool): The flag below controls to allow Tensor core TF32 computation on CUBLAS and the
+              default value is ``False``.
 
     Raises:
         ValueError: If input key is not an attribute in context.
@@ -1346,6 +1434,8 @@ def set_context(**kwargs):
         >>> ms.set_context(graph_kernel_flags="--opt_level=2 --dump_as_text")
         >>> ms.set_context(reserve_class_name_in_scope=True)
         >>> ms.set_context(variable_memory_max_size="6GB")
+        >>> ms.set_context(aoe_tune_mode="online")
+        >>> ms.set_context(aoe_config={"job_type": "2"})
         >>> ms.set_context(check_bprop=True)
         >>> ms.set_context(max_device_memory="3.5GB")
         >>> ms.set_context(mempool_block_size="1GB")
@@ -1364,7 +1454,8 @@ def set_context(**kwargs):
         >>> ms.set_context(ascend_config={"precision_mode": "force_fp16", "jit_compile": True,
         ...                "atomic_clean_policy": 1, "op_precision_mode": "./op_precision_config_file"})
         >>> ms.set_context(jit_syntax_level=ms.STRICT)
-        >>> ms.set_context(gpu_config={"conv_fprop_algo": "performance"})
+        >>> ms.set_context(gpu_config={"conv_fprop_algo": "performance", "conv_allow_tf32": True,
+        ...                "matmul_allow_tf32": True})
     """
     ctx = _context()
     # set device target first
@@ -1390,7 +1481,7 @@ def set_context(**kwargs):
                 value = 0
             if value > 3:
                 raise ValueError(f"value for save_graphs should be 0-3 but got '{value}'")
-        if key == 'jit_syntax_level' and value != STRICT and value != COMPATIBLE and value != LAX:
+        if key == 'jit_syntax_level' and value not in (STRICT, COMPATIBLE, LAX):
             raise ValueError(f"For 'jit_syntax_level', the value should be context.STRICT"
                              f" or context.LAX, but got {value}.")
         if not _check_target_specific_cfgs(device, key):
@@ -1454,16 +1545,16 @@ class ParallelMode:
     """
     Parallel mode options.
 
-    There are five kinds of parallel modes, "STAND_ALONE", "DATA_PARALLEL",
-    "HYBRID_PARALLEL", "SEMI_AUTO_PARALLEL" and "AUTO_PARALLEL". Default: "STAND_ALONE".
+    There are five kinds of parallel modes, ``STAND_ALONE``, ``DATA_PARALLEL``,
+    ``HYBRID_PARALLEL``, ``SEMI_AUTO_PARALLEL`` and ``AUTO_PARALLEL``. Default: ``STAND_ALONE``.
 
-    - STAND_ALONE: Only one processor is working.
-    - DATA_PARALLEL: Distributes the data across different processors.
-    - HYBRID_PARALLEL: Achieves data parallelism and model parallelism manually.
-    - SEMI_AUTO_PARALLEL: Achieves data parallelism and model parallelism by setting parallel strategies.
-    - AUTO_PARALLEL: Achieves parallelism automatically.
+    - ``STAND_ALONE``: Only one processor is working.
+    - ``DATA_PARALLEL``: Distributes the data across different processors.
+    - ``HYBRID_PARALLEL``: Achieves data parallelism and model parallelism manually.
+    - ``SEMI_AUTO_PARALLEL``: Achieves data parallelism and model parallelism by setting parallel strategies.
+    - ``AUTO_PARALLEL``: Achieves parallelism automatically.
 
-    MODE_LIST: The list of all supported parallel modes.
+    ``MODE_LIST``: The list of all supported parallel modes.
     """
 
     STAND_ALONE = "stand_alone"

mindspore/dataset/__init__.py

@@ -21,7 +21,7 @@ Besides, this module provides APIs to sample data while loading.
 
 We can enable cache in most of the dataset with its key arguments 'cache'. Please notice that cache is not supported
 on Windows platform yet. Do not use it while loading and processing data on Windows. More introductions and limitations
-can refer `Single-Node Tensor Cache <https://www.mindspore.cn/tutorials/experts/en/r2.1/dataset/cache.html>`_ .
+can refer `Single-Node Tensor Cache <https://www.mindspore.cn/tutorials/experts/en/r2.2/dataset/cache.html>`_ .
 
 Common imported modules in corresponding API examples are as follows:
 
@@ -55,11 +55,11 @@ The specific steps are as follows:
 - Dataset operation: The user uses the dataset object method `.shuffle` / `.filter` / `.skip` / `.split` /
   `.take` / ... to further shuffle, filter, skip, and obtain the maximum number of samples of datasets;
 - Dataset sample transform operation: The user can add data transform operations
-  ( `vision transform <https://mindspore.cn/docs/en/r2.1/api_python/mindspore.\
+  ( `vision transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
   dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
-  `NLP transform <https://mindspore.cn/docs/en/r2.1/api_python/mindspore.\
+  `NLP transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
   dataset.transforms.html#module-mindspore.dataset.text>`_ ,
-  `audio transform <https://mindspore.cn/docs/en/r2.1/api_python/mindspore.\
+  `audio transform <https://mindspore.cn/docs/en/r2.2/api_python/mindspore.\
   dataset.transforms.html#module-mindspore.dataset.audio>`_ ) to the map
   operation to perform transformations. During data preprocessing, multiple map operations can be defined to
   perform different transform operations to different fields. The data transform operation can also be a
@@ -73,7 +73,7 @@ Quick start of Dataset Pipeline
 -------------------------------
 
 For a quick start of using Dataset Pipeline, download `Load & Process Data With Dataset Pipeline
-<https://www.mindspore.cn/docs/en/r2.1/api_python/samples/dataset/dataset_gallery.html>`_
+<https://www.mindspore.cn/docs/en/r2.2/api_python/samples/dataset/dataset_gallery.html>`_
 to local and run in sequence.
 
 """

mindspore/dataset/audio/__init__.py

@@ -40,10 +40,10 @@ Descriptions of common data processing terms are as follows:
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.1/api_python/
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.2/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.1/advanced/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.2/advanced/dataset/eager.html>`_ .
 """
 from __future__ import absolute_import